diff --git a/control_plane/examples/audit_clickhouse_corpus.rs b/control_plane/examples/audit_clickhouse_corpus.rs index e7497b8d1..dc34920b7 100644 --- a/control_plane/examples/audit_clickhouse_corpus.rs +++ b/control_plane/examples/audit_clickhouse_corpus.rs @@ -56,7 +56,7 @@ fn publication_inputs(schema: &Schema, sql: String) -> ClickHouseSqlWorkload { let mut precompute_plan = PrecomputePlan::build_backend_local(envelope.clone(), vec![materialization.clone()]) .unwrap(); - let mut transmission_plan = control_plane::physical::compiler::compile_transmission_plan( + let mut transmission_plan = control_plane::physical::compiler::build_transmission_plan( envelope, &precompute_plan, &Default::default(), @@ -72,7 +72,7 @@ fn publication_inputs(schema: &Schema, sql: String) -> ClickHouseSqlWorkload { precompute_plan.summary_catalog = Some(reference.clone()); transmission_plan.summary_catalog = Some(reference); ClickHouseSqlWorkload { - sds, + summary_catalog: sds, precompute_plan, transmission_plan, tables: std::collections::HashMap::from([("raw_samples".into(), schema.clone())]), diff --git a/control_plane/examples/calibration_candidates.rs b/control_plane/examples/calibration_candidates.rs index 65fcb1369..f2894c4fb 100644 --- a/control_plane/examples/calibration_candidates.rs +++ b/control_plane/examples/calibration_candidates.rs @@ -1,6 +1,6 @@ //! Export every bindable candidate for isolated measurement, without selecting a winner. use control_plane::physical::{ - compiler::{BackendLocalPlanningSnapshot, PhysicalCompiler}, + compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}, workload_cost, }; use planner_types::post_asap::{SummaryExpr, SummaryNode}; @@ -9,7 +9,7 @@ use std::{collections::BTreeMap, rc::Rc}; // Planner IR does not implement Serialize. Preserve actual DAG identity and // typed variant/edges; leaf metadata uses explicitly labelled Debug encoding. -fn planner_forest(queries: &[control_plane::physical::compiler::PlanningQuery]) -> Value { +fn planner_forest(queries: &[control_plane::physical::compiler::QueryCompilationInput]) -> Value { fn visit( node: &Rc, seen: &mut BTreeMap, @@ -118,7 +118,7 @@ fn planner_forest(queries: &[control_plane::physical::compiler::PlanningQuery]) } let mut seen = BTreeMap::new(); let mut nodes = BTreeMap::new(); - let roots:Vec<_>=queries.iter().map(|q|json!({"query_id":q.query_id,"original_promql":q.query_string,"root":visit(&q.post_asap,&mut seen,&mut nodes)})).collect(); + let roots:Vec<_>=queries.iter().map(|q|json!({"query_id":q.query_id,"original_promql":q.query_string,"root":visit(&q.selected_plan_root,&mut seen,&mut nodes)})).collect(); json!({"encoding":"structured_graph_with_debug_metadata_v1","scope":"actual candidate Planner post-ASAP input before physical binding; not reconstructed from installed nodes","roots":roots,"nodes":nodes}) } @@ -127,26 +127,26 @@ fn main() -> Result<(), Box> { .nth(1) .ok_or("usage: calibration_candidates SNAPSHOT.json [--metricsql]")?; let metricsql = std::env::args().skip(2).any(|arg| arg == "--metricsql"); - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_slice(&std::fs::read(path)?)?; - let (request, environment) = snapshot.planning_request()?; + let snapshot: BackendLocalPlanningInput = serde_json::from_slice(&std::fs::read(path)?)?; + let (request, environment) = snapshot.into_physical_compilation_request()?; let mut results = Vec::new(); - for (index, candidate) in workload_cost::with_exact_alternative(request)? + for (index, candidate) in workload_cost::enumerate_exact_and_materialized_candidates(request)? .into_iter() .enumerate() { let queries = candidate.queries.clone(); - let materialization_policy = candidate.materialization_policy.clone(); + let enabled_materialization_keys = candidate.enabled_materialization_keys.clone(); let planner_selected_queries = planner_forest(&queries); let compiled = if metricsql { - PhysicalCompiler.compile_metricsql(candidate, environment.clone()) + PhysicalPlanCompiler.compile_metricsql(candidate, environment.clone()) } else { - PhysicalCompiler.compile(candidate, environment.clone()) + PhysicalPlanCompiler.compile_promql(candidate, environment.clone()) }; let plan = match compiled { Ok(plan) => plan, Err(error) => { results.push( - json!({"candidate_index": index, "materialization_policy": materialization_policy, "planner_selected_queries": planner_selected_queries, "unavailable_reason": error.to_string()}), + json!({"candidate_index": index, "materialization_policy": enabled_materialization_keys, "planner_selected_queries": planner_selected_queries, "unavailable_reason": error.to_string()}), ); continue; } @@ -155,14 +155,14 @@ fn main() -> Result<(), Box> { Ok(manifest) => manifest, Err(error) => { results.push( - json!({"candidate_index": index, "materialization_policy": materialization_policy, "planner_selected_queries": planner_selected_queries, "unavailable_reason": error.to_string()}), + json!({"candidate_index": index, "materialization_policy": enabled_materialization_keys, "planner_selected_queries": planner_selected_queries, "unavailable_reason": error.to_string()}), ); continue; } }; results.push(json!({ "candidate_index": index, - "materialization_policy": materialization_policy, + "materialization_policy": enabled_materialization_keys, "planner_selected_queries": planner_selected_queries, "manifest": manifest, "lifecycle_estimates": plan.lifecycle_estimates, diff --git a/control_plane/examples/compile_workload_artifact.rs b/control_plane/examples/compile_workload_artifact.rs index fa6ceed58..392ba0292 100644 --- a/control_plane/examples/compile_workload_artifact.rs +++ b/control_plane/examples/compile_workload_artifact.rs @@ -1,5 +1,5 @@ //! Control-plane entry point: cost-select a workload and emit its atomic install request. -use control_plane::physical::compiler::BackendLocalPlanningSnapshot; +use control_plane::physical::compiler::BackendLocalPlanningInput; use serde_json::json; fn main() -> Result<(), Box> { @@ -15,12 +15,12 @@ fn main() -> Result<(), Box> { if args.next().is_some() { return Err("unexpected arguments".into()); } - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_slice(&std::fs::read(path)?)?; + let snapshot: BackendLocalPlanningInput = serde_json::from_slice(&std::fs::read(path)?)?; let start = std::time::Instant::now(); let plan = if metricsql { snapshot.compile_metricsql()? } else { - snapshot.compile()? + snapshot.compile_promql()? }; let elapsed = start.elapsed().as_nanos(); let comparison = plan @@ -33,7 +33,7 @@ fn main() -> Result<(), Box> { "planning_elapsed_ns": elapsed, "envelope": plan.envelope, "cost_comparison": comparison, - "logical_selection": plan.logical_selection, + "logical_selection": plan.planner_selection_trace, "backend_revision": control_plane::physical::compiler::BACKEND_REVISION, "planner_revision": control_plane::physical::compiler::PLANNER_REVISION, "lifecycle_estimates": plan.lifecycle_estimates, diff --git a/control_plane/examples/workload_cost_manifest.rs b/control_plane/examples/workload_cost_manifest.rs index 723fe8ed2..142f4a07c 100644 --- a/control_plane/examples/workload_cost_manifest.rs +++ b/control_plane/examples/workload_cost_manifest.rs @@ -1,21 +1,21 @@ //! Emit pricing requirements; never fabricate quotes or publish a plan. use control_plane::physical::{ - compiler::BackendLocalPlanningSnapshot, compiler::PhysicalCompiler, workload_cost, + compiler::BackendLocalPlanningInput, compiler::PhysicalPlanCompiler, workload_cost, }; fn main() -> Result<(), Box> { let path = std::env::args() .nth(1) .ok_or("usage: workload_cost_manifest SNAPSHOT.json")?; - let snapshot: BackendLocalPlanningSnapshot = + let snapshot: BackendLocalPlanningInput = serde_json::from_str(&std::fs::read_to_string(path)?)?; - let (request, environment) = snapshot.planning_request()?; - let manifests = workload_cost::with_exact_alternative(request)? + let (request, environment) = snapshot.into_physical_compilation_request()?; + let manifests = workload_cost::enumerate_exact_and_materialized_candidates(request)? .into_iter() .filter_map(|candidate| { let queries = candidate.queries.clone(); - PhysicalCompiler - .compile(candidate, environment.clone()) + PhysicalPlanCompiler + .compile_promql(candidate, environment.clone()) .and_then(|plan| workload_cost::manifest(&plan, &queries)) .ok() }) diff --git a/control_plane/src/backend_client.rs b/control_plane/src/backend_client.rs index 3ff478560..595f97321 100644 --- a/control_plane/src/backend_client.rs +++ b/control_plane/src/backend_client.rs @@ -549,16 +549,18 @@ mod tests { #[tokio::test] async fn catalog_publication_posts_canonical_document_without_legacy_bytes() { - let snapshot: crate::physical::compiler::BackendLocalPlanningSnapshot = - serde_json::from_str(include_str!( - "../../docs/examples/asapquery-planning-snapshot.json" - )) - .unwrap(); - let publication = crate::physical::compiler::tests::quoted_snapshot(snapshot, false) - .compile() - .unwrap() - .publication() - .unwrap(); + let snapshot: crate::physical::compiler::BackendLocalPlanningInput = serde_json::from_str( + include_str!("../../docs/examples/asapquery-planning-snapshot.json"), + ) + .unwrap(); + let publication = crate::physical::compiler::tests::quoted_snapshot( + snapshot, + crate::physical::compiler::QueryFrontend::PromQl, + ) + .compile_promql() + .unwrap() + .to_publication_artifact() + .unwrap(); let hits: StdArc>> = StdArc::new(Mutex::new(Vec::new())); let route_hits = hits.clone(); let app = Router::new().route( diff --git a/control_plane/src/clickhouse.rs b/control_plane/src/clickhouse.rs index d45510303..442a8d1a3 100644 --- a/control_plane/src/clickhouse.rs +++ b/control_plane/src/clickhouse.rs @@ -205,7 +205,8 @@ pub use asap_frontend_sql::SqlCatalog as ClickHouseSqlCatalog; #[derive(Debug, Deserialize)] pub struct ClickHouseSqlWorkload { - pub sds: SummaryCatalog, + #[serde(rename = "sds", alias = "summary_catalog")] + pub summary_catalog: SummaryCatalog, pub precompute_plan: PrecomputePlan, pub transmission_plan: TransmissionPlan, pub tables: HashMap, @@ -301,7 +302,7 @@ pub async fn compile_automatic_clickhouse_workload( .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?, ); precompute.executable_dags = installed_dags; - let mut transmission = crate::physical::compiler::compile_transmission_plan( + let mut transmission = crate::physical::compiler::build_transmission_plan( request.envelope.clone(), &precompute, &std::collections::BTreeMap::new(), @@ -410,7 +411,7 @@ pub async fn compile_clickhouse_workload( ) -> Result { request .precompute_plan - .validate_against_catalog(&request.sds) + .validate_against_catalog(&request.summary_catalog) .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; request .transmission_plan @@ -441,13 +442,13 @@ pub async fn compile_clickhouse_workload( let mut precompute_plan = request.precompute_plan.clone(); precompute_plan.executable_dags = installed_dags; let publication = crate::physical::publication::PhysicalPlanPublication { - summary_catalog: request.sds.clone(), + summary_catalog: request.summary_catalog.clone(), precompute_plan, collector_plans: Vec::new(), transmission_plan: request.transmission_plan.clone(), query_plan: QueryPlan { - plan_id: request.sds.plan_id, - plan_version: request.sds.plan_version, + plan_id: request.summary_catalog.plan_id, + plan_version: request.summary_catalog.plan_version, clickhouse_context: Some(ClickHousePlanningContext { window_templates, tables: request.tables.clone(), @@ -1405,7 +1406,7 @@ mod tests { let mut precompute = PrecomputePlan::build_backend_local(envelope.clone(), vec![config]).unwrap(); precompute.summary_catalog = Some(sds.reference().unwrap()); - let mut transmission = crate::physical::compiler::compile_transmission_plan( + let mut transmission = crate::physical::compiler::build_transmission_plan( envelope, &precompute, &std::collections::BTreeMap::new(), @@ -1423,7 +1424,7 @@ mod tests { ) }; let mut request = ClickHouseSqlWorkload { - sds, + summary_catalog: sds, precompute_plan: precompute, transmission_plan: transmission, tables: HashMap::from([ @@ -1453,7 +1454,7 @@ mod tests { let simple_sql = "SELECT sum(value) FROM telemetry WHERE timestamp_ms >= 0 AND timestamp_ms < 2000"; let simple = compile_clickhouse_workload(&ClickHouseSqlWorkload { - sds: request.sds.clone(), + summary_catalog: request.summary_catalog.clone(), precompute_plan: request.precompute_plan.clone(), transmission_plan: request.transmission_plan.clone(), tables: request.tables.clone(), @@ -1498,7 +1499,7 @@ mod tests { ] }; let multiple = compile_clickhouse_workload(&ClickHouseSqlWorkload { - sds: request.sds.clone(), + summary_catalog: request.summary_catalog.clone(), precompute_plan: request.precompute_plan.clone(), transmission_plan: request.transmission_plan.clone(), tables: request.tables.clone(), @@ -1711,18 +1712,21 @@ mod tests { value: planner_types::pre_asap::ScalarValue::Utf8("requests".into()), }], }); - request.sds = SummaryCatalog::from_materializations(71, 1, &[config.clone()]).unwrap(); + request.summary_catalog = + SummaryCatalog::from_materializations(71, 1, &[config.clone()]).unwrap(); let envelope = request.precompute_plan.envelope.clone(); request.precompute_plan = PrecomputePlan::build_backend_local(envelope.clone(), vec![config]).unwrap(); - request.precompute_plan.summary_catalog = Some(request.sds.reference().unwrap()); - request.transmission_plan = crate::physical::compiler::compile_transmission_plan( + request.precompute_plan.summary_catalog = + Some(request.summary_catalog.reference().unwrap()); + request.transmission_plan = crate::physical::compiler::build_transmission_plan( envelope, &request.precompute_plan, &std::collections::BTreeMap::new(), ) .unwrap(); - request.transmission_plan.summary_catalog = Some(request.sds.reference().unwrap()); + request.transmission_plan.summary_catalog = + Some(request.summary_catalog.reference().unwrap()); assert!(compile_clickhouse_workload(&request).await.is_ok()); } } diff --git a/control_plane/src/emit/backend_push.rs b/control_plane/src/emit/backend_push.rs index 5efc89971..8ad4ef35c 100644 --- a/control_plane/src/emit/backend_push.rs +++ b/control_plane/src/emit/backend_push.rs @@ -231,7 +231,7 @@ async fn push_documents_coupled( warn!(%error, "failed to bind compatibility PrecomputePlan to SummaryCatalog"); return (false, false, 0); } - let transmission_plan = match crate::physical::compiler::compile_transmission_plan( + let transmission_plan = match crate::physical::compiler::build_transmission_plan( precompute_plan.envelope.clone(), &precompute_plan, &Default::default(), diff --git a/control_plane/src/emit/mod.rs b/control_plane/src/emit/mod.rs index bd270e4c8..5ae0bfb12 100644 --- a/control_plane/src/emit/mod.rs +++ b/control_plane/src/emit/mod.rs @@ -341,19 +341,18 @@ pub fn collect_metric_to_family( // by several capabilities accumulates several families, so its // samples fan into each per-family pipeline at the agent and the // backend serves every (metric, capability) the workload needs. - for (_, workload, _wc) in workload_store.get_all_for_metric(&entry.metric_name) { + for (_, workload) in workload_store.get_all_for_metric(&entry.metric_name) { // If this metric declares an `item_label` (its inner // high-cardinality dimension, e.g. "endpoint") and the // parsed query's own label filters name a value for it (e.g. // `{endpoint="checkout"}`), thread that through as the // `Frequency` intent's actual per-item filter -- see // `bind_workload_typed_with_item_filter`'s doc. - let item_filter = entry.item_label.as_deref().and_then(|label| { - workload - .label_filters - .get(label) - .map(|v| (label, v.as_str())) - }); + let filters = workload.label_filters(); + let item_filter = entry + .item_label + .as_deref() + .and_then(|label| filters.get(label).map(|v| (label, v.as_str()))); let Some(deployment_expr) = crate::physical::workload_planner::bind_workload_typed_with_item_filter( &workload, @@ -404,12 +403,15 @@ pub fn collect_metric_to_grouping_labels( // pre-B2 semantics ("the entry the controller pre-popped first // wins") in the common case AND lets a multi-role metric still // emit a single keep_keys OTTL processor per metric. - if let Some((_, workload, _)) = workload_store + if let Some((_, workload)) = workload_store .get_all_for_metric(&entry.metric_name) .into_iter() .next() { - out.insert(entry.metric_name.clone(), workload.group_by_labels.clone()); + out.insert( + entry.metric_name.clone(), + workload.group_by_labels().clone(), + ); } } out @@ -603,7 +605,7 @@ pub fn collect_cumulative_counter_metrics( let mut seen: std::collections::BTreeSet = std::collections::BTreeSet::new(); for entry in registry.entries() { // `derive_agg_role` reads the WorkloadEntry directly (query - // string + family override), not the lowered QueryWorkload, so + // string + family override), not the lowered RegisteredWorkload, so // we classify the registry entry. We still consult the // workload_store to confirm the metric was successfully // pre-populated (matching the contract of the sibling @@ -866,24 +868,18 @@ mod runtime_tests { // Reproduces the live demo gap: 3 of 6 (HLL, CountSketch, CMS) silently // drop because the analyzer pre-population path doesn't propagate // `sketch_family_override` from the workload YAML into - // `QueryWorkload::sketch_type_override`. + // `RegisteredWorkload::sketch_type_override`. /// Mimics the pre-population loop in `main()` — turns each - /// `WorkloadEntry` into a `QueryWorkload` via the shared `Analyzer`. + /// `WorkloadEntry` into a `RegisteredWorkload` via the shared `Analyzer`. fn populate_store_from_registry(registry: &WorkloadRegistry, store: &WorkloadStore) { use crate::pipeline::Analyzer; - use crate::types; let analyzer = Analyzer::new(); for entry in registry.entries() { let spec = crate::workload::query_spec_for_entry(entry); if let Ok(wl) = analyzer.analyze(spec) { let role = crate::workload::derive_agg_role(entry); - store.set( - &entry.metric_name, - role, - wl, - types::WorkloadCharacteristics::default(), - ); + store.set(&entry.metric_name, role, wl); } } } @@ -1090,12 +1086,12 @@ mod runtime_tests { /// fan into three pipelines. /// /// We populate the store directly with three `(metric, role)` - /// `QueryWorkload`s — one per capability — so the test pins + /// `RegisteredWorkload`s — one per capability — so the test pins /// `collect_metric_to_family`'s union semantics independently of the /// analyzer's query-string → AggType parsing. #[test] fn collect_metric_to_family_unions_multiple_capabilities_per_metric() { - use crate::types::{AggType, QueryWorkload, SketchType, WorkloadCharacteristics}; + use crate::types::{AggType, RegisteredWorkload, SketchType}; use crate::workload::AggRole; use planner_types::post_asap::SketchAlgorithm; use std::collections::BTreeSet; @@ -1122,35 +1118,34 @@ mod runtime_tests { let mk = |agg: AggType, override_family: Option, quantiles: Vec| - -> QueryWorkload { - QueryWorkload { + -> RegisteredWorkload { + crate::registered_workload::fixtures::WorkloadFixture { metric_name: METRIC.to_string(), label_filters: Default::default(), group_by_labels: Vec::new(), aggregations: vec![agg], time_window: Duration::from_secs(60), repeat_every: None, - accuracy_sla: 0.01, + accuracy: crate::types::AccuracyTarget::Epsilon(0.01), latency_sla: None, sketch_type_override: override_family, exact_required: false, quantiles, } + .build() }; // Quantile → DDSketch (explicit override valid for Quantile). store.set( METRIC, AggRole::Quantile, mk(AggType::Quantile, Some(SketchType::DDSketch), vec![0.99]), - WorkloadCharacteristics::default(), ); // Cardinality → HLL (override valid for the Cardinality class). store.set( METRIC, AggRole::Count, mk(AggType::Cardinality, Some(SketchType::HLL), Vec::new()), - WorkloadCharacteristics::default(), ); // Frequency → CMS. This deployment's capability catalog exposes // CountSketch only for TopK, so the incompatible override is ignored; @@ -1163,7 +1158,6 @@ mod runtime_tests { Some(SketchType::CountSketch), Vec::new(), ), - WorkloadCharacteristics::default(), ); let map = collect_metric_to_family(®istry, &store); @@ -1187,7 +1181,7 @@ mod runtime_tests { // Pre-B3 the WorkloadEntry YAML had no way to declare grouping // labels — the analyzer pulled them only from PromQL `by (...)` // clauses. Bare `quantile_over_time(0.99, metric[30s])` carries no - // `by`, so `QueryWorkload.group_by_labels` ended up empty, so + // `by`, so `RegisteredWorkload.group_by_labels` ended up empty, so // `collect_metric_to_grouping_labels` returned `{metric: vec![]}`, // so the 5-sketch routing emitter wrote // `keep_keys(datapoint.attributes, [])` — stripping ALL attrs @@ -1196,7 +1190,7 @@ mod runtime_tests { // // Post-B3 a declarative `grouping_labels: [zone]` on WorkloadEntry // is threaded through the pre-pop QuerySpec → analyzer → - // QueryWorkload.group_by_labels → collect_metric_to_grouping_labels + // RegisteredWorkload.group_by_labels → collect_metric_to_grouping_labels // → the emitter's keep_keys list. Without this round-trip the // end-to-end test's sid catalog stays empty-per-zone. #[test] @@ -1222,7 +1216,7 @@ mod runtime_tests { populate_store_from_registry(®istry, &store); // The analyzer must have threaded grouping_labels into - // QueryWorkload.group_by_labels. + // RegisteredWorkload.group_by_labels. let map = collect_metric_to_grouping_labels(®istry, &store); assert_eq!( map.get("http_requests_total_latency_ms"), diff --git a/control_plane/src/lib.rs b/control_plane/src/lib.rs index e93d3c0f4..905effbf8 100644 --- a/control_plane/src/lib.rs +++ b/control_plane/src/lib.rs @@ -1,7 +1,14 @@ -//! Control-plane planning, configuration emission, and deployment services. +//! Backend control-plane library and standalone service. //! -//! These modules support in-process integration and the standalone control-plane -//! binary. Internal module paths are not a wire-protocol compatibility contract. +//! ASAPPlanner owns canonical semantic IR and legal logical selection. This +//! crate adapts workload/evidence inputs, compiles physical candidates, checks +//! provider quotes, and publishes one consistent catalog-backed generation. +//! Shared execution and installation contracts live in `asap_types`. +//! +//! `physical::compiler`, `physical::workload_cost`, and `clickhouse` are the +//! current compilation paths. Metric stage emission consumes the same canonical +//! workload model through a registration adapter. Public modules are integration APIs, +//! not an independent wire schema or a second semantic planner. #![allow( clippy::collapsible_match, @@ -29,6 +36,7 @@ pub mod pipeline; pub mod planner_selection; pub mod query_parser; pub mod query_plan; +pub mod registered_workload; pub mod replan; pub mod runtime_samples; pub mod store; diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index 7e8833e64..8785e56f7 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -8,7 +8,6 @@ use control_plane::monitor; use control_plane::opamp; use control_plane::physical; use control_plane::pipeline; -use control_plane::query_parser; use control_plane::replan; use control_plane::runtime_samples; use control_plane::store; @@ -41,7 +40,6 @@ use physical::deployment_cost::tco; use physical::deployment_cost::DeploymentCostPlanner; use physical::plan_cache::CachedDeploymentPlanner; use pipeline::{Analyzer, QuerySpec}; -use query_parser::parse_query_expr_canonical; use replan::Replanner; use store::{PlanStore, WorkloadStore}; use types::AgentCollectorConfig; @@ -252,7 +250,7 @@ async fn main() { // // Critical: thread `sketch_family_override` from each registry entry // into the QuerySpec's `sketch_type` field — that's what populates - // `QueryWorkload::sketch_type_override`, which the typed planner + // `RegisteredWorkload::sketch_type_override`, which the typed planner // (`bind_workload_typed`) reads to honour MVP §46 entries 5–8 (HLL / // CountSketch / CountMinSketch). Without this stitch the workloads // round-trip through the analyzer with a None override and the @@ -283,11 +281,10 @@ async fn main() { let role = control_plane::workload::derive_agg_role(entry); match analyzer.analyze(spec) { Ok(wl) => { - let wc = types::WorkloadCharacteristics::default(); - let plan = planner.plan(&wl, Some(&wc)); - let metric_name = wl.metric_name.clone(); + let plan = planner.plan(&wl); + let metric_name = wl.metric_name().clone(); plan_store.set(&metric_name, role, plan); - workload_store.set(&metric_name, role, wl, wc); + workload_store.set(&metric_name, role, wl); } Err(e) => { warn!(metric = %entry.metric_name, role = %role, error = %e, @@ -512,7 +509,7 @@ struct PhysicalPlanQueryRequest { #[serde(default)] group_by: Vec, accuracy: types::AccuracyTarget, - lifecycle: physical::compiler::LifecyclePlanningInput, + lifecycle: physical::compiler::SummaryLifecyclePlanningInputs, window_cost_model: physical::compiler::WindowCostModel, evaluation_phase_ms: u64, #[serde(default)] @@ -530,7 +527,8 @@ struct CompileAndPublishPhysicalPlanRequest { #[serde(default)] workload_cost_evidence: Option, queries: Vec, - collector_ids: Vec, + #[serde(rename = "collector_ids", alias = "target_collector_ids")] + target_collector_ids: Vec, capability_snapshot_id: String, #[serde(default)] evidence: HashMap, @@ -559,48 +557,19 @@ fn default_physical_plan_timeout_ms() -> u64 { 10_000 } -#[derive(Clone, Copy)] -enum PhysicalQueryFrontend { - PromQl, - MetricsQl, -} - -impl PhysicalQueryFrontend { - fn parse( - self, - query: &str, - accuracy: types::AccuracyTarget, - ) -> Result { - match self { - Self::PromQl => parse_query_expr_canonical(query, accuracy) - .map_err(|e| format!("frontend.promql: {e}")), - Self::MetricsQl => parse_query_expr_canonical(query, accuracy) - .map_err(|e| format!("victoriametrics.promql_subset: {e}")), - } - } - fn compile( - self, - request: physical::compiler::PlanningRequest, - environment: physical::compiler::DeploymentEnvironment, - ) -> Result { - match self { - Self::PromQl => physical::compiler::PhysicalCompiler.compile(request, environment), - Self::MetricsQl => { - physical::compiler::PhysicalCompiler.compile_metricsql(request, environment) - } - } - } -} +use physical::compiler::QueryFrontend; #[derive(Debug, Serialize)] struct CompileAndPublishPhysicalPlanResponse { - cost_comparison: Option, - logical_selection: Vec, + cost_comparison: Option, + #[serde(rename = "logical_selection", alias = "planner_selection_trace")] + planner_selection_trace: Vec, plan_id: u64, plan_version: u64, status: &'static str, generated_at_unix_ms: u64, - collector_ids: Vec, + #[serde(rename = "collector_ids", alias = "target_collector_ids")] + target_collector_ids: Vec, lifecycle_estimates: Vec, } @@ -610,20 +579,20 @@ async fn handle_compile_and_publish_physical_plan( State(st): State, Json(request): Json, ) -> Response { - compile_and_publish_physical_plan(st, request, PhysicalQueryFrontend::PromQl).await + compile_and_publish_physical_plan(st, request, QueryFrontend::PromQl).await } async fn handle_compile_and_publish_metricsql_physical_plan( State(st): State, Json(request): Json, ) -> Response { - compile_and_publish_physical_plan(st, request, PhysicalQueryFrontend::MetricsQl).await + compile_and_publish_physical_plan(st, request, QueryFrontend::MetricsQl).await } async fn compile_and_publish_physical_plan( st: AppState, mut request: CompileAndPublishPhysicalPlanRequest, - frontend: PhysicalQueryFrontend, + frontend: QueryFrontend, ) -> Response { // Serialize typed activations so an older response cannot overwrite the // catalog recorded after a newer backend activation. @@ -635,7 +604,7 @@ async fn compile_and_publish_physical_plan( let catalog = active_catalog.clone(); erp.resolve_population_data_descriptor(catalog.as_deref()); } - let (bundle, collector_ids, apply_timeout, adaptation_evidence, _) = + let (bundle, target_collector_ids, apply_timeout, adaptation_evidence, _) = match compile_physical_plan_request(request, false, frontend) { Ok((Some(bundle), ids, timeout, adaptation, manifests)) => { (bundle, ids, timeout, adaptation, manifests) @@ -668,7 +637,7 @@ async fn compile_and_publish_physical_plan( ) .into_response(); } - let publication = match bundle.publication() { + let publication = match bundle.to_publication_artifact() { Ok(publication) => publication, Err(error) => { return ( @@ -736,12 +705,12 @@ async fn compile_and_publish_physical_plan( Json(CompileAndPublishPhysicalPlanResponse { cost_comparison: bundle.cost_comparison, - logical_selection: bundle.logical_selection, + planner_selection_trace: bundle.planner_selection_trace, plan_id: bundle.envelope.plan_id, plan_version: bundle.envelope.plan_version, status: "active", generated_at_unix_ms: bundle.envelope.generated_at_unix_ms, - collector_ids, + target_collector_ids, lifecycle_estimates: bundle.lifecycle_estimates, }) .into_response() @@ -812,16 +781,16 @@ async fn publish_clickhouse_plan( fn compile_physical_plan_request( request: CompileAndPublishPhysicalPlanRequest, manifests_only: bool, - frontend: PhysicalQueryFrontend, + frontend: QueryFrontend, ) -> Result< ( - Option, + Option, Vec, Duration, Vec, ( Vec, - Vec, + Vec, Vec, ), ), @@ -829,9 +798,9 @@ fn compile_physical_plan_request( > { if request.queries.is_empty() || (request.target == physical::compiler::PhysicalDeploymentTarget::DistributedCollectors - && request.collector_ids.is_empty()) + && request.target_collector_ids.is_empty()) || (request.target == physical::compiler::PhysicalDeploymentTarget::BackendLocalRemoteWrite - && !request.collector_ids.is_empty()) + && !request.target_collector_ids.is_empty()) { return Err(( StatusCode::UNPROCESSABLE_ENTITY, @@ -913,23 +882,23 @@ fn compile_physical_plan_request( ..Default::default() }, }); - queries.push(physical::compiler::PlanningQuery { + queries.push(physical::compiler::QueryCompilationInput { query_id: query.query_id, query_string: query.query_string, - post_asap, - source: planner_types::pre_asap::Source::TimeSeries { + selected_plan_root: post_asap, + legacy_query_source: planner_types::pre_asap::Source::TimeSeries { metric: query.metric, }, - window_secs: query.window_secs, - group_by: query.group_by, - accuracy: query.accuracy, - lifecycle: query.lifecycle, - window_implementations: Vec::new(), - runtime_policy: query.runtime_policy, + query_lookback_seconds: query.window_secs, + group_by_labels: query.group_by, + accuracy_target: query.accuracy, + summary_lifecycle_inputs: query.lifecycle, + window_realization_candidates: Vec::new(), + materialization_runtime_policy: query.runtime_policy, }); } - let logical_selection = match physical::compiler::select_workload_roots_with_trace( + let planner_selection_trace = match physical::compiler::select_logical_roots_with_trace( &mut queries, canonical_roots, &request.evidence, @@ -944,8 +913,8 @@ fn compile_physical_plan_request( physical::compiler::prepare_window_implementations(query, &model, request.target, 0) .map_err(|error| (StatusCode::UNPROCESSABLE_ENTITY, error.to_string().into()))?; } - let planning_request = physical::compiler::PlanningRequest { - logical_selection, + let compilation_request = physical::compiler::PhysicalCompilationRequest { + planner_selection_trace, query_workload: Some(planner_types::workload::QueryWorkload { language: planner_types::workload::QueryLanguage::PromQL, query_batch: None, @@ -953,20 +922,20 @@ fn compile_physical_plan_request( data_workload: None, }), queries, - hybrid_execution: request.target + allow_mixed_summary_and_exact_execution: request.target == physical::compiler::PhysicalDeploymentTarget::BackendLocalRemoteWrite, - materialization_policy: None, - evidence: request.evidence, + enabled_materialization_keys: None, + topk_membership_evidence_by_query_id: request.evidence, exact_composition_costs: request.exact_composition_costs, erp: request.erp, planner_revision: request.planner_revision, source_sample_interval_ms: None, - query_staleness_margin_ms: 0, + query_retention_margin_ms: 0, retained_summary_memory_budget_bytes: None, }; - let environment = physical::compiler::DeploymentEnvironment { + let environment = physical::compiler::PhysicalDeploymentContext { target: request.target, - collector_ids: request.collector_ids.clone(), + target_collector_ids: request.target_collector_ids.clone(), capability_snapshot_id: request.capability_snapshot_id, observed_at_unix_ms: now, max_evidence_age_ms: request.max_evidence_age_ms, @@ -975,13 +944,15 @@ fn compile_physical_plan_request( expiry_unix_ms: request.expiry_unix_ms, backend_compat: request.backend_compat, }; - let candidates = physical::workload_cost::with_exact_alternative(planning_request.clone()) - .map_err(|error| (StatusCode::UNPROCESSABLE_ENTITY, error.to_string().into()))?; - let logical_selection = planning_request.logical_selection.clone(); - let (manifests, alternatives) = physical::workload_cost::prepare_manifests( + let candidates = physical::workload_cost::enumerate_exact_and_materialized_candidates( + compilation_request.clone(), + ) + .map_err(|error| (StatusCode::UNPROCESSABLE_ENTITY, error.to_string().into()))?; + let planner_selection_trace = compilation_request.planner_selection_trace.clone(); + let (manifests, alternatives) = physical::workload_cost::compile_candidates_for_pricing( candidates.clone(), environment.clone(), - matches!(frontend, PhysicalQueryFrontend::MetricsQl), + frontend, ); let apply_timeout = Duration::from_millis(request.apply_timeout_ms); // Quote preparation enumerates feasible bindings; it does not select the @@ -991,27 +962,33 @@ fn compile_physical_plan_request( return Err(( StatusCode::UNPROCESSABLE_ENTITY, serde_json::json!({"status": "all_infeasible", "alternatives": alternatives, - "logical_selection": planning_request.logical_selection}), + "logical_selection": compilation_request.planner_selection_trace}), )); } return Ok(( None, - request.collector_ids, + request.target_collector_ids, apply_timeout, request.runtime_adaptation_evidence, - (manifests, alternatives, logical_selection), + (manifests, alternatives, planner_selection_trace), )); } let compiled = match request.workload_cost_evidence { Some(evidence) => match frontend { - PhysicalQueryFrontend::PromQl => { - physical::workload_cost::select(candidates, environment, &evidence) - } - PhysicalQueryFrontend::MetricsQl => { - physical::workload_cost::select_metricsql(candidates, environment, &evidence) + QueryFrontend::PromQl => physical::workload_cost::select_lowest_cost_candidate( + candidates, + environment, + &evidence, + ), + QueryFrontend::MetricsQl => { + physical::workload_cost::select_lowest_cost_metricsql_candidate( + candidates, + environment, + &evidence, + ) } }, - None => frontend.compile(planning_request, environment), + None => frontend.compile(compilation_request, environment), }; let bundle = match compiled { Ok(bundle) => bundle, @@ -1022,10 +999,10 @@ fn compile_physical_plan_request( }; Ok(( Some(bundle), - request.collector_ids, + request.target_collector_ids, apply_timeout, request.runtime_adaptation_evidence, - (manifests, alternatives, logical_selection), + (manifests, alternatives, planner_selection_trace), )) } @@ -1033,13 +1010,13 @@ fn compile_physical_plan_request( async fn handle_workload_cost_manifests( Json(request): Json, ) -> impl IntoResponse { - workload_cost_manifests(request, PhysicalQueryFrontend::PromQl) + workload_cost_manifests(request, QueryFrontend::PromQl) } async fn handle_metricsql_workload_cost_manifests( Json(request): Json, ) -> impl IntoResponse { - workload_cost_manifests(request, PhysicalQueryFrontend::MetricsQl) + workload_cost_manifests(request, QueryFrontend::MetricsQl) } fn physical_compile_failure((status, report): (StatusCode, serde_json::Value)) -> Response { @@ -1051,7 +1028,7 @@ fn physical_compile_failure((status, report): (StatusCode, serde_json::Value)) - fn workload_cost_manifests( request: CompileAndPublishPhysicalPlanRequest, - frontend: PhysicalQueryFrontend, + frontend: QueryFrontend, ) -> Response { if request.workload_cost_evidence.is_some() { return ( @@ -1062,9 +1039,9 @@ fn workload_cost_manifests( } let explain = request.explain; match compile_physical_plan_request(request, true, frontend) { - Ok((_, _, _, _, (manifests, alternatives, logical_selection))) => { + Ok((_, _, _, _, (manifests, alternatives, planner_selection_trace))) => { if explain { - Json(serde_json::json!({"manifests": manifests, "alternatives": alternatives, "logical_selection": logical_selection})) + Json(serde_json::json!({"manifests": manifests, "alternatives": alternatives, "logical_selection": planner_selection_trace})) .into_response() } else { Json(manifests).into_response() @@ -1077,35 +1054,19 @@ fn workload_cost_manifests( // ── Handlers ────────────────────────────────────────────────────────────────── async fn handle_plan(State(st): State, Json(spec): Json) -> impl IntoResponse { - let wc = spec.workload.clone(); - let query_string = spec.query_string.clone(); let workload = match st.analyzer.analyze(spec) { Ok(w) => w, Err(e) => return (StatusCode::UNPROCESSABLE_ENTITY, e.to_string()).into_response(), }; - let plan = st.planner.plan(&workload, Some(&wc)); + let query_string = Some(workload.entry().query.0); + + let plan = st.planner.plan(&workload); // Derive deployment configs from the bound query. Keep its Rc-backed DAG // scoped before any await so the handler future remains Send. let stage_configs = { - let mut bound_physical: Option = None; - if let Some(ref qs) = query_string { - match parse_query_expr_canonical(qs, workload.accuracy.clone()) { - Err(e) => { - warn!(query = %qs, error = %e, "parse_query_expr_canonical failed; skipping algebra pipeline") - } - Ok(qe) => { - // L4 sketch binding: lower the optimised L3 tree to the - // sketch-bound `PhysicalExpr` IR — the typed L5's input. - bound_physical = control_plane::physical::post_asap::bind_query_expr( - &qe, - workload.accuracy.clone(), - ) - .ok(); - } - } - } + let bound_physical = physical::workload_planner::bind_registered_query(&workload).ok(); let stage_configs: Option< std::collections::HashMap< @@ -1130,13 +1091,13 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> // (Quantile vs Sum) would silently overwrite the prior plan. let role = { let entry = control_plane::workload::WorkloadEntry { - metric_name: workload.metric_name.clone(), + metric_name: workload.metric_name().clone(), query_string: query_string.clone(), - accuracy_sla: workload.accuracy_sla, + accuracy_sla: 1.0 - workload.error_bound(), assign_to_role: String::from("agent"), - sketch_family_override: workload.sketch_type_override.clone(), + sketch_family_override: workload.deployment.sketch_type_override.clone(), target_path: None, - grouping_labels: workload.group_by_labels.clone(), + grouping_labels: workload.group_by_labels().clone(), // Role derivation does not depend on sampling; default 1.0. sample_p: 1.0, // Role derivation does not depend on the cardinality hint. @@ -1149,10 +1110,10 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> }; control_plane::workload::derive_agg_role(&entry) }; - st.store.set(&workload.metric_name, role, plan.clone()); + st.store.set(workload.metric_name(), role, plan.clone()); // Persist workload so the replanner can re-run plan() without the original spec. st.workload_store - .set(&workload.metric_name, role, workload.clone(), wc); + .set(workload.metric_name(), role, workload.clone()); // ── Push agent config to agent-role collectors ──────────────────────────── if let Ok(agent_yaml) = generate_agent_collector_config(&plan.agent_config, &st.opamp_endpoint) @@ -1187,8 +1148,8 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> // `agg.grouping = workload.group_by_labels` // patch on the Backend stage below). edge.metric_to_grouping_labels.insert( - workload.metric_name.clone(), - workload.group_by_labels.clone(), + workload.metric_name().clone(), + workload.group_by_labels().clone(), ); // Issue #2: broadcast push — no single agent id // in scope, so emit `$AGENT_ID` placeholder and @@ -1258,7 +1219,7 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> // columns (open-set label naming is // a Step γ TODO in // `intent_algebra::column_resolution`). - // `QueryWorkload` carries both unambiguously, + // `RegisteredWorkload` carries both unambiguously, // and every aggregation under one workload // shares them — so the patch is uniform. let item_labels = emit::collect_metric_to_item_label( @@ -1267,12 +1228,12 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> ); for agg in &mut be.aggregations { if agg.metric_name.is_empty() { - agg.metric_name = workload.metric_name.clone(); + agg.metric_name = workload.metric_name().clone(); } if agg.window_secs == 0 { - agg.window_secs = workload.time_window.as_secs(); + agg.window_secs = workload.time_window().as_secs(); } - agg.grouping = workload.group_by_labels.clone(); + agg.grouping = workload.group_by_labels().clone(); agg.item_label = item_labels.get(&agg.metric_name).cloned(); } // Option B unification: every typed cumulative @@ -1289,7 +1250,7 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> post_typed_backend_for_role( st.backend_client.as_ref(), &st.backend_routing_cache, - &workload.metric_name, + &workload.metric_name(), role, be, &monitors, @@ -1305,7 +1266,7 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> } } else if physical::stage_split::typed_stage_split_enabled() { warn!( - metric = %workload.metric_name, + metric = %workload.metric_name(), "[USE_TYPED_STAGE_SPLIT] split_typed_three_stage returned None; \ legacy plan output unaffected" ); @@ -1318,7 +1279,7 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> .set_sketch_type(&agent_id, sketch_type.clone()) .await; st.replanner - .register_agent(&agent_id, &workload.metric_name, role) + .register_agent(&agent_id, &workload.metric_name(), role) .await; } @@ -1327,7 +1288,7 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> ( StatusCode::OK, Json(json!({ - "metric": workload.metric_name, + "metric": workload.metric_name(), "sketch_type": plan.agent_config.sketch_type.to_string(), "mode": plan.agent_config.mode.to_string(), "aggregate_by": plan.agent_config.aggregate_by, @@ -1569,7 +1530,7 @@ async fn emit_bootstrap_typed( // stays the source of the edge config. let mut chosen: Option<(String, crate::physical::post_asap::PhysicalExpr)> = None; 'outer: for cand in &candidates { - for (_, wl, _) in st.workload_store.get_all_for_metric(cand) { + for (_, wl) in st.workload_store.get_all_for_metric(cand) { if let Some(expr) = physical::workload_planner::bind_workload_typed(&wl) { chosen = Some((cand.clone(), expr)); break 'outer; @@ -1834,11 +1795,14 @@ mod api_tests { // A missing warm implementation must not hide the executable exact quote. #[tokio::test] async fn cost_manifests_survive_unavailable_warm_candidate() { - let snapshot: physical::compiler::BackendLocalPlanningSnapshot = serde_json::from_str( + let snapshot: physical::compiler::BackendLocalPlanningInput = serde_json::from_str( include_str!("../../docs/examples/asapquery-planning-snapshot.json"), ) .unwrap(); - let (planning, _) = snapshot.clone().planning_request().unwrap(); + let (planning, _) = snapshot + .clone() + .into_physical_compilation_request() + .unwrap(); let query = &planning.queries[0]; let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -1847,8 +1811,8 @@ mod api_tests { let mut request_body = serde_json::json!({ "queries": [{ "query_id": query.query_id, "query_string": query.query_string, - "metric": "m", "window_secs": 60, "accuracy": query.accuracy, - "lifecycle": query.lifecycle, "evaluation_phase_ms": 0, "window_cost_model": snapshot.implementation.window_cost_model + "metric": "m", "window_secs": 60, "accuracy": query.accuracy_target, + "lifecycle": query.summary_lifecycle_inputs, "evaluation_phase_ms": 0, "window_cost_model": snapshot.physical_inputs.window_cost_model }], "collector_ids": ["test"], "capability_snapshot_id": "test", "planner_revision": physical::compiler::PLANNER_REVISION, @@ -1898,29 +1862,33 @@ mod api_tests { #[test] fn backend_local_typed_request_compiles_without_collectors() { - let snapshot: physical::compiler::BackendLocalPlanningSnapshot = serde_json::from_str( + let snapshot: physical::compiler::BackendLocalPlanningInput = serde_json::from_str( include_str!("../../docs/examples/asapquery-compatibility-demo-snapshot.json"), ) .unwrap(); - let (planning, _) = snapshot.clone().planning_request().unwrap(); + let (planning, _) = snapshot + .clone() + .into_physical_compilation_request() + .unwrap(); let mut query = planning.queries[0].clone(); let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_millis() as u64; - query.lifecycle.evidence_observed_at_unix_ms = now; - for implementation in &mut query.window_implementations { + query.summary_lifecycle_inputs.evidence_observed_at_unix_ms = now; + for implementation in &mut query.window_realization_candidates { implementation.cost.observed_at_unix_ms = now; } - let planner_types::pre_asap::Source::TimeSeries { metric } = &query.source else { + let planner_types::pre_asap::Source::TimeSeries { metric } = &query.legacy_query_source + else { panic!("expected time series fixture"); }; let value = serde_json::json!({ "target": "backend_local_remote_write", "queries": [{ "query_id": query.query_id, "query_string": query.query_string, - "metric": metric, "window_secs": query.window_secs, "accuracy": query.accuracy, - "lifecycle": query.lifecycle, "evaluation_phase_ms": 0, "window_cost_model": { "implementation_id": "test", "cost": query.window_implementations[0].cost } + "metric": metric, "window_secs": query.query_lookback_seconds, "accuracy": query.accuracy_target, + "lifecycle": query.summary_lifecycle_inputs, "evaluation_phase_ms": 0, "window_cost_model": { "implementation_id": "test", "cost": query.window_realization_candidates[0].cost } }], "collector_ids": [], "capability_snapshot_id": "test", "planner_revision": physical::compiler::PLANNER_REVISION, @@ -1929,7 +1897,7 @@ mod api_tests { }); let request = serde_json::from_value(value.clone()).unwrap(); let (plan, collectors, _, _, _) = - compile_physical_plan_request(request, false, PhysicalQueryFrontend::PromQl).unwrap(); + compile_physical_plan_request(request, false, QueryFrontend::PromQl).unwrap(); let plan = plan.unwrap(); assert!(collectors.is_empty()); assert!(plan.collector_plans.is_empty()); @@ -1943,7 +1911,7 @@ mod api_tests { assert!(compile_physical_plan_request( serde_json::from_value(distributed).unwrap(), false, - PhysicalQueryFrontend::PromQl + QueryFrontend::PromQl ) .is_err()); } @@ -2044,7 +2012,7 @@ mod api_tests { .workload_store .get_all_for_metric("typed_accuracy_metric"); assert_eq!(stored.len(), 1); - assert_eq!(stored[0].1.accuracy, target); + assert_eq!(stored[0].1.accuracy(), target); let plans = state.store.get_all_for_metric("typed_accuracy_metric"); assert_eq!(plans.len(), 1); if matches!(target, AccuracyTarget::Epsilon(_)) { @@ -2287,8 +2255,8 @@ mod api_tests { data: types::DataShape::default(), }; let wl = analyzer.analyze(spec).unwrap(); - let wc = types::WorkloadCharacteristics::default(); - let plan = planner.plan(&wl, Some(&wc)); + + let plan = planner.plan(&wl); // B2 (metric, role): pre-populate using the same role the // on_connect callback's `derive_agg_role(entry)` will compute // for this test's workloads.yaml entry (no query_string + no @@ -2300,12 +2268,7 @@ mod api_tests { control_plane::workload::AggRole::Other, plan, ); - workload_store.set( - "http_latency", - control_plane::workload::AggRole::Other, - wl, - wc, - ); + workload_store.set("http_latency", control_plane::workload::AggRole::Other, wl); // Build replanner and late-binding cells. let replanner_cell: Arc>>> = @@ -2432,15 +2395,10 @@ mod api_tests { data: types::DataShape::default(), }; let wl = analyzer.analyze(spec).unwrap(); - let wc = types::WorkloadCharacteristics::default(); - let plan = planner.plan(&wl, Some(&wc)); + + let plan = planner.plan(&wl); plan_store.set("metric_a", control_plane::workload::AggRole::Quantile, plan); - workload_store.set( - "metric_a", - control_plane::workload::AggRole::Quantile, - wl, - wc, - ); + workload_store.set("metric_a", control_plane::workload::AggRole::Quantile, wl); let replanner = Arc::new(Replanner::new( Arc::clone(&planner), @@ -2697,14 +2655,14 @@ mod api_tests { data: types::DataShape::default(), }; let wl = analyzer.analyze(spec).expect("analyze"); - let wc = types::WorkloadCharacteristics::default(); - let plan = state.planner.plan(&wl, Some(&wc)); + + let plan = state.planner.plan(&wl); state .store .set(metric, control_plane::workload::AggRole::Quantile, plan); state .workload_store - .set(metric, control_plane::workload::AggRole::Quantile, wl, wc); + .set(metric, control_plane::workload::AggRole::Quantile, wl); // 4. Swap in the populated registry. state.workload_registry = registry; @@ -2922,14 +2880,14 @@ mod api_tests { data: types::DataShape::default(), }; let wl = analyzer.analyze(spec).expect("analyze"); - let wc = types::WorkloadCharacteristics::default(); - let plan = state.planner.plan(&wl, Some(&wc)); + + let plan = state.planner.plan(&wl); state .store .set(*m, control_plane::workload::AggRole::Quantile, plan); state .workload_store - .set(*m, control_plane::workload::AggRole::Quantile, wl, wc); + .set(*m, control_plane::workload::AggRole::Quantile, wl); } // 4. Swap in the populated registry. @@ -3092,12 +3050,11 @@ mod api_tests { for entry in registry.entries() { let spec = control_plane::workload::query_spec_for_entry(entry); if let Ok(wl) = analyzer.analyze(spec) { - let wc = types::WorkloadCharacteristics::default(); - let plan = state.planner.plan(&wl, Some(&wc)); - let metric_name = wl.metric_name.clone(); + let plan = state.planner.plan(&wl); + let metric_name = wl.metric_name().clone(); let role = control_plane::workload::derive_agg_role(entry); state.store.set(&metric_name, role, plan); - state.workload_store.set(&metric_name, role, wl, wc); + state.workload_store.set(&metric_name, role, wl); } } diff --git a/control_plane/src/opamp/mod.rs b/control_plane/src/opamp/mod.rs index cb9318bc8..33759262a 100644 --- a/control_plane/src/opamp/mod.rs +++ b/control_plane/src/opamp/mod.rs @@ -1018,7 +1018,7 @@ mod tests { window_secs: 60, abstract_window_framework: planner_types::post_asap::SummaryWindowFramework::Tumbling, - window_implementation_id: "collector-tumbling-v1".into(), + window_realization_id: "collector-tumbling-v1".into(), slide_secs: 60, pane_origin_ms: Some(0), window_layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 60 }, diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 660ec1c07..a365ce220 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -59,38 +59,38 @@ fn is_default_retained_summary_memory_budget_bytes(value: &u64) -> bool { } #[derive(Debug, Clone)] -pub struct PlanningQuery { +pub struct QueryCompilationInput { pub query_id: String, /// Catalog expression used only to build the stable QueryPlan identity. - /// The selected implementation comes from `post_asap`, never this text. + /// The selected implementation comes from `selected_plan_root`, never this text. pub query_string: String, /// Planner-selected post-ASAP DAG. The physical compiler must not /// re-select a summary family from pre-ASAP input. - pub post_asap: Rc, + pub selected_plan_root: Rc, /// Legacy catalog source retained for request compatibility. Physical /// materialization sources are derived from each post-ASAP SummaryAgg; - /// this field is only used to reject table execution in the MVP. - pub source: Source, - pub window_secs: u64, + /// it also remains part of the cost manifest workload identity. + pub legacy_query_source: Source, + pub query_lookback_seconds: u64, /// Label names are deployment metadata because Planner's canonical IR /// currently carries positional column IDs at this boundary. - pub group_by: Vec, - pub accuracy: AccuracyTarget, - pub lifecycle: LifecyclePlanningInput, + pub group_by_labels: Vec, + pub accuracy_target: AccuracyTarget, + pub summary_lifecycle_inputs: SummaryLifecyclePlanningInputs, /// Executor-feasible concrete realizations offered to Planner for its /// abstract window-framework decision. The compiler retains physical /// identities and exposes only framework + complete weighted cost to /// Planner. An empty or stale set fails closed. - pub window_implementations: Vec, + pub window_realization_candidates: Vec, /// Physical runtime policy selected for this Planner materialization. /// It is validated against the selected summary family during physical /// compilation and becomes part of the immutable plan generation. - pub runtime_policy: RuntimeRulePolicy, + pub materialization_runtime_policy: RuntimeRulePolicy, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] -pub struct ImplementationCostEvidence { +pub struct WindowRealizationCostQuote { pub model_version: String, pub workload_fingerprint: String, pub observed_at_unix_ms: u64, @@ -112,14 +112,15 @@ pub struct ImplementationCostEvidence { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] -pub struct WindowImplementationCandidate { +pub struct WindowRealizationCandidate { /// Backend-owned identity; never copied into Planner IR. - pub implementation_id: String, + #[serde(rename = "implementation_id", alias = "realization_id")] + pub realization_id: String, pub framework: SummaryWindowFramework, pub window_secs: u64, pub slide_secs: u64, pub layout: asap_types::WindowMaterializationLayout, - pub cost: ImplementationCostEvidence, + pub cost: WindowRealizationCostQuote, /// Only compiler-generated quotes may be repriced after changing their layout. /// Serialized input always becomes provider evidence. #[serde(skip)] @@ -132,7 +133,7 @@ pub struct WindowImplementationCandidate { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] -pub struct LifecycleCostEvidence { +pub struct LifecycleUnitCosts { pub build: f64, pub maintenance_per_update: f64, pub read: f64, @@ -142,31 +143,32 @@ pub struct LifecycleCostEvidence { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] -pub struct LifecyclePlanningInput { +pub struct SummaryLifecyclePlanningInputs { pub evaluation_interval_ms: u32, pub ingestion_rate_per_second: f64, pub evidence_observed_at_unix_ms: u64, pub evidence_valid_for_ms: u64, pub horizon_seconds: f64, - pub costs: LifecycleCostEvidence, + pub costs: LifecycleUnitCosts, } #[derive(Debug, Clone, Default)] -pub struct PlanningRequest { +pub struct PhysicalCompilationRequest { /// Diagnostic projections of the original Planner search; never consumed by selection. - pub logical_selection: Vec, + pub planner_selection_trace: Vec, /// Enable a composable DAG with SummaryStore materializations and Prometheus exact subtrees. - pub hybrid_execution: bool, - /// Allowed materialization leaf contracts; None enables every eligible leaf. - pub materialization_policy: Option>, + pub allow_mixed_summary_and_exact_execution: bool, + /// Enabled optional candidate keys: None enables all eligible keys; an + /// explicitly empty set enables none. These are not catalog definition IDs. + pub enabled_materialization_keys: Option>, /// Original dashboard demand, in the same order as queries. None is legacy input. pub query_workload: Option, - pub queries: Vec, - pub evidence: HashMap, + pub queries: Vec, + pub topk_membership_evidence_by_query_id: HashMap, /// Fresh measured costs for Planner exact/summary composition sites, /// scoped to query IDs just like accuracy evidence. pub exact_composition_costs: HashMap>, - /// Optional distribution-conditioned empirical sizing policy. Hybrid + /// Optional Error–Resource Profile inputs and distribution-conditioned sizing. Hybrid /// mode falls back to theoretical sizing and then exact execution. pub erp: Option, pub planner_revision: String, @@ -176,7 +178,7 @@ pub struct PlanningRequest { /// How far behind the newest ingested sample an admitted query may be /// evaluated. This extends physical retention only; it never changes the /// PromQL range selector used for readout. - pub query_staleness_margin_ms: u64, + pub query_retention_margin_ms: u64, /// Maximum aggregate encoded footprint of every retained summary pane. /// `None` uses the backend's default persistence-memory limit. pub retained_summary_memory_budget_bytes: Option, @@ -193,9 +195,10 @@ pub struct TopKMembershipEvidence { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] -pub struct DeploymentEnvironment { +pub struct PhysicalDeploymentContext { pub target: PhysicalDeploymentTarget, - pub collector_ids: Vec, + #[serde(rename = "collector_ids", alias = "target_collector_ids")] + pub target_collector_ids: Vec, pub capability_snapshot_id: String, pub observed_at_unix_ms: u64, pub max_evidence_age_ms: u64, @@ -219,21 +222,23 @@ pub enum PhysicalDeploymentTarget { /// identity required to choose a concrete physical realization. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] -pub struct BackendLocalPlanningSnapshot { - pub snapshot_version: u32, +pub struct BackendLocalPlanningInput { + #[serde(rename = "snapshot_version", alias = "schema_version")] + pub schema_version: u32, /// May be absent during candidate discovery, never during deployment. #[serde(default, skip_serializing_if = "Option::is_none")] pub workload_cost_evidence: Option, pub query_workload: QueryWorkload, pub data_workload: DataWorkload, - pub implementation: BackendLocalImplementation, - pub environment: DeploymentEnvironment, + #[serde(rename = "implementation", alias = "physical_inputs")] + pub physical_inputs: BackendLocalPhysicalInputs, + pub environment: PhysicalDeploymentContext, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] -pub struct BackendLocalImplementation { - pub lifecycle_costs: LifecycleCostEvidence, +pub struct BackendLocalPhysicalInputs { + pub lifecycle_costs: LifecycleUnitCosts, pub evidence_observed_at_unix_ms: u64, pub evidence_valid_for_ms: u64, pub horizon_seconds: f64, @@ -241,7 +246,11 @@ pub struct BackendLocalImplementation { #[serde(default, skip_serializing_if = "Option::is_none")] pub source_sample_interval_ms: Option, #[serde(default, skip_serializing_if = "u64_is_zero")] - pub query_staleness_margin_ms: u64, + #[serde( + rename = "query_staleness_margin_ms", + alias = "query_retention_margin_ms" + )] + pub query_retention_margin_ms: u64, /// Admission budget for all retained panes and estimated partitions. /// Missing legacy snapshots inherit the backend default. #[serde( @@ -249,7 +258,11 @@ pub struct BackendLocalImplementation { alias = "maxRetainedSummaryBytes", skip_serializing_if = "is_default_retained_summary_memory_budget_bytes" )] - pub max_retained_summary_bytes: u64, + #[serde( + rename = "max_retained_summary_bytes", + alias = "retained_summary_memory_budget_bytes" + )] + pub retained_summary_memory_budget_bytes: u64, /// Certificates keyed by exact registered PromQL; converted to root IDs /// before workload selection so one query cannot borrow another's evidence. #[serde(default, skip_serializing_if = "HashMap::is_empty")] @@ -342,7 +355,7 @@ pub fn gos_policy_from_accuracy_budget( }) } -pub fn compile_transmission_plan( +pub fn build_transmission_plan( envelope: PlanEnvelope, precompute: &PrecomputePlan, runtime_policies: &BTreeMap, @@ -409,7 +422,7 @@ pub fn compile_transmission_plan( /// Complete physical projection of one post-ASAP planning decision. /// All three child plans share the same envelope and are compiled together. #[derive(Debug, Clone)] -pub struct PhysicalPlan { +pub struct CompiledPhysicalPlan { pub envelope: PlanEnvelope, pub summary_catalog: super::summary_catalog::SummaryCatalog, pub collector_plans: Vec, @@ -423,15 +436,16 @@ pub struct PhysicalPlan { pub storage_routing: serde_json::Value, /// Lifecycle component only, not a complete physical-plan comparison. pub lifecycle_estimates: Vec, - pub cost_comparison: Option, - pub logical_selection: Vec, + pub cost_comparison: Option, + pub planner_selection_trace: Vec, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct MaterializationLifecycleEstimate { pub materialization: asap_types::sds::SummaryDefinitionId, pub consumer_query_ids: Vec, - pub window_implementation_id: String, + #[serde(rename = "window_implementation_id", alias = "window_realization_id")] + pub window_realization_id: String, pub horizon_seconds: f64, pub expected_reads: f64, pub expected_updates: f64, @@ -483,44 +497,52 @@ impl AccuracyEvidenceProvider for QueryEvidence<'_> { } #[derive(Debug, Default)] -pub struct PhysicalCompiler; +pub struct PhysicalPlanCompiler; -impl BackendLocalPlanningSnapshot { +impl BackendLocalPlanningInput { /// Invoke the pinned Planner from canonical startup workloads and compile - /// one backend-local PhysicalPlan. No CollectorPlan is produced and no + /// one backend-local CompiledPhysicalPlan. No CollectorPlan is produced and no /// precompiled serving artifact is accepted at this boundary. - pub fn compile(self) -> Result { - self.compile_frontend(false) + pub fn compile_promql(self) -> Result { + self.compile_frontend(QueryFrontend::PromQl) } /// Use the shared parser subset with MetricsQL serving and exact routing. - pub fn compile_metricsql(self) -> Result { - self.compile_frontend(true) + pub fn compile_metricsql(self) -> Result { + self.compile_frontend(QueryFrontend::MetricsQl) } - fn compile_frontend(self, metricsql: bool) -> Result { + fn compile_frontend( + self, + frontend: QueryFrontend, + ) -> Result { let evidence = self.workload_cost_evidence.clone().ok_or_else(|| { CompileError::Snapshot( "deployment requires complete workload cost evidence; export candidates and price them before compiling".into(), ) })?; - let (request, environment) = self.planning_request()?; - let candidates = super::workload_cost::with_exact_alternative(request)?; - if metricsql { - super::workload_cost::select_metricsql(candidates, environment, &evidence) + let (request, environment) = self.into_physical_compilation_request()?; + let candidates = + super::workload_cost::enumerate_exact_and_materialized_candidates(request)?; + if frontend == QueryFrontend::MetricsQl { + super::workload_cost::select_lowest_cost_metricsql_candidate( + candidates, + environment, + &evidence, + ) } else { - super::workload_cost::select(candidates, environment, &evidence) + super::workload_cost::select_lowest_cost_candidate(candidates, environment, &evidence) } } /// Build Planner-authorized candidates for evidence collection without publishing. - pub fn planning_request( + pub fn into_physical_compilation_request( self, - ) -> Result<(PlanningRequest, DeploymentEnvironment), CompileError> { - if self.snapshot_version != 2 { + ) -> Result<(PhysicalCompilationRequest, PhysicalDeploymentContext), CompileError> { + if self.schema_version != 2 { return Err(CompileError::Snapshot(format!( "unsupported workload snapshot version {}; only version 2 is supported", - self.snapshot_version + self.schema_version ))); } if self.environment.target != PhysicalDeploymentTarget::BackendLocalRemoteWrite { @@ -595,40 +617,40 @@ impl BackendLocalPlanningSnapshot { let source_hint = source_metrics.iter().next().cloned().ok_or_else(|| { CompileError::Snapshot(format!("query {index} has no named time-series source")) })?; - let lifecycle = LifecyclePlanningInput { + let lifecycle = SummaryLifecyclePlanningInputs { evaluation_interval_ms, ingestion_rate_per_second: ingestion_rate.0, - evidence_observed_at_unix_ms: self.implementation.evidence_observed_at_unix_ms, - evidence_valid_for_ms: self.implementation.evidence_valid_for_ms, - horizon_seconds: self.implementation.horizon_seconds, - costs: self.implementation.lifecycle_costs.clone(), + evidence_observed_at_unix_ms: self.physical_inputs.evidence_observed_at_unix_ms, + evidence_valid_for_ms: self.physical_inputs.evidence_valid_for_ms, + horizon_seconds: self.physical_inputs.horizon_seconds, + costs: self.physical_inputs.lifecycle_costs.clone(), }; let post_asap = crate::planner_selection::keep_pre_asap(&parsed) .map_err(|error| CompileError::Snapshot(format!("query {index}: {error}")))?; canonical_roots.push(Rc::new(parsed)); let query_id = format!("compat-query-{index}"); - if let Some(evidence) = self.implementation.topk_evidence.get(&query_string) { + if let Some(evidence) = self.physical_inputs.topk_evidence.get(&query_string) { topk_evidence_by_id.insert(query_id.clone(), evidence.clone()); } - queries.push(PlanningQuery { + queries.push(QueryCompilationInput { query_id, query_string: query_string.clone(), - post_asap, - source: Source::TimeSeries { + selected_plan_root: post_asap, + legacy_query_source: Source::TimeSeries { metric: source_hint, }, - window_secs: lookback_ms / 1_000, - group_by: metadata.group_by_labels, - accuracy, - lifecycle, - window_implementations: Vec::new(), - runtime_policy: RuntimeRulePolicy::default(), + query_lookback_seconds: lookback_ms / 1_000, + group_by_labels: metadata.group_by_labels, + accuracy_target: accuracy, + summary_lifecycle_inputs: lifecycle, + window_realization_candidates: Vec::new(), + materialization_runtime_policy: RuntimeRulePolicy::default(), }); } let mut exact_costs_by_id = HashMap::new(); for (index, entry) in workload.entries().enumerate() { if let Some(rows) = self - .implementation + .physical_inputs .exact_composition_costs .get(&entry.query.0) { @@ -646,37 +668,37 @@ impl BackendLocalPlanningSnapshot { exact_costs_by_id.insert(format!("compat-query-{index}"), rows.clone()); } } - let logical_selection = select_workload_roots_with_trace( + let planner_selection_trace = select_logical_roots_with_trace( &mut queries, canonical_roots, &topk_evidence_by_id, &exact_costs_by_id, - self.implementation.erp.as_ref(), + self.physical_inputs.erp.as_ref(), )?; for query in &mut queries { prepare_window_implementations( query, - &self.implementation.window_cost_model, + &self.physical_inputs.window_cost_model, self.environment.target, - self.implementation.query_staleness_margin_ms, + self.physical_inputs.query_retention_margin_ms, )?; } // Composable lowering residualizes unsafe leaves individually; retain Planner siblings. Ok(( - PlanningRequest { - logical_selection, - hybrid_execution: true, - materialization_policy: None, + PhysicalCompilationRequest { + planner_selection_trace, + allow_mixed_summary_and_exact_execution: true, + enabled_materialization_keys: None, query_workload: Some(workload), queries, - evidence: topk_evidence_by_id, + topk_membership_evidence_by_query_id: topk_evidence_by_id, exact_composition_costs: exact_costs_by_id, - erp: self.implementation.erp, + erp: self.physical_inputs.erp, planner_revision: PLANNER_REVISION.into(), - source_sample_interval_ms: self.implementation.source_sample_interval_ms, - query_staleness_margin_ms: self.implementation.query_staleness_margin_ms, + source_sample_interval_ms: self.physical_inputs.source_sample_interval_ms, + query_retention_margin_ms: self.physical_inputs.query_retention_margin_ms, retained_summary_memory_budget_bytes: Some( - self.implementation.max_retained_summary_bytes, + self.physical_inputs.retained_summary_memory_budget_bytes, ), }, self.environment, @@ -750,14 +772,14 @@ fn has_unsafe_raw_entity_leaf( } /// Preserve native execution for raw states that cannot preserve source semantics. -fn preserve_native_unsafe_raw_roots(queries: &mut [PlanningQuery]) -> Result<(), CompileError> { +fn preserve_native_unsafe_raw_roots( + queries: &mut [QueryCompilationInput], +) -> Result<(), CompileError> { for query in queries { - let selected = - collect_selected_materializations(&query.post_asap, false).map_err(|reason| { - CompileError::Query { - query_id: query.query_id.clone(), - reason, - } + let selected = collect_selected_materializations(&query.selected_plan_root, false) + .map_err(|reason| CompileError::Query { + query_id: query.query_id.clone(), + reason, })?; if selected.is_empty() { continue; @@ -766,17 +788,18 @@ fn preserve_native_unsafe_raw_roots(queries: &mut [PlanningQuery]) -> Result<(), .iter() .map(|state| Rc::clone(&state.node)) .collect::>(); - let unsafe_entities = has_unsafe_raw_entity_leaf(&query.post_asap, &selected_nodes, false); + let unsafe_entities = + has_unsafe_raw_entity_leaf(&query.selected_plan_root, &selected_nodes, false); if unsafe_entities { let parsed = crate::query_parser::parse_query_expr_canonical( &query.query_string, - query.accuracy.clone(), + query.accuracy_target.clone(), ) .map_err(|error| CompileError::Query { query_id: query.query_id.clone(), reason: error.to_string(), })?; - query.post_asap = + query.selected_plan_root = crate::planner_selection::keep_pre_asap(&parsed).map_err(|error| { CompileError::Query { query_id: query.query_id.clone(), @@ -793,29 +816,29 @@ fn preserve_native_unsafe_raw_roots(queries: &mut [PlanningQuery]) -> Result<(), /// intentionally unsupported (and therefore invalid as an executable /// maintenance DAG), but the original query remains a valid exact plan. fn preserve_invalid_exact_fallback_roots( - queries: &mut [PlanningQuery], + queries: &mut [QueryCompilationInput], composable: bool, ) -> Result<(), CompileError> { for query in queries { - let selected = - collect_selected_materializations(&query.post_asap, composable).map_err(|reason| { - CompileError::Query { - query_id: query.query_id.clone(), - reason, - } + let selected = collect_selected_materializations(&query.selected_plan_root, composable) + .map_err(|reason| CompileError::Query { + query_id: query.query_id.clone(), + reason, })?; let invalid_executable = - selected.is_empty() && validate_executable_subdag(&query.post_asap).is_err(); - if invalid_executable && !matches!(query.post_asap.expr, SummaryExpr::KeepPreAsap(_)) { + selected.is_empty() && validate_executable_subdag(&query.selected_plan_root).is_err(); + if invalid_executable + && !matches!(query.selected_plan_root.expr, SummaryExpr::KeepPreAsap(_)) + { let parsed = crate::query_parser::parse_query_expr_canonical( &query.query_string, - query.accuracy.clone(), + query.accuracy_target.clone(), ) .map_err(|error| CompileError::Query { query_id: query.query_id.clone(), reason: error.to_string(), })?; - query.post_asap = + query.selected_plan_root = crate::planner_selection::keep_pre_asap(&parsed).map_err(|error| { CompileError::Query { query_id: query.query_id.clone(), @@ -832,16 +855,14 @@ fn preserve_invalid_exact_fallback_roots( /// query as one native exact root. Mixed queries retain their other selected /// summaries and let residual lowering cut only the counter branches. fn preserve_metricsql_counter_only_roots( - queries: &mut [PlanningQuery], + queries: &mut [QueryCompilationInput], composable: bool, ) -> Result<(), CompileError> { for query in queries { - let selected = - collect_selected_materializations(&query.post_asap, composable).map_err(|reason| { - CompileError::Query { - query_id: query.query_id.clone(), - reason, - } + let selected = collect_selected_materializations(&query.selected_plan_root, composable) + .map_err(|reason| CompileError::Query { + query_id: query.query_id.clone(), + reason, })?; if selected.is_empty() || !selected.iter().all(|state| { @@ -859,46 +880,47 @@ fn preserve_metricsql_counter_only_roots( } let parsed = crate::query_parser::parse_query_expr_canonical( &query.query_string, - query.accuracy.clone(), + query.accuracy_target.clone(), ) .map_err(|error| CompileError::Query { query_id: query.query_id.clone(), reason: error.to_string(), })?; - query.post_asap = crate::planner_selection::keep_pre_asap(&parsed).map_err(|error| { - CompileError::Query { - query_id: query.query_id.clone(), - reason: error.to_string(), - } - })?; + query.selected_plan_root = + crate::planner_selection::keep_pre_asap(&parsed).map_err(|error| { + CompileError::Query { + query_id: query.query_id.clone(), + reason: error.to_string(), + } + })?; } Ok(()) } -impl PhysicalCompiler { - pub fn compile( +impl PhysicalPlanCompiler { + pub fn compile_promql( &self, - request: PlanningRequest, - environment: DeploymentEnvironment, - ) -> Result { - self.compile_language(request, environment, false) + request: PhysicalCompilationRequest, + environment: PhysicalDeploymentContext, + ) -> Result { + self.compile_for_frontend(request, environment, QueryFrontend::PromQl) } pub fn compile_metricsql( &self, - request: PlanningRequest, - environment: DeploymentEnvironment, - ) -> Result { - self.compile_language(request, environment, true) + request: PhysicalCompilationRequest, + environment: PhysicalDeploymentContext, + ) -> Result { + self.compile_for_frontend(request, environment, QueryFrontend::MetricsQl) } - fn compile_language( + pub fn compile_for_frontend( &self, - mut request: PlanningRequest, - environment: DeploymentEnvironment, - metricsql: bool, - ) -> Result { - if request.hybrid_execution + mut request: PhysicalCompilationRequest, + environment: PhysicalDeploymentContext, + frontend: QueryFrontend, + ) -> Result { + if request.allow_mixed_summary_and_exact_execution && environment.target != PhysicalDeploymentTarget::BackendLocalRemoteWrite { return Err(CompileError::Snapshot( @@ -912,7 +934,7 @@ impl PhysicalCompiler { }); } if environment.target == PhysicalDeploymentTarget::BackendLocalRemoteWrite - && !environment.collector_ids.is_empty() + && !environment.target_collector_ids.is_empty() { return Err(CompileError::Query { query_id: "deployment-target".into(), @@ -942,13 +964,19 @@ impl PhysicalCompiler { } } - if metricsql { - preserve_metricsql_counter_only_roots(&mut request.queries, request.hybrid_execution)?; + if frontend == QueryFrontend::MetricsQl { + preserve_metricsql_counter_only_roots( + &mut request.queries, + request.allow_mixed_summary_and_exact_execution, + )?; } - preserve_invalid_exact_fallback_roots(&mut request.queries, request.hybrid_execution)?; + preserve_invalid_exact_fallback_roots( + &mut request.queries, + request.allow_mixed_summary_and_exact_execution, + )?; if environment.target == PhysicalDeploymentTarget::BackendLocalRemoteWrite - && !request.hybrid_execution + && !request.allow_mixed_summary_and_exact_execution { preserve_native_unsafe_raw_roots(&mut request.queries)?; } @@ -957,10 +985,10 @@ impl PhysicalCompiler { .queries .iter() .enumerate() - .map(|(id, query)| (id, Rc::clone(&query.post_asap))) + .map(|(id, query)| (id, Rc::clone(&query.selected_plan_root))) .collect(); for (id, root) in planner_types::post_asap::share_common_summary_subtrees(roots) { - request.queries[id].post_asap = root; + request.queries[id].selected_plan_root = root; } let mut compiled_materializations = Vec::with_capacity(request.queries.len()); let mut collector_materializations = Vec::with_capacity(request.queries.len()); @@ -979,7 +1007,7 @@ impl PhysicalCompiler { let consumers = materialization_consumers( &request.queries, environment.target, - request.hybrid_execution, + request.allow_mixed_summary_and_exact_execution, )?; let mut lifecycle_estimates = BTreeMap::::new(); @@ -991,16 +1019,21 @@ impl PhysicalCompiler { BTreeMap::>::new(); for (query_index, query) in request.queries.iter().enumerate() { - let evidence = request.evidence.get(&query.query_id); + let evidence = request + .topk_membership_evidence_by_query_id + .get(&query.query_id); if let Some(e) = evidence { validate_evidence(&query.query_id, e, &environment)?; } - let node = query.post_asap.clone(); - let selected = collect_selected_materializations(&node, request.hybrid_execution) - .map_err(|reason| CompileError::Query { - query_id: query.query_id.clone(), - reason, - })?; + let node = query.selected_plan_root.clone(); + let selected = collect_selected_materializations( + &node, + request.allow_mixed_summary_and_exact_execution, + ) + .map_err(|reason| CompileError::Query { + query_id: query.query_id.clone(), + reason, + })?; let selected = selected .into_iter() .filter(|state| { @@ -1008,14 +1041,14 @@ impl PhysicalCompiler { // MetricsQL includes boundary samples differently, so retain // these leaves as external exact dependencies until its // counter semantics have a dedicated implementation. - !(metricsql && matches!(state.family, + !(frontend == QueryFrontend::MetricsQl && matches!(state.family, SummaryFamilyType::ExactAggregate( planner_types::post_asap::ExactKind::Rate | planner_types::post_asap::ExactKind::Increase, _))) - && (!request.hybrid_execution + && (!request.allow_mixed_summary_and_exact_execution || state.window_secs.is_none_or(|window| { query - .window_implementations + .window_realization_candidates .iter() .any(|candidate| candidate.window_secs == window) })) @@ -1025,22 +1058,22 @@ impl PhysicalCompiler { planner_types::post_asap::ExactKind::MinMax, _ ) - ) || crate::query_plan::logical::selected_range_max_materialization( + ) || crate::query_plan::residual::selected_range_max_materialization( &query.query_string, &state.node, ) .ok() .flatten() .is_some()) - && request.materialization_policy.as_ref().is_none_or(|policy| { - let key = crate::query_plan::logical::selected_counter_materialization( + && request.enabled_materialization_keys.as_ref().is_none_or(|policy| { + let key = crate::query_plan::residual::selected_counter_materialization( &query.query_string, &state.node, ) .ok() .flatten() .or_else(|| { - crate::query_plan::logical::selected_range_max_materialization( + crate::query_plan::residual::selected_range_max_materialization( &query.query_string, &state.node, ) @@ -1058,14 +1091,15 @@ impl PhysicalCompiler { if selected.is_empty() { continue; } - let executable = - planner_types::post_asap::compile_executable_dag_with_node_ids(&query.post_asap) - .map_err(|error| CompileError::Query { - query_id: query.query_id.clone(), - reason: format!("invalid executable subDAG: {error}"), - })?; + let executable = planner_types::post_asap::compile_executable_dag_with_node_ids( + &query.selected_plan_root, + ) + .map_err(|error| CompileError::Query { + query_id: query.query_id.clone(), + reason: format!("invalid executable subDAG: {error}"), + })?; executable_dags[query_index] = Some(executable); - validate_lifecycle_input(&query.query_id, &query.lifecycle)?; + validate_lifecycle_input(&query.query_id, &query.summary_lifecycle_inputs)?; if environment.target == PhysicalDeploymentTarget::DistributedCollectors && selected.iter().any(|state| { matches!( @@ -1083,7 +1117,7 @@ impl PhysicalCompiler { .into(), }); } - match &query.source { + match &query.legacy_query_source { Source::TimeSeries { .. } => {} Source::Table { .. } => { return Err(CompileError::Query { @@ -1113,35 +1147,40 @@ impl PhysicalCompiler { let cohort_nodes = windows::cohort_nodes(&selected); for (ordinal, selected) in selected.into_iter().enumerate() { let mut branch_query = query.clone(); - branch_query.window_secs = selected.window_secs.unwrap_or(query.window_secs); - branch_query.group_by = selected + branch_query.query_lookback_seconds = + selected.window_secs.unwrap_or(query.query_lookback_seconds); + branch_query.group_by_labels = selected .group_by .clone() - .unwrap_or_else(|| query.group_by.clone()); - branch_query.window_implementations.retain(|candidate| { - candidate.window_secs == branch_query.window_secs - && if cohort_nodes.contains(&(Rc::as_ptr(&selected.node) as usize)) { - windows::is_full_cohort(candidate) - } else { - !candidate.cohort_only - } - }); + .unwrap_or_else(|| query.group_by_labels.clone()); + branch_query + .window_realization_candidates + .retain(|candidate| { + candidate.window_secs == branch_query.query_lookback_seconds + && if cohort_nodes.contains(&(Rc::as_ptr(&selected.node) as usize)) { + windows::is_full_cohort(candidate) + } else { + !candidate.cohort_only + } + }); let query = &branch_query; let lifecycle_costs = SummaryMaintenanceLifecycleCostInputs { - build_cost: Some(Cost(query.lifecycle.costs.build)), + build_cost: Some(Cost(query.summary_lifecycle_inputs.costs.build)), maintenance_cost_per_update: Some(Cost( - query.lifecycle.costs.maintenance_per_update, + query.summary_lifecycle_inputs.costs.maintenance_per_update, + )), + summary_read_cost: Some(Cost(query.summary_lifecycle_inputs.costs.read)), + retention_cost_rate: Some(CostRate( + query.summary_lifecycle_inputs.costs.retention_per_second, )), - summary_read_cost: Some(Cost(query.lifecycle.costs.read)), - retention_cost_rate: Some(CostRate(query.lifecycle.costs.retention_per_second)), - retirement_cost: Some(Cost(query.lifecycle.costs.retirement)), + retirement_cost: Some(Cost(query.summary_lifecycle_inputs.costs.retirement)), }; let window_costs = super::realization::RealizationProvider::windows( &super::realization::ExistingRealizations, query, &environment, )?; - let model = ControlPlaneCostModel::new(query.accuracy.clone()) + let model = ControlPlaneCostModel::new(query.accuracy_target.clone()) .with_summary_maintenance( lifecycle_costs, SummaryMaintenanceCapabilities { @@ -1187,8 +1226,8 @@ impl PhysicalCompiler { return true; }; let other = &request.queries[index]; - if other.lifecycle.evaluation_interval_ms - != query.lifecycle.evaluation_interval_ms + if other.summary_lifecycle_inputs.evaluation_interval_ms + != query.summary_lifecycle_inputs.evaluation_interval_ms { return false; } @@ -1204,8 +1243,10 @@ impl PhysicalCompiler { .collect::>(); match (phases[query_index], phases[index]) { (Some(a), Some(b)) => { - let cadence = u64::from(query.lifecycle.evaluation_interval_ms); - let window = query.window_secs.saturating_mul(1_000); + let cadence = u64::from( + query.summary_lifecycle_inputs.evaluation_interval_ms, + ); + let window = query.query_lookback_seconds.saturating_mul(1_000); a % cadence == b % cadence && a % window == b % window } _ => index == query_index, @@ -1227,8 +1268,8 @@ impl PhysicalCompiler { .as_ref() .map(|workload| (workload, consumer_indices.clone())), )?; - let window_implementation = query.window_implementations.iter() - .find(|candidate| candidate.implementation_id == planner_selection.window_implementation_id + let window_implementation = query.window_realization_candidates.iter() + .find(|candidate| candidate.realization_id == planner_selection.window_realization_id && candidate.framework == planner_selection.window_framework) .ok_or_else(|| CompileError::Lifecycle { query_id: query.query_id.clone(), @@ -1385,8 +1426,8 @@ impl PhysicalCompiler { .or_insert_with(|| MaterializationLifecycleEstimate { materialization: materialization.into(), consumer_query_ids, - window_implementation_id: window_implementation.implementation_id.clone(), - horizon_seconds: query.lifecycle.horizon_seconds, + window_realization_id: window_implementation.realization_id.clone(), + horizon_seconds: query.summary_lifecycle_inputs.horizon_seconds, expected_reads: planner_selection.expected_reads, expected_updates: planner_selection.expected_updates, lifecycle_cost: planner_selection.lifecycle_cost, @@ -1411,10 +1452,11 @@ impl PhysicalCompiler { }); } } - if let Some(existing) = - runtime_policies.insert(materialization, query.runtime_policy.clone()) - { - if existing != query.runtime_policy { + if let Some(existing) = runtime_policies.insert( + materialization, + query.materialization_runtime_policy.clone(), + ) { + if existing != query.materialization_runtime_policy { return Err(CompileError::Query { query_id: query.query_id.clone(), reason: "queries sharing one materialization specify different runtime policies" @@ -1433,7 +1475,7 @@ impl PhysicalCompiler { group_by: physical_group_by, window_secs: runtime_materialization.window_size, abstract_window_framework: planner_selection.window_framework.clone(), - window_implementation_id: window_implementation.implementation_id.clone(), + window_realization_id: window_implementation.realization_id.clone(), slide_secs: runtime_materialization.slide_interval, pane_origin_ms: runtime_materialization.pane_origin_ms, window_layout: window_implementation.layout.clone(), @@ -1479,14 +1521,14 @@ impl PhysicalCompiler { ); } - let plan_id = if request.hybrid_execution { + let plan_id = if request.allow_mixed_summary_and_exact_execution { use std::hash::{Hash, Hasher}; let mut hash = std::collections::hash_map::DefaultHasher::new(); stable_workload_plan_id(&plan_materializations, &request.queries).hash(&mut hash); "typed-local-residual-v3-counter-index".hash(&mut hash); - request.materialization_policy.hash(&mut hash); + request.enabled_materialization_keys.hash(&mut hash); for query in &request.queries { - format!("{:?}", query.post_asap).hash(&mut hash); + format!("{:?}", query.selected_plan_root).hash(&mut hash); } hash.finish() } else { @@ -1503,7 +1545,9 @@ impl PhysicalCompiler { capability_snapshot_id: environment.capability_snapshot_id, }; let producer_ids = match environment.target { - PhysicalDeploymentTarget::DistributedCollectors => environment.collector_ids.clone(), + PhysicalDeploymentTarget::DistributedCollectors => { + environment.target_collector_ids.clone() + } PhysicalDeploymentTarget::BackendLocalRemoteWrite => Vec::new(), }; // Several queries/readouts may intentionally share one maintained @@ -1587,7 +1631,7 @@ impl PhysicalCompiler { let window_ms = materialization.window_size.saturating_mul(1_000); if materialization_family != physical_materialization_family(node_family) || window_ms == 0 - || source_window.unwrap_or(query.window_secs).saturating_mul(1_000) + || source_window.unwrap_or(query.query_lookback_seconds).saturating_mul(1_000) % window_ms != 0 { return Err(crate::query_plan::QueryPlanError::Invalid(format!( @@ -1609,25 +1653,25 @@ impl PhysicalCompiler { }) }; let instant = InstantExecution { - lookback_ms: query.window_secs.saturating_mul(1_000), + lookback_ms: query.query_lookback_seconds.saturating_mul(1_000), full_history: false, cumulative_readout: true, }; // A whole-query native fallback need not be expressible in the local // residual algebra (for example an ERP-rejected entropy readout). // Retain its native boundary without discarding other workload roots. - let native_root = request.hybrid_execution - && if let SummaryExpr::KeepPreAsap(expr) = &query.post_asap.expr { + let native_root = request.allow_mixed_summary_and_exact_execution + && if let SummaryExpr::KeepPreAsap(expr) = &query.selected_plan_root.expr { let original = crate::query_parser::parse_query_expr_canonical( &query.query_string, - query.accuracy.clone(), + query.accuracy_target.clone(), ) .map_err(|error| CompileError::Query { query_id: query.query_id.clone(), reason: error.to_string(), })?; expr.as_ref() == &original - && crate::query_plan::logical::compile_logical( + && crate::query_plan::residual::compile_logical( query.query_id.clone(), canonical.clone(), instant, @@ -1637,11 +1681,11 @@ impl PhysicalCompiler { } else { false }; - let mut entry = if request.hybrid_execution && !native_root { + let mut entry = if request.allow_mixed_summary_and_exact_execution && !native_root { crate::query_plan::compile_bound_composable_mapped( query.query_id.clone(), canonical.clone(), - &query.post_asap, + &query.selected_plan_root, instant, FallbackPolicy::ExactBackend, binding, @@ -1658,7 +1702,7 @@ impl PhysicalCompiler { crate::query_plan::compile_bound_mapped( query.query_id.clone(), canonical.clone(), - &query.post_asap, + &query.selected_plan_root, instant, FallbackPolicy::ExactBackend, binding, @@ -1672,13 +1716,13 @@ impl PhysicalCompiler { }, ) }?; - if request.hybrid_execution { + if request.allow_mixed_summary_and_exact_execution { // Any Planner-selected leaf without a physical summary binding // is an exact subtree boundary. Deployed plans never retain a // backend-local range index leaf. - crate::query_plan::logical::finalize_residuals(&mut entry)?; + crate::query_plan::residual::finalize_residuals(&mut entry)?; } - if metricsql { + if frontend == QueryFrontend::MetricsQl { entry.language = crate::query_plan::QueryLanguage::MetricsQl; } let catalog_key = QueryPlan::catalog_key(entry.language, &canonical); @@ -1735,7 +1779,7 @@ impl PhysicalCompiler { if let Some(lookback_ms) = max_lookback_ms { materialization.num_aggregates_to_retain = Some(retained_state_count( lookback_ms, - request.query_staleness_margin_ms, + request.query_retention_margin_ms, materialization.slide_interval.saturating_mul(1_000), &materialization.window_layout, )); @@ -1795,7 +1839,7 @@ impl PhysicalCompiler { query_id: "precompute-plan".into(), reason: error.to_string(), })?; - let mut transmission_plan = crate::physical::compiler::compile_transmission_plan( + let mut transmission_plan = crate::physical::compiler::build_transmission_plan( envelope.clone(), &precompute_plan, &runtime_policies, @@ -1857,7 +1901,7 @@ impl PhysicalCompiler { crate::emit::stage_config::DEFAULT_TENANT, &routed_algorithms.into_iter().collect::>(), ); - Ok(PhysicalPlan { + Ok(CompiledPhysicalPlan { envelope, summary_catalog, collector_plans, @@ -1867,7 +1911,7 @@ impl PhysicalCompiler { storage_routing, lifecycle_estimates: lifecycle_estimates.into_values().collect(), cost_comparison: None, - logical_selection: request.logical_selection, + planner_selection_trace: request.planner_selection_trace, }) } } @@ -1923,13 +1967,13 @@ fn summary_agg_metric(node: &SummaryNode) -> Option { /// Shared selection boundary for canonical startup and compile-and-publish. /// Certificate-bearing roots stay isolated: equal certificate values do not /// establish that the certificate's source scope covers another query. -pub fn select_workload_roots( - queries: &mut [PlanningQuery], +pub fn select_logical_roots_for_queries( + queries: &mut [QueryCompilationInput], roots: Vec>, evidence: &HashMap, exact_costs: &HashMap>, ) -> Result<(), CompileError> { - select_workload_roots_with_erp(queries, roots, evidence, exact_costs, None) + select_logical_roots_with_error_resource_profiles(queries, roots, evidence, exact_costs, None) } fn observed_population_matches_root( @@ -1979,18 +2023,18 @@ fn observed_population_matches_root( == i64::try_from(window.saturating_mul(1000)).ok() } -pub fn select_workload_roots_with_erp( - queries: &mut [PlanningQuery], +pub fn select_logical_roots_with_error_resource_profiles( + queries: &mut [QueryCompilationInput], roots: Vec>, evidence: &HashMap, exact_costs: &HashMap>, erp: Option<&super::erp::ErpPlanningInput>, ) -> Result<(), CompileError> { - select_workload_roots_with_trace(queries, roots, evidence, exact_costs, erp).map(|_| ()) + select_logical_roots_with_trace(queries, roots, evidence, exact_costs, erp).map(|_| ()) } -pub fn select_workload_roots_with_trace( - queries: &mut [PlanningQuery], +pub fn select_logical_roots_with_trace( + queries: &mut [QueryCompilationInput], roots: Vec>, evidence: &HashMap, exact_costs: &HashMap>, @@ -2006,7 +2050,7 @@ pub fn select_workload_roots_with_trace( Vec::new(); let original_roots = roots.clone(); for (index, root) in roots.into_iter().enumerate() { - let accuracy = &queries[index].accuracy; + let accuracy = &queries[index].accuracy_target; let certificate_scope = (evidence.contains_key(&queries[index].query_id) || exact_costs.contains_key(&queries[index].query_id)) .then(|| queries[index].query_id.clone()); @@ -2081,7 +2125,7 @@ pub fn select_workload_roots_with_trace( let selected_indices = selected.iter().map(|(index, _)| *index).collect::>(); for (index, node) in selected { if erp.is_some_and(|policy| { - requires_exact_erp_fallback(&node, &queries[index].accuracy, policy) + requires_exact_erp_fallback(&node, &queries[index].accuracy_target, policy) }) { if let Some(values) = trace["deployment_overrides"].as_array_mut() { values.push( @@ -2089,16 +2133,16 @@ pub fn select_workload_roots_with_trace( "reason": "ERP requires exact fallback"}), ); } - queries[index].post_asap = + queries[index].selected_plan_root = crate::planner_selection::keep_pre_asap(&original_roots[index]) .map_err(|error| CompileError::Snapshot(error.to_string()))?; } else { - queries[index].post_asap = node; + queries[index].selected_plan_root = node; } } trace["committed_roots"] = serde_json::json!(selected_indices.into_iter().map(|index| serde_json::json!({"query_index": index, - "logical_root_id": crate::planner_selection::explained_root_id(&queries[index].post_asap, &queries[index].accuracy) + "logical_root_id": crate::planner_selection::explained_root_id(&queries[index].selected_plan_root, &queries[index].accuracy_target) })).collect::>()); traces.push(trace); } @@ -2196,11 +2240,11 @@ fn requires_exact_erp_fallback( #[cfg(test)] /// Planner-adapter selection step used before physical compilation. Keeping /// this separate makes the ownership boundary explicit: callers supply the -/// selected post-ASAP DAG to [`PhysicalCompiler::compile`]. +/// selected post-ASAP DAG to [`PhysicalPlanCompiler::compile`]. pub fn select_post_asap( expr: &QueryExpr, accuracy: AccuracyTarget, - lifecycle: &LifecyclePlanningInput, + lifecycle: &SummaryLifecyclePlanningInputs, evidence: Option<&TopKMembershipEvidence>, ) -> Result, crate::planner_selection::SelectionError> { let model = ControlPlaneCostModel::new(accuracy).with_summary_maintenance( @@ -2229,7 +2273,7 @@ pub fn select_post_asap( fn validate_evidence( query_id: &str, evidence: &TopKMembershipEvidence, - env: &DeploymentEnvironment, + env: &PhysicalDeploymentContext, ) -> Result<(), CompileError> { let age = env .observed_at_unix_ms @@ -2253,7 +2297,7 @@ fn validate_evidence( fn validate_lifecycle_input( query_id: &str, - input: &LifecyclePlanningInput, + input: &SummaryLifecyclePlanningInputs, ) -> Result<(), CompileError> { let costs = [ input.costs.build, @@ -2280,12 +2324,12 @@ fn validate_lifecycle_input( /// Price one derived layout from the snapshot's own lifecycle unit costs. /// -/// `ImplementationCostEvidence` is normally measured evidence, and its +/// `WindowRealizationCostQuote` is normally measured evidence, and its /// `weighted_cost` doc puts pricing update CPU, query-time merges, retained /// memory, storage, scans and network on the evidence producer. When a /// snapshot prices no candidate, the control plane becomes that producer for /// the derived shapes — and it does so without inventing a single magnitude. -/// Every unit cost below is supplied evidence (`LifecycleCostEvidence`, from +/// Every unit cost below is supplied evidence (`LifecycleUnitCosts`, from /// `implementation.lifecycle_costs`); every multiplier is a structural count /// that follows from the layout's definition. Nothing here is a measurement. /// @@ -2315,13 +2359,13 @@ fn validate_lifecycle_input( /// by `validate_window_implementations` and by the `min_by` that ranks /// candidates. `model_version` records that the quote is derived. pub(super) fn derived_window_cost( - template: &ImplementationCostEvidence, - lifecycle: &LifecyclePlanningInput, + template: &WindowRealizationCostQuote, + lifecycle: &SummaryLifecyclePlanningInputs, window_secs: u64, slide_secs: u64, layout: &asap_types::WindowMaterializationLayout, staleness_margin_ms: u64, -) -> ImplementationCostEvidence { +) -> WindowRealizationCostQuote { let costs = &lifecycle.costs; let horizon = lifecycle.horizon_seconds.max(0.0); let window = window_secs.max(1) as f64; @@ -2361,7 +2405,7 @@ pub(super) fn derived_window_cost( let cpu_cost = build + maintenance; let weighted_cost = cpu_cost + read + retention + retirement; - ImplementationCostEvidence { + WindowRealizationCostQuote { model_version: format!( "{}+derived-window-layout-v1", template @@ -2432,10 +2476,10 @@ fn derived_window_candidates( expr: &QueryExpr, lookback_ms: u64, evaluation_interval_ms: u32, - cost: ImplementationCostEvidence, - lifecycle: &LifecyclePlanningInput, + cost: WindowRealizationCostQuote, + lifecycle: &SummaryLifecyclePlanningInputs, staleness_margin_ms: u64, -) -> Vec { +) -> Vec { let mut windows = range_selector_windows_secs(expr); if windows.is_empty() { windows.insert(lookback_ms / 1_000); @@ -2462,30 +2506,31 @@ fn derived_window_candidates( } pub(super) fn validate_window_implementations( - query: &PlanningQuery, - environment: &DeploymentEnvironment, + query: &QueryCompilationInput, + environment: &PhysicalDeploymentContext, ) -> Result, CompileError> { let mut ids = BTreeSet::new(); let mut candidates = Vec::new(); - for candidate in &query.window_implementations { + for candidate in &query.window_realization_candidates { let evidence = &candidate.cost; let age = environment .observed_at_unix_ms .saturating_sub(evidence.observed_at_unix_ms); let valid = evidence.observed_at_unix_ms <= environment.observed_at_unix_ms - && !candidate.implementation_id.trim().is_empty() - && ids.insert(candidate.implementation_id.clone()) + && !candidate.realization_id.trim().is_empty() + && ids.insert(candidate.realization_id.clone()) && !evidence.model_version.trim().is_empty() && !evidence.workload_fingerprint.trim().is_empty() && evidence.valid_for_ms != 0 && age <= environment.max_evidence_age_ms.min(evidence.valid_for_ms) && evidence.horizon_seconds.is_finite() - && (evidence.horizon_seconds - query.lifecycle.horizon_seconds).abs() <= f64::EPSILON + && (evidence.horizon_seconds - query.summary_lifecycle_inputs.horizon_seconds).abs() + <= f64::EPSILON && evidence.cpu_cost.is_finite() && evidence.cpu_cost >= 0.0 && evidence.weighted_cost.is_finite() && evidence.weighted_cost >= 0.0 - && candidate.window_secs == query.window_secs + && candidate.window_secs == query.query_lookback_seconds && candidate .layout .validate(candidate.window_secs, candidate.slide_secs) @@ -2496,12 +2541,12 @@ pub(super) fn validate_window_implementations( query_id: query.query_id.clone(), reason: format!( "window implementation `{}` has incomplete, stale, incompatible, or duplicate physical evidence", - candidate.implementation_id + candidate.realization_id ), }); } candidates.push(( - candidate.implementation_id.clone(), + candidate.realization_id.clone(), candidate.framework.clone(), Cost(evidence.weighted_cost), )); @@ -2637,7 +2682,7 @@ fn validate_retained_summary_footprint( } struct PlannerPhysicalSelection { - window_implementation_id: String, + window_realization_id: String, lifecycle: CollectorLifecycle, window_framework: SummaryWindowFramework, expected_reads: f64, @@ -2646,20 +2691,20 @@ struct PlannerPhysicalSelection { } fn select_lifecycle( - query: &PlanningQuery, + query: &QueryCompilationInput, node: &SummaryNode, model: &ControlPlaneCostModel, - environment: &DeploymentEnvironment, - consumers: &[&PlanningQuery], + environment: &PhysicalDeploymentContext, + consumers: &[&QueryCompilationInput], original_workload: Option<(&QueryWorkload, Vec)>, ) -> Result { // Current lifecycle evidence is per producer with one unit read cost. // Conflicting source/rate/horizon/cost snapshots cannot be averaged into // invented evidence. Only recurrence may differ between consumers. - let mut common = query.lifecycle.clone(); + let mut common = query.summary_lifecycle_inputs.clone(); common.evaluation_interval_ms = 0; for consumer in consumers { - let mut input = consumer.lifecycle.clone(); + let mut input = consumer.summary_lifecycle_inputs.clone(); input.evaluation_interval_ms = 0; if input != common { return Err(CompileError::Lifecycle { @@ -2677,10 +2722,10 @@ fn select_lifecycle( .map(|query| RepeatingEntry { query: Query(query.query_id.clone()), demand: RepeatedDemand::FixedInterval(RepetitionInterval( - query.lifecycle.evaluation_interval_ms, + query.summary_lifecycle_inputs.evaluation_interval_ms, )), requirements: QueryRequirements { - accuracy: AccuracyRequirement::Explicit(query.accuracy.clone()), + accuracy: AccuracyRequirement::Explicit(query.accuracy_target.clone()), ..QueryRequirements::default() }, predictability: Predictability::Predictable { known_at: None }, @@ -2689,10 +2734,10 @@ fn select_lifecycle( // claim deletion support for moving-window retractions. scope: QueryTimeScope::Unknown, lookback: Some(DurationMs( - materialization_leaf_contract(node) + raw_materialization_input_contract(node) .ok() .and_then(|(_, window, _)| window) - .unwrap_or(query.window_secs) + .unwrap_or(query.query_lookback_seconds) .saturating_mul(1_000), )), as_of: None, @@ -2703,10 +2748,12 @@ fn select_lifecycle( data_workload: Some(DataWorkload { arrival: DataArrival::ContinuouslyIngesting, ingestion_rate: Evidence { - value: Some(Rate(query.lifecycle.ingestion_rate_per_second)), + value: Some(Rate( + query.summary_lifecycle_inputs.ingestion_rate_per_second, + )), source: EvidenceSource::Observed, - observed_at_ms: Some(query.lifecycle.evidence_observed_at_unix_ms), - valid_for_ms: Some(query.lifecycle.evidence_valid_for_ms), + observed_at_ms: Some(query.summary_lifecycle_inputs.evidence_observed_at_unix_ms), + valid_for_ms: Some(query.summary_lifecycle_inputs.evidence_valid_for_ms), }, ..DataWorkload::default() }), @@ -2727,7 +2774,7 @@ fn select_lifecycle( Rc::new(node.clone()), WorkloadDemand::new(&workload, &indices), environment.observed_at_unix_ms, - Some(Horizon(query.lifecycle.horizon_seconds)), + Some(Horizon(query.summary_lifecycle_inputs.horizon_seconds)), SummaryMaintenanceLifecycleCapabilities { supports_ephemeral: false, supports_prepared: false, @@ -2757,12 +2804,13 @@ fn select_lifecycle( reason: "latest ASAPPlanner selected no window framework from the supplied physical evidence".into(), })?; Ok(PlannerPhysicalSelection { - window_implementation_id: plan.selected_window_implementation_id.clone().ok_or_else( - || CompileError::Lifecycle { + window_realization_id: plan + .selected_window_implementation_id + .clone() + .ok_or_else(|| CompileError::Lifecycle { query_id: query.query_id.clone(), reason: "Planner returned no concrete window implementation identity".into(), - }, - )?, + })?, expected_reads: plan.expected_reads.ok_or_else(|| CompileError::Lifecycle { query_id: query.query_id.clone(), reason: "missing joint read demand".into(), @@ -2774,7 +2822,7 @@ fn select_lifecycle( reason: "missing source update demand".into(), })? .0 - * query.lifecycle.horizon_seconds, + * query.summary_lifecycle_inputs.horizon_seconds, lifecycle_cost: plan.deployments[0] .alternatives .iter() @@ -2821,7 +2869,7 @@ fn select_lifecycle( /// A warm producer may consume only a source whose semantics its precompute accumulator /// implements. Predicates and shifted ranges remain executable residual nodes. -pub(crate) fn materialization_leaf_contract( +pub(crate) fn raw_materialization_input_contract( node: &SummaryNode, ) -> Result<(String, Option, String), String> { let SummaryExpr::SummaryAgg { child, .. } = &node.expr else { @@ -2972,9 +3020,9 @@ fn immutable_materialization_sources(node: &SummaryNode) -> Option Result<(String, Option, String), String> { if let Some(source) = immutable_materialization_sources(node) { - materialization_leaf_contract(&source[0]) + raw_materialization_input_contract(&source[0]) } else { - materialization_leaf_contract(node) + raw_materialization_input_contract(node) } } @@ -3005,7 +3053,7 @@ fn validate_executable_subdag(node: &Rc) -> Result<(), String> { } fn physical_aggregation( - query: &PlanningQuery, + query: &QueryCompilationInput, selected: &SelectedMaterialization, aggregation_id: String, target: PhysicalDeploymentTarget, @@ -3014,12 +3062,12 @@ fn physical_aggregation( aggregation_id, metric_name: selected.metric.clone(), family: physical_materialization_family(&selected.family), - window_secs: selected.window_secs.unwrap_or(query.window_secs), + window_secs: selected.window_secs.unwrap_or(query.query_lookback_seconds), spatial_filter: selected.spatial_filter.clone(), grouping: selected .group_by .clone() - .unwrap_or_else(|| query.group_by.clone()), + .unwrap_or_else(|| query.group_by_labels.clone()), item_label: selected.item_label.clone(), heap_update_mode: selected.parameters.get("weight_mode").and_then(|mode| { match mode.as_str() { @@ -3090,7 +3138,7 @@ pub(crate) fn aggregation_config_for_materialization( } fn materialization_consumers( - queries: &[PlanningQuery], + queries: &[QueryCompilationInput], target: PhysicalDeploymentTarget, composable: bool, ) -> Result>, CompileError> { @@ -3100,18 +3148,16 @@ fn materialization_consumers( let mut cohort_programs = BTreeMap::>::new(); for (index, query) in queries.iter().enumerate() { - let states = - collect_selected_materializations(&query.post_asap, composable).map_err(|reason| { - CompileError::Query { - query_id: query.query_id.clone(), - reason, - } + let states = collect_selected_materializations(&query.selected_plan_root, composable) + .map_err(|reason| CompileError::Query { + query_id: query.query_id.clone(), + reason, })?; for state in states { if composable && state.window_secs.is_some_and(|window| { !query - .window_implementations + .window_realization_candidates .iter() .any(|candidate| candidate.window_secs == window) }) @@ -3154,7 +3200,7 @@ fn shared_pane_origin_ms( let pane_width = i64::try_from(pane_width_ms) .map_err(|_| "materialized pane width exceeds runtime timestamp range".to_string())?; let Some(workload) = workload else { - // Compatibility-only PlanningRequest callers do not claim a certified + // Compatibility-only PhysicalCompilationRequest callers do not claim a certified // phase. The read path rejects this binding and uses exact fallback. return Ok(None); }; @@ -3484,7 +3530,7 @@ fn stable_plan_id(materializations: &[CollectorMaterialization]) -> u64 { fn stable_workload_plan_id( materializations: &[CollectorMaterialization], - queries: &[PlanningQuery], + queries: &[QueryCompilationInput], ) -> u64 { use std::hash::{Hash, Hasher}; let mut hasher = std::collections::hash_map::DefaultHasher::new(); @@ -3496,6 +3542,89 @@ fn stable_workload_plan_id( hasher.finish() } +// Compatibility imports; new callers use the domain names above. +#[deprecated(note = "Use BackendLocalPhysicalInputs")] +pub use BackendLocalPhysicalInputs as BackendLocalImplementation; +#[deprecated(note = "Use BackendLocalPlanningInput")] +pub use BackendLocalPlanningInput as BackendLocalPlanningSnapshot; +#[deprecated(note = "Use CompiledPhysicalPlan")] +pub use CompiledPhysicalPlan as PhysicalPlan; +#[deprecated(note = "Use LifecycleUnitCosts")] +pub use LifecycleUnitCosts as LifecycleCostEvidence; +#[deprecated(note = "Use PhysicalCompilationRequest")] +pub use PhysicalCompilationRequest as PlanningRequest; +#[deprecated(note = "Use PhysicalDeploymentContext")] +pub use PhysicalDeploymentContext as DeploymentEnvironment; +#[deprecated(note = "Use PhysicalPlanCompiler")] +pub use PhysicalPlanCompiler as PhysicalCompiler; +#[deprecated(note = "Use QueryCompilationInput")] +pub use QueryCompilationInput as PlanningQuery; +#[deprecated(note = "Use SummaryLifecyclePlanningInputs")] +pub use SummaryLifecyclePlanningInputs as LifecyclePlanningInput; +#[deprecated(note = "Use WindowRealizationCandidate")] +pub use WindowRealizationCandidate as WindowImplementationCandidate; +#[deprecated(note = "Use WindowRealizationCostQuote")] +pub use WindowRealizationCostQuote as ImplementationCostEvidence; + +/// Parser/executor frontend for the time-series physical compiler. SQL has its +/// own compilation input and must not silently enter this path. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum QueryFrontend { + PromQl, + MetricsQl, +} +impl QueryFrontend { + pub fn parse( + self, + query: &str, + accuracy: crate::types::AccuracyTarget, + ) -> Result { + crate::query_parser::parse_query_expr_canonical(query, accuracy).map_err(|error| match self + { + Self::PromQl => format!("frontend.promql: {error}"), + Self::MetricsQl => format!("victoriametrics.promql_subset: {error}"), + }) + } + pub fn compile( + self, + request: PhysicalCompilationRequest, + environment: PhysicalDeploymentContext, + ) -> Result { + PhysicalPlanCompiler.compile_for_frontend(request, environment, self) + } +} +impl BackendLocalPlanningInput { + #[deprecated(note = "Use compile_promql")] + pub fn compile(self) -> Result { + self.compile_promql() + } + #[deprecated(note = "Use into_physical_compilation_request")] + pub fn planning_request( + self, + ) -> Result<(PhysicalCompilationRequest, PhysicalDeploymentContext), CompileError> { + self.into_physical_compilation_request() + } +} +impl PhysicalPlanCompiler { + #[deprecated(note = "Use compile_promql")] + pub fn compile( + &self, + request: PhysicalCompilationRequest, + environment: PhysicalDeploymentContext, + ) -> Result { + self.compile_promql(request, environment) + } +} + +#[deprecated(note = "Use build_transmission_plan")] +pub use build_transmission_plan as compile_transmission_plan; +#[deprecated(note = "Use select_logical_roots_for_queries")] +pub use select_logical_roots_for_queries as select_workload_roots; +#[deprecated(note = "Use select_logical_roots_with_error_resource_profiles")] +pub use select_logical_roots_with_error_resource_profiles as select_workload_roots_with_erp; +#[deprecated(note = "Use select_logical_roots_with_trace")] +pub use select_logical_roots_with_trace as select_workload_roots_with_trace; + #[cfg(test)] pub(crate) mod tests { use super::*; @@ -3505,8 +3634,8 @@ pub(crate) mod tests { // miss. Publication passes this document straight to the data plane. #[test] fn compiled_plan_routes_every_materialized_metric() { - let plan = quoted_snapshot(planning_snapshot(), false) - .compile() + let plan = quoted_snapshot(planning_snapshot(), QueryFrontend::PromQl) + .compile_promql() .unwrap(); let routing = &plan.storage_routing; assert_eq!(routing["default_engine"], "asap_query"); @@ -3544,7 +3673,9 @@ pub(crate) mod tests { let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; entry.query = Query("sum_over_time(a[1m]) / sum_over_time(a[10m])".into()); entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let plan = quoted_snapshot(snapshot, false).compile().unwrap(); + let plan = quoted_snapshot(snapshot, crate::physical::compiler::QueryFrontend::PromQl) + .compile_promql() + .unwrap(); assert!(plan.cost_comparison.is_some()); assert_eq!(plan.precompute_plan.materializations.len(), 1); let bindings = plan @@ -3566,22 +3697,26 @@ pub(crate) mod tests { // Synthetic quotes exercise deployment selection in tests, never production defaults. pub(crate) fn quoted_snapshot( - mut snapshot: BackendLocalPlanningSnapshot, - metricsql: bool, - ) -> BackendLocalPlanningSnapshot { + mut snapshot: BackendLocalPlanningInput, + frontend: QueryFrontend, + ) -> BackendLocalPlanningInput { use super::super::workload_cost::{ - manifest, with_exact_alternative, WorkloadCostEvidence, WorkloadQuote, + enumerate_exact_and_materialized_candidates, manifest, WorkloadCostEvidence, + WorkloadQuote, }; - let (request, environment) = snapshot.clone().planning_request().unwrap(); - let quotes = with_exact_alternative(request) + let (request, environment) = snapshot + .clone() + .into_physical_compilation_request() + .unwrap(); + let quotes = enumerate_exact_and_materialized_candidates(request) .unwrap() .into_iter() .enumerate() .filter_map(|(index, candidate)| { - let plan = if metricsql { - PhysicalCompiler.compile_metricsql(candidate.clone(), environment.clone()) + let plan = if frontend == QueryFrontend::MetricsQl { + PhysicalPlanCompiler.compile_metricsql(candidate.clone(), environment.clone()) } else { - PhysicalCompiler.compile(candidate.clone(), environment.clone()) + PhysicalPlanCompiler.compile_promql(candidate.clone(), environment.clone()) } .ok()?; let manifest = manifest(&plan, &candidate.queries).unwrap(); @@ -3611,11 +3746,13 @@ pub(crate) mod tests { /// Optional counter masks must retain the workload's mandatory sketch bindings. #[test] fn costed_mixed_workload_retains_sketches_and_counter_readouts() { - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + let snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-compatibility-demo-snapshot.json" )) .unwrap(); - let plan = quoted_snapshot(snapshot, false).compile().unwrap(); + let plan = quoted_snapshot(snapshot, crate::physical::compiler::QueryFrontend::PromQl) + .compile_promql() + .unwrap(); assert!(!plan.precompute_plan.materializations.is_empty()); for entry in plan.query_plan.entries.values() { assert!(!entry.materialization_bindings().is_empty(), "{entry:#?}"); @@ -3625,31 +3762,31 @@ pub(crate) mod tests { /// A schema marker cannot opt into a legacy deployment policy. #[test] fn only_current_snapshot_schema_is_accepted() { - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + let snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); for version in [0, 1, 3] { let mut old = snapshot.clone(); - old.snapshot_version = version; + old.schema_version = version; assert!(old .clone() - .planning_request() + .into_physical_compilation_request() .unwrap_err() .to_string() .contains("only version 2")); - assert!(old.compile().is_err()); + assert!(old.compile_promql().is_err()); } - assert!(snapshot.planning_request().is_ok()); + assert!(snapshot.into_physical_compilation_request().is_ok()); } #[test] fn installed_partition_must_match_the_bound_dag_reduction() { let mut env = environment(10_000); env.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - env.collector_ids.clear(); - let mut plan = PhysicalCompiler - .compile(request("scope", "sum_over_time(m[1m])"), env) + env.target_collector_ids.clear(); + let mut plan = PhysicalPlanCompiler + .compile_promql(request("scope", "sum_over_time(m[1m])"), env) .unwrap(); let installed = plan .precompute_plan @@ -3696,9 +3833,9 @@ pub(crate) mod tests { ] { let mut environment = environment(10_000); environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - environment.collector_ids.clear(); - let plan = PhysicalCompiler - .compile(request("per-entity", query), environment) + environment.target_collector_ids.clear(); + let plan = PhysicalPlanCompiler + .compile_promql(request("per-entity", query), environment) .unwrap(); assert!( plan.precompute_plan @@ -3715,9 +3852,9 @@ pub(crate) mod tests { ] { let mut environment = environment(10_000); environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - environment.collector_ids.clear(); - let plan = PhysicalCompiler - .compile(request("reduced", query), environment) + environment.target_collector_ids.clear(); + let plan = PhysicalPlanCompiler + .compile_promql(request("reduced", query), environment) .unwrap(); assert!( !plan.precompute_plan.materializations.is_empty(), @@ -3731,8 +3868,10 @@ pub(crate) mod tests { let request = request("counter", "sum(rate(m[1m]))"); let mut environment = environment(10_000); environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - environment.collector_ids.clear(); - let plan = PhysicalCompiler.compile(request, environment).unwrap(); + environment.target_collector_ids.clear(); + let plan = PhysicalPlanCompiler + .compile_promql(request, environment) + .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); assert_eq!( plan.precompute_plan.materializations[0].num_aggregates_to_retain, @@ -3754,8 +3893,8 @@ pub(crate) mod tests { #[test] fn raw_counter_artifact_is_valid_for_backend_precompute() { - let plan = PhysicalCompiler - .compile(request("counter", "rate(m[1m])"), environment(10_000)) + let plan = PhysicalPlanCompiler + .compile_promql(request("counter", "rate(m[1m])"), environment(10_000)) .unwrap(); plan.precompute_plan.validate().unwrap(); let catalog = plan.summary_catalog.clone(); @@ -3780,8 +3919,10 @@ pub(crate) mod tests { .extend(request("gauge", "sum(sum_over_time(g[1m]))").queries); let mut environment = environment(10_000); environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - environment.collector_ids.clear(); - let plan = PhysicalCompiler.compile(workload, environment).unwrap(); + environment.target_collector_ids.clear(); + let plan = PhysicalPlanCompiler + .compile_promql(workload, environment) + .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 2); assert_eq!( plan.query_plan @@ -3804,16 +3945,16 @@ pub(crate) mod tests { // Capability normalization precedes candidate enumeration, avoiding duplicate exact quotes. #[test] fn counter_only_snapshot_has_distinct_local_and_native_cost_alternatives() { - let mut snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; entry.query = Query("rate(m[1m])".into()); entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let (request, _) = snapshot.planning_request().unwrap(); + let (request, _) = snapshot.into_physical_compilation_request().unwrap(); assert_eq!( - super::super::workload_cost::with_exact_alternative(request) + super::super::workload_cost::enumerate_exact_and_materialized_candidates(request) .unwrap() .len(), 3 @@ -3835,8 +3976,8 @@ pub(crate) mod tests { source: "unit-fixture".into(), }; let request = request_with_evidence("topk", query, Some(evidence)).unwrap(); - let plan = PhysicalCompiler - .compile(request, environment(10000)) + let plan = PhysicalPlanCompiler + .compile_promql(request, environment(10000)) .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1, "{query}"); assert_eq!( @@ -3857,8 +3998,8 @@ pub(crate) mod tests { source: "unit-fixture".into(), }; let request = request_with_evidence("topk-rate", query, Some(evidence)).unwrap(); - let plan = PhysicalCompiler - .compile(request, environment(10_000)) + let plan = PhysicalPlanCompiler + .compile_promql(request, environment(10_000)) .unwrap(); let entry = plan.query_plan.entries.values().next().unwrap(); let crate::query_plan::QueryPlanNode::CandidateTopK { inputs, .. } = @@ -3935,7 +4076,7 @@ pub(crate) mod tests { #[test] fn hybrid_weighted_topk_installs_only_candidates_and_delegates_filtered_exact_values() { use crate::query_plan::{ - logical::LogicalOperator, ExternalExactInput, ExternalExactOutput, QueryPlanNode, + logical::ResidualQueryOperator, ExternalExactInput, ExternalExactOutput, QueryPlanNode, }; let query = "topk(2, sum by (job) (rate(m[1m])))"; let evidence = TopKMembershipEvidence { @@ -3946,12 +4087,14 @@ pub(crate) mod tests { source: "unit-fixture".into(), }; let mut request = request_with_evidence("topk-rate", query, Some(evidence)).unwrap(); - request.hybrid_execution = true; + request.allow_mixed_summary_and_exact_execution = true; let mut environment = environment(10_000); environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - environment.collector_ids.clear(); + environment.target_collector_ids.clear(); - let plan = PhysicalCompiler.compile(request, environment).unwrap(); + let plan = PhysicalPlanCompiler + .compile_promql(request, environment) + .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); assert_eq!( @@ -3979,7 +4122,7 @@ pub(crate) mod tests { node, QueryPlanNode::ExactReadout { .. } | QueryPlanNode::Logical { - operator: LogicalOperator::Scan { .. }, + operator: ResidualQueryOperator::Scan { .. }, .. } ))); @@ -4014,8 +4157,8 @@ pub(crate) mod tests { observed_at_unix_ms: 9_500, source: "unit-fixture".into(), }; - let plan = PhysicalCompiler - .compile( + let plan = PhysicalPlanCompiler + .compile_promql( request_with_evidence( "topk-rate", "topk(2, sum by (job) (rate(m[1m])))", @@ -4064,8 +4207,8 @@ pub(crate) mod tests { let mut request = request("bounded", "sum(sum_over_time(m[1m]))"); request.retained_summary_memory_budget_bytes = Some(1); - let error = PhysicalCompiler - .compile(request, environment(10_000)) + let error = PhysicalPlanCompiler + .compile_promql(request, environment(10_000)) .unwrap_err(); assert!(error.to_string().contains("retained summary footprint")); } @@ -4073,22 +4216,29 @@ pub(crate) mod tests { #[test] fn legacy_backend_snapshot_gets_explicit_retained_memory_default_and_alias() { let source = include_str!("../../../docs/examples/asapquery-planning-snapshot.json"); - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(source).unwrap(); + let snapshot: BackendLocalPlanningInput = serde_json::from_str(source).unwrap(); assert_eq!( - snapshot.implementation.max_retained_summary_bytes, + snapshot + .physical_inputs + .retained_summary_memory_budget_bytes, DEFAULT_RETAINED_SUMMARY_MEMORY_BUDGET_BYTES ); let mut value: Value = serde_json::from_str(source).unwrap(); value["implementation"]["maxRetainedSummaryBytes"] = json!(123_456); - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_value(value).unwrap(); - assert_eq!(snapshot.implementation.max_retained_summary_bytes, 123_456); + let snapshot: BackendLocalPlanningInput = serde_json::from_value(value).unwrap(); + assert_eq!( + snapshot + .physical_inputs + .retained_summary_memory_budget_bytes, + 123_456 + ); } - fn environment(now: u64) -> DeploymentEnvironment { - DeploymentEnvironment { + fn environment(now: u64) -> PhysicalDeploymentContext { + PhysicalDeploymentContext { target: PhysicalDeploymentTarget::DistributedCollectors, - collector_ids: vec!["edge-a".into(), "edge-b".into()], + target_collector_ids: vec!["edge-a".into(), "edge-b".into()], capability_snapshot_id: "caps-7".into(), observed_at_unix_ms: now, max_evidence_age_ms: 60_000, @@ -4103,20 +4253,20 @@ pub(crate) mod tests { query_id: &str, promql: &str, evidence: Option, - ) -> Result { + ) -> Result { let accuracy = AccuracyTarget::EpsilonDelta { epsilon: 0.01, delta: 0.01, }; let parsed = crate::query_parser::parse_query_expr_canonical(promql, accuracy.clone()) .expect("canonical query"); - let lifecycle = LifecyclePlanningInput { + let lifecycle = SummaryLifecyclePlanningInputs { evaluation_interval_ms: 10_000, ingestion_rate_per_second: 100.0, evidence_observed_at_unix_ms: 9_500, evidence_valid_for_ms: 60_000, horizon_seconds: 300.0, - costs: LifecycleCostEvidence { + costs: LifecycleUnitCosts { build: 10.0, maintenance_per_update: 0.001, read: 0.1, @@ -4129,29 +4279,29 @@ pub(crate) mod tests { if let Some(evidence) = evidence { evidence_by_query.insert(query_id.to_string(), evidence); } - Ok(PlanningRequest { - logical_selection: Vec::new(), - hybrid_execution: false, - materialization_policy: None, + Ok(PhysicalCompilationRequest { + planner_selection_trace: Vec::new(), + allow_mixed_summary_and_exact_execution: false, + enabled_materialization_keys: None, query_workload: None, - queries: vec![PlanningQuery { + queries: vec![QueryCompilationInput { query_id: query_id.into(), query_string: promql.into(), - post_asap, - source: Source::TimeSeries { metric: "m".into() }, - window_secs: 60, - group_by: vec![], - accuracy, - lifecycle, - window_implementations: vec![WindowImplementationCandidate { + selected_plan_root: post_asap, + legacy_query_source: Source::TimeSeries { metric: "m".into() }, + query_lookback_seconds: 60, + group_by_labels: vec![], + accuracy_target: accuracy, + summary_lifecycle_inputs: lifecycle, + window_realization_candidates: vec![WindowRealizationCandidate { derived: false, cohort_only: false, - implementation_id: "collector-tumbling-v1".into(), + realization_id: "collector-tumbling-v1".into(), framework: SummaryWindowFramework::Tumbling, window_secs: 60, slide_secs: 60, layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 60 }, - cost: ImplementationCostEvidence { + cost: WindowRealizationCostQuote { model_version: "test-cost-v1".into(), workload_fingerprint: "test-workload".into(), observed_at_unix_ms: 9_500, @@ -4165,19 +4315,19 @@ pub(crate) mod tests { weighted_cost: 1.0, }, }], - runtime_policy: RuntimeRulePolicy::default(), + materialization_runtime_policy: RuntimeRulePolicy::default(), }], - evidence: evidence_by_query, + topk_membership_evidence_by_query_id: evidence_by_query, erp: None, exact_composition_costs: HashMap::new(), planner_revision: PLANNER_REVISION.into(), source_sample_interval_ms: None, - query_staleness_margin_ms: 0, + query_retention_margin_ms: 0, retained_summary_memory_budget_bytes: None, }) } - fn request(query_id: &str, promql: &str) -> PlanningRequest { + fn request(query_id: &str, promql: &str) -> PhysicalCompilationRequest { request_with_evidence(query_id, promql, None).expect("post-ASAP selection") } @@ -4203,16 +4353,19 @@ pub(crate) mod tests { #[test] fn immutable_nested_summary_keeps_actual_source_and_derived_bindings() { let mut workload = request("nested", "quantile(0.9, sum_over_time(m[1m]))"); - workload.hybrid_execution = true; + workload.allow_mixed_summary_and_exact_execution = true; let states = - collect_selected_materializations(&workload.queries[0].post_asap, true).unwrap(); + collect_selected_materializations(&workload.queries[0].selected_plan_root, true) + .unwrap(); assert_eq!(states.len(), 2, "source and consumer must both be selected"); assert!(immutable_materialization_sources(&states[0].node).is_none()); assert!(immutable_materialization_sources(&states[1].node).is_some()); let mut deployment = environment(10_000); deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - deployment.collector_ids.clear(); - let plan = PhysicalCompiler.compile(workload, deployment).unwrap(); + deployment.target_collector_ids.clear(); + let plan = PhysicalPlanCompiler + .compile_promql(workload, deployment) + .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 2); let derived = plan .precompute_plan @@ -4246,16 +4399,19 @@ pub(crate) mod tests { "nested", "quantile(0.9, sum_over_time(m[1m]) + sum_over_time(n[1m]))", ); - workload.hybrid_execution = true; + workload.allow_mixed_summary_and_exact_execution = true; let states = - collect_selected_materializations(&workload.queries[0].post_asap, true).unwrap(); + collect_selected_materializations(&workload.queries[0].selected_plan_root, true) + .unwrap(); assert_eq!(states.len(), 3, "source and consumer must both be selected"); assert!(immutable_materialization_sources(&states[0].node).is_none()); assert!(immutable_materialization_sources(&states[2].node).is_some()); let mut deployment = environment(10_000); deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - deployment.collector_ids.clear(); - let plan = PhysicalCompiler.compile(workload, deployment).unwrap(); + deployment.target_collector_ids.clear(); + let plan = PhysicalPlanCompiler + .compile_promql(workload, deployment) + .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 3); assert!(plan .precompute_plan @@ -4305,19 +4461,24 @@ pub(crate) mod tests { fn distinct_range_compiles_to_partitioned_hll() { let mut deployment = environment(10_000); deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - deployment.collector_ids.clear(); + deployment.target_collector_ids.clear(); let mut workload = request("distinct", "distinct_over_time(m{job=\"api\"}[1m])"); let query = &mut workload.queries[0]; // HLL's modeled RSE does not certify a failure probability. - query.accuracy = AccuracyTarget::Epsilon(0.05); + query.accuracy_target = AccuracyTarget::Epsilon(0.05); let parsed = crate::query_parser::parse_query_expr_canonical( &query.query_string, - query.accuracy.clone(), + query.accuracy_target.clone(), + ) + .unwrap(); + query.selected_plan_root = select_post_asap( + &parsed, + query.accuracy_target.clone(), + &query.summary_lifecycle_inputs, + None, ) .unwrap(); - query.post_asap = - select_post_asap(&parsed, query.accuracy.clone(), &query.lifecycle, None).unwrap(); - let plan = PhysicalCompiler + let plan = PhysicalPlanCompiler .compile_metricsql(workload, deployment) .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); @@ -4349,20 +4510,25 @@ pub(crate) mod tests { fn unsupported_cardinality_family_fails_admission() { let mut deployment = environment(10_000); deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - deployment.collector_ids.clear(); + deployment.target_collector_ids.clear(); let mut workload = request("confidence", "distinct_over_time(m[1m])"); - workload.hybrid_execution = true; - let result = PhysicalCompiler.compile_metricsql(workload, deployment); + workload.allow_mixed_summary_and_exact_execution = true; + let result = PhysicalPlanCompiler.compile_metricsql(workload, deployment); assert!(matches!(result, Err(CompileError::QueryPlan(_)))); } #[test] fn snapshot_metricsql_entry_uses_the_shared_serving_language_contract() { - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + let snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-compatibility-demo-snapshot.json" )) .unwrap(); - let plan = quoted_snapshot(snapshot, true).compile_metricsql().unwrap(); + let plan = quoted_snapshot( + snapshot, + crate::physical::compiler::QueryFrontend::MetricsQl, + ) + .compile_metricsql() + .unwrap(); assert!(!plan.query_plan.entries.is_empty()); assert!(plan .query_plan @@ -4378,8 +4544,8 @@ pub(crate) mod tests { second.query_string = "max_over_time(b[1m])".into(); workload.queries.push(second); - let error = PhysicalCompiler - .compile(workload, environment(10_000)) + let error = PhysicalPlanCompiler + .compile_promql(workload, environment(10_000)) .unwrap_err(); assert!(matches!( error, @@ -4395,11 +4561,11 @@ pub(crate) mod tests { "increase(counter_probe{case=\"reset\"}[5s])", ] { let mut workload = request("counter", text); - workload.hybrid_execution = true; + workload.allow_mixed_summary_and_exact_execution = true; let mut deployment = environment(10_000); deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - deployment.collector_ids.clear(); - let plan = PhysicalCompiler + deployment.target_collector_ids.clear(); + let plan = PhysicalPlanCompiler .compile_metricsql(workload, deployment) .unwrap(); assert!(plan.precompute_plan.materializations.is_empty()); @@ -4412,11 +4578,11 @@ pub(crate) mod tests { #[test] fn metricsql_counter_gate_preserves_an_independent_summary_sibling() { let mut workload = request("mixed", "max_over_time(m[1m]) + rate(m[1m])"); - workload.hybrid_execution = true; + workload.allow_mixed_summary_and_exact_execution = true; let mut deployment = environment(10_000); deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - deployment.collector_ids.clear(); - let plan = PhysicalCompiler + deployment.target_collector_ids.clear(); + let plan = PhysicalPlanCompiler .compile_metricsql(workload, deployment) .unwrap(); assert!(!plan.precompute_plan.materializations.is_empty()); @@ -4431,26 +4597,29 @@ pub(crate) mod tests { ))); let entry = plan.query_plan.entries.values().next().unwrap(); assert!(!entry.materialization_bindings().is_empty()); - assert!(entry.nodes.values().any(|node| matches!( - node, - crate::query_plan::QueryPlanNode::ExternalExact { .. } - | crate::query_plan::QueryPlanNode::Logical { - operator: crate::query_plan::logical::LogicalOperator::ExactSubquery { .. }, - .. - } - ))); + assert!( + entry.nodes.values().any(|node| matches!( + node, + crate::query_plan::QueryPlanNode::ExternalExact { .. } + | crate::query_plan::QueryPlanNode::Logical { + operator: + crate::query_plan::residual::ResidualQueryOperator::ExactSubquery { .. }, + .. + } + )) + ); } #[test] fn metricsql_compilation_publishes_a_language_tagged_query_entry() { let query = "mad_over_time(m[1m])"; let mut workload = request("vm-q", "last_over_time(m[1m])"); - let accuracy = workload.queries[0].accuracy.clone(); + let accuracy = workload.queries[0].accuracy_target.clone(); let canonical = asap_frontend_promql::lower_promql(query, accuracy.clone()).unwrap(); workload.queries[0].query_string = query.into(); - workload.queries[0].post_asap = + workload.queries[0].selected_plan_root = crate::planner_selection::keep_pre_asap(&canonical).unwrap(); - let plan = PhysicalCompiler + let plan = PhysicalPlanCompiler .compile_metricsql(workload, environment(10_000)) .unwrap(); let identity = canonical_promql(query).unwrap(); @@ -4468,7 +4637,7 @@ pub(crate) mod tests { let roots = vec![Rc::new( crate::query_parser::parse_query_expr_canonical( &workload.queries[0].query_string, - workload.queries[0].accuracy.clone(), + workload.queries[0].accuracy_target.clone(), ) .unwrap(), )]; @@ -4499,16 +4668,16 @@ pub(crate) mod tests { max_memory_bytes: None, }, }; - select_workload_roots_with_erp( + select_logical_roots_with_error_resource_profiles( &mut workload.queries, roots, - &workload.evidence, + &workload.topk_membership_evidence_by_query_id, &workload.exact_composition_costs, Some(&erp), ) .unwrap(); assert!(matches!( - workload.queries[0].post_asap.expr, + workload.queries[0].selected_plan_root.expr, SummaryExpr::KeepPreAsap(_) )); } @@ -4562,18 +4731,18 @@ pub(crate) mod tests { ); let mut workload = request("q", "quantile_over_time(0.9, m[1m])"); - workload.queries[0].accuracy = AccuracyTarget::Epsilon(0.06); + workload.queries[0].accuracy_target = AccuracyTarget::Epsilon(0.06); let root = Rc::new( crate::query_parser::parse_query_expr_canonical( &workload.queries[0].query_string, - workload.queries[0].accuracy.clone(), + workload.queries[0].accuracy_target.clone(), ) .unwrap(), ); - select_workload_roots_with_erp( + select_logical_roots_with_error_resource_profiles( &mut workload.queries, vec![root], - &workload.evidence, + &workload.topk_membership_evidence_by_query_id, &workload.exact_composition_costs, Some(&erp), ) @@ -4595,15 +4764,19 @@ pub(crate) mod tests { } } assert!( - contains_measured_kll(&workload.queries[0].post_asap), + contains_measured_kll(&workload.queries[0].selected_plan_root), "ERP hit was lost before physical compilation: {:#?}", - workload.queries[0].post_asap + workload.queries[0].selected_plan_root ); - let guarantee = workload.queries[0].post_asap.guarantee.as_ref().unwrap(); + let guarantee = workload.queries[0] + .selected_plan_root + .guarantee + .as_ref() + .unwrap(); assert_eq!(guarantee.failure_probability.evaluate(), None); assert!(!guarantee.is_exact()); - let plan = PhysicalCompiler - .compile(workload, environment(10000)) + let plan = PhysicalPlanCompiler + .compile_promql(workload, environment(10000)) .unwrap(); assert_eq!(plan.precompute_plan.materializations[0].parameters["k"], 32); let empirical_identity = plan.precompute_plan.materializations[0].policy_fingerprint(); @@ -4628,7 +4801,7 @@ pub(crate) mod tests { (unsupported, AccuracyTarget::Epsilon(0.06), true), ] { let mut workload = request("q", "quantile_over_time(0.9, m[1m])"); - workload.queries[0].accuracy = accuracy.clone(); + workload.queries[0].accuracy_target = accuracy.clone(); let root = Rc::new( crate::query_parser::parse_query_expr_canonical( &workload.queries[0].query_string, @@ -4636,23 +4809,25 @@ pub(crate) mod tests { ) .unwrap(), ); - select_workload_roots_with_erp( + select_logical_roots_with_error_resource_profiles( &mut workload.queries, vec![root], - &workload.evidence, + &workload.topk_membership_evidence_by_query_id, &workload.exact_composition_costs, Some(&policy), ) .unwrap(); if exact { assert!(matches!( - workload.queries[0].post_asap.expr, + workload.queries[0].selected_plan_root.expr, SummaryExpr::KeepPreAsap(_) )); } else { - assert!(!contains_measured_kll(&workload.queries[0].post_asap)); - let plan = PhysicalCompiler - .compile(workload, environment(10000)) + assert!(!contains_measured_kll( + &workload.queries[0].selected_plan_root + )); + let plan = PhysicalPlanCompiler + .compile_promql(workload, environment(10000)) .unwrap(); let state = &plan.precompute_plan.materializations[0]; assert!(state.parameters["k"].as_u64().unwrap() > 32); @@ -4731,11 +4906,11 @@ pub(crate) mod tests { fn sum_rate_uses_summary_child_only_with_measured_composition_costs() { let promql = "sum by (job) (rate(m[1m]))"; let mut with_evidence = request("topk-rate", promql); - with_evidence.hybrid_execution = true; + with_evidence.allow_mixed_summary_and_exact_execution = true; let root = Rc::new( crate::query_parser::parse_query_expr_canonical( promql, - with_evidence.queries[0].accuracy.clone(), + with_evidence.queries[0].accuracy_target.clone(), ) .unwrap(), ); @@ -4743,17 +4918,19 @@ pub(crate) mod tests { "topk-rate".into(), measured_exact_composition_rows(&root, 9_500), ); - select_workload_roots( + select_logical_roots_for_queries( &mut with_evidence.queries, vec![root], - &with_evidence.evidence, + &with_evidence.topk_membership_evidence_by_query_id, &with_evidence.exact_composition_costs, ) .unwrap(); let mut backend = environment(10_000); backend.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - backend.collector_ids.clear(); - let plan = PhysicalCompiler.compile(with_evidence, backend).unwrap(); + backend.target_collector_ids.clear(); + let plan = PhysicalPlanCompiler + .compile_promql(with_evidence, backend) + .unwrap(); assert!( !plan.summary_catalog.materializations.is_empty(), "measured exact-composition evidence must expose the rate child as a SummaryStore binding" @@ -4764,25 +4941,27 @@ pub(crate) mod tests { fn sum_rate_without_composition_costs_does_not_invent_a_composition_cost() { let promql = "sum by (job) (rate(m[1m]))"; let mut unavailable = request("topk-rate", promql); - unavailable.hybrid_execution = true; + unavailable.allow_mixed_summary_and_exact_execution = true; let root = Rc::new( crate::query_parser::parse_query_expr_canonical( promql, - unavailable.queries[0].accuracy.clone(), + unavailable.queries[0].accuracy_target.clone(), ) .unwrap(), ); - select_workload_roots( + select_logical_roots_for_queries( &mut unavailable.queries, vec![root], - &unavailable.evidence, + &unavailable.topk_membership_evidence_by_query_id, &unavailable.exact_composition_costs, ) .unwrap(); let mut backend = environment(10_000); backend.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - backend.collector_ids.clear(); - let plan = PhysicalCompiler.compile(unavailable, backend).unwrap(); + backend.target_collector_ids.clear(); + let plan = PhysicalPlanCompiler + .compile_promql(unavailable, backend) + .unwrap(); // Planner 54f can realize this particular shape directly as an exact // counter readout plus a query-time reduce; it does not require an // ExactComposition candidate. The absence of evidence must therefore @@ -4883,21 +5062,21 @@ pub(crate) mod tests { Rc::new( crate::query_parser::parse_query_expr_canonical( &query.query_string, - query.accuracy.clone(), + query.accuracy_target.clone(), ) .unwrap(), ) }) .collect(); - select_workload_roots( + select_logical_roots_for_queries( &mut workload.queries, roots, - &workload.evidence, + &workload.topk_membership_evidence_by_query_id, &workload.exact_composition_costs, ) .unwrap(); - let bundle = PhysicalCompiler - .compile(workload, environment(10000)) + let bundle = PhysicalPlanCompiler + .compile_promql(workload, environment(10000)) .unwrap(); assert_eq!(bundle.query_plan.entries.len(), 2); assert_eq!(bundle.collector_plans[0].materializations.len(), 1); @@ -4916,10 +5095,10 @@ pub(crate) mod tests { #[test] fn shared_selection_rejects_incomplete_root_mapping() { let mut workload = request("q", "quantile_over_time(0.9, m[1m])"); - assert!(select_workload_roots( + assert!(select_logical_roots_for_queries( &mut workload.queries, vec![], - &workload.evidence, + &workload.topk_membership_evidence_by_query_id, &workload.exact_composition_costs, ) .is_err()); @@ -4928,8 +5107,8 @@ pub(crate) mod tests { // Adding another readout adds recurring reads, not another update stream. #[test] fn joint_lifecycle_charges_shared_updates_once() { - let baseline = PhysicalCompiler - .compile( + let baseline = PhysicalPlanCompiler + .compile_promql( request("q90", "quantile_over_time(0.9, m[1m])"), environment(10000), ) @@ -4938,10 +5117,10 @@ pub(crate) mod tests { let mut second = request("q99", "quantile_over_time(0.99, m[1m])") .queries .remove(0); - second.lifecycle.evaluation_interval_ms = 20000; + second.summary_lifecycle_inputs.evaluation_interval_ms = 20000; workload.queries.push(second); - let shared = PhysicalCompiler - .compile(workload, environment(10000)) + let shared = PhysicalPlanCompiler + .compile_promql(workload, environment(10000)) .unwrap(); assert_eq!(shared.lifecycle_estimates.len(), 1); let estimate = &shared.lifecycle_estimates[0]; @@ -4966,10 +5145,10 @@ pub(crate) mod tests { let mut second = request("q99", "quantile_over_time(0.99, m[1m])") .queries .remove(0); - second.lifecycle.ingestion_rate_per_second = 200.0; + second.summary_lifecycle_inputs.ingestion_rate_per_second = 200.0; workload.queries.push(second); assert!(matches!( - PhysicalCompiler.compile(workload, environment(10000)), + PhysicalPlanCompiler.compile_promql(workload, environment(10000)), Err(CompileError::Lifecycle { .. }) )); } @@ -4993,10 +5172,10 @@ pub(crate) mod tests { let mut env = environment(10_000); env.target = target; if target == PhysicalDeploymentTarget::BackendLocalRemoteWrite { - env.collector_ids.clear(); + env.target_collector_ids.clear(); } - let bundle = PhysicalCompiler - .compile(workload, env) + let bundle = PhysicalPlanCompiler + .compile_promql(workload, env) .expect("shared compile"); assert_eq!(bundle.query_plan.entries.len(), 2); assert_eq!(bundle.summary_catalog.materializations.len(), 1); @@ -5019,15 +5198,15 @@ pub(crate) mod tests { #[test] fn adding_shared_consumer_changes_plan_identity() { let workload = request("q90", "quantile_over_time(0.90, m[1m])"); - let one = PhysicalCompiler - .compile(workload, environment(10_000)) + let one = PhysicalPlanCompiler + .compile_promql(workload, environment(10_000)) .unwrap(); let mut workload = request("q90", "quantile_over_time(0.90, m[1m])"); workload .queries .extend(request("q99", "quantile_over_time(0.99, m[1m])").queries); - let two = PhysicalCompiler - .compile(workload, environment(10_000)) + let two = PhysicalPlanCompiler + .compile_promql(workload, environment(10_000)) .unwrap(); assert_ne!(one.envelope.plan_id, two.envelope.plan_id); assert_eq!(two.collector_plans[0].materializations.len(), 1); @@ -5037,10 +5216,10 @@ pub(crate) mod tests { fn different_sources_do_not_share_materializations() { let mut workload = request("qm", "quantile_over_time(0.90, m[1m])"); let mut other = request("qn", "quantile_over_time(0.90, n[1m])"); - other.queries[0].source = Source::TimeSeries { metric: "n".into() }; + other.queries[0].legacy_query_source = Source::TimeSeries { metric: "n".into() }; workload.queries.extend(other.queries); - let bundle = PhysicalCompiler - .compile(workload, environment(10_000)) + let bundle = PhysicalPlanCompiler + .compile_promql(workload, environment(10_000)) .unwrap(); assert_eq!(bundle.summary_catalog.materializations.len(), 2); assert_eq!(bundle.precompute_plan.materializations.len(), 2); @@ -5055,8 +5234,8 @@ pub(crate) mod tests { workload .queries .extend(request("increase", "increase(m[1m])").queries); - let bundle = PhysicalCompiler - .compile(workload, environment(10_000)) + let bundle = PhysicalPlanCompiler + .compile_promql(workload, environment(10_000)) .unwrap(); assert_eq!(bundle.query_plan.entries.len(), 2); assert_eq!(bundle.precompute_plan.materializations.len(), 1); @@ -5070,11 +5249,11 @@ pub(crate) mod tests { fn shared_materialization_rejects_conflicting_deployment_contracts() { let mut workload = request("q90", "quantile_over_time(0.90, m[1m])"); let mut other = request("q99", "quantile_over_time(0.99, m[1m])"); - other.queries[0].window_implementations[0].implementation_id = + other.queries[0].window_realization_candidates[0].realization_id = "another-implementation".into(); workload.queries.extend(other.queries); - let error = PhysicalCompiler - .compile(workload, environment(10_000)) + let error = PhysicalPlanCompiler + .compile_promql(workload, environment(10_000)) .expect_err("conflicting shared state must fail before publication"); assert!(error .to_string() @@ -5084,7 +5263,7 @@ pub(crate) mod tests { #[test] fn exact_dashboard_binds_sum_and_count_to_one_local_producer() { // Both dashboard roots use one packed raw accumulator, with explicit readouts. - let mut snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); @@ -5097,8 +5276,8 @@ pub(crate) mod tests { .into(), ); entries.push(mean); - let (request, env) = snapshot.planning_request().unwrap(); - let bundle = PhysicalCompiler.compile(request, env).unwrap(); + let (request, env) = snapshot.into_physical_compilation_request().unwrap(); + let bundle = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!(bundle.precompute_plan.materializations.len(), 1); assert_eq!(bundle.query_plan.entries.len(), 2); for entry in bundle.query_plan.entries.values() { @@ -5118,7 +5297,7 @@ pub(crate) mod tests { .any(|entry| entry.nodes.values().any(|node| matches!( node, crate::query_plan::QueryPlanNode::Logical { - operator: crate::query_plan::logical::LogicalOperator::Binary { .. }, + operator: crate::query_plan::residual::ResidualQueryOperator::Binary { .. }, .. } )))); @@ -5126,7 +5305,7 @@ pub(crate) mod tests { #[test] fn invalid_unselected_candidate_is_canonicalized_to_exact_fallback() { - let mut snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); @@ -5134,16 +5313,16 @@ pub(crate) mod tests { entry.query = Query("sum by (service) (sum_over_time(m[1m]) / count_over_time(m[1m]))".into()); entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let (mut request, _) = snapshot.planning_request().unwrap(); + let (mut request, _) = snapshot.into_physical_compilation_request().unwrap(); assert!(!matches!( - request.queries[0].post_asap.expr, + request.queries[0].selected_plan_root.expr, SummaryExpr::KeepPreAsap(_) )); preserve_invalid_exact_fallback_roots(&mut request.queries, true).unwrap(); assert!(matches!( - request.queries[0].post_asap.expr, + request.queries[0].selected_plan_root.expr, SummaryExpr::KeepPreAsap(_) )); } @@ -5151,8 +5330,8 @@ pub(crate) mod tests { #[test] fn selected_maintenance_dependency_still_requires_a_valid_executable_dag() { let mut request = request("invalid-dependency", "sum(sum_over_time(m[1m]))"); - let selected = request.queries[0].post_asap.clone(); - request.queries[0].post_asap = Rc::new(SummaryNode { + let selected = request.queries[0].selected_plan_root.clone(); + request.queries[0].selected_plan_root = Rc::new(SummaryNode { expr: SummaryExpr::BinaryOp { timing: planner_types::post_asap::ExecutionTiming::ReadTime, lhs: selected.clone(), @@ -5167,12 +5346,14 @@ pub(crate) mod tests { schema: selected.schema.clone(), guarantee: None, }); - request.hybrid_execution = true; + request.allow_mixed_summary_and_exact_execution = true; let mut environment = environment(10_000); environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - environment.collector_ids.clear(); + environment.target_collector_ids.clear(); - let error = PhysicalCompiler.compile(request, environment).unwrap_err(); + let error = PhysicalPlanCompiler + .compile_promql(request, environment) + .unwrap_err(); assert!(error.to_string().contains("invalid executable subDAG")); } @@ -5188,16 +5369,18 @@ pub(crate) mod tests { "sum_over_time(m[1m]) / count_over_time(m[5m])", "sum_over_time(m[1m] offset 1h) / count_over_time(m[1m] offset 1h)", ] { - let mut snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); let entries = snapshot.query_workload.repeating_queries.as_mut().unwrap(); entries[0].query = Query(query.into()); entries[0].requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let (mut request, environment) = snapshot.planning_request().unwrap(); - request.hybrid_execution = false; - let bundle = PhysicalCompiler.compile(request, environment).unwrap(); + let (mut request, environment) = snapshot.into_physical_compilation_request().unwrap(); + request.allow_mixed_summary_and_exact_execution = false; + let bundle = PhysicalPlanCompiler + .compile_promql(request, environment) + .unwrap(); assert!(bundle.precompute_plan.materializations.is_empty()); assert!(bundle.query_plan.entries.values().all(|entry| matches!( entry.nodes[&entry.root], @@ -5209,7 +5392,7 @@ pub(crate) mod tests { // Local composition must evaluate per-series division before the outer sum. #[test] fn composable_non_additive_rollup_does_not_pool_raw_producers() { - let mut snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); @@ -5217,16 +5400,18 @@ pub(crate) mod tests { entry.query = Query("sum by (service) (sum_over_time(m[1m]) / count_over_time(m[1m]))".into()); entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let (request, environment) = snapshot.planning_request().unwrap(); - let candidates = super::super::workload_cost::with_exact_alternative(request).unwrap(); - match PhysicalCompiler.compile(candidates[0].clone(), environment.clone()) { + let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); + let candidates = + super::super::workload_cost::enumerate_exact_and_materialized_candidates(request) + .unwrap(); + match PhysicalPlanCompiler.compile_promql(candidates[0].clone(), environment.clone()) { Ok(plan) => assert!(plan.precompute_plan.materializations.is_empty()), Err(error) => assert!(error .to_string() .contains("semantically identical original subtree witness")), } - let native = PhysicalCompiler - .compile(candidates.last().unwrap().clone(), environment) + let native = PhysicalPlanCompiler + .compile_promql(candidates.last().unwrap().clone(), environment) .unwrap(); assert!(native.precompute_plan.materializations.is_empty()); } @@ -5234,15 +5419,15 @@ pub(crate) mod tests { #[test] fn composable_per_entity_window_installs_isolated_state() { use crate::query_plan::QueryPlanNode; - let mut snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; entry.query = Query("sum_over_time(m[1m])".into()); entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let (request, env) = snapshot.planning_request().unwrap(); - let plan = PhysicalCompiler.compile(request, env).unwrap(); + let (request, env) = snapshot.into_physical_compilation_request().unwrap(); + let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); assert_eq!( plan.precompute_plan.materializations[0].partitioning, @@ -5259,7 +5444,7 @@ pub(crate) mod tests { // remains valid for both readouts. #[test] fn composable_binary_binds_independent_source_windows() { - let mut snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); @@ -5268,8 +5453,8 @@ pub(crate) mod tests { entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); // The derivation now covers both range selectors, so this no longer // needs a hand-supplied 5m candidate to keep `b` from falling back. - let (request, env) = snapshot.planning_request().unwrap(); - let plan = PhysicalCompiler.compile(request, env).unwrap(); + let (request, env) = snapshot.into_physical_compilation_request().unwrap(); + let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); let bindings = plan .query_plan .entries @@ -5312,9 +5497,9 @@ pub(crate) mod tests { json!("quantile(0.9, sum_over_time(m[1m]))"); value["data_workload"]["ingestion_rate"]["value"] = json!(rate); value["query_workload"]["data_workload"]["ingestion_rate"]["value"] = json!(rate); - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_value(value).unwrap(); - let (request, env) = snapshot.planning_request().unwrap(); - let plan = PhysicalCompiler.compile(request, env).unwrap(); + let snapshot: BackendLocalPlanningInput = serde_json::from_value(value).unwrap(); + let (request, env) = snapshot.into_physical_compilation_request().unwrap(); + let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); assert!(plan .precompute_plan .materializations @@ -5332,7 +5517,7 @@ pub(crate) mod tests { #[test] fn raw_leaf_keeps_its_cadence_beside_a_same_window_derived_cohort() { let mut snapshot = planning_snapshot(); - let model = snapshot.implementation.window_cost_model.clone(); + let model = snapshot.physical_inputs.window_cost_model.clone(); let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; entry.query = Query("quantile(0.9, sum_over_time(m[1m]))".into()); entry.demand = RepeatedDemand::FixedIntervalAt { @@ -5342,10 +5527,10 @@ pub(crate) mod tests { let mut right = snapshot.clone(); right.query_workload.repeating_queries.as_mut().unwrap()[0].query = Query("sum_over_time(n[1m])".into()); - let (mut request, env) = snapshot.planning_request().unwrap(); - let (right, _) = right.planning_request().unwrap(); - let left = request.queries[0].post_asap.clone(); - let right = right.queries[0].post_asap.clone(); + let (mut request, env) = snapshot.into_physical_compilation_request().unwrap(); + let (right, _) = right.into_physical_compilation_request().unwrap(); + let left = request.queries[0].selected_plan_root.clone(); + let right = right.queries[0].selected_plan_root.clone(); let right = Rc::new(SummaryNode { expr: SummaryExpr::ValueOperation { timing: planner_types::post_asap::ExecutionTiming::ReadTime, @@ -5355,7 +5540,7 @@ pub(crate) mod tests { schema: right.schema.clone(), guarantee: None, }); - request.queries[0].post_asap = Rc::new(SummaryNode { + request.queries[0].selected_plan_root = Rc::new(SummaryNode { expr: SummaryExpr::BinaryOp { timing: planner_types::post_asap::ExecutionTiming::ReadTime, lhs: left.clone(), @@ -5382,14 +5567,14 @@ pub(crate) mod tests { .query = Query(text.into()); prepare_window_implementations(&mut request.queries[0], &model, env.target, 0).unwrap(); request.queries[0] - .window_implementations + .window_realization_candidates .retain(|candidate| { matches!( candidate.layout, asap_types::WindowMaterializationLayout::Pane { .. } ) }); - let plan = PhysicalCompiler.compile(request, env).unwrap(); + let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); assert!(plan .precompute_plan .materializations @@ -5408,9 +5593,9 @@ pub(crate) mod tests { // A full-window producer keeps overlapping accumulators alive even before publication. #[test] fn derived_window_regression_resident_cost() { - let template = planning_snapshot().implementation.window_cost_model.cost; + let template = planning_snapshot().physical_inputs.window_cost_model.cost; let mut lifecycle = planning_lifecycle(); - lifecycle.costs = LifecycleCostEvidence { + lifecycle.costs = LifecycleUnitCosts { build: 0.0, maintenance_per_update: 0.0, read: 0.0, @@ -5442,8 +5627,8 @@ pub(crate) mod tests { interval: RepetitionInterval(interval), evaluation_phase: planner_types::workload::TimestampMs(0), }; - let (request, env) = snapshot.planning_request().unwrap(); - let plan = PhysicalCompiler.compile(request, env).unwrap(); + let (request, env) = snapshot.into_physical_compilation_request().unwrap(); + let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), expected_states); let entry = plan.query_plan.entries.values().next().unwrap(); let bindings = entry.materialization_bindings(); @@ -5480,8 +5665,8 @@ pub(crate) mod tests { evaluation_phase: planner_types::workload::TimestampMs(phase), }; entries.push(second); - let (request, env) = snapshot.planning_request().unwrap(); - let plan = PhysicalCompiler.compile(request, env).unwrap(); + let (request, env) = snapshot.into_physical_compilation_request().unwrap(); + let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!( plan.precompute_plan.materializations.len(), if phase == 0 { 1 } else { 2 } @@ -5500,11 +5685,11 @@ pub(crate) mod tests { #[test] fn derived_cost_overflow_is_rejected() { let snapshot = planning_snapshot(); - let (mut request, env) = snapshot.planning_request().unwrap(); + let (mut request, env) = snapshot.into_physical_compilation_request().unwrap(); let query = &mut request.queries[0]; - let mut lifecycle = query.lifecycle.clone(); + let mut lifecycle = query.summary_lifecycle_inputs.clone(); lifecycle.costs.build = f64::MAX; - let candidate = &mut query.window_implementations[0]; + let candidate = &mut query.window_realization_candidates[0]; candidate.cost = derived_window_cost( &candidate.cost, &lifecycle, @@ -5524,25 +5709,32 @@ pub(crate) mod tests { let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; entry.query = Query(query.into()); entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let (derived, _) = snapshot.clone().planning_request().unwrap(); - snapshot.implementation.window_cost_model.quotes = - derived.queries[0].window_implementations.clone(); - let (request, env) = snapshot.planning_request().unwrap(); + let (derived, _) = snapshot + .clone() + .into_physical_compilation_request() + .unwrap(); + snapshot.physical_inputs.window_cost_model.quotes = + derived.queries[0].window_realization_candidates.clone(); + let (request, env) = snapshot.into_physical_compilation_request().unwrap(); assert!(request.queries[0] - .window_implementations + .window_realization_candidates .iter() .any(|c| !c.derived)); - let plan = PhysicalCompiler.compile(request, env).unwrap(); + let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 2); } - fn planning_lifecycle() -> LifecyclePlanningInput { - planning_snapshot().planning_request().unwrap().0.queries[0] - .lifecycle + fn planning_lifecycle() -> SummaryLifecyclePlanningInputs { + planning_snapshot() + .into_physical_compilation_request() + .unwrap() + .0 + .queries[0] + .summary_lifecycle_inputs .clone() } - fn planning_snapshot() -> BackendLocalPlanningSnapshot { + fn planning_snapshot() -> BackendLocalPlanningInput { serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) @@ -5554,7 +5746,7 @@ pub(crate) mod tests { // answers with results that only change once per window. #[test] fn derived_window_candidate_follows_the_evaluation_cadence() { - let cost = planning_snapshot().implementation.window_cost_model.cost; + let cost = planning_snapshot().physical_inputs.window_cost_model.cost; let expr = crate::query_parser::parse_query_expr_canonical( "quantile_over_time(0.5, data[5m])", AccuracyTarget::Exact, @@ -5567,7 +5759,7 @@ pub(crate) mod tests { assert_eq!( derived .iter() - .map(|c| (c.implementation_id.as_str(), c.layout.clone())) + .map(|c| (c.realization_id.as_str(), c.layout.clone())) .collect::>(), vec![ ( @@ -5597,7 +5789,7 @@ pub(crate) mod tests { // not the magnitudes. #[test] fn derived_window_layout_prices_write_against_read_amplification() { - let cost = planning_snapshot().implementation.window_cost_model.cost; + let cost = planning_snapshot().physical_inputs.window_cost_model.cost; let expr = crate::query_parser::parse_query_expr_canonical( "quantile_over_time(0.5, data[5m])", AccuracyTarget::Exact, @@ -5645,7 +5837,7 @@ pub(crate) mod tests { // framework/layout table, so there is no alternative to price against it. #[test] fn tumbling_shapes_have_no_layout_alternative_to_rank() { - let cost = planning_snapshot().implementation.window_cost_model.cost; + let cost = planning_snapshot().physical_inputs.window_cost_model.cost; let expr = crate::query_parser::parse_query_expr_canonical( "quantile_over_time(0.5, data[5m])", AccuracyTarget::Exact, @@ -5661,7 +5853,7 @@ pub(crate) mod tests { 0, ); assert_eq!(derived.len(), 1); - assert_eq!(derived[0].implementation_id, "id-300s-slide-300s-pane-300s"); + assert_eq!(derived[0].realization_id, "id-300s-slide-300s-pane-300s"); assert_eq!(derived[0].framework, SummaryWindowFramework::Tumbling); } @@ -5670,8 +5862,8 @@ pub(crate) mod tests { #[test] fn derived_window_candidate_shapes_are_accepted_by_validation() { let snapshot = planning_snapshot(); - let (request, environment) = snapshot.planning_request().unwrap(); - let cost = planning_snapshot().implementation.window_cost_model.cost; + let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); + let cost = planning_snapshot().physical_inputs.window_cost_model.cost; for (lookback_ms, evaluation_ms) in [ (300_000, 30_000), (300_000, 300_000), @@ -5679,13 +5871,13 @@ pub(crate) mod tests { (60_000, 90_000), ] { let mut query = request.queries[0].clone(); - query.window_secs = lookback_ms / 1_000; + query.query_lookback_seconds = lookback_ms / 1_000; let expr = crate::query_parser::parse_query_expr_canonical( &format!("quantile_over_time(0.5, data[{}s])", lookback_ms / 1_000), AccuracyTarget::Exact, ) .unwrap(); - query.window_implementations = derived_window_candidates( + query.window_realization_candidates = derived_window_candidates( "derived", &expr, lookback_ms, @@ -5718,14 +5910,16 @@ pub(crate) mod tests { interval: RepetitionInterval(evaluation * 1_000), evaluation_phase: planner_types::workload::TimestampMs(phase), }; - let (mut request, env) = snapshot.planning_request().unwrap(); - request.queries[0].window_implementations.retain(|c| { - matches!( - c.layout, - asap_types::WindowMaterializationLayout::FullWindow - ) == full - }); - let plan = PhysicalCompiler.compile(request, env).unwrap(); + let (mut request, env) = snapshot.into_physical_compilation_request().unwrap(); + request.queries[0] + .window_realization_candidates + .retain(|c| { + matches!( + c.layout, + asap_types::WindowMaterializationLayout::FullWindow + ) == full + }); + let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); let config = &plan.precompute_plan.materializations[0]; assert_eq!(config.slide_interval, u64::from(evaluation)); assert_eq!(config.window_size, 60); @@ -5753,9 +5947,9 @@ pub(crate) mod tests { fn different_cadences_share_common_panes_only_when_cheaper() { for read_cost in [0.0, 1_000_000.0] { let mut snapshot = planning_snapshot(); - snapshot.implementation.lifecycle_costs.read = read_cost; + snapshot.physical_inputs.lifecycle_costs.read = read_cost; snapshot - .implementation + .physical_inputs .lifecycle_costs .maintenance_per_update = 1.0; let entries = snapshot.query_workload.repeating_queries.as_mut().unwrap(); @@ -5772,16 +5966,16 @@ pub(crate) mod tests { evaluation_phase: planner_types::workload::TimestampMs(0), }; entries.push(second); - let (mut request, env) = snapshot.planning_request().unwrap(); + let (mut request, env) = snapshot.into_physical_compilation_request().unwrap(); for query in &mut request.queries { - query.window_implementations.retain(|c| { + query.window_realization_candidates.retain(|c| { matches!( c.layout, asap_types::WindowMaterializationLayout::Pane { .. } ) }); } - let plan = PhysicalCompiler.compile(request, env).unwrap(); + let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!( plan.precompute_plan.materializations.len(), if read_cost == 0.0 { 1 } else { 2 } @@ -5803,10 +5997,10 @@ pub(crate) mod tests { for (third_phase, third_interval) in [(5_000, 20_000), (0, 1_000)] { let mut snapshot = planning_snapshot(); snapshot - .implementation + .physical_inputs .lifecycle_costs .maintenance_per_update = 1.0; - snapshot.implementation.lifecycle_costs.read = + snapshot.physical_inputs.lifecycle_costs.read = if third_interval == 1_000 { 100.0 } else { 0.0 }; let entries = snapshot.query_workload.repeating_queries.as_mut().unwrap(); let template = entries[0].clone(); @@ -5825,16 +6019,16 @@ pub(crate) mod tests { }; entries.push(entry); } - let (mut request, env) = snapshot.planning_request().unwrap(); + let (mut request, env) = snapshot.into_physical_compilation_request().unwrap(); for query in &mut request.queries { - query.window_implementations.retain(|c| { + query.window_realization_candidates.retain(|c| { matches!( c.layout, asap_types::WindowMaterializationLayout::Pane { .. } ) }); } - let plan = PhysicalCompiler.compile(request, env).unwrap(); + let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 2); assert!(plan .lifecycle_estimates @@ -5853,9 +6047,9 @@ pub(crate) mod tests { interval: RepetitionInterval(1_500), evaluation_phase: planner_types::workload::TimestampMs(0), }; - let (request, env) = snapshot.planning_request().unwrap(); - assert!(request.queries[0].window_implementations.is_empty()); - let plan = PhysicalCompiler.compile(request, env).unwrap(); + let (request, env) = snapshot.into_physical_compilation_request().unwrap(); + assert!(request.queries[0].window_realization_candidates.is_empty()); + let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); assert!(plan.precompute_plan.materializations.is_empty()); } @@ -5873,7 +6067,7 @@ pub(crate) mod tests { &expr, 60_000, evaluation_secs * 1_000, - planning_snapshot().implementation.window_cost_model.cost, + planning_snapshot().physical_inputs.window_cost_model.cost, &planning_lifecycle(), 0, ); @@ -5902,20 +6096,25 @@ pub(crate) mod tests { #[test] fn measured_window_quote_preserves_shape_and_price() { let mut snapshot = planning_snapshot(); - let (derived, _) = snapshot.clone().planning_request().unwrap(); - let mut quote = derived.queries[0].window_implementations[0].clone(); - quote.implementation_id = "measured-pane".into(); + let (derived, _) = snapshot + .clone() + .into_physical_compilation_request() + .unwrap(); + let mut quote = derived.queries[0].window_realization_candidates[0].clone(); + quote.realization_id = "measured-pane".into(); quote.cost.weighted_cost = 8.0; quote.derived = false; snapshot - .implementation + .physical_inputs .window_cost_model .quotes .push(quote.clone()); - let (request, _) = snapshot.planning_request().unwrap(); - assert!(request.queries[0].window_implementations.contains("e)); + let (request, _) = snapshot.into_physical_compilation_request().unwrap(); assert!(request.queries[0] - .window_implementations + .window_realization_candidates + .contains("e)); + assert!(request.queries[0] + .window_realization_candidates .iter() .any(|candidate| candidate.derived)); } @@ -5926,8 +6125,10 @@ pub(crate) mod tests { #[test] fn retained_state_count_follows_the_derived_pane_width() { let snapshot = planning_snapshot(); - let (request, environment) = snapshot.planning_request().unwrap(); - let plan = PhysicalCompiler.compile(request, environment).unwrap(); + let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); + let plan = PhysicalPlanCompiler + .compile_promql(request, environment) + .unwrap(); assert_eq!( plan.precompute_plan.materializations[0].num_aggregates_to_retain, Some(7) @@ -5950,8 +6151,10 @@ pub(crate) mod tests { evaluation_phase: planner_types::workload::TimestampMs(0), }; } - let (request, environment) = snapshot.planning_request().unwrap(); - let plan = PhysicalCompiler.compile(request, environment).unwrap(); + let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); + let plan = PhysicalPlanCompiler + .compile_promql(request, environment) + .unwrap(); let materialization = &plan.precompute_plan.materializations[0]; assert_eq!( ( @@ -5976,8 +6179,8 @@ pub(crate) mod tests { // with each operand keeping its own range. #[test] fn composable_binary_summarizes_each_prometheus_filtered_operand() { - use crate::query_plan::{logical::LogicalOperator, QueryPlanNode}; - let mut snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + use crate::query_plan::{logical::ResidualQueryOperator, QueryPlanNode}; + let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); @@ -5985,8 +6188,8 @@ pub(crate) mod tests { entry.query = Query("sum(sum_over_time(a[1m])) / sum(sum_over_time(b{job!=\"x\"}[5m]))".into()); entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let (request, env) = snapshot.planning_request().unwrap(); - let plan = PhysicalCompiler.compile(request, env).unwrap(); + let (request, env) = snapshot.into_physical_compilation_request().unwrap(); + let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); let query = plan.query_plan.entries.values().next().unwrap(); let bindings = query.materialization_bindings(); // Both operands now hold a summary. The filtered denominator is no @@ -6021,7 +6224,7 @@ pub(crate) mod tests { assert!(!query.nodes.values().any(|node| matches!( node, QueryPlanNode::Logical { - operator: LogicalOperator::Scan { .. }, + operator: ResidualQueryOperator::Scan { .. }, .. } ))); @@ -6037,13 +6240,15 @@ pub(crate) mod tests { "sum_over_time(m{job!~\"a.*\"}[1m])", ] { let request = request("scope", query); - let selected = collect_selected_materializations(&request.queries[0].post_asap, false); + let selected = + collect_selected_materializations(&request.queries[0].selected_plan_root, false); let selected = selected.unwrap(); assert_eq!(selected.len(), 1, "{query}"); assert!(!selected[0].spatial_filter.is_empty(), "{query}"); } let request = request("scope", "sum_over_time(m[1m] offset 1h)"); - let selected = collect_selected_materializations(&request.queries[0].post_asap, false); + let selected = + collect_selected_materializations(&request.queries[0].selected_plan_root, false); assert!(selected.is_err() || selected.unwrap().is_empty()); } @@ -6056,19 +6261,22 @@ pub(crate) mod tests { "sum(avg_over_time(m{job=~\".+\"}[6h]))", "sum(rate(m[5m] offset 1h))", ] { - let mut snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; entry.query = Query(query.into()); entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let (mut request, environment) = snapshot.planning_request().unwrap(); - request = super::super::workload_cost::with_exact_alternative(request) - .unwrap() - .pop() + let (mut request, environment) = snapshot.into_physical_compilation_request().unwrap(); + request = + super::super::workload_cost::enumerate_exact_and_materialized_candidates(request) + .unwrap() + .pop() + .unwrap(); + let bundle = PhysicalPlanCompiler + .compile_promql(request, environment) .unwrap(); - let bundle = PhysicalCompiler.compile(request, environment).unwrap(); assert!(bundle.precompute_plan.materializations.is_empty()); assert!(bundle.query_plan.entries.values().all(|entry| matches!( entry.nodes[&entry.root], @@ -6080,11 +6288,13 @@ pub(crate) mod tests { // Projections reference one immutable snapshot and reject drift or foreign state. #[test] fn catalog_projection_rejects_missing_stale_and_foreign_references() { - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + let snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); - let bundle = quoted_snapshot(snapshot, false).compile().unwrap(); + let bundle = quoted_snapshot(snapshot, crate::physical::compiler::QueryFrontend::PromQl) + .compile_promql() + .unwrap(); let catalog = &bundle.summary_catalog; let mut transmission = bundle.transmission_plan.clone(); transmission.validate_against_catalog(catalog).unwrap(); @@ -6143,7 +6353,7 @@ pub(crate) mod tests { #[test] fn canonical_snapshot_preserves_shared_bindings_after_serialization() { // Two different registered readouts survive publication with one state. - let mut snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); @@ -6152,7 +6362,9 @@ pub(crate) mod tests { let mut second = entries[0].clone(); second.query = Query("sum(sum_over_time(m[1m])) * 2".into()); entries.push(second); - let bundle = quoted_snapshot(snapshot, false).compile().unwrap(); + let bundle = quoted_snapshot(snapshot, crate::physical::compiler::QueryFrontend::PromQl) + .compile_promql() + .unwrap(); assert_eq!(bundle.query_plan.entries.len(), 2); assert_eq!(bundle.precompute_plan.materializations.len(), 1); let query_plan: QueryPlan = @@ -6169,8 +6381,8 @@ pub(crate) mod tests { #[test] fn precompute_catalog_validates_without_backend_projection() { - let bundle = PhysicalCompiler - .compile( + let bundle = PhysicalPlanCompiler + .compile_promql( request("catalog", "quantile_over_time(0.99, m[1m])"), environment(10_000), ) @@ -6226,13 +6438,13 @@ pub(crate) mod tests { #[test] fn publication_is_catalog_authoritative_and_round_trips() { - let bundle = PhysicalCompiler - .compile( + let bundle = PhysicalPlanCompiler + .compile_promql( request("publication", "quantile_over_time(0.99, m[1m])"), environment(10_000), ) .unwrap(); - let publication = bundle.publication().unwrap(); + let publication = bundle.to_publication_artifact().unwrap(); let json = serde_json::to_value(&publication).unwrap(); let mut decoded: super::super::publication::PhysicalPlanPublication = serde_json::from_value(json).unwrap(); @@ -6251,8 +6463,8 @@ pub(crate) mod tests { #[test] fn compiles_one_decision_into_matching_collector_and_backend_views() { - let bundle = PhysicalCompiler - .compile( + let bundle = PhysicalPlanCompiler + .compile_promql( request("q-quantile", "quantile_over_time(0.99, m[1m])"), environment(10_000), ) @@ -6368,7 +6580,7 @@ pub(crate) mod tests { SummaryWindowFramework::Tumbling ); assert_eq!( - plan.materializations[0].window_implementation_id, + plan.materializations[0].window_realization_id, "collector-tumbling-v1" ); assert_eq!(plan.materializations[0].slide_secs, 60); @@ -6390,8 +6602,8 @@ pub(crate) mod tests { #[test] fn backend_local_hll_and_envelope_ingest_are_supported() { - let bundle = PhysicalCompiler - .compile( + let bundle = PhysicalPlanCompiler + .compile_promql( request("q", "quantile_over_time(0.99, m[1m])"), environment(10_000), ) @@ -6420,8 +6632,8 @@ pub(crate) mod tests { #[test] fn backend_local_precompute_contract_has_no_collector_producers() { - let bundle = PhysicalCompiler - .compile( + let bundle = PhysicalPlanCompiler + .compile_promql( request("q-quantile", "quantile_over_time(0.99, m[1m])"), environment(10_000), ) @@ -6440,7 +6652,7 @@ pub(crate) mod tests { assert_eq!(plan.ingest.timestamp_unit, TimestampUnit::UnixMilliseconds); assert!(plan.producers.is_empty()); plan.validate().expect("valid backend-local projection"); - crate::physical::compiler::compile_transmission_plan( + crate::physical::compiler::build_transmission_plan( bundle.envelope, &plan, &BTreeMap::new(), @@ -6489,28 +6701,28 @@ pub(crate) mod tests { }; let mut environment = environment(10_000); environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - environment.collector_ids.clear(); + environment.target_collector_ids.clear(); let template = request("template", "sum(sum_over_time(m[1m]))") .queries .remove(0); - let snapshot = BackendLocalPlanningSnapshot { - snapshot_version: 2, + let snapshot = BackendLocalPlanningInput { + schema_version: 2, workload_cost_evidence: None, query_workload, data_workload, - implementation: BackendLocalImplementation { - lifecycle_costs: template.lifecycle.costs, + physical_inputs: BackendLocalPhysicalInputs { + lifecycle_costs: template.summary_lifecycle_inputs.costs, evidence_observed_at_unix_ms: 9_500, evidence_valid_for_ms: 60_000, horizon_seconds: 300.0, window_cost_model: WindowCostModel { implementation_id: "backend-tumbling-v1".into(), - cost: template.window_implementations[0].cost.clone(), + cost: template.window_realization_candidates[0].cost.clone(), quotes: Vec::new(), }, source_sample_interval_ms: None, - query_staleness_margin_ms: 0, - max_retained_summary_bytes: DEFAULT_RETAINED_SUMMARY_MEMORY_BUDGET_BYTES, + query_retention_margin_ms: 0, + retained_summary_memory_budget_bytes: DEFAULT_RETAINED_SUMMARY_MEMORY_BUDGET_BYTES, topk_evidence: HashMap::new(), exact_composition_costs: HashMap::new(), erp: None, @@ -6520,19 +6732,25 @@ pub(crate) mod tests { assert_eq!( snapshot .clone() - .planning_request() + .into_physical_compilation_request() .unwrap() .0 .query_workload .as_ref(), Some(&snapshot.query_workload) ); - let first = quoted_snapshot(snapshot.clone(), false) - .compile() - .expect("first deterministic plan"); - let second = quoted_snapshot(snapshot.clone(), false) - .compile() - .expect("second deterministic plan"); + let first = quoted_snapshot( + snapshot.clone(), + crate::physical::compiler::QueryFrontend::PromQl, + ) + .compile_promql() + .expect("first deterministic plan"); + let second = quoted_snapshot( + snapshot.clone(), + crate::physical::compiler::QueryFrontend::PromQl, + ) + .compile_promql() + .expect("second deterministic plan"); assert_eq!(first.envelope, second.envelope); assert_eq!(first.summary_catalog, second.summary_catalog); assert_eq!(first.query_plan, second.query_plan); @@ -6543,10 +6761,10 @@ pub(crate) mod tests { ); let encoded = serde_json::to_vec(&snapshot).expect("serialize startup snapshot"); - let decoded: BackendLocalPlanningSnapshot = + let decoded: BackendLocalPlanningInput = serde_json::from_slice(&encoded).expect("deserialize startup snapshot"); - let bundle = quoted_snapshot(decoded, false) - .compile() + let bundle = quoted_snapshot(decoded, crate::physical::compiler::QueryFrontend::PromQl) + .compile_promql() .expect("canonical startup planning"); assert!(bundle.collector_plans.is_empty()); @@ -6601,8 +6819,9 @@ pub(crate) mod tests { ] { let mut deployment = environment(10_000); deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - deployment.collector_ids.clear(); - let compiled = PhysicalCompiler.compile(request(query_id, promql), deployment); + deployment.target_collector_ids.clear(); + let compiled = + PhysicalPlanCompiler.compile_promql(request(query_id, promql), deployment); let plan = compiled.unwrap_or_else(|error| panic!("{promql} must compile: {error}")); assert_eq!(plan.summary_catalog.materializations.len(), 1, "{promql}"); assert_eq!(plan.query_plan.entries.len(), 1, "{promql}"); @@ -6629,31 +6848,35 @@ pub(crate) mod tests { #[test] fn checked_in_per_entity_snapshot_preserves_native_alternative() { let source = include_str!("../../../docs/examples/asapquery-planning-snapshot.json"); - let snapshot: BackendLocalPlanningSnapshot = + let snapshot: BackendLocalPlanningInput = serde_json::from_str(source).expect("strict canonical workload fixture"); let encoded = serde_json::to_value(&snapshot).expect("canonical snapshot value"); let fixture: serde_json::Value = serde_json::from_str(source).expect("fixture JSON"); assert_eq!(encoded, fixture); assert!( - snapshot.clone().compile().is_err(), + snapshot.clone().compile_promql().is_err(), "discovery fixtures must be priced before deployment" ); - let (local, env) = snapshot.clone().planning_request().unwrap(); - let isolated = PhysicalCompiler.compile(local, env).unwrap(); + let (local, env) = snapshot + .clone() + .into_physical_compilation_request() + .unwrap(); + let isolated = PhysicalPlanCompiler.compile_promql(local, env).unwrap(); assert!(!isolated.precompute_plan.materializations.is_empty()); assert!(isolated .precompute_plan .materializations .iter() .all(|state| state.partitioning.is_some())); - let (request, environment) = snapshot.planning_request().unwrap(); - let native = crate::physical::workload_cost::with_exact_alternative(request) - .unwrap() - .pop() - .unwrap(); - let plan = PhysicalCompiler - .compile(native, environment) + let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); + let native = + crate::physical::workload_cost::enumerate_exact_and_materialized_candidates(request) + .unwrap() + .pop() + .unwrap(); + let plan = PhysicalPlanCompiler + .compile_promql(native, environment) .expect("native fixture compiles"); assert!(plan.precompute_plan.materializations.is_empty()); assert!(plan.collector_plans.is_empty()); @@ -6669,27 +6892,31 @@ pub(crate) mod tests { fn compatibility_demo_preserves_complete_native_query_matrix() { let source = include_str!("../../../docs/examples/asapquery-compatibility-demo-snapshot.json"); - let snapshot: BackendLocalPlanningSnapshot = + let snapshot: BackendLocalPlanningInput = serde_json::from_str(source).expect("strict compatibility demo fixture"); assert!( - snapshot.clone().compile().is_err(), + snapshot.clone().compile_promql().is_err(), "discovery fixtures must be priced before deployment" ); - let (local, env) = snapshot.clone().planning_request().unwrap(); - let isolated = PhysicalCompiler.compile(local, env).unwrap(); + let (local, env) = snapshot + .clone() + .into_physical_compilation_request() + .unwrap(); + let isolated = PhysicalPlanCompiler.compile_promql(local, env).unwrap(); assert!(!isolated.precompute_plan.materializations.is_empty()); assert!(isolated .precompute_plan .materializations .iter() .all(|state| state.partitioning.is_some())); - let (request, environment) = snapshot.planning_request().unwrap(); - let native = crate::physical::workload_cost::with_exact_alternative(request) - .unwrap() - .pop() - .unwrap(); - let plan = PhysicalCompiler - .compile(native, environment) + let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); + let native = + crate::physical::workload_cost::enumerate_exact_and_materialized_candidates(request) + .unwrap() + .pop() + .unwrap(); + let plan = PhysicalPlanCompiler + .compile_promql(native, environment) .expect("native demo compiles"); assert!(plan.collector_plans.is_empty()); @@ -6710,15 +6937,15 @@ pub(crate) mod tests { #[test] fn multiple_readouts_share_one_precompute_materialization() { - let mut planning_request = request("q-p90", "quantile_over_time(0.90, m[1m])"); + let mut compilation_request = request("q-p90", "quantile_over_time(0.90, m[1m])"); let second = request("q-p99", "quantile_over_time(0.99, m[1m])") .queries .into_iter() .next() .unwrap(); - planning_request.queries.push(second); - let bundle = PhysicalCompiler - .compile(planning_request, environment(10_000)) + compilation_request.queries.push(second); + let bundle = PhysicalPlanCompiler + .compile_promql(compilation_request, environment(10_000)) .unwrap(); assert_eq!(bundle.query_plan.entries.len(), 2); @@ -6730,10 +6957,10 @@ pub(crate) mod tests { #[test] fn compiles_every_materialization_leaf_in_a_merge_dag() { - let mut planning_request = request("q-merge", "quantile_over_time(0.90, m[1m])"); + let mut compilation_request = request("q-merge", "quantile_over_time(0.90, m[1m])"); let right_request = request("q-right", "quantile_over_time(0.90, n[1m])"); - let left_root = planning_request.queries[0].post_asap.clone(); - let right_root = right_request.queries[0].post_asap.clone(); + let left_root = compilation_request.queries[0].selected_plan_root.clone(); + let right_root = right_request.queries[0].selected_plan_root.clone(); let (left, query) = match &left_root.expr { SummaryExpr::SummaryEstimate { summary_input, @@ -6752,7 +6979,7 @@ pub(crate) mod tests { schema: left.schema.clone(), guarantee: None, }); - planning_request.queries[0].post_asap = Rc::new(SummaryNode { + compilation_request.queries[0].selected_plan_root = Rc::new(SummaryNode { expr: SummaryExpr::SummaryEstimate { summary_input: merge, query, @@ -6761,8 +6988,8 @@ pub(crate) mod tests { guarantee: left_root.guarantee.clone(), }); - let bundle = PhysicalCompiler - .compile(planning_request, environment(10_000)) + let bundle = PhysicalPlanCompiler + .compile_promql(compilation_request, environment(10_000)) .expect("compile merged post-ASAP DAG"); assert_eq!(bundle.summary_catalog.materializations.len(), 2); assert_eq!(bundle.precompute_plan.materializations.len(), 2); @@ -6811,8 +7038,8 @@ pub(crate) mod tests { #[test] fn precompute_schema_must_match_materialization_semantics() { - let bundle = PhysicalCompiler - .compile( + let bundle = PhysicalPlanCompiler + .compile_promql( request("q-quantile", "quantile_over_time(0.99, m[1m])"), environment(10_000), ) @@ -6834,16 +7061,16 @@ pub(crate) mod tests { { let mut request = request("q", "sum(sum_over_time(m[1m]))"); let query = &mut request.queries[0]; - let mut small = query.window_implementations[0].clone(); - small.implementation_id = "small".into(); + let mut small = query.window_realization_candidates[0].clone(); + small.realization_id = "small".into(); small.layout = asap_types::WindowMaterializationLayout::Pane { pane_secs: 10 }; small.cost.weighted_cost = small_cost; - query.window_implementations[0].implementation_id = "large".into(); - query.window_implementations.push(small); + query.window_realization_candidates[0].realization_id = "large".into(); + query.window_realization_candidates.push(small); let mut env = environment(10_000); env.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - env.collector_ids.clear(); - let bundle = PhysicalCompiler.compile(request, env).unwrap(); + env.target_collector_ids.clear(); + let bundle = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); let materialization = bundle.precompute_plan.materializations.first().unwrap(); assert_eq!(materialization.window_size, 60); assert_eq!( @@ -6860,7 +7087,7 @@ pub(crate) mod tests { expected_pane_secs * 1000 ); assert_eq!( - bundle.lifecycle_estimates[0].window_implementation_id, + bundle.lifecycle_estimates[0].window_realization_id, expected_id ); } @@ -6874,8 +7101,8 @@ pub(crate) mod tests { ] { let mut request = request("q", "sum(sum_over_time(m[1m]))"); let query = &mut request.queries[0]; - let mut panes = query.window_implementations[0].clone(); - panes.implementation_id = "mergeable-panes".into(); + let mut panes = query.window_realization_candidates[0].clone(); + panes.realization_id = "mergeable-panes".into(); panes.framework = SummaryWindowFramework::Sliding; panes.slide_secs = 10; panes.layout = asap_types::WindowMaterializationLayout::Pane { pane_secs: 10 }; @@ -6884,7 +7111,7 @@ pub(crate) mod tests { panes.cost.storage_bytes = 1_024; let mut full = panes.clone(); - full.implementation_id = "full-window".into(); + full.realization_id = "full-window".into(); full.layout = asap_types::WindowMaterializationLayout::FullWindow; full.cost.weighted_cost = full_cost; // Full windows spend more update CPU and retained bytes, while @@ -6892,14 +7119,14 @@ pub(crate) mod tests { // includes the workload's measured read frequency and cardinality. full.cost.cpu_cost = 20.0; full.cost.storage_bytes = 64 * 1_024; - query.window_implementations = vec![panes, full]; + query.window_realization_candidates = vec![panes, full]; let mut env = environment(10_000); env.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - env.collector_ids.clear(); - let bundle = PhysicalCompiler.compile(request, env).unwrap(); + env.target_collector_ids.clear(); + let bundle = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!( - bundle.lifecycle_estimates[0].window_implementation_id, + bundle.lifecycle_estimates[0].window_realization_id, expected_id ); assert_eq!( @@ -6911,7 +7138,11 @@ pub(crate) mod tests { ); // Publication validates stored extent independently of slide cadence. // In particular the full-window candidate stores 60s at a 10s slide. - bundle.publication().unwrap().validate().unwrap(); + bundle + .to_publication_artifact() + .unwrap() + .validate() + .unwrap(); assert_eq!(bundle.precompute_plan.materializations[0].window_size, 60); assert_eq!( bundle.precompute_plan.materializations[0].slide_interval, @@ -6928,23 +7159,23 @@ pub(crate) mod tests { let mut second = request("q40", "sum(sum_over_time(m[40s]))") .queries .remove(0); - workload.queries[0].window_secs = 20; - workload.queries[0].window_implementations[0].window_secs = 20; - second.window_secs = 40; - second.lifecycle.evaluation_interval_ms = 20_000; - second.window_implementations[0].window_secs = 40; + workload.queries[0].query_lookback_seconds = 20; + workload.queries[0].window_realization_candidates[0].window_secs = 20; + second.query_lookback_seconds = 40; + second.summary_lifecycle_inputs.evaluation_interval_ms = 20_000; + second.window_realization_candidates[0].window_secs = 40; workload.queries.push(second); for query in &mut workload.queries { - query.window_implementations[0].framework = SummaryWindowFramework::Sliding; - query.window_implementations[0].slide_secs = 10; - query.window_implementations[0].layout = + query.window_realization_candidates[0].framework = SummaryWindowFramework::Sliding; + query.window_realization_candidates[0].slide_secs = 10; + query.window_realization_candidates[0].layout = asap_types::WindowMaterializationLayout::Pane { pane_secs: 10 }; - query.window_implementations[0].implementation_id = "shared-ten-second-pane".into(); + query.window_realization_candidates[0].realization_id = "shared-ten-second-pane".into(); } let mut env = environment(10_000); env.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - env.collector_ids.clear(); - let bundle = PhysicalCompiler.compile(workload, env).unwrap(); + env.target_collector_ids.clear(); + let bundle = PhysicalPlanCompiler.compile_promql(workload, env).unwrap(); assert_eq!(bundle.precompute_plan.materializations.len(), 2); } @@ -6952,27 +7183,27 @@ pub(crate) mod tests { #[test] fn tumbling_sizes_reject_non_divisors() { let mut request = request("q", "sum(sum_over_time(m[1m]))"); - request.queries[0].window_implementations[0].layout = + request.queries[0].window_realization_candidates[0].layout = asap_types::WindowMaterializationLayout::Pane { pane_secs: 7 }; let mut env = environment(10_000); env.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - assert!(PhysicalCompiler.compile(request, env).is_err()); + assert!(PhysicalPlanCompiler.compile_promql(request, env).is_err()); } #[test] fn missing_window_implementation_evidence_fails_closed() { let mut request = request("q-window", "quantile_over_time(0.99, m[1m])"); - request.queries[0].window_implementations.clear(); - let error = PhysicalCompiler - .compile(request, environment(10_000)) + request.queries[0].window_realization_candidates.clear(); + let error = PhysicalPlanCompiler + .compile_promql(request, environment(10_000)) .expect_err("Planner must not receive a zero-cost invented window"); assert!(matches!(error, CompileError::Lifecycle { .. })); } #[test] fn precompute_plan_rejects_schema_or_producer_drift() { - let bundle = PhysicalCompiler - .compile( + let bundle = PhysicalPlanCompiler + .compile_promql( request("q-quantile", "quantile_over_time(0.99, m[1m])"), environment(10_000), ) @@ -6995,8 +7226,8 @@ pub(crate) mod tests { #[test] fn precompute_plan_rejects_empty_or_duplicate_schema_ids() { - let bundle = PhysicalCompiler - .compile( + let bundle = PhysicalPlanCompiler + .compile_promql( request("q-quantile", "quantile_over_time(0.99, m[1m])"), environment(10_000), ) @@ -7036,8 +7267,8 @@ pub(crate) mod tests { }), ) .expect("selection accepts evidence before freshness validation"); - let error = PhysicalCompiler - .compile(request, environment(100_000)) + let error = PhysicalPlanCompiler + .compile_promql(request, environment(100_000)) .expect_err("stale certificate must fail"); assert!(matches!(error, CompileError::InvalidEvidence { .. })); } @@ -7057,16 +7288,16 @@ pub(crate) mod tests { ) .expect("selection occurs before deployment-time freshness validation"); assert!(matches!( - PhysicalCompiler.compile(topk, environment(10_000)), + PhysicalPlanCompiler.compile_promql(topk, environment(10_000)), Err(CompileError::InvalidEvidence { .. }) )); let mut window = request("q-window", "quantile_over_time(0.99, m[1m])"); - window.queries[0].window_implementations[0] + window.queries[0].window_realization_candidates[0] .cost .observed_at_unix_ms = 10_001; assert!(matches!( - PhysicalCompiler.compile(window, environment(10_000)), + PhysicalPlanCompiler.compile_promql(window, environment(10_000)), Err(CompileError::Lifecycle { .. }) )); } @@ -7085,8 +7316,8 @@ pub(crate) mod tests { }), ) .expect("selection accepts valid evidence"); - let bundle = PhysicalCompiler - .compile(request, environment(10_000)) + let bundle = PhysicalPlanCompiler + .compile_promql(request, environment(10_000)) .expect("certified TopK compiles"); assert_eq!(bundle.summary_catalog.materializations.len(), 1); assert_eq!( @@ -7102,7 +7333,7 @@ pub(crate) mod tests { let mut request = request("q", "quantile_over_time(0.9, m[1m])"); request.planner_revision = "different".into(); assert!(matches!( - PhysicalCompiler.compile(request, environment(10_000)), + PhysicalPlanCompiler.compile_promql(request, environment(10_000)), Err(CompileError::PlannerRevision { .. }) )); } @@ -7110,10 +7341,14 @@ pub(crate) mod tests { #[test] fn stale_lifecycle_evidence_fails_closed() { let mut request = request("q", "quantile_over_time(0.9, m[1m])"); - request.queries[0].lifecycle.evidence_observed_at_unix_ms = 1; - request.queries[0].lifecycle.evidence_valid_for_ms = 10; + request.queries[0] + .summary_lifecycle_inputs + .evidence_observed_at_unix_ms = 1; + request.queries[0] + .summary_lifecycle_inputs + .evidence_valid_for_ms = 10; assert!(matches!( - PhysicalCompiler.compile(request, environment(10_000)), + PhysicalPlanCompiler.compile_promql(request, environment(10_000)), Err(CompileError::Lifecycle { .. }) )); } @@ -7139,8 +7374,8 @@ pub(crate) mod tests { #[test] fn runtime_policy_encoding_is_checked() { - let bundle = PhysicalCompiler - .compile( + let bundle = PhysicalPlanCompiler + .compile_promql( request("q", "quantile_over_time(0.99, m[1m])"), environment(10_000), ) @@ -7167,12 +7402,12 @@ pub(crate) mod tests { }), ) .expect("frequency selection"); - request.queries[0].runtime_policy.delta = Some(DeltaPolicy { + request.queries[0].materialization_runtime_policy.delta = Some(DeltaPolicy { absolute_threshold: 0.0, gos: None, }); - let bundle = PhysicalCompiler - .compile(request, environment(10_000)) + let bundle = PhysicalPlanCompiler + .compile_promql(request, environment(10_000)) .expect("delta-capable physical plan"); let rule = &bundle.transmission_plan.rules[0]; assert_eq!(rule.mode, TransmissionMode::Delta); @@ -7206,8 +7441,8 @@ pub(crate) mod tests { #[test] fn runtime_adaptation_requires_fresh_exact_evidence_and_successor_version() { - let bundle = PhysicalCompiler - .compile( + let bundle = PhysicalPlanCompiler + .compile_promql( request("q", "quantile_over_time(0.99, m[1m])"), environment(10_000), ) diff --git a/control_plane/src/physical/compiler/windows.rs b/control_plane/src/physical/compiler/windows.rs index cc8134ff7..d135ab17d 100644 --- a/control_plane/src/physical/compiler/windows.rs +++ b/control_plane/src/physical/compiler/windows.rs @@ -8,9 +8,9 @@ use asap_types::WindowMaterializationLayout; #[serde(deny_unknown_fields)] pub struct WindowCostModel { pub implementation_id: String, - pub cost: ImplementationCostEvidence, + pub cost: WindowRealizationCostQuote, #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub quotes: Vec, + pub quotes: Vec, } pub(in crate::physical) fn gcd(mut a: u64, mut b: u64) -> u64 { @@ -20,7 +20,7 @@ pub(in crate::physical) fn gcd(mut a: u64, mut b: u64) -> u64 { a } -pub(super) fn is_full_cohort(candidate: &WindowImplementationCandidate) -> bool { +pub(super) fn is_full_cohort(candidate: &WindowRealizationCandidate) -> bool { candidate.slide_secs == candidate.window_secs && candidate.layout == WindowMaterializationLayout::Pane { @@ -40,7 +40,7 @@ pub(super) fn cohort_nodes(states: &[SelectedMaterialization]) -> BTreeSet bool { candidate @@ -66,12 +66,12 @@ pub(super) fn supported( pub(super) fn derive( model: &WindowCostModel, - lifecycle: &LifecyclePlanningInput, + lifecycle: &SummaryLifecyclePlanningInputs, window_secs: u64, full_cohort: bool, target: PhysicalDeploymentTarget, staleness_margin_ms: u64, -) -> Vec { +) -> Vec { let evaluation_ms = u64::from(lifecycle.evaluation_interval_ms); // Runtime layouts have second precision. Never truncate a fractional cadence. if window_secs == 0 || evaluation_ms == 0 || evaluation_ms % 1_000 != 0 { @@ -99,8 +99,8 @@ pub(super) fn derive( WindowMaterializationLayout::Pane { pane_secs } => format!("pane-{pane_secs}s"), _ => "full-window".into(), }; - let candidate = WindowImplementationCandidate { - implementation_id: format!( + let candidate = WindowRealizationCandidate { + realization_id: format!( "{}-{window_secs}s-slide-{slide_secs}s-{suffix}", model.implementation_id ), @@ -129,7 +129,7 @@ pub(super) fn derive( } pub fn prepare_window_implementations( - query: &mut PlanningQuery, + query: &mut QueryCompilationInput, model: &WindowCostModel, target: PhysicalDeploymentTarget, staleness_margin_ms: u64, @@ -143,9 +143,9 @@ pub fn prepare_window_implementations( let mut model = model.clone(); let fingerprint = canonical_promql(&query.query_string).map_err(CompileError::QueryPlan)?; model.cost.workload_fingerprint = fingerprint.clone(); - model.cost.horizon_seconds = query.lifecycle.horizon_seconds; + model.cost.horizon_seconds = query.summary_lifecycle_inputs.horizon_seconds; let states = collect_selected_materializations( - &query.post_asap, + &query.selected_plan_root, target == PhysicalDeploymentTarget::BackendLocalRemoteWrite, ) .map_err(|reason| CompileError::Query { @@ -157,7 +157,7 @@ pub fn prepare_window_implementations( .iter() .map(|state| { ( - state.window_secs.unwrap_or(query.window_secs), + state.window_secs.unwrap_or(query.query_lookback_seconds), cohorts.contains(&(Rc::as_ptr(&state.node) as usize)), ) }) @@ -167,7 +167,7 @@ pub fn prepare_window_implementations( .flat_map(|&(window, cohort)| { derive( &model, - &query.lifecycle, + &query.summary_lifecycle_inputs, window, cohort, target, @@ -198,7 +198,7 @@ pub fn prepare_window_implementations( is_full_cohort(quote) } else { quote.slide_secs.saturating_mul(1_000) - == u64::from(query.lifecycle.evaluation_interval_ms) + == u64::from(query.summary_lifecycle_inputs.evaluation_interval_ms) } }); if !applicable || !supported(quote, target) { @@ -206,7 +206,7 @@ pub fn prepare_window_implementations( query_id: query.query_id.clone(), reason: format!( "window quote `{}` does not match an executable state layout", - quote.implementation_id + quote.realization_id ), }); } @@ -218,12 +218,12 @@ pub fn prepare_window_implementations( let mut quote = quote.clone(); quote.derived = false; quote.cohort_only = quote.slide_secs.saturating_mul(1_000) - != u64::from(query.lifecycle.evaluation_interval_ms); + != u64::from(query.summary_lifecycle_inputs.evaluation_interval_ms); candidates.push(quote); } let mut unique = BTreeMap::new(); for candidate in &candidates { - if let Some(previous) = unique.insert(&candidate.implementation_id, candidate) { + if let Some(previous) = unique.insert(&candidate.realization_id, candidate) { if previous != candidate { return Err(CompileError::Lifecycle { query_id: query.query_id.clone(), @@ -233,8 +233,8 @@ pub fn prepare_window_implementations( } } let mut ids = BTreeSet::new(); - candidates.retain(|candidate| ids.insert(candidate.implementation_id.clone())); - query.window_implementations = candidates; + candidates.retain(|candidate| ids.insert(candidate.realization_id.clone())); + query.window_realization_candidates = candidates; Ok(()) } @@ -242,7 +242,7 @@ pub fn prepare_window_implementations( mod tests { use super::*; - fn snapshot() -> BackendLocalPlanningSnapshot { + fn snapshot() -> BackendLocalPlanningInput { serde_json::from_str(include_str!( "../../../../docs/examples/asapquery-planning-snapshot.json" )) @@ -253,9 +253,9 @@ mod tests { #[test] fn collector_generation_excludes_partial_panes_and_sparse_full_windows() { let snapshot = snapshot(); - let model = snapshot.implementation.window_cost_model.clone(); - let (request, _) = snapshot.planning_request().unwrap(); - let mut lifecycle = request.queries[0].lifecycle.clone(); + let model = snapshot.physical_inputs.window_cost_model.clone(); + let (request, _) = snapshot.into_physical_compilation_request().unwrap(); + let mut lifecycle = request.queries[0].summary_lifecycle_inputs.clone(); for (interval, expected_count) in [ (20_000, 1), (45_000, 1), @@ -284,10 +284,10 @@ mod tests { #[test] fn quotes_cannot_bypass_layout_or_runtime_constraints() { let snapshot = snapshot(); - let mut model = snapshot.implementation.window_cost_model.clone(); - let (request, _) = snapshot.planning_request().unwrap(); + let mut model = snapshot.physical_inputs.window_cost_model.clone(); + let (request, _) = snapshot.into_physical_compilation_request().unwrap(); let mut query = request.queries[0].clone(); - query.lifecycle.evaluation_interval_ms = 20_000; + query.summary_lifecycle_inputs.evaluation_interval_ms = 20_000; prepare_window_implementations( &mut query, &model, @@ -295,7 +295,7 @@ mod tests { 0, ) .unwrap(); - let mut quote = query.window_implementations[0].clone(); + let mut quote = query.window_realization_candidates[0].clone(); quote.layout = WindowMaterializationLayout::Pane { pane_secs: 30 }; model.quotes = vec![quote.clone()]; assert!(prepare_window_implementations( @@ -325,11 +325,13 @@ mod tests { #[test] fn conflicting_quote_ids_and_duplicate_shapes_are_rejected() { let snapshot = snapshot(); - let mut model = snapshot.implementation.window_cost_model.clone(); - let (request, _) = snapshot.planning_request().unwrap(); + let mut model = snapshot.physical_inputs.window_cost_model.clone(); + let (request, _) = snapshot.into_physical_compilation_request().unwrap(); let mut query = request.queries[0].clone(); - let mut quote = query.window_implementations[0].clone(); - quote.implementation_id = query.window_implementations[1].implementation_id.clone(); + let mut quote = query.window_realization_candidates[0].clone(); + quote.realization_id = query.window_realization_candidates[1] + .realization_id + .clone(); model.quotes = vec![quote.clone()]; assert!(prepare_window_implementations( &mut query, @@ -338,7 +340,7 @@ mod tests { 0 ) .is_err()); - quote.implementation_id = "duplicate-shape".into(); + quote.realization_id = "duplicate-shape".into(); model.quotes.push(quote); assert!(prepare_window_implementations( &mut query, @@ -352,12 +354,12 @@ mod tests { // External serialization never grants permission to reinterpret measured prices. #[test] fn serialized_generated_quote_loses_compiler_provenance() { - let (request, _) = snapshot().planning_request().unwrap(); - let candidate = &request.queries[0].window_implementations[0]; + let (request, _) = snapshot().into_physical_compilation_request().unwrap(); + let candidate = &request.queries[0].window_realization_candidates[0]; assert!(candidate.derived); let value = serde_json::to_value(candidate).unwrap(); assert!(value.get("derived").is_none()); - let decoded: WindowImplementationCandidate = serde_json::from_value(value).unwrap(); + let decoded: WindowRealizationCandidate = serde_json::from_value(value).unwrap(); assert!(!decoded.derived); } } diff --git a/control_plane/src/physical/deployment_cost/delta.rs b/control_plane/src/physical/deployment_cost/delta.rs index 0a1df2d1f..51e7649f1 100644 --- a/control_plane/src/physical/deployment_cost/delta.rs +++ b/control_plane/src/physical/deployment_cost/delta.rs @@ -129,12 +129,12 @@ pub fn delta_benchmark_table() -> HashMap { /// A shorter flush period means: /// • fewer inserts accumulate per period → lower fill rate → better delta /// • more flushes per second → higher total CPU overhead -pub fn flush_period_secs(plan: &CollectionPlan, w: &QueryWorkload) -> f64 { +pub fn flush_period_secs(plan: &CollectionPlan, w: &RegisteredWorkload) -> f64 { if let Some(wd) = plan.agent_config.window_duration { return wd.as_secs_f64(); } // Batch mode: use repeat_every as a proxy for the batch arrival interval. - w.repeat_every + w.repeat_every() .unwrap_or(Duration::from_secs(1)) .as_secs_f64() .max(0.001) // guard against zero @@ -196,7 +196,7 @@ fn estimate_distinct_keys(inserts_per_flush: f64, wc: &WorkloadCharacteristics) pub fn estimate_fill_rate( wc: &WorkloadCharacteristics, plan: &CollectionPlan, - w: &QueryWorkload, + w: &RegisteredWorkload, ) -> f64 { let flush_secs = flush_period_secs(plan, w); let inserts_per_flush = wc.samples_per_sec_per_series * wc.series_count as f64 * flush_secs; @@ -313,7 +313,7 @@ pub const RAW_PASSTHROUGH_SAMPLE_RATE_THRESHOLD: f64 = 10.0; /// 7. Otherwise → UseDelta. pub fn decide_delta( plan: &CollectionPlan, - w: &QueryWorkload, + w: &RegisteredWorkload, wc: &WorkloadCharacteristics, bytes_per_series_per_sec: f64, ) -> (DeltaDecision, TransmissionCostSummary) { @@ -474,21 +474,22 @@ mod tests { use chrono::Utc; use std::collections::HashMap; - fn workload_for(agg: AggType) -> QueryWorkload { - QueryWorkload { + fn workload_for(agg: AggType) -> RegisteredWorkload { + crate::registered_workload::fixtures::WorkloadFixture { metric_name: "m".into(), label_filters: HashMap::new(), group_by_labels: vec![], aggregations: vec![agg], time_window: Duration::from_secs(300), repeat_every: Some(Duration::from_secs(10)), - accuracy_sla: 0.01, + accuracy: crate::types::AccuracyTarget::Epsilon(0.01), latency_sla: None, sketch_type_override: None, exact_required: false, quantiles: vec![], } + .build() } fn make_plan(st: SketchType, window: Option) -> CollectionPlan { @@ -549,7 +550,7 @@ mod tests { #[test] fn flush_period_uses_repeat_every_in_batch_mode() { let mut w = workload_for(AggType::Frequency); - w.repeat_every = Some(Duration::from_secs(15)); + w.set_repeat_every(Some(Duration::from_secs(15))); let plan = make_plan(SketchType::CountMinSketch, None); assert_eq!(flush_period_secs(&plan, &w), 15.0); } @@ -557,7 +558,7 @@ mod tests { #[test] fn flush_period_batch_fallback_is_one_second() { let mut w = workload_for(AggType::Frequency); - w.repeat_every = None; + w.set_repeat_every(None); let plan = make_plan(SketchType::CountMinSketch, None); assert_eq!(flush_period_secs(&plan, &w), 1.0); } diff --git a/control_plane/src/physical/deployment_cost/mod.rs b/control_plane/src/physical/deployment_cost/mod.rs index 1f294fea9..22f977267 100644 --- a/control_plane/src/physical/deployment_cost/mod.rs +++ b/control_plane/src/physical/deployment_cost/mod.rs @@ -97,7 +97,7 @@ pub struct PlanScore { /// Estimates resource costs for a given plan + workload using the provided cost table. pub fn score_with( plan: &CollectionPlan, - w: &QueryWorkload, + w: &RegisteredWorkload, table: &HashMap, ) -> PlanScore { let st = &plan.agent_config.sketch_type; @@ -126,13 +126,13 @@ pub fn score_with( cpu_micros_per_sample: costs.cpu_micros_per_sample, memory_bytes: memory, estimated_error: err, - meets_sla: matches!(w.accuracy, crate::types::AccuracyTarget::Epsilon(epsilon) if epsilon > 0.0) + meets_sla: matches!(w.accuracy(), crate::types::AccuracyTarget::Epsilon(epsilon) if epsilon > 0.0) && err <= sla, } } /// Estimates resource costs for a given plan + workload. -pub fn score(plan: &CollectionPlan, w: &QueryWorkload) -> PlanScore { +pub fn score(plan: &CollectionPlan, w: &RegisteredWorkload) -> PlanScore { let table = benchmark_table(); let st = &plan.agent_config.sketch_type; @@ -164,7 +164,7 @@ pub fn score(plan: &CollectionPlan, w: &QueryWorkload) -> PlanScore { cpu_micros_per_sample: costs.cpu_micros_per_sample, memory_bytes: memory, estimated_error: err, - meets_sla: matches!(w.accuracy, crate::types::AccuracyTarget::Epsilon(epsilon) if epsilon > 0.0) + meets_sla: matches!(w.accuracy(), crate::types::AccuracyTarget::Epsilon(epsilon) if epsilon > 0.0) && err <= sla, } } @@ -233,25 +233,25 @@ impl DeploymentCostPlanner { /// /// `wc` drives the delta transmission decision: fill rate, flush rate, /// CPU / memory overhead, and raw vs. sketch bandwidth comparison. - /// Pass `None` to use conservative defaults (1 000 series, 100 Hz, - /// 100 B/sample, Zipf distribution, no memory budget). - pub fn plan(&self, w: &QueryWorkload, wc: Option<&WorkloadCharacteristics>) -> CollectionPlan { - if !matches!(w.accuracy, crate::types::AccuracyTarget::Epsilon(epsilon) if epsilon > 0.0) { + /// `None` means no usable data evidence: retain the legal rule-based plan + /// without estimating rate-dependent costs from fabricated defaults. + pub fn plan( + &self, + w: &RegisteredWorkload, + wc: Option<&WorkloadCharacteristics>, + ) -> CollectionPlan { + if !matches!(w.accuracy(), crate::types::AccuracyTarget::Epsilon(epsilon) if epsilon > 0.0) + { return self.inner.plan(w); } - let default_wc; - let wc = match wc { - Some(c) => c, - None => { - default_wc = WorkloadCharacteristics::default(); - &default_wc - } + let Some(wc) = wc else { + return self.inner.plan(w); }; let table = self.cost_table(); // If a specific sketch type is pinned, use it directly. - if let Some(st) = &w.sketch_type_override { + if let Some(st) = &w.deployment.sketch_type_override { let params = default_sketch_params(st, w.error_bound()); let (mode, window_duration) = select_window_strategy(w); let mut plan = self.inner.plan(w); @@ -263,7 +263,8 @@ impl DeploymentCostPlanner { return plan; } - let candidates = crate::physical::sketch_catalog::candidates_for_workload(&w.aggregations); + let candidates = + crate::physical::sketch_catalog::candidates_for_workload(&w.aggregations()); // Start with the rule-based plan as the baseline. let baseline = self.inner.plan(w); @@ -301,7 +302,7 @@ impl DeploymentCostPlanner { /// Runs the delta cost model and writes the decision into the plan using a provided cost table. fn apply_delta_decision_with( plan: &mut CollectionPlan, - w: &QueryWorkload, + w: &RegisteredWorkload, wc: &WorkloadCharacteristics, table: &HashMap, ) { @@ -345,21 +346,22 @@ mod tests { use std::collections::HashMap; use std::time::Duration; - fn workload(aggs: Vec) -> QueryWorkload { - QueryWorkload { + fn workload(aggs: Vec) -> RegisteredWorkload { + crate::registered_workload::fixtures::WorkloadFixture { metric_name: "test".into(), label_filters: HashMap::new(), group_by_labels: vec![], aggregations: aggs, time_window: Duration::from_secs(300), repeat_every: None, - accuracy_sla: 0.01, + accuracy: crate::types::AccuracyTarget::Epsilon(0.01), latency_sla: None, sketch_type_override: None, exact_required: false, quantiles: vec![], } + .build() } fn dummy_plan(st: SketchType) -> CollectionPlan { @@ -403,10 +405,10 @@ mod tests { #[test] fn ddsketch_fails_tight_sla() { - let w = QueryWorkload { - accuracy_sla: 0.001, - accuracy: crate::types::AccuracyTarget::Epsilon(0.001), - ..workload(vec![AggType::Quantile]) + let w = { + let mut w = workload(vec![AggType::Quantile]); + w.set_accuracy(crate::types::AccuracyTarget::Epsilon(0.001)); + w }; // Force 1% params despite tighter SLA. let mut plan = dummy_plan(SketchType::DDSketch); @@ -428,18 +430,20 @@ mod tests { #[test] fn dim_multiplier_increases_bandwidth() { - let w_few = QueryWorkload { - group_by_labels: vec!["host".into()], - ..workload(vec![AggType::Quantile]) + let w_few = { + let mut w = workload(vec![AggType::Quantile]); + w.deployment.retained_labels = vec!["host".into()]; + w }; - let w_many = QueryWorkload { - group_by_labels: vec![ + let w_many = { + let mut w = workload(vec![AggType::Quantile]); + w.deployment.retained_labels = vec![ "host".into(), "service".into(), "zone".into(), "region".into(), - ], - ..workload(vec![AggType::Quantile]) + ]; + w }; let pl = DeploymentPlanCompiler::new(); let s_few = score(&pl.plan(&w_few), &w_few); @@ -449,10 +453,10 @@ mod tests { #[test] fn kll_error_formula() { - let w = QueryWorkload { - accuracy_sla: 0.02, - accuracy: crate::types::AccuracyTarget::Epsilon(0.02), - ..workload(vec![AggType::Quantile]) + let w = { + let mut w = workload(vec![AggType::Quantile]); + w.set_accuracy(crate::types::AccuracyTarget::Epsilon(0.02)); + w }; let mut plan = dummy_plan(SketchType::KLL); plan.agent_config.sketch_params = SketchParams::KLL { @@ -471,27 +475,28 @@ mod tests { (AggType::Cardinality, 0.01), (AggType::Frequency, 0.02), ] { - let w = QueryWorkload { - accuracy_sla: sla, - accuracy: crate::types::AccuracyTarget::Epsilon(sla), - ..workload(vec![agg]) + let w = { + let mut w = workload(vec![agg]); + w.set_accuracy(crate::types::AccuracyTarget::Epsilon(sla)); + w }; let plan = pl.plan(&w, None); let s = score(&plan, &w); assert!( s.meets_sla, "agg={} sla={sla}: plan does not meet SLA (error={})", - w.aggregations[0], s.estimated_error + w.aggregations()[0], + s.estimated_error ); } } #[test] fn cost_model_prefers_lower_bandwidth_for_cardinality() { - let w = QueryWorkload { - accuracy_sla: 0.02, - accuracy: crate::types::AccuracyTarget::Epsilon(0.02), - ..workload(vec![AggType::Cardinality]) + let w = { + let mut w = workload(vec![AggType::Cardinality]); + w.set_accuracy(crate::types::AccuracyTarget::Epsilon(0.02)); + w }; let plan = DeploymentCostPlanner::new().plan(&w, None); assert_eq!( diff --git a/control_plane/src/physical/erp.rs b/control_plane/src/physical/erp.rs index 5240a7edc..a298e29a2 100644 --- a/control_plane/src/physical/erp.rs +++ b/control_plane/src/physical/erp.rs @@ -1457,11 +1457,14 @@ mod tests { query["query"] = "distinct_over_time(asap_demo_latency_ms[5s])".into(); query["requirements"]["accuracy"] = serde_json::json!({"explicit":{"Epsilon":0.05}}); fixture["query_workload"]["repeating_queries"] = serde_json::json!([query]); - let snapshot: crate::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: crate::physical::compiler::BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); - let plan = crate::physical::compiler::tests::quoted_snapshot(snapshot, false) - .compile() - .unwrap(); + let plan = crate::physical::compiler::tests::quoted_snapshot( + snapshot, + crate::physical::compiler::QueryFrontend::PromQl, + ) + .compile_promql() + .unwrap(); let (mut policy, mut observed) = online_population_fixture(); observed.catalog_generation = plan.summary_catalog.reference().unwrap(); observed.summary_definition_id = diff --git a/control_plane/src/physical/pane_reuse.rs b/control_plane/src/physical/pane_reuse.rs index 713624be2..b76cdbf00 100644 --- a/control_plane/src/physical/pane_reuse.rs +++ b/control_plane/src/physical/pane_reuse.rs @@ -1,6 +1,6 @@ use super::compiler::{ derived_window_cost, gcd, retained_state_count, CollectorMaterialization, - MaterializationLifecycleEstimate, PlanningRequest, RuntimeRulePolicy, + MaterializationLifecycleEstimate, PhysicalCompilationRequest, RuntimeRulePolicy, }; use asap_types::WindowMaterializationLayout; use planner_types::post_asap::{PostAsapNodeId, SummaryWindowFramework}; @@ -10,7 +10,7 @@ use std::collections::{BTreeMap, BTreeSet}; /// Only raw additive states are eligible; derived cohorts retain their full-window identity. #[allow(clippy::too_many_arguments)] pub(super) fn share_additive_panes( - request: &PlanningRequest, + request: &PhysicalCompilationRequest, materializations: &mut [asap_types::PrecomputeMaterialization], producers: &mut Vec, plan_producers: &mut [CollectorMaterialization], @@ -62,11 +62,10 @@ pub(super) fn share_additive_panes( // Provenance belongs to the selected layout, not to its request entry point. if consumers.iter().any(|index| { !request.queries[*index] - .window_implementations + .window_realization_candidates .iter() .any(|candidate| { - candidate.implementation_id == estimate.window_implementation_id - && candidate.derived + candidate.realization_id == estimate.window_realization_id && candidate.derived }) }) { continue; @@ -74,12 +73,12 @@ pub(super) fn share_additive_panes( let Some(policy) = policies.get(&old) else { continue; }; - let mut lifecycle = query.lifecycle.clone(); + let mut lifecycle = query.summary_lifecycle_inputs.clone(); lifecycle.evaluation_interval_ms = 0; if consumers.iter().any(|index| { - let mut other = request.queries[*index].lifecycle.clone(); + let mut other = request.queries[*index].summary_lifecycle_inputs.clone(); other.evaluation_interval_ms = 0; - other != lifecycle || request.queries[*index].accuracy != query.accuracy + other != lifecycle || request.queries[*index].accuracy_target != query.accuracy_target }) { continue; } @@ -95,7 +94,7 @@ pub(super) fn share_additive_panes( canonical.policy_fingerprint(), serde_json::to_string(&lifecycle).unwrap(), serde_json::to_string(policy).unwrap(), - serde_json::to_string(&query.accuracy).unwrap(), + serde_json::to_string(&query.accuracy_target).unwrap(), ); let index = physical.len(); if let Some(group) = groups.iter_mut().find(|group| keys[group[0]] == key) { @@ -131,7 +130,7 @@ pub(super) fn share_additive_panes( let mut canonical = physical[group.members[0]].1.clone(); canonical.num_aggregates_to_retain = Some(retained_state_count( group.lookback_ms, - request.query_staleness_margin_ms, + request.query_retention_margin_ms, canonical.slide_interval * 1000, &canonical.window_layout, )); @@ -157,7 +156,7 @@ pub(super) fn share_additive_panes( combined.expected_reads = 0.0; combined.expected_updates = 0.0; combined.lifecycle_cost = group.cost; - combined.window_implementation_id = format!("shared-pane-{}", new.0); + combined.window_realization_id = format!("shared-pane-{}", new.0); for index in group.members { let old = physical[index].0; if let Some(estimate) = estimates.remove(&old) { @@ -199,7 +198,7 @@ pub(super) fn share_additive_panes( producer.window_layout = canonical.window_layout.clone(); producer.pane_origin_ms = canonical.pane_origin_ms; producer.abstract_window_framework = SummaryWindowFramework::Tumbling; - producer.window_implementation_id = estimates[&new].window_implementation_id.clone(); + producer.window_realization_id = estimates[&new].window_realization_id.clone(); } } let mut seen = BTreeSet::new(); @@ -216,7 +215,7 @@ struct SharedGroup { } fn select_shared_groups( - request: &PlanningRequest, + request: &PhysicalCompilationRequest, physical: &[( asap_types::PolicyFingerprint, asap_types::PrecomputeMaterialization, @@ -257,14 +256,14 @@ fn select_shared_groups( let (old, m) = &physical[index]; let consumers = &member_consumers[index]; let query = &request.queries[*consumers.first().unwrap()]; - let selected_id = &estimates[old].window_implementation_id; + let selected_id = &estimates[old].window_realization_id; let template = &query - .window_implementations + .window_realization_candidates .iter() - .find(|candidate| &candidate.implementation_id == selected_id) + .find(|candidate| &candidate.realization_id == selected_id) .unwrap() .cost; - let mut maintenance = query.lifecycle.clone(); + let mut maintenance = query.summary_lifecycle_inputs.clone(); maintenance.costs.read = 0.0; independent += derived_window_cost( template, @@ -272,7 +271,7 @@ fn select_shared_groups( m.window_size, m.slide_interval, &m.window_layout, - request.query_staleness_margin_ms, + request.query_retention_margin_ms, ) .weighted_cost; producer = producer.max( @@ -282,12 +281,12 @@ fn select_shared_groups( m.window_size, m.slide_interval, &WindowMaterializationLayout::Pane { pane_secs }, - request.query_staleness_margin_ms, + request.query_retention_margin_ms, ) .weighted_cost, ); for &consumer in consumers { - let lifecycle = &request.queries[consumer].lifecycle; + let lifecycle = &request.queries[consumer].summary_lifecycle_inputs; let unit_reads = lifecycle.costs.read * lifecycle.horizon_seconds / (f64::from(lifecycle.evaluation_interval_ms) / 1_000.0); independent += diff --git a/control_plane/src/physical/plan_cache.rs b/control_plane/src/physical/plan_cache.rs index a032c878b..553edf035 100644 --- a/control_plane/src/physical/plan_cache.rs +++ b/control_plane/src/physical/plan_cache.rs @@ -17,7 +17,7 @@ use std::collections::HashMap; use std::sync::{Arc, RwLock}; use crate::physical::deployment_cost::DeploymentCostPlanner; -use crate::types::{CollectionPlan, QueryWorkload, WorkloadCharacteristics}; +use crate::types::{CollectionPlan, RegisteredWorkload}; pub struct CachedDeploymentPlanner { inner: DeploymentCostPlanner, @@ -34,12 +34,8 @@ impl CachedDeploymentPlanner { /// Return the baseline plan for this metric, or run the cost model and /// establish a new baseline if this is the first request for the metric. - pub fn plan( - &self, - workload: &QueryWorkload, - wc: Option<&WorkloadCharacteristics>, - ) -> CollectionPlan { - let key = &workload.metric_name; + pub fn plan(&self, workload: &RegisteredWorkload) -> CollectionPlan { + let key = &workload.metric_name(); // Fast path: return the cached plan if one exists. { @@ -50,7 +46,8 @@ impl CachedDeploymentPlanner { } // Slow path: first request for this metric — run cost optimisation. - let plan = self.inner.plan(workload, wc); + let facts = workload.characteristics_at(chrono::Utc::now().timestamp_millis() as u64); + let plan = self.inner.plan(workload, facts.as_ref()); self.cache .write() .unwrap() @@ -81,21 +78,22 @@ mod tests { use std::collections::HashMap; use std::time::Duration; - fn workload(metric: &str) -> QueryWorkload { - QueryWorkload { + fn workload(metric: &str) -> RegisteredWorkload { + crate::registered_workload::fixtures::WorkloadFixture { metric_name: metric.into(), label_filters: HashMap::new(), group_by_labels: vec![], aggregations: vec![AggType::Quantile], time_window: Duration::from_secs(300), repeat_every: None, - accuracy_sla: 0.01, + accuracy: crate::types::AccuracyTarget::Epsilon(0.01), latency_sla: None, sketch_type_override: None, exact_required: false, quantiles: vec![0.99], } + .build() } fn planner() -> CachedDeploymentPlanner { @@ -105,7 +103,7 @@ mod tests { #[test] fn first_call_produces_a_plan() { let p = planner(); - let plan = p.plan(&workload("latency"), None); + let plan = p.plan(&workload("latency")); // Cost model picks the cheapest sketch that meets the SLA; verify // we got a valid plan. transmit_sketch defaults to false (enabled // by DeploymentCostPlanner when appropriate). @@ -115,11 +113,11 @@ mod tests { #[test] fn second_call_returns_same_plan() { let p = planner(); - let first = p.plan(&workload("latency"), None); + let first = p.plan(&workload("latency")); // Change the workload — the baseline planner must ignore it. let mut w2 = workload("latency"); - w2.aggregations = vec![AggType::Cardinality]; - let second = p.plan(&w2, None); + w2.set_accuracy(crate::types::AccuracyTarget::Exact); + let second = p.plan(&w2); assert_eq!( first.agent_config.sketch_type, second.agent_config.sketch_type, "baseline plan must not change even when workload changes" @@ -129,8 +127,8 @@ mod tests { #[test] fn different_metrics_get_independent_plans() { let p = planner(); - let a = p.plan(&workload("metric_a"), None); - let b = p.plan(&workload("metric_b"), None); + let a = p.plan(&workload("metric_a")); + let b = p.plan(&workload("metric_b")); // Both plans are valid (exact sketch type may differ by cost model // internals, but we just check they are independently produced). let _ = (a, b); @@ -140,12 +138,12 @@ mod tests { #[test] fn reset_allows_re_plan() { let p = planner(); - let first = p.plan(&workload("latency"), None); + let first = p.plan(&workload("latency")); p.reset("latency"); assert!(p.baseline_metrics().is_empty()); // After reset the planner will run the cost model again on the same // workload and should produce an equivalent plan. - let second = p.plan(&workload("latency"), None); + let second = p.plan(&workload("latency")); assert_eq!( first.agent_config.sketch_type, second.agent_config.sketch_type, "same workload after reset should produce the same sketch type" @@ -155,9 +153,9 @@ mod tests { #[test] fn baseline_metrics_lists_all_seen_metrics() { let p = planner(); - p.plan(&workload("cpu"), None); - p.plan(&workload("mem"), None); - p.plan(&workload("cpu"), None); // repeat — should not double-count + p.plan(&workload("cpu")); + p.plan(&workload("mem")); + p.plan(&workload("cpu")); // repeat — should not double-count let mut metrics = p.baseline_metrics(); metrics.sort(); assert_eq!(metrics, vec!["cpu", "mem"]); diff --git a/control_plane/src/physical/post_asap/mod.rs b/control_plane/src/physical/post_asap/mod.rs index d446bc264..469db23e4 100644 --- a/control_plane/src/physical/post_asap/mod.rs +++ b/control_plane/src/physical/post_asap/mod.rs @@ -13,10 +13,10 @@ pub mod deployment_expr; pub mod lower; pub mod matcher; -#[cfg(test)] -mod tests; - // Re-exports — `crate::physical::post_asap::*` for downstream callers. pub use deployment_expr::{PhysicalExpr, PostAsapPlan}; pub use lower::{bind_query_expr, bind_query_expr_with_cost_model, BindingError}; pub use matcher::SummaryFamilyMatcher; + +#[cfg(test)] +mod tests; diff --git a/control_plane/src/physical/post_asap/tests.rs b/control_plane/src/physical/post_asap/tests.rs index 15b19d88b..f74f5afe4 100644 --- a/control_plane/src/physical/post_asap/tests.rs +++ b/control_plane/src/physical/post_asap/tests.rs @@ -738,7 +738,7 @@ fn phase_b_e2e_rate_falls_through_to_logical() { /// The legacy single-expression binder cannot choose a frequency sketch for /// value-ranked `topk(sum(rate(...)))` without membership evidence. The /// workload planner handles this query as query-time Sort+Limit over its -/// recursively planned child; its coverage lives in `query_plan::logical`. +/// recursively planned child; its coverage lives in `query_plan::residual`. #[test] fn phase_b_legacy_topk_requires_membership_evidence() { let query = "topk(10, sum by (instance) (rate(http_requests_total[5m])))"; diff --git a/control_plane/src/physical/publication.rs b/control_plane/src/physical/publication.rs index 63081fa5b..b12ac10ce 100644 --- a/control_plane/src/physical/publication.rs +++ b/control_plane/src/physical/publication.rs @@ -1,9 +1,9 @@ //! Control-plane construction of the shared catalog publication contract. -use super::compiler::PhysicalPlan; +use super::compiler::CompiledPhysicalPlan; pub use asap_types::plan_publication::{PhysicalPlanInstallRequest, PhysicalPlanPublication}; -impl PhysicalPlan { - pub fn publication(&self) -> Result { +impl CompiledPhysicalPlan { + pub fn to_publication_artifact(&self) -> Result { let artifact = PhysicalPlanPublication { summary_catalog: self.summary_catalog.clone(), precompute_plan: self.precompute_plan.clone(), @@ -15,3 +15,10 @@ impl PhysicalPlan { Ok(artifact) } } + +impl CompiledPhysicalPlan { + #[deprecated(note = "Use to_publication_artifact")] + pub fn publication(&self) -> Result { + self.to_publication_artifact() + } +} diff --git a/control_plane/src/physical/realization.rs b/control_plane/src/physical/realization.rs index 74d3a8fa6..61672dfa7 100644 --- a/control_plane/src/physical/realization.rs +++ b/control_plane/src/physical/realization.rs @@ -3,8 +3,8 @@ //! Providers may validate and price physical implementations, never rewrite //! selected logical roots or infer a pane width from a query's slide. use super::compiler::{ - CompileError, DeploymentEnvironment, PhysicalCompiler, PhysicalPlan, PlanningQuery, - PlanningRequest, + CompileError, CompiledPhysicalPlan, PhysicalCompilationRequest, PhysicalDeploymentContext, + PhysicalPlanCompiler, QueryCompilationInput, }; use super::workload_cost::{PricedComponents, WorkloadCostEvidence, WorkloadCostManifest}; use asap_aware_mapping::cost_model::Cost; @@ -21,22 +21,22 @@ pub(crate) trait RealizationProvider { fn windows( &self, - query: &PlanningQuery, - environment: &DeploymentEnvironment, + query: &QueryCompilationInput, + environment: &PhysicalDeploymentContext, ) -> Result, CompileError>; fn compile( &self, - request: PlanningRequest, - environment: DeploymentEnvironment, - metricsql: bool, - ) -> Result; + request: PhysicalCompilationRequest, + environment: PhysicalDeploymentContext, + frontend: super::compiler::QueryFrontend, + ) -> Result; fn price( &self, evidence: &WorkloadCostEvidence, manifest: &WorkloadCostManifest, - ) -> Result; + ) -> Result; } /// Only the currently implemented deployment paths. Capability validation @@ -58,22 +58,22 @@ impl RealizationProvider for ExistingRealizations { fn windows( &self, - query: &PlanningQuery, - environment: &DeploymentEnvironment, + query: &QueryCompilationInput, + environment: &PhysicalDeploymentContext, ) -> Result, CompileError> { super::compiler::validate_window_implementations(query, environment) } fn compile( &self, - request: PlanningRequest, - environment: DeploymentEnvironment, - metricsql: bool, - ) -> Result { - if metricsql { - PhysicalCompiler.compile_metricsql(request, environment) + request: PhysicalCompilationRequest, + environment: PhysicalDeploymentContext, + frontend: super::compiler::QueryFrontend, + ) -> Result { + if frontend == super::compiler::QueryFrontend::MetricsQl { + PhysicalPlanCompiler.compile_metricsql(request, environment) } else { - PhysicalCompiler.compile(request, environment) + PhysicalPlanCompiler.compile_promql(request, environment) } } @@ -81,7 +81,7 @@ impl RealizationProvider for ExistingRealizations { &self, evidence: &WorkloadCostEvidence, manifest: &WorkloadCostManifest, - ) -> Result { + ) -> Result { evidence.price(manifest) } } diff --git a/control_plane/src/physical/stage_split.rs b/control_plane/src/physical/stage_split.rs index 95b26d6ee..c75ec2a4c 100644 --- a/control_plane/src/physical/stage_split.rs +++ b/control_plane/src/physical/stage_split.rs @@ -88,25 +88,26 @@ mod l5_walk_propagation_tests { //! TODO in `intent_algebra::column_resolution`). use crate::physical::colored_dag::StageConfig; - use crate::types::{AggType, QueryWorkload}; + use crate::types::{AggType, RegisteredWorkload}; use std::collections::HashMap; use std::time::Duration; - fn workload(metric: &str, group_by: Vec, window: Duration) -> QueryWorkload { - QueryWorkload { + fn workload(metric: &str, group_by: Vec, window: Duration) -> RegisteredWorkload { + crate::registered_workload::fixtures::WorkloadFixture { metric_name: metric.to_string(), label_filters: HashMap::new(), group_by_labels: group_by, aggregations: vec![AggType::Quantile], time_window: window, repeat_every: None, - accuracy_sla: 0.01, + accuracy: crate::types::AccuracyTarget::Epsilon(0.01), latency_sla: None, sketch_type_override: None, exact_required: false, quantiles: vec![0.99], } + .build() } #[test] diff --git a/control_plane/src/physical/workload_cost.rs b/control_plane/src/physical/workload_cost.rs index 735660188..c41a395a6 100644 --- a/control_plane/src/physical/workload_cost.rs +++ b/control_plane/src/physical/workload_cost.rs @@ -1,13 +1,15 @@ -//! Complete, provider-priced comparisons of already-bound workload alternatives. +//! Compile physical candidates, export quoteable manifests, and select the +//! lowest-cost feasible candidate from the supplied bounded inventory. //! -//! This is an evidence manifest over the existing physical projection, not a -//! second semantic DAG. Planner supplies legal alternatives; deployment quotes -//! price every reachable operation, and the backend commits one complete plan. +//! Planner owns semantic legality. A manifest describes the exact physical +//! demand to price; provider quotes and candidate evaluations are separate. mod materialization_candidates; +mod status; +pub use status::{CandidateEvaluationStatus, CandidateSearchScope}; #[cfg(test)] -use super::compiler::PhysicalCompiler; +use super::compiler::PhysicalPlanCompiler; use std::collections::{BTreeMap, BTreeSet}; @@ -16,18 +18,21 @@ use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use super::compiler::{ - CompileError, DeploymentEnvironment, PhysicalPlan, PlanningQuery, PlanningRequest, + CompileError, CompiledPhysicalPlan, PhysicalCompilationRequest, PhysicalDeploymentContext, + QueryCompilationInput, }; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] -pub struct CostDemand { +pub struct CostComponentDemand { /// Exact implementation/configuration being priced, not merely a family. pub implementation: Value, /// `horizon` includes all work in the manifest's source/time scope; /// `query_evaluation` is one execution of this bound query operator. - pub unit: String, - pub multiplicity: f64, + #[serde(rename = "unit", alias = "pricing_basis")] + pub pricing_basis: String, + #[serde(rename = "multiplicity", alias = "occurrences_per_horizon")] + pub occurrences_per_horizon: f64, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] @@ -41,7 +46,7 @@ pub struct WorkloadCostManifest { pub horizon_seconds: f64, /// Canonical roots, requirements and demand must match across alternatives. pub workload: BTreeMap, - pub components: BTreeMap, + pub components: BTreeMap, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] @@ -72,14 +77,17 @@ pub struct WorkloadCostEvidence { } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct AlternativeCost { - pub alternative_id: Option, +/// A candidate diagnostic can precede pricing or record compilation failure. +pub struct CandidatePlanEvaluation { + #[serde(rename = "alternative_id", alias = "candidate_id")] + pub candidate_id: Option, #[serde(default)] pub logical_root_ids: Vec, - pub physical_alternative_id: Option, + #[serde(rename = "physical_alternative_id", alias = "physical_candidate_id")] + pub physical_candidate_id: Option, pub identity_unavailable_reason: Option, #[serde(default)] - pub status: String, + pub status: CandidateEvaluationStatus, pub plan_id: Option, pub total_cost: Option, pub unavailable_reason: Option, @@ -87,16 +95,23 @@ pub struct AlternativeCost { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct MaterializationSearchCoverage { - pub eligible_leaves: usize, - pub enumerated_local_masks: usize, + #[serde(rename = "eligible_leaves", alias = "eligible_materialization_count")] + pub eligible_materialization_count: usize, + #[serde( + rename = "enumerated_local_masks", + alias = "enumerated_candidate_key_sets" + )] + pub enumerated_candidate_key_sets: usize, pub exhaustive: bool, - pub scope: String, + #[serde(rename = "scope", alias = "search_scope")] + pub search_scope: CandidateSearchScope, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct WorkloadCostComparison { +pub struct CandidatePlanSelectionReport { #[serde(default)] - pub logical_selection: Vec, + #[serde(rename = "logical_selection", alias = "planner_selection_trace")] + pub planner_selection_trace: Vec, #[serde(default, alias = "index_search_coverage")] pub materialization_search_coverage: Option, pub data_snapshot_id: String, @@ -104,7 +119,8 @@ pub struct WorkloadCostComparison { pub selected_plan_id: u64, pub selected_manifest: WorkloadCostManifest, pub component_costs: BTreeMap, - pub alternatives: Vec, + #[serde(rename = "alternatives", alias = "candidate_evaluations")] + pub candidate_evaluations: Vec, } fn invalid(reason: impl Into) -> CompileError { @@ -112,13 +128,13 @@ fn invalid(reason: impl Into) -> CompileError { } pub fn manifest( - plan: &PhysicalPlan, - queries: &[PlanningQuery], + plan: &CompiledPhysicalPlan, + queries: &[QueryCompilationInput], ) -> Result { let horizon = queries .first() .ok_or_else(|| invalid("empty workload"))? - .lifecycle + .summary_lifecycle_inputs .horizon_seconds; if !horizon.is_finite() || horizon <= 0.0 { return Err(invalid("invalid horizon")); @@ -126,7 +142,8 @@ pub fn manifest( let mut workload = BTreeMap::new(); let mut reads = BTreeMap::new(); for query in queries { - if query.lifecycle.horizon_seconds != horizon || query.lifecycle.evaluation_interval_ms == 0 + if query.summary_lifecycle_inputs.horizon_seconds != horizon + || query.summary_lifecycle_inputs.evaluation_interval_ms == 0 { return Err(invalid("mixed horizons or unknown recurrence")); } @@ -135,9 +152,9 @@ pub fn manifest( .insert( query.query_id.clone(), json!({ - "query": canonical, "accuracy": query.accuracy, - "evaluation_interval_ms": query.lifecycle.evaluation_interval_ms, - "source": query.source, + "query": canonical, "accuracy": query.accuracy_target, + "evaluation_interval_ms": query.summary_lifecycle_inputs.evaluation_interval_ms, + "source": query.legacy_query_source, }), ) .is_some() @@ -146,20 +163,21 @@ pub fn manifest( } reads.insert( query.query_id.clone(), - horizon * 1000.0 / f64::from(query.lifecycle.evaluation_interval_ms), + horizon * 1000.0 / f64::from(query.summary_lifecycle_inputs.evaluation_interval_ms), ); } let mut components = BTreeMap::new(); - let mut add = |id: String, implementation: Value, unit: &str, multiplicity: f64| { - components.insert( - id, - CostDemand { - implementation, - unit: unit.into(), - multiplicity, - }, - ); - }; + let mut add = + |id: String, implementation: Value, pricing_basis: &str, occurrences_per_horizon: f64| { + components.insert( + id, + CostComponentDemand { + implementation, + pricing_basis: pricing_basis.into(), + occurrences_per_horizon, + }, + ); + }; // Backend merge/update, storage, and edge maintenance are separate work. // The raw input is read once per source partition, not once per consumer. for schema in &plan.precompute_plan.schemas { @@ -185,7 +203,7 @@ pub fn manifest( .find(|m| m.policy_fingerprint() == schema.materialization.fingerprint()) .ok_or_else(|| invalid("state has no physical implementation"))?; let identity = json!({"schema": schema, "location": location, "physical": physical, - "window_implementation": plan.lifecycle_estimates.iter().find(|e| e.materialization == schema.materialization).map(|e| &e.window_implementation_id)}); + "window_implementation": plan.lifecycle_estimates.iter().find(|e| e.materialization == schema.materialization).map(|e| &e.window_realization_id)}); for operation in ["build", "update", "residency", "retire"] { add( format!("state:{location}:{}:{operation}", schema.materialization.0), @@ -222,7 +240,7 @@ pub fn manifest( // quoted as already-provisioned/non-incremental for this decision. let parsed = crate::query_parser::parse_query_expr_canonical( &query.query_string, - query.accuracy.clone(), + query.accuracy_target.clone(), ) .map_err(|error| invalid(error.to_string()))?; for metric in exact_source_metrics(&parsed)? { @@ -236,7 +254,7 @@ pub fn manifest( if matches!( node, crate::query_plan::QueryPlanNode::Logical { - operator: crate::query_plan::logical::LogicalOperator::Scan { .. }, + operator: crate::query_plan::residual::ResidualQueryOperator::Scan { .. }, .. } ) { @@ -246,9 +264,10 @@ pub fn manifest( } if let crate::query_plan::QueryPlanNode::Logical { operator: - crate::query_plan::logical::LogicalOperator::ExactSubquery { query } - | crate::query_plan::logical::LogicalOperator::CandidateExactSubquery { - query, .. + crate::query_plan::residual::ResidualQueryOperator::ExactSubquery { query } + | crate::query_plan::residual::ResidualQueryOperator::CandidateExactSubquery { + query, + .. }, .. } = node @@ -355,7 +374,7 @@ pub(crate) fn exact_source_metrics( pub(super) type PricedComponents = (Cost, BTreeMap); impl WorkloadCostEvidence { - fn validate(&self, env: &DeploymentEnvironment) -> Result<(), CompileError> { + fn validate(&self, env: &PhysicalDeploymentContext) -> Result<(), CompileError> { if self.backend_revision != super::compiler::BACKEND_REVISION || self.planner_revision != super::compiler::PLANNER_REVISION { @@ -382,7 +401,7 @@ impl WorkloadCostEvidence { pub(super) fn price( &self, manifest: &WorkloadCostManifest, - ) -> Result { + ) -> Result { let quotes = self .quotes .iter() @@ -390,20 +409,20 @@ impl WorkloadCostEvidence { .collect::>(); if quotes.len() != 1 { return Err(( - "evidence_missing", + CandidateEvaluationStatus::EvidenceMissing, "missing or ambiguous quote for exact manifest".into(), )); } let quote = quotes[0]; if !quote.executable { return Err(( - "rejected", + CandidateEvaluationStatus::ProviderRejected, "provider reports unavailable implementation".into(), )); } if !quote.unit_costs.keys().eq(manifest.components.keys()) { return Err(( - "evidence_invalid", + CandidateEvaluationStatus::EvidenceInvalid, "incomplete or extraneous component evidence".into(), )); } @@ -411,64 +430,82 @@ impl WorkloadCostEvidence { let mut components = BTreeMap::new(); for (id, demand) in &manifest.components { let unit = quote.unit_costs[id]; - let cost = unit * demand.multiplicity; + let cost = unit * demand.occurrences_per_horizon; if !unit.is_finite() || unit < 0.0 || !cost.is_finite() || cost < 0.0 { - return Err(("evidence_invalid", format!("invalid cost for {id}"))); + return Err(( + CandidateEvaluationStatus::EvidenceInvalid, + format!("invalid cost for {id}"), + )); } total += cost; components.insert(id.clone(), cost); } if !total.is_finite() { - return Err(("evidence_invalid", "cost overflow".into())); + return Err(( + CandidateEvaluationStatus::EvidenceInvalid, + "cost overflow".into(), + )); } Ok((Cost(total), components)) } } -fn alternative_description(candidate: &PlanningRequest) -> AlternativeCost { +fn alternative_description(candidate: &PhysicalCompilationRequest) -> CandidatePlanEvaluation { let root_ids = candidate .queries .iter() - .map(|query| crate::planner_selection::explained_root_id(&query.post_asap, &query.accuracy)) + .map(|query| { + crate::planner_selection::explained_root_id( + &query.selected_plan_root, + &query.accuracy_target, + ) + }) .collect::>(); let complete = root_ids.iter().all(Option::is_some); let mut logical_root_ids = root_ids.into_iter().flatten().collect::>(); logical_root_ids.sort(); logical_root_ids.dedup(); - AlternativeCost { - alternative_id: complete.then(|| { + CandidatePlanEvaluation { + candidate_id: complete.then(|| { crate::planner_selection::explain_identity( "alternative", &( &logical_root_ids, - candidate.hybrid_execution, - &candidate.materialization_policy, + candidate.allow_mixed_summary_and_exact_execution, + &candidate.enabled_materialization_keys, ), ) }), logical_root_ids, identity_unavailable_reason: (!complete) .then(|| "lossless canonical executable export unavailable for a logical root".into()), - physical_alternative_id: None, - status: "bind_failed".into(), + physical_candidate_id: None, + status: CandidateEvaluationStatus::CompilationFailed, plan_id: None, total_cost: None, unavailable_reason: None, } } -fn bind_alternative( - candidate: PlanningRequest, - env: DeploymentEnvironment, - metricsql: bool, -) -> Result<(PhysicalPlan, WorkloadCostManifest, AlternativeCost), Box> { +fn compile_candidate_for_pricing( + candidate: PhysicalCompilationRequest, + env: PhysicalDeploymentContext, + frontend: super::compiler::QueryFrontend, +) -> Result< + ( + CompiledPhysicalPlan, + WorkloadCostManifest, + CandidatePlanEvaluation, + ), + Box, +> { let mut description = alternative_description(&candidate); let queries = candidate.queries.clone(); let compiled = super::realization::RealizationProvider::compile( &super::realization::ExistingRealizations, candidate, env, - metricsql, + frontend, ); let plan = match compiled { Ok(plan) => plan, @@ -483,7 +520,7 @@ fn bind_alternative( let mut bindings = plan .lifecycle_estimates .iter() - .map(|item| (item.materialization, item.window_implementation_id.clone())) + .map(|item| (item.materialization, item.window_realization_id.clone())) .collect::>(); bindings.sort(); let mut placement = plan @@ -500,21 +537,25 @@ fn bind_alternative( }) .collect::>(); placement.sort(); - description.physical_alternative_id = description.alternative_id.as_ref().map(|alternative| { - crate::planner_selection::explain_identity( - "physical", - &( - alternative, - bindings, - placement, - &plan.precompute_plan.ingest, - ), - ) - }); + description.physical_candidate_id = + description + .candidate_id + .as_ref() + .map(|logical_candidate_id| { + crate::planner_selection::explain_identity( + "physical", + &( + logical_candidate_id, + bindings, + placement, + &plan.precompute_plan.ingest, + ), + ) + }); match manifest(&plan, &queries) { Ok(manifest) => { - description.status = "bound".into(); + description.status = CandidateEvaluationStatus::AwaitingQuote; Ok((plan, manifest, description)) } Err(error) => { @@ -524,84 +565,99 @@ fn bind_alternative( } } -/// Preserve failed bindings alongside quoteable manifests. This does not select or publish. -pub fn prepare_manifests( - candidates: Vec, - env: DeploymentEnvironment, - metricsql: bool, -) -> (Vec, Vec) { +/// Preserve failed bindings alongside quoteable manifests. This does not select_lowest_cost_candidate or publish. +pub fn compile_candidates_for_pricing( + candidates: Vec, + env: PhysicalDeploymentContext, + frontend: super::compiler::QueryFrontend, +) -> (Vec, Vec) { let mut manifests = Vec::new(); - let mut alternatives = Vec::new(); + let mut candidate_evaluations = Vec::new(); for candidate in candidates { - match bind_alternative(candidate, env.clone(), metricsql) { + match compile_candidate_for_pricing(candidate, env.clone(), frontend) { Ok((_, manifest, description)) => { manifests.push(manifest); - alternatives.push(description); + candidate_evaluations.push(description); } - Err(description) => alternatives.push(*description), + Err(description) => candidate_evaluations.push(*description), } } - (manifests, alternatives) + (manifests, candidate_evaluations) } /// Compare complete Planner-authorized forests after binding. Infeasible or /// uncosted alternatives are retained as unavailable, never assigned zero. -pub fn select( - candidates: Vec, - env: DeploymentEnvironment, +pub fn select_lowest_cost_candidate( + candidates: Vec, + env: PhysicalDeploymentContext, evidence: &WorkloadCostEvidence, -) -> Result { - select_with_frontend(candidates, env, evidence, false) +) -> Result { + select_candidates( + candidates, + env, + evidence, + super::compiler::QueryFrontend::PromQl, + ) } -pub fn select_metricsql( - candidates: Vec, - env: DeploymentEnvironment, +pub fn select_lowest_cost_metricsql_candidate( + candidates: Vec, + env: PhysicalDeploymentContext, evidence: &WorkloadCostEvidence, -) -> Result { - select_with_frontend(candidates, env, evidence, true) +) -> Result { + select_candidates( + candidates, + env, + evidence, + super::compiler::QueryFrontend::MetricsQl, + ) } -fn select_with_frontend( - candidates: Vec, - env: DeploymentEnvironment, +fn select_candidates( + candidates: Vec, + env: PhysicalDeploymentContext, evidence: &WorkloadCostEvidence, - metricsql: bool, -) -> Result { + frontend: super::compiler::QueryFrontend, +) -> Result { evidence.validate(&env)?; if candidates.is_empty() || candidates.len() > 64 { return Err(invalid( "candidate inventory must contain 1..=64 alternatives", )); } - let policies: BTreeSet<_> = candidates + let candidate_key_sets: BTreeSet<_> = candidates .iter() - .filter(|c| c.hybrid_execution) - .filter_map(|c| c.materialization_policy.clone()) + .filter(|c| c.allow_mixed_summary_and_exact_execution) + .filter_map(|c| c.enabled_materialization_keys.clone()) .collect(); - let leaves: BTreeSet<_> = policies.iter().flat_map(|p| p.iter().cloned()).collect(); - let materialization_search_coverage = (!policies.is_empty()).then(|| MaterializationSearchCoverage { - eligible_leaves: leaves.len(), - enumerated_local_masks: policies.len(), - exhaustive: leaves.len() < usize::BITS as usize && policies.len() == (1usize << leaves.len()), - scope: "Backend materialization versus Prometheus exact-subquery masks over Planner-authorized leaves; native alternative separate; bounded inventory does not claim an unenumerated optimum".into(), - }); - let logical_selection = candidates[0].logical_selection.clone(); + let eligible_keys: BTreeSet<_> = candidate_key_sets + .iter() + .flat_map(|p| p.iter().cloned()) + .collect(); + let materialization_search_coverage = + (!candidate_key_sets.is_empty()).then(|| MaterializationSearchCoverage { + eligible_materialization_count: eligible_keys.len(), + enumerated_candidate_key_sets: candidate_key_sets.len(), + exhaustive: eligible_keys.len() < usize::BITS as usize + && candidate_key_sets.len() == (1usize << eligible_keys.len()), + search_scope: CandidateSearchScope::PlannerAuthorizedMaterializations, + }); + let planner_selection_trace = candidates[0].planner_selection_trace.clone(); let mut comparison_workload = None; - let mut alternatives = Vec::new(); + let mut candidate_evaluations = Vec::new(); let mut best_index = 0; let mut best: Option<( Cost, - PhysicalPlan, + CompiledPhysicalPlan, WorkloadCostManifest, BTreeMap, )> = None; for candidate in candidates { let (plan, manifest, mut description) = - match bind_alternative(candidate, env.clone(), metricsql) { + match compile_candidate_for_pricing(candidate, env.clone(), frontend) { Ok(bound) => bound, Err(description) => { - alternatives.push(*description); + candidate_evaluations.push(*description); continue; } }; @@ -621,80 +677,82 @@ fn select_with_frontend( &manifest, ) { Ok((cost, components)) => { - description.status = "unselected".into(); + description.status = CandidateEvaluationStatus::Unselected; description.total_cost = Some(cost.0); - alternatives.push(description); + candidate_evaluations.push(description); if best.as_ref().is_none_or(|(previous, ..)| cost < *previous) { - best_index = alternatives.len() - 1; + best_index = candidate_evaluations.len() - 1; best = Some((cost, plan, manifest, components)); } } Err((status, reason)) => { - description.status = status.into(); + description.status = status; description.unavailable_reason = Some(reason); - alternatives.push(description); + candidate_evaluations.push(description); } } } let (_, mut plan, selected_manifest, component_costs) = best.ok_or_else(|| { CompileError::Alternatives( - json!({"status": "all_infeasible", "logical_selection": logical_selection, - "alternatives": alternatives}), + json!({"status": "all_infeasible", "logical_selection": planner_selection_trace, + "alternatives": candidate_evaluations}), ) })?; // Exactly the winner retained by the existing strict-less-than selector. - alternatives[best_index].status = "selected".into(); - plan.cost_comparison = Some(WorkloadCostComparison { - logical_selection, + candidate_evaluations[best_index].status = CandidateEvaluationStatus::Selected; + plan.cost_comparison = Some(CandidatePlanSelectionReport { + planner_selection_trace, materialization_search_coverage, data_snapshot_id: evidence.data_snapshot_id.clone(), model_version: evidence.model_version.clone(), selected_plan_id: plan.envelope.plan_id, selected_manifest, component_costs, - alternatives, + candidate_evaluations, }); Ok(plan) } /// The current executor exposes continuously maintained state and the native -/// exact backend. Additional Planner-produced forests can use `select` directly. -pub fn with_exact_alternative( - request: PlanningRequest, -) -> Result, CompileError> { +/// exact backend. Additional Planner-produced forests can use `select_lowest_cost_candidate` directly. +pub fn enumerate_exact_and_materialized_candidates( + request: PhysicalCompilationRequest, +) -> Result, CompileError> { let mut exact = request.clone(); - exact.hybrid_execution = false; - exact.materialization_policy = None; + exact.allow_mixed_summary_and_exact_execution = false; + exact.enabled_materialization_keys = None; for query in &mut exact.queries { let parsed = crate::query_parser::parse_query_expr_canonical( &query.query_string, - query.accuracy.clone(), + query.accuracy_target.clone(), ) .map_err(|error| invalid(error.to_string()))?; - query.post_asap = crate::planner_selection::keep_pre_asap(&parsed) + query.selected_plan_root = crate::planner_selection::keep_pre_asap(&parsed) .map_err(|error| invalid(error.to_string()))?; } - if !request.hybrid_execution + if !request.allow_mixed_summary_and_exact_execution && request .queries .iter() .zip(&exact.queries) - .all(|(a, b)| a.post_asap == b.post_asap) + .all(|(a, b)| a.selected_plan_root == b.selected_plan_root) { Ok(vec![request]) } else { - if !request.hybrid_execution || request.materialization_policy.is_some() { + if !request.allow_mixed_summary_and_exact_execution + || request.enabled_materialization_keys.is_some() + { return Ok(vec![request, exact]); } let mut keys = BTreeSet::new(); for query in &request.queries { - match crate::query_plan::logical::materialization_candidate_keys( + match crate::query_plan::residual::eligible_materialization_keys( &query.query_string, - &query.post_asap, + &query.selected_plan_root, ) { Ok(found) => keys.extend(found), // A failed local projection must not make the native alternative - // disappear. Compile/select retains its concrete unavailability. + // disappear. Compile/select_lowest_cost_candidate retains its concrete unavailability. Err(_) => return Ok(vec![request, exact]), } } @@ -702,28 +760,62 @@ pub fn with_exact_alternative( return Ok(vec![request, exact]); } let inventory = materialization_candidates::enumerate(keys); - debug_assert_eq!(inventory.exhaustive, inventory.eligible_leaves <= 4); - let mut alternatives: Vec<_> = inventory - .masks + debug_assert_eq!( + inventory.exhaustive, + inventory.eligible_materialization_count <= 4 + ); + let mut candidate_requests: Vec<_> = inventory + .candidate_key_sets .into_iter() - .map(|mask| { + .map(|enabled_keys| { let mut candidate = request.clone(); - candidate.materialization_policy = Some(mask); + candidate.enabled_materialization_keys = Some(enabled_keys); candidate }) .collect(); - alternatives.push(exact); - Ok(alternatives) + candidate_requests.push(exact); + Ok(candidate_requests) } } +// Compatibility imports; new callers use the domain names above. +#[deprecated(note = "Use CandidatePlanEvaluation")] +pub use CandidatePlanEvaluation as AlternativeCost; +#[deprecated(note = "Use CandidatePlanSelectionReport")] +pub use CandidatePlanSelectionReport as WorkloadCostComparison; +#[deprecated(note = "Use CostComponentDemand")] +pub use CostComponentDemand as CostDemand; + +#[deprecated(note = "Use enumerate_exact_and_materialized_candidates")] +pub use enumerate_exact_and_materialized_candidates as with_exact_alternative; +#[deprecated(note = "Use select_lowest_cost_candidate")] +pub use select_lowest_cost_candidate as select; +#[deprecated(note = "Use select_lowest_cost_metricsql_candidate")] +pub use select_lowest_cost_metricsql_candidate as select_metricsql; +#[deprecated(note = "Use compile_candidates_for_pricing with QueryFrontend")] +pub fn prepare_manifests( + candidates: Vec, + env: PhysicalDeploymentContext, + metricsql: bool, +) -> (Vec, Vec) { + compile_candidates_for_pricing( + candidates, + env, + if metricsql { + super::compiler::QueryFrontend::MetricsQl + } else { + super::compiler::QueryFrontend::PromQl + }, + ) +} + #[cfg(test)] mod tests { - use super::super::compiler::BackendLocalPlanningSnapshot; + use super::super::compiler::BackendLocalPlanningInput; use super::*; - fn fixture() -> BackendLocalPlanningSnapshot { - let mut snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + fn fixture() -> BackendLocalPlanningInput { + let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); @@ -732,46 +824,144 @@ mod tests { snapshot } + // New input aliases must produce the same candidate identities and manifests + // while serialization continues to serve existing evidence producers. + #[test] + fn renamed_inputs_preserve_candidate_manifests_and_wire_names() { + let legacy = serde_json::to_value(fixture()).unwrap(); + assert!(legacy.get("snapshot_version").is_some()); + assert!(legacy.get("physical_inputs").is_none()); + let mut renamed = legacy.clone(); + let root = renamed.as_object_mut().unwrap(); + let version = root.remove("snapshot_version").unwrap(); + root.insert("schema_version".into(), version); + let inputs = root.remove("implementation").unwrap(); + // Upstream window planning now consumes a cost model; removed default + // window fields are no longer part of the naming compatibility contract. + assert!(inputs.get("window_cost_model").is_some()); + assert!(inputs.get("window_implementation_id").is_none()); + assert!(inputs.get("implementation_cost").is_none()); + root.insert("physical_inputs".into(), inputs); + let environment = root + .get_mut("environment") + .unwrap() + .as_object_mut() + .unwrap(); + let collectors = environment.remove("collector_ids").unwrap(); + environment.insert("target_collector_ids".into(), collectors); + + let old: BackendLocalPlanningInput = serde_json::from_value(legacy.clone()).unwrap(); + let new: BackendLocalPlanningInput = serde_json::from_value(renamed).unwrap(); + assert_eq!(old, new); + assert_eq!(serde_json::to_value(&new).unwrap(), legacy); + // Shared publication fields already had domain names: renaming the + // streaming accessor must not change their wire keys or catalog hash. + let (request, environment) = new.clone().into_physical_compilation_request().unwrap(); + let plan = PhysicalPlanCompiler + .compile_promql(request, environment) + .unwrap(); + for document in [ + serde_json::to_value(&plan.summary_catalog).unwrap(), + serde_json::to_value(&plan.precompute_plan).unwrap(), + ] { + assert!(document.get("materializations").is_some()); + assert!(document.get("get_all_aggregation_configs").is_none()); + } + let compile = |input: BackendLocalPlanningInput| { + let (request, environment) = input.into_physical_compilation_request().unwrap(); + compile_candidates_for_pricing( + enumerate_exact_and_materialized_candidates(request).unwrap(), + environment, + super::super::compiler::QueryFrontend::PromQl, + ) + }; + let old_candidates = compile(old); + let new_candidates = compile(new); + assert!(!old_candidates.0.is_empty()); + assert_eq!(old_candidates, new_candidates); + } + + // A renamed demand remains readable by old quote providers, including + // fractional recurrence; missing and future statuses keep round-tripping. + #[test] + fn demand_and_evaluation_keep_legacy_wire_contracts() { + let old = json!({"implementation": {"op": "read"}, "unit": "query_evaluation", "multiplicity": 2.5}); + let demand: CostComponentDemand = serde_json::from_value(old.clone()).unwrap(); + assert_eq!(demand.pricing_basis, "query_evaluation"); + assert_eq!(demand.occurrences_per_horizon, 2.5); + assert_eq!(serde_json::to_value(demand).unwrap(), old); + let row = json!({ + "alternative_id": null, "physical_alternative_id": null, + "identity_unavailable_reason": null, "plan_id": null, + "total_cost": null, "unavailable_reason": null + }); + let evaluation: CandidatePlanEvaluation = serde_json::from_value(row).unwrap(); + assert_eq!(evaluation.status, CandidateEvaluationStatus::Unspecified); + let encoded = serde_json::to_value(evaluation).unwrap(); + assert_eq!(encoded["status"], ""); + assert!(encoded.get("alternative_id").is_some()); + assert!(encoded.get("candidate_id").is_none()); + } + // IDs describe semantics; activation/version changes do not create new alternatives. #[test] fn explain_identity_is_stable_across_activations_and_distinguishes_native() { - let (request, mut env) = fixture().planning_request().unwrap(); - let candidates = with_exact_alternative(request).unwrap(); - let (_, first) = prepare_manifests(candidates.clone(), env.clone(), false); + let (request, mut env) = fixture().into_physical_compilation_request().unwrap(); + let candidates = enumerate_exact_and_materialized_candidates(request).unwrap(); + let (_, first) = compile_candidates_for_pricing( + candidates.clone(), + env.clone(), + crate::physical::compiler::QueryFrontend::PromQl, + ); env.plan_version += 1; env.activation_unix_ms += 1; - let (_, second) = prepare_manifests(candidates, env, false); + let (_, second) = compile_candidates_for_pricing( + candidates, + env, + crate::physical::compiler::QueryFrontend::PromQl, + ); assert_eq!(first.len(), second.len()); for (a, b) in first.iter().zip(&second) { - assert!(a.alternative_id.is_some()); - assert!(a.physical_alternative_id.is_some()); - assert_eq!(a.alternative_id, b.alternative_id); - assert_eq!(a.physical_alternative_id, b.physical_alternative_id); + assert!(a.candidate_id.is_some()); + assert!(a.physical_candidate_id.is_some()); + assert_eq!(a.candidate_id, b.candidate_id); + assert_eq!(a.physical_candidate_id, b.physical_candidate_id); } assert_ne!( - first.first().unwrap().alternative_id, - first.last().unwrap().alternative_id + first.first().unwrap().candidate_id, + first.last().unwrap().candidate_id ); } // Bind failures remain visible even when the native manifest is usable. #[test] fn explain_retains_failed_bindings_and_all_missing_quotes() { - let (mut request, env) = fixture().planning_request().unwrap(); - request.hybrid_execution = false; - request.queries[0].window_implementations.clear(); - let candidates = with_exact_alternative(request).unwrap(); - let (manifests, explanations) = prepare_manifests(candidates, env, false); + let (mut request, env) = fixture().into_physical_compilation_request().unwrap(); + request.allow_mixed_summary_and_exact_execution = false; + request.queries[0].window_realization_candidates.clear(); + let candidates = enumerate_exact_and_materialized_candidates(request).unwrap(); + let (manifests, explanations) = compile_candidates_for_pricing( + candidates, + env, + crate::physical::compiler::QueryFrontend::PromQl, + ); assert_eq!(manifests.len(), 1); assert_eq!(explanations.len(), 2); - assert_eq!(explanations[0].status, "bind_failed"); + assert_eq!( + explanations[0].status, + CandidateEvaluationStatus::CompilationFailed + ); assert!(explanations[0].unavailable_reason.is_some()); - assert_eq!(explanations[1].status, "bound"); + assert_eq!( + explanations[1].status, + CandidateEvaluationStatus::AwaitingQuote + ); let (candidates, env, mut evidence) = quoted(); let count = candidates.len(); evidence.quotes.clear(); - let CompileError::Alternatives(report) = select(candidates, env, &evidence).unwrap_err() + let CompileError::Alternatives(report) = + select_lowest_cost_candidate(candidates, env, &evidence).unwrap_err() else { panic!("expected structured all-infeasible report") }; @@ -797,19 +987,22 @@ mod tests { }) .min_by(|a, b| a.0.total_cmp(&b.0)) .unwrap(); - let plan = select(candidates, env, &evidence).unwrap(); + let plan = select_lowest_cost_candidate(candidates, env, &evidence).unwrap(); assert_eq!(plan.envelope.plan_id, expected.1); let comparison = plan.cost_comparison.unwrap(); let selected = comparison - .alternatives + .candidate_evaluations .iter() - .filter(|item| item.status == "selected") + .filter(|item| item.status == CandidateEvaluationStatus::Selected) .collect::>(); assert_eq!(selected.len(), 1); assert_eq!(selected[0].total_cost, Some(expected.0)); - assert!(selected[0].alternative_id.is_some()); - assert!(selected[0].physical_alternative_id.is_some()); - assert_eq!(comparison.logical_selection, plan.logical_selection); + assert!(selected[0].candidate_id.is_some()); + assert!(selected[0].physical_candidate_id.is_some()); + assert_eq!( + comparison.planner_selection_trace, + plan.planner_selection_trace + ); } #[test] @@ -828,8 +1021,10 @@ mod tests { "max_over_time(service_retry_queue_depth{job=\"order-service\"}[6h])".into(), ); entries.push(second); - let (request, env) = snapshot.planning_request().unwrap(); - let plan = PhysicalCompiler.compile(request.clone(), env).unwrap(); + let (request, env) = snapshot.into_physical_compilation_request().unwrap(); + let plan = PhysicalPlanCompiler + .compile_promql(request.clone(), env) + .unwrap(); let costs = manifest(&plan, &request.queries).unwrap(); assert_eq!( costs @@ -870,14 +1065,22 @@ mod tests { q.requirements.accuracy = planner_types::workload::AccuracyRequirement::Explicit( crate::types::AccuracyTarget::Exact, ); - let (request, environment) = snapshot.planning_request().unwrap(); - let candidates = with_exact_alternative(request).unwrap(); - assert_eq!(candidates.len(), 5, "four legal masks plus native"); + let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); + let candidates = enumerate_exact_and_materialized_candidates(request).unwrap(); + assert_eq!( + candidates.len(), + 5, + "four legal candidate key sets plus native" + ); let mut identities = BTreeSet::new(); for candidate in &candidates[..4] { - let enabled = candidate.materialization_policy.as_ref().unwrap().len(); - let plan = PhysicalCompiler - .compile(candidate.clone(), environment.clone()) + let enabled = candidate + .enabled_materialization_keys + .as_ref() + .unwrap() + .len(); + let plan = PhysicalPlanCompiler + .compile_promql(candidate.clone(), environment.clone()) .unwrap(); assert!(identities.insert(plan.envelope.plan_id)); let cost = manifest(&plan, &candidate.queries).unwrap(); @@ -898,7 +1101,7 @@ mod tests { assert_eq!( cost.components .values() - .filter(|v| v.unit == "horizon" + .filter(|v| v.pricing_basis == "horizon" && v.implementation.get("location").and_then(Value::as_str) == Some("exact_backend")) .count(), @@ -913,21 +1116,26 @@ mod tests { crate::query_plan::QueryPlanNode::ExactFallback { .. } )))); } - assert!(!candidates.last().unwrap().hybrid_execution); + assert!( + !candidates + .last() + .unwrap() + .allow_mixed_summary_and_exact_execution + ); } fn quoted() -> ( - Vec, - DeploymentEnvironment, + Vec, + PhysicalDeploymentContext, WorkloadCostEvidence, ) { - let (request, env) = fixture().planning_request().unwrap(); - let candidates = with_exact_alternative(request).unwrap(); + let (request, env) = fixture().into_physical_compilation_request().unwrap(); + let candidates = enumerate_exact_and_materialized_candidates(request).unwrap(); let quotes = candidates .iter() .map(|candidate| { - let plan = PhysicalCompiler - .compile(candidate.clone(), env.clone()) + let plan = PhysicalPlanCompiler + .compile_promql(candidate.clone(), env.clone()) .unwrap(); let manifest = manifest(&plan, &candidate.queries).unwrap(); let unit_costs = manifest @@ -963,14 +1171,14 @@ mod tests { entry.query = Query("sum(rate(a{job=\"x\"}[1m])) / sum(rate(a{job!=\"x\"}[5m]))".into()); entry.requirements.accuracy = AccuracyRequirement::Explicit(crate::types::AccuracyTarget::Exact); - let (request, environment) = snapshot.planning_request().unwrap(); - let candidates = with_exact_alternative(request).unwrap(); + let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); + let candidates = enumerate_exact_and_materialized_candidates(request).unwrap(); assert!(candidates.len() >= 2); - let local = PhysicalCompiler - .compile(candidates[0].clone(), environment.clone()) + let local = PhysicalPlanCompiler + .compile_promql(candidates[0].clone(), environment.clone()) .unwrap(); - let native = PhysicalCompiler - .compile(candidates.last().unwrap().clone(), environment) + let native = PhysicalPlanCompiler + .compile_promql(candidates.last().unwrap().clone(), environment) .unwrap(); assert_ne!(local.envelope.plan_id, native.envelope.plan_id); let manifest = manifest(&local, &candidates[0].queries).unwrap(); @@ -1011,7 +1219,7 @@ mod tests { ), ("sum_over_time(m[1m]) + count_over_time(m[1m])", vec!["m"]), ] { - let (mut request, env) = fixture().planning_request().unwrap(); + let (mut request, env) = fixture().into_physical_compilation_request().unwrap(); request.queries[0].query_string = query.into(); request .query_workload @@ -1021,9 +1229,12 @@ mod tests { .as_mut() .unwrap()[0] .query = planner_types::workload::Query(query.into()); - let exact = with_exact_alternative(request).unwrap().pop().unwrap(); - let plan = PhysicalCompiler - .compile(exact.clone(), env.clone()) + let exact = enumerate_exact_and_materialized_candidates(request) + .unwrap() + .pop() + .unwrap(); + let plan = PhysicalPlanCompiler + .compile_promql(exact.clone(), env.clone()) .unwrap(); let manifest = manifest(&plan, &exact.queries).unwrap(); let sources: Vec<_> = manifest @@ -1054,7 +1265,9 @@ mod tests { executable: true, }], }; - assert!(select(vec![exact.clone()], env.clone(), &evidence).is_ok()); + assert!( + select_lowest_cost_candidate(vec![exact.clone()], env.clone(), &evidence).is_ok() + ); let source_id = evidence.quotes[0] .unit_costs .keys() @@ -1063,7 +1276,7 @@ mod tests { .clone(); evidence.quotes[0].unit_costs.remove(&source_id); assert!( - select(vec![exact], env, &evidence).is_err(), + select_lowest_cost_candidate(vec![exact], env, &evidence).is_err(), "missing input upkeep must fail closed" ); } @@ -1072,8 +1285,12 @@ mod tests { // Hidden or unresolved sources must not yield a partially priced manifest. #[test] fn exact_source_discovery_rejects_unresolved_inputs() { - let accuracy = fixture().planning_request().unwrap().0.queries[0] - .accuracy + let accuracy = fixture() + .into_physical_compilation_request() + .unwrap() + .0 + .queries[0] + .accuracy_target .clone(); for query in ["info(m)", "{job=\"api\"}"] { let parsed = @@ -1099,21 +1316,25 @@ mod tests { for cost in evidence.quotes[1].unit_costs.values_mut() { *cost = 1000.0; } - let warm = select(candidates.clone(), env.clone(), &evidence).unwrap(); + let warm = + select_lowest_cost_candidate(candidates.clone(), env.clone(), &evidence).unwrap(); assert_eq!(warm.envelope.plan_id, evidence.quotes[0].manifest.plan_id); let report = warm.cost_comparison.unwrap(); assert_eq!( report.component_costs.len(), report.selected_manifest.components.len() ); - assert_eq!(report.alternatives.len(), 2); - assert!(report.alternatives.iter().all(|a| a.total_cost.is_some())); + assert_eq!(report.candidate_evaluations.len(), 2); + assert!(report + .candidate_evaluations + .iter() + .all(|a| a.total_cost.is_some())); for (id, cost) in &mut evidence.quotes[0].unit_costs { if id.ends_with(":residency") { *cost = 1e9; } } - let raw = select(candidates, env, &evidence).unwrap(); + let raw = select_lowest_cost_candidate(candidates, env, &evidence).unwrap(); assert_eq!(raw.envelope.plan_id, evidence.quotes[1].manifest.plan_id); } @@ -1121,48 +1342,51 @@ mod tests { fn incomplete_unavailable_and_wrong_generation_quotes_are_not_free() { let (candidates, env, mut evidence) = quoted(); evidence.quotes[0].unit_costs.pop_first(); - let plan = select(candidates.clone(), env.clone(), &evidence).unwrap(); - assert!(plan.cost_comparison.unwrap().alternatives[0] + let plan = + select_lowest_cost_candidate(candidates.clone(), env.clone(), &evidence).unwrap(); + assert!(plan.cost_comparison.unwrap().candidate_evaluations[0] .unavailable_reason .is_some()); evidence.quotes[1].executable = false; - assert!(select(candidates.clone(), env.clone(), &evidence).is_err()); + assert!(select_lowest_cost_candidate(candidates.clone(), env.clone(), &evidence).is_err()); let (_, _, mut evidence) = quoted(); evidence .quotes .iter_mut() .for_each(|quote| quote.manifest.capability_snapshot_id.push_str("-wrong")); - assert!(select(candidates.clone(), env.clone(), &evidence).is_err()); + assert!(select_lowest_cost_candidate(candidates.clone(), env.clone(), &evidence).is_err()); let (_, _, mut evidence) = quoted(); evidence.observed_at_unix_ms = env.observed_at_unix_ms + 1; - assert!(select(candidates, env, &evidence).is_err()); + assert!(select_lowest_cost_candidate(candidates, env, &evidence).is_err()); } #[test] fn evidence_from_a_different_compiler_build_is_rejected_before_matching_quotes() { let (candidates, env, mut evidence) = quoted(); evidence.backend_revision = "stale-backend-build".into(); - let error = select(candidates, env, &evidence).unwrap_err().to_string(); + let error = select_lowest_cost_candidate(candidates, env, &evidence) + .unwrap_err() + .to_string(); assert!(error.contains("cost evidence compiler mismatch"), "{error}"); } #[test] fn snapshot_requires_quotes_and_roundtrips_selection() { let mut snapshot = fixture(); - assert!(snapshot.clone().compile().is_err()); + assert!(snapshot.clone().compile_promql().is_err()); let (_, _, evidence) = quoted(); snapshot.workload_cost_evidence = Some(evidence); - let snapshot: BackendLocalPlanningSnapshot = + let snapshot: BackendLocalPlanningInput = serde_json::from_str(&serde_json::to_string(&snapshot).unwrap()).unwrap(); - assert!(snapshot.compile().unwrap().cost_comparison.is_some()); + assert!(snapshot.compile_promql().unwrap().cost_comparison.is_some()); } #[test] fn second_consumer_adds_reads_not_another_shared_state() { - let (request, env) = fixture().planning_request().unwrap(); + let (request, env) = fixture().into_physical_compilation_request().unwrap(); let first = manifest( - &PhysicalCompiler - .compile(request.clone(), env.clone()) + &PhysicalPlanCompiler + .compile_promql(request.clone(), env.clone()) .unwrap(), &request.queries, ) @@ -1189,20 +1413,22 @@ mod tests { std::rc::Rc::new( crate::query_parser::parse_query_expr_canonical( &query.query_string, - query.accuracy.clone(), + query.accuracy_target.clone(), ) .unwrap(), ) }) .collect(); - super::super::compiler::select_workload_roots( + super::super::compiler::select_logical_roots_for_queries( &mut shared.queries, roots, - &shared.evidence, + &shared.topk_membership_evidence_by_query_id, &shared.exact_composition_costs, ) .unwrap(); - let plan = PhysicalCompiler.compile(shared.clone(), env).unwrap(); + let plan = PhysicalPlanCompiler + .compile_promql(shared.clone(), env) + .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); let second = manifest(&plan, &shared.queries).unwrap(); let states = |m: &WorkloadCostManifest| { @@ -1220,9 +1446,14 @@ mod tests { fn exact_alternative_does_not_require_unused_state_implementation_evidence() { let (candidates, env, evidence) = quoted(); let mut exact = candidates[1].clone(); - assert_eq!(with_exact_alternative(exact.clone()).unwrap().len(), 1); - exact.queries[0].window_implementations.clear(); - assert!(select(vec![exact], env, &evidence).is_ok()); + assert_eq!( + enumerate_exact_and_materialized_candidates(exact.clone()) + .unwrap() + .len(), + 1 + ); + exact.queries[0].window_realization_candidates.clear(); + assert!(select_lowest_cost_candidate(vec![exact], env, &evidence).is_ok()); } #[test] @@ -1232,14 +1463,14 @@ mod tests { .quotes .iter_mut() .for_each(|quote| quote.manifest.horizon_seconds += 1.0); - assert!(select(candidates.clone(), env.clone(), &evidence).is_err()); + assert!(select_lowest_cost_candidate(candidates.clone(), env.clone(), &evidence).is_err()); let (_, _, mut evidence) = quoted(); evidence.quotes.extend(evidence.quotes.clone()); - assert!(select(candidates.clone(), env.clone(), &evidence).is_err()); + assert!(select_lowest_cost_candidate(candidates.clone(), env.clone(), &evidence).is_err()); let (_, _, mut evidence) = quoted(); for quote in &mut evidence.quotes { *quote.unit_costs.values_mut().next().unwrap() = -1.0; } - assert!(select(candidates, env, &evidence).is_err()); + assert!(select_lowest_cost_candidate(candidates, env, &evidence).is_err()); } } diff --git a/control_plane/src/physical/workload_cost/materialization_candidates.rs b/control_plane/src/physical/workload_cost/materialization_candidates.rs index 48c9efa71..2344192f2 100644 --- a/control_plane/src/physical/workload_cost/materialization_candidates.rs +++ b/control_plane/src/physical/workload_cost/materialization_candidates.rs @@ -3,53 +3,53 @@ use std::collections::BTreeSet; #[derive(Debug)] -pub(super) struct MaterializationMasks { - pub masks: Vec>, +pub(super) struct MaterializationCandidateSets { + pub candidate_key_sets: Vec>, pub exhaustive: bool, - pub eligible_leaves: usize, + pub eligible_materialization_count: usize, } -/// Enumerate every mask for up to four leaves. Larger forests retain all-materialized, +/// Enumerate every enabled_keys for up to four leaves. Larger forests retain all-materialized, /// all-exact, then singleton/complement pairs in stable key order. The caller must /// disclose bounded coverage; no unenumerated optimum is claimed. Reserve one /// of the selector's 64 candidate slots for native execution. -pub(super) fn enumerate(keys: BTreeSet) -> MaterializationMasks { - let eligible_leaves = keys.len(); +pub(super) fn enumerate(keys: BTreeSet) -> MaterializationCandidateSets { + let eligible_materialization_count = keys.len(); let ordered: Vec<_> = keys.iter().cloned().collect(); - let exhaustive = eligible_leaves <= 4; - let mut masks = vec![keys.clone()]; + let exhaustive = eligible_materialization_count <= 4; + let mut candidate_key_sets = vec![keys.clone()]; if exhaustive { - for bits in 0..(1usize << eligible_leaves) { - let mask = ordered + for bits in 0..(1usize << eligible_materialization_count) { + let enabled_keys = ordered .iter() .enumerate() .filter(|(i, _)| bits & (1 << i) != 0) .map(|(_, key)| key.clone()) .collect(); - if !masks.contains(&mask) { - masks.push(mask); + if !candidate_key_sets.contains(&enabled_keys) { + candidate_key_sets.push(enabled_keys); } } } else { - masks.push(BTreeSet::new()); + candidate_key_sets.push(BTreeSet::new()); for key in ordered { - for mask in [ + for enabled_keys in [ BTreeSet::from([key.clone()]), keys.difference(&BTreeSet::from([key])).cloned().collect(), ] { - if masks.len() >= 63 { + if candidate_key_sets.len() >= 63 { break; } - if !masks.contains(&mask) { - masks.push(mask); + if !candidate_key_sets.contains(&enabled_keys) { + candidate_key_sets.push(enabled_keys); } } } } - MaterializationMasks { - masks, + MaterializationCandidateSets { + candidate_key_sets, exhaustive, - eligible_leaves, + eligible_materialization_count, } } @@ -60,24 +60,45 @@ mod tests { fn small_forests_cover_every_mixed_path_once() { let result = enumerate(BTreeSet::from(["a".into(), "b".into()])); assert!(result.exhaustive); - assert_eq!(result.eligible_leaves, 2); - assert_eq!(result.masks.len(), 4); - assert_eq!(result.masks.iter().collect::>().len(), 4); - assert!(result.masks.contains(&BTreeSet::from(["a".into()]))); - assert!(result.masks.contains(&BTreeSet::from(["b".into()]))); + assert_eq!(result.eligible_materialization_count, 2); + assert_eq!(result.candidate_key_sets.len(), 4); + assert_eq!( + result + .candidate_key_sets + .iter() + .collect::>() + .len(), + 4 + ); + assert!(result + .candidate_key_sets + .contains(&BTreeSet::from(["a".into()]))); + assert!(result + .candidate_key_sets + .contains(&BTreeSet::from(["b".into()]))); } #[test] fn large_inventory_reserves_native_slot_and_discloses_truncation() { let keys = (0..100).map(|i| format!("{i:03}")).collect(); let result = enumerate(keys); assert!(!result.exhaustive); - assert_eq!(result.masks.len(), 63); - assert_eq!(result.masks[0].len(), 100); - assert!(result.masks[1].is_empty()); - assert_eq!(result.masks.iter().collect::>().len(), 63); + assert_eq!(result.candidate_key_sets.len(), 63); + assert_eq!(result.candidate_key_sets[0].len(), 100); + assert!(result.candidate_key_sets[1].is_empty()); + assert_eq!( + result + .candidate_key_sets + .iter() + .collect::>() + .len(), + 63 + ); } #[test] fn no_materialization_has_one_exact_implementation() { - assert_eq!(enumerate(BTreeSet::new()).masks, vec![BTreeSet::new()]); + assert_eq!( + enumerate(BTreeSet::new()).candidate_key_sets, + vec![BTreeSet::new()] + ); } } diff --git a/control_plane/src/physical/workload_cost/status.rs b/control_plane/src/physical/workload_cost/status.rs new file mode 100644 index 000000000..e73051e8a --- /dev/null +++ b/control_plane/src/physical/workload_cost/status.rs @@ -0,0 +1,124 @@ +//! Typed candidate diagnostics with the existing string wire representation. +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(from = "String", into = "String")] +pub enum CandidateEvaluationStatus { + #[default] + Unspecified, + CompilationFailed, + AwaitingQuote, + EvidenceMissing, + ProviderRejected, + EvidenceInvalid, + Unselected, + Selected, + /// Preserve diagnostics from other producer versions during migration. + Other(String), +} + +impl CandidateEvaluationStatus { + pub fn as_str(&self) -> &str { + match self { + Self::Unspecified => "", + Self::CompilationFailed => "bind_failed", + Self::AwaitingQuote => "bound", + Self::EvidenceMissing => "evidence_missing", + Self::ProviderRejected => "rejected", + Self::EvidenceInvalid => "evidence_invalid", + Self::Unselected => "unselected", + Self::Selected => "selected", + Self::Other(value) => value, + } + } +} + +impl From for CandidateEvaluationStatus { + fn from(value: String) -> Self { + match value.as_str() { + "" => Self::Unspecified, + "bind_failed" => Self::CompilationFailed, + "bound" => Self::AwaitingQuote, + "evidence_missing" => Self::EvidenceMissing, + "rejected" => Self::ProviderRejected, + "evidence_invalid" => Self::EvidenceInvalid, + "unselected" => Self::Unselected, + "selected" => Self::Selected, + _ => Self::Other(value), + } + } +} + +impl From for String { + fn from(value: CandidateEvaluationStatus) -> Self { + value.as_str().to_owned() + } +} + +// Preserve the existing report text, including historical terminology, on the +// wire. The typed variant describes the actual scope for new Rust consumers. +const MATERIALIZATION_SEARCH_SCOPE: &str = "Backend materialization versus Prometheus exact-subquery masks over Planner-authorized leaves; native alternative separate; bounded inventory does not claim an unenumerated optimum"; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(from = "String", into = "String")] +pub enum CandidateSearchScope { + PlannerAuthorizedMaterializations, + Other(String), +} + +impl From for CandidateSearchScope { + fn from(value: String) -> Self { + if value == MATERIALIZATION_SEARCH_SCOPE { + Self::PlannerAuthorizedMaterializations + } else { + Self::Other(value) + } + } +} + +impl From for String { + fn from(value: CandidateSearchScope) -> Self { + match value { + CandidateSearchScope::PlannerAuthorizedMaterializations => { + MATERIALIZATION_SEARCH_SCOPE.to_owned() + } + CandidateSearchScope::Other(value) => value, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Renaming diagnostic variants does not change known or future wire values. + #[test] + fn status_wire_values_round_trip() { + for value in [ + "", + "bind_failed", + "bound", + "evidence_missing", + "rejected", + "evidence_invalid", + "unselected", + "selected", + "future_status", + ] { + let status: CandidateEvaluationStatus = + serde_json::from_value(serde_json::json!(value)).unwrap(); + assert_eq!(status.as_str(), value); + assert_eq!(serde_json::to_value(status).unwrap(), value); + } + } + + /// Old search descriptions retain their exact serialized representation. + #[test] + fn scope_wire_values_round_trip() { + for value in [MATERIALIZATION_SEARCH_SCOPE, "future_scope"] { + let scope: CandidateSearchScope = + serde_json::from_value(serde_json::json!(value)).unwrap(); + assert_eq!(serde_json::to_value(scope).unwrap(), value); + } + } +} diff --git a/control_plane/src/physical/workload_planner.rs b/control_plane/src/physical/workload_planner.rs index a2531861e..dc5385570 100644 --- a/control_plane/src/physical/workload_planner.rs +++ b/control_plane/src/physical/workload_planner.rs @@ -1,4 +1,4 @@ -//! Compatibility compiler from legacy flat workloads to physical deployment +//! Compiler from registered canonical workloads to physical deployment //! plans. Summary selection delegates to ASAPPlanner. use chrono::Utc; @@ -6,6 +6,28 @@ use std::time::Duration; use crate::types::*; +/// Bind the complete registered expression used by the HTTP deployment path. +pub fn bind_registered_query( + w: &RegisteredWorkload, +) -> anyhow::Result { + let query = crate::query_parser::parse_query_expr_canonical(&w.entry().query.0, w.accuracy())?; + if let Some(sketch) = &w.deployment.sketch_type_override { + let cost_model = crate::physical::post_asap::cost_model::ForcedFamilyCostModel::new( + w.accuracy(), + planner_types::post_asap::SketchAlgorithm::from(sketch.clone()), + ); + Ok(crate::physical::post_asap::bind_query_expr_with_cost_model( + &query, + &cost_model, + )?) + } else { + Ok(crate::physical::post_asap::bind_query_expr( + &query, + w.accuracy(), + )?) + } +} + pub const DEFAULT_VALID_FOR: Duration = Duration::from_secs(10 * 60); /// MVP fixture classification used only to translate the demo workload into @@ -36,13 +58,15 @@ fn mvp_deployment_policy( }) } -/// Bind a flat workload to a typed physical expression for stage emission. +/// Derive a typed collector expression from a registered canonical workload. /// /// An explicit sketch override takes precedence when valid for the statistic. /// Otherwise deployment contract rows select the family, with aggregation-type /// defaults for other metrics. Unsupported and raw-passthrough workloads /// return `None`. -pub fn bind_workload_typed(w: &QueryWorkload) -> Option { +pub fn bind_workload_typed( + w: &RegisteredWorkload, +) -> Option { bind_workload_typed_with_evidence(w, None, None) } @@ -51,7 +75,7 @@ pub fn bind_workload_typed(w: &QueryWorkload) -> Option Option { bind_workload_typed_with_evidence(w, None, Some(evidence)) @@ -62,21 +86,21 @@ pub fn bind_workload_typed_with_topk_evidence( /// query's actual per-item filter value through to the bound /// `SketchQuery::PointCount` -- `None` (what `bind_workload_typed` itself /// passes) gives the bare bucket total, same as before this parameter -/// existed. `QueryWorkload` itself carries no `item_label` field (adding +/// existed. `RegisteredWorkload` itself carries no `item_label` field (adding /// one would break its 30+ struct-literal construction sites across the /// crate), so callers that know a metric's item_label -- e.g. /// `emit::collect_metric_to_family`'s loop, which already has `entry: /// &WorkloadEntry` and `workload.label_filters` in scope -- pass it in /// directly instead. pub fn bind_workload_typed_with_item_filter( - w: &QueryWorkload, + w: &RegisteredWorkload, item_filter: Option<(&str, &str)>, ) -> Option { bind_workload_typed_with_evidence(w, item_filter, None) } fn bind_workload_typed_with_evidence( - w: &QueryWorkload, + w: &RegisteredWorkload, item_filter: Option<(&str, &str)>, topk_evidence: Option<&crate::physical::compiler::TopKMembershipEvidence>, ) -> Option { @@ -96,12 +120,12 @@ fn bind_workload_typed_with_evidence( // `top_endpoint_qps` / `endpoint_request_freq`) parse to // `exact_required: true` and the typed binder declines, so the // 5-sketch routing emitter never sees them. - let metric_is_contract_row = mvp_deployment_policy(&w.metric_name).is_some(); - let operator_pinned_sketch = w.sketch_type_override.is_some(); - if w.exact_required && !metric_is_contract_row && !operator_pinned_sketch { + let metric_is_contract_row = mvp_deployment_policy(&w.metric_name()).is_some(); + let operator_pinned_sketch = w.deployment.sketch_type_override.is_some(); + if w.exact_required() && !metric_is_contract_row && !operator_pinned_sketch { return None; } - if w.aggregations.len() != 1 { + if w.aggregations().len() != 1 && !metric_is_contract_row { return None; } @@ -111,7 +135,7 @@ fn bind_workload_typed_with_evidence( // default. The metric-name match owns the demo contract rows; the // AggType fallback covers everything else. let (statistic, default_kind) = - mvp_deployment_policy(&w.metric_name).unwrap_or_else(|| match w.aggregations[0] { + mvp_deployment_policy(&w.metric_name()).unwrap_or_else(|| match w.aggregations()[0] { AggType::Quantile => (DeploymentIntent::Quantile, SketchAlgorithm::DDSketch), AggType::Cardinality => (DeploymentIntent::Cardinality, SketchAlgorithm::Hll), AggType::Frequency => (DeploymentIntent::Frequency, SketchAlgorithm::Cms), @@ -139,6 +163,7 @@ fn bind_workload_typed_with_evidence( // instead so the binding never produces a nonsense (sketch, stat) // pair. let override_kind: Option = w + .deployment .sketch_type_override .as_ref() .map(|st| SketchAlgorithm::from(st.clone())); @@ -146,7 +171,7 @@ fn bind_workload_typed_with_evidence( // the intent. An invalid override produces no candidate below. let kind = override_kind.unwrap_or(default_kind); - let accuracy = w.accuracy.clone(); + let accuracy = w.accuracy().clone(); let intent_accuracy = accuracy.clone(); // Build the matching L3 `AggIntent` for the picked statistic class. @@ -157,7 +182,7 @@ fn bind_workload_typed_with_evidence( let intent = match statistic { DeploymentIntent::Quantile => L3AggIntent::Quantile { col: None, - q: w.quantiles.first().copied().unwrap_or(0.99), + q: w.quantiles().first().copied().unwrap_or(0.99), accuracy: intent_accuracy, }, DeploymentIntent::Cardinality => L3AggIntent::Cardinality { @@ -178,7 +203,7 @@ fn bind_workload_typed_with_evidence( let scan = QueryExpr::Scan { source: Source::TimeSeries { - metric: w.metric_name.clone(), + metric: w.metric_name().clone(), }, // This synthetic scan only exists to drive `Bind*` rule dispatch // against a representative `Aggregate` shape — the rules key off @@ -213,7 +238,7 @@ fn bind_workload_typed_with_evidence( ), }; let windowed = QueryExpr::TimeRange { - range: w.time_window, + range: w.time_window(), child: Box::new(scan).into(), }; // Planner's weighted Top-K contract deliberately accepts only an @@ -348,30 +373,46 @@ impl DeploymentPlanCompiler { } } - pub fn plan(&self, w: &QueryWorkload) -> CollectionPlan { + pub fn with_defaults(defaults: SketchDefaults) -> Self { + Self { + valid_for: DEFAULT_VALID_FOR, + sketch_defaults: defaults, + } + } + + pub fn plan(&self, w: &RegisteredWorkload) -> CollectionPlan { // This legacy scalar cost path cannot certify a failure probability. // Exact/zero-error and EpsilonDelta use raw; the typed binder independently // checks the full requirement against Planner's family guarantees. - if w.exact_required - || !matches!(w.accuracy, crate::types::AccuracyTarget::Epsilon(epsilon) if epsilon > 0.0) + if w.exact_required() + || !matches!(w.accuracy(), crate::types::AccuracyTarget::Epsilon(epsilon) if epsilon > 0.0) { return self.raw_passthrough_plan(w); } - let sketch_type = crate::physical::sketch_catalog::sketch_type_for_agg(&w.aggregations); + if w.deployment.sketch_type_override.is_some() && bind_workload_typed(w).is_none() { + return self.raw_passthrough_plan(w); + } + let sketch_type = w + .deployment + .sketch_type_override + .clone() + .unwrap_or_else(|| { + crate::physical::sketch_catalog::sketch_type_for_agg(&w.aggregations()) + }); let sketch_params = crate::physical::sketch_catalog::build_sketch_params( &self.sketch_defaults, &sketch_type, w.error_bound(), - &w.quantiles, + &w.quantiles(), ); let (mode, window_duration) = select_window_strategy(w); - let mut aggregate_by = w.group_by_labels.clone(); + let mut aggregate_by = w.group_by_labels().clone(); aggregate_by.sort(); let mut label_matchers: Vec = w - .label_filters + .label_filters() .iter() .map(|(k, v)| format!("{k}={v}")) .collect(); @@ -410,11 +451,11 @@ impl DeploymentPlanCompiler { /// Returns a raw-passthrough plan for queries that require exact per-sample /// computation (RSI, MACD, stochastic oscillator, etc.). - fn raw_passthrough_plan(&self, w: &QueryWorkload) -> CollectionPlan { + fn raw_passthrough_plan(&self, w: &RegisteredWorkload) -> CollectionPlan { let valid_until = Utc::now() + chrono::Duration::seconds(self.valid_for.as_secs() as i64); let mut label_matchers: Vec = w - .label_filters + .label_filters() .iter() .map(|(k, v)| format!("{k}={v}")) .collect(); @@ -458,10 +499,10 @@ pub use crate::physical::sketch_catalog::{build_sketch_params, default_sketch_pa /// /// Rule: if `latency_sla >= time_window` (or unset) → window mode. /// otherwise → batch mode (gateway/backend merges on query). -pub fn select_window_strategy(w: &QueryWorkload) -> (ProcessorMode, Option) { - match w.latency_sla { - None => (ProcessorMode::Window, Some(w.time_window)), - Some(ls) if ls >= w.time_window => (ProcessorMode::Window, Some(w.time_window)), +pub fn select_window_strategy(w: &RegisteredWorkload) -> (ProcessorMode, Option) { + match w.latency_sla() { + None => (ProcessorMode::Window, Some(w.time_window())), + Some(ls) if ls >= w.time_window() => (ProcessorMode::Window, Some(w.time_window())), _ => (ProcessorMode::Batch, None), } } @@ -473,21 +514,22 @@ mod tests { use super::*; use std::collections::HashMap; - fn workload(aggs: Vec) -> QueryWorkload { - QueryWorkload { + fn workload(aggs: Vec) -> RegisteredWorkload { + crate::registered_workload::fixtures::WorkloadFixture { metric_name: "test".into(), label_filters: HashMap::new(), group_by_labels: vec![], aggregations: aggs, time_window: Duration::from_secs(300), repeat_every: None, - accuracy_sla: 0.01, + accuracy: crate::types::AccuracyTarget::Epsilon(0.01), latency_sla: None, sketch_type_override: None, exact_required: false, quantiles: vec![], } + .build() } #[test] @@ -522,7 +564,7 @@ mod tests { (SketchType::CountMinSketch, SketchAlgorithm::DDSketch), ] { let mut w = workload(vec![AggType::Quantile]); - w.sketch_type_override = Some(ov.clone()); + w.deployment.sketch_type_override = Some(ov.clone()); let pe = bind_workload_typed(&w) .unwrap_or_else(|| panic!("bind declined for override {ov:?}")); assert_eq!( @@ -534,20 +576,19 @@ mod tests { } #[test] - fn quantile_priority_wins() { - let plan = DeploymentPlanCompiler::new() - .plan(&workload(vec![AggType::Quantile, AggType::Cardinality])); - assert_eq!( - plan.agent_config.sketch_type, - SketchType::DDSketch, - "quantile should take priority over cardinality" - ); + fn mixed_field_aggregations_require_explicit_queries() { + let spec = serde_json::from_value(serde_json::json!({ + "metric_name": "test", "time_window": "5m", "accuracy_sla": 0.99, + "aggregations": ["quantile", "cardinality"] + })) + .unwrap(); + assert!(crate::pipeline::Analyzer::new().analyze(spec).is_err()); } #[test] fn window_mode_when_latency_geq_time_window() { let mut w = workload(vec![AggType::Quantile]); - w.latency_sla = Some(Duration::from_secs(600)); // 10m >= 5m + w.set_latency_sla(Some(Duration::from_secs(600))); // 10m >= 5m let plan = DeploymentPlanCompiler::new().plan(&w); assert_eq!(plan.agent_config.mode, ProcessorMode::Window); assert_eq!( @@ -559,7 +600,7 @@ mod tests { #[test] fn batch_mode_when_latency_lt_time_window() { let mut w = workload(vec![AggType::Quantile]); - w.latency_sla = Some(Duration::from_secs(60)); // 1m < 5m + w.set_latency_sla(Some(Duration::from_secs(60))); // 1m < 5m let plan = DeploymentPlanCompiler::new().plan(&w); assert_eq!(plan.agent_config.mode, ProcessorMode::Batch); assert_eq!(plan.agent_config.window_duration, None); @@ -568,7 +609,7 @@ mod tests { #[test] fn no_latency_sla_defaults_to_window() { let mut w = workload(vec![AggType::Quantile]); - w.latency_sla = None; + w.set_latency_sla(None); let plan = DeploymentPlanCompiler::new().plan(&w); assert_eq!(plan.agent_config.mode, ProcessorMode::Window); } @@ -576,7 +617,7 @@ mod tests { #[test] fn aggregate_by_sorted() { let mut w = workload(vec![AggType::Quantile]); - w.group_by_labels = vec!["zone".into(), "host.name".into(), "service".into()]; + w.deployment.retained_labels = vec!["zone".into(), "host.name".into(), "service".into()]; let plan = DeploymentPlanCompiler::new().plan(&w); assert_eq!( plan.agent_config.aggregate_by, @@ -587,11 +628,13 @@ mod tests { #[test] fn label_matchers_from_filters() { let mut w = workload(vec![AggType::Quantile]); - w.label_filters = [ - ("env".into(), "prod".into()), - ("service".into(), "web".into()), - ] - .into(); + w.set_label_filters( + [ + ("env".into(), "prod".into()), + ("service".into(), "web".into()), + ] + .into(), + ); let plan = DeploymentPlanCompiler::new().plan(&w); assert_eq!(plan.agent_config.label_matchers.len(), 2); } @@ -599,7 +642,7 @@ mod tests { #[test] fn ddsketch_accuracy_params() { let mut w = workload(vec![AggType::Quantile]); - w.accuracy = crate::types::AccuracyTarget::Epsilon(0.005); + w.set_accuracy(crate::types::AccuracyTarget::Epsilon(0.005)); let plan = DeploymentPlanCompiler::new().plan(&w); match &plan.agent_config.sketch_params { SketchParams::DDSketch { @@ -612,7 +655,7 @@ mod tests { #[test] fn hll_precision_coarse_sla() { let mut w = workload(vec![AggType::Cardinality]); - w.accuracy = crate::types::AccuracyTarget::Epsilon(0.03); + w.set_accuracy(crate::types::AccuracyTarget::Epsilon(0.03)); let plan = DeploymentPlanCompiler::new().plan(&w); match &plan.agent_config.sketch_params { SketchParams::HLL { precision } => { @@ -689,21 +732,22 @@ mod tests { /// contract rows; the AggType still has to be a valid one (the enum /// has no `TopK` variant, so for `top_endpoint_qps` we pass /// `Frequency` and rely on the metric-name reclassification). - fn workload_for(metric: &str, agg: AggType) -> QueryWorkload { - QueryWorkload { + fn workload_for(metric: &str, agg: AggType) -> RegisteredWorkload { + crate::registered_workload::fixtures::WorkloadFixture { metric_name: metric.into(), label_filters: HashMap::new(), group_by_labels: vec![], aggregations: vec![agg], time_window: Duration::from_secs(300), repeat_every: None, - accuracy_sla: 0.01, + accuracy: crate::types::AccuracyTarget::Epsilon(0.01), latency_sla: None, sketch_type_override: None, exact_required: false, quantiles: vec![], } + .build() } fn topk_evidence() -> crate::physical::compiler::TopKMembershipEvidence { @@ -797,6 +841,31 @@ mod tests { )); } + /// Canonical binding must preserve pins for both field-only and string requests. + #[test] + fn canonical_binding_preserves_field_only_sketch_pins() { + for sketch in [SketchType::KLL, SketchType::DDSketch] { + for use_query_string in [false, true] { + let mut input = serde_json::json!({ + "metric_name": "latency", "aggregations": ["quantile"], "time_window": "5m", + "accuracy_sla": 0.99, "sketch_type": sketch, + }); + if use_query_string { + input["query_string"] = + serde_json::json!("quantile_over_time(0.99, latency[5m])"); + } + let workload = crate::pipeline::Analyzer::new() + .analyze(serde_json::from_value(input).unwrap()) + .unwrap(); + let bound = bind_registered_query(&workload).unwrap(); + assert_eq!( + extract_family(&bound), + Some(SketchAlgorithm::from(sketch.clone())) + ); + } + } + } + #[test] fn typed_binding_endpoint_request_freq_binds_cms() { // Contract: `endpoint_request_freq` → CMS (Frequency). `Frequency` @@ -860,7 +929,7 @@ mod tests { // for Quantile per the capability matrix, so the override is // honoured. let mut w = workload_for("http_latency_ms", AggType::Quantile); - w.sketch_type_override = Some(SketchType::KLL); + w.deployment.sketch_type_override = Some(SketchType::KLL); let bound = bind_workload_typed(&w).expect("override should still bind"); assert_eq!( extract_family(&bound), @@ -874,7 +943,7 @@ mod tests { // `request_size_bytes`'s contract row is KLL (rank-err); a // workload override of `DDSketch` flips it back to DDSketch. let mut w = workload_for("request_size_bytes", AggType::Quantile); - w.sketch_type_override = Some(SketchType::DDSketch); + w.deployment.sketch_type_override = Some(SketchType::DDSketch); let bound = bind_workload_typed(&w).expect("override should still bind"); assert_eq!( extract_family(&bound), @@ -891,7 +960,7 @@ mod tests { // metric, the planner should accept it instead of falling // back to the canonical CountSketch default. let mut w = workload_for("top_endpoint_qps", AggType::Frequency); - w.sketch_type_override = Some(SketchType::CountMinSketch); + w.deployment.sketch_type_override = Some(SketchType::CountMinSketch); let bound = bind_workload_typed_with_topk_evidence(&w, &topk_evidence()) .expect("CMS Top-K override must bind with evidence"); assert_eq!(extract_family(&bound), Some(SketchAlgorithm::CmsWithHeap)); @@ -917,7 +986,7 @@ mod tests { // matrix rejects the override, and the planner falls back to // the contract-row default (DDSketch for `http_latency_ms`). let mut w = workload_for("http_latency_ms", AggType::Quantile); - w.sketch_type_override = Some(SketchType::HLL); + w.deployment.sketch_type_override = Some(SketchType::HLL); let bound = bind_workload_typed(&w).expect("fallback should bind"); assert_eq!( extract_family(&bound), diff --git a/control_plane/src/pipeline.rs b/control_plane/src/pipeline.rs index cc4a6d5b2..6a4b1b9ee 100644 --- a/control_plane/src/pipeline.rs +++ b/control_plane/src/pipeline.rs @@ -5,7 +5,7 @@ use std::time::Duration; use crate::query_parser; use crate::types::{AccuracyTarget, DataShape, QueryId, QueryLanguage, QueryShape}; -use crate::types::{AggType, QueryWorkload, SketchType, WorkloadCharacteristics}; +use crate::types::{AggType, RegisteredWorkload, SketchType, WorkloadCharacteristics}; // ── Public API ──────────────────────────────────────────────────────────────── @@ -19,8 +19,8 @@ use crate::types::{AggType, QueryWorkload, SketchType, WorkloadCharacteristics}; /// 2. **Query string** — supply a raw PromQL string in /// `query_string`. The analyzer parses it and fills in `metric_name`, /// `aggregations`, `group_by_labels`, `label_filters`, and `time_window` -/// automatically. Any explicit fields that are non-empty / non-default -/// **override** the parsed values, so the two approaches compose. +/// automatically. Explicit semantic fields must agree with the expression; +/// conflicting overrides are rejected before registration. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct QuerySpec { /// Raw PromQL query string to parse (SP-1 automatic extraction). @@ -29,49 +29,38 @@ pub struct QuerySpec { #[serde(default)] pub query_string: Option, - /// Metric name override. Required when `query_string` is absent. + /// Metric identity. Required without a query; must match a supplied query. #[serde(default)] pub metric_name: String, #[serde(default)] pub label_filters: HashMap, + /// Additional labels the collector must retain; does not rewrite query GROUP BY. #[serde(default)] pub group_by_labels: Vec, - /// Aggregation type overrides ("quantile", "cardinality", "frequency"). + /// Field-only aggregation ("quantile", "cardinality", "frequency"). /// Required when `query_string` is absent. #[serde(default)] pub aggregations: Vec, - /// Time window override (e.g. "5m"). Required when `query_string` is absent. + /// Time window (e.g. "5m"). Required without a query; otherwise must agree. #[serde(default)] pub time_window: String, #[serde(default)] pub repeat_every: Option, pub accuracy_sla: f64, pub latency_sla: Option, - /// Optional: pin a specific sketch type, bypassing the cost-model planner. + /// Optional implementation constraint, still subject to Planner legality. pub sketch_type: Option, /// Observable data-stream characteristics used for delta / raw-vs-sketch /// bandwidth comparison. Omit to use conservative defaults. #[serde(default)] pub workload: WorkloadCharacteristics, - // ── design.md alignment: new fields, defaulted for back-compat ──────── - // - // These fields converge `QuerySpec` toward the typed schema in - // `control_plane/docs/design.md` §6 `core::workload`. Each is defaulted - // so the existing JSON API surface (POST /api/v1/plan handlers, - // pre-population from `workloads.yaml`, the test fixtures elsewhere - // in the control plane) keeps working without supplying them. The - // accuracy target is consumed by parsing and binding. The other fields - // retain the partial support described in `Analyzer::analyze`. - /// Stable identifier preserved across replan cycles. Optional; - /// auto-derived from `metric_name + accuracy_sla` if omitted - /// (existing API callers don't supply this). + /// Optional stable registration identifier, preserved as metadata. #[serde(default)] pub id: Option, - /// Source language. Inferred from `query_string` syntax / parser - /// dispatch when omitted (existing API callers default to PromQL - /// behavior, which matches today's `query_parser::parse_query`). + /// This metric-registration endpoint accepts PromQL; other languages use + /// their dedicated compilation paths. #[serde(default)] pub language: Option, @@ -82,8 +71,8 @@ pub struct QuerySpec { #[serde(default)] pub accuracy: Option, - /// Per-evaluation $ budget. Optional; the cost model picks freely - /// when unset. + /// Reserved compatibility field. Explicit dollar constraints are rejected + /// because metric registration does not implement them. #[serde(default)] pub dollars: Option, @@ -122,7 +111,7 @@ impl Analyzer { Self } - pub fn analyze(&self, spec: QuerySpec) -> anyhow::Result { + pub fn analyze(&self, spec: QuerySpec) -> anyhow::Result { let accuracy = crate::types::resolve_accuracy_target(spec.accuracy.as_ref(), spec.accuracy_sla) .map_err(|error| anyhow!(error))?; @@ -133,8 +122,8 @@ impl Analyzer { // hard rejections are at L1 because they have no semantically // valid plan: a streaming query over a static dataset, and a // streaming query over a mutable relation (no retraction-aware - // sketches in the catalog yet). Everything else is accepted - // here — downstream rule firing can still narrow further. + // sketches in the catalog yet). Canonical conversion below also checks + // which recurrence and data shapes this deployment path can represent. match (&spec.shape, &spec.data) { (QueryShape::Streaming, DataShape::Batch) => { return Err(anyhow!( @@ -153,18 +142,6 @@ impl Analyzer { _ => {} } - // Compatibility reporting only: all semantic consumers use `accuracy`. - let accuracy_sla = if spec.accuracy.is_none() { - spec.accuracy_sla - } else { - match accuracy { - AccuracyTarget::Exact => 1.0, - AccuracyTarget::Epsilon(epsilon) | AccuracyTarget::EpsilonDelta { epsilon, .. } => { - 1.0 - epsilon - } - } - }; - // ── Step 1: parse query_string if provided ───────────────────────── // Parsing and downstream binding receive this same resolved target. let parsed = spec @@ -187,12 +164,6 @@ impl Analyzer { let aggregations = if !spec.aggregations.is_empty() { parse_agg_types(&spec.aggregations)? } else if let Some(ref p) = parsed { - if p.aggregations.is_empty() && !p.exact_required { - return Err(anyhow!( - "could not infer aggregation type from query_string; \ - provide explicit aggregations" - )); - } p.aggregations.clone() } else { return Err(anyhow!("at least one aggregation is required")); @@ -212,12 +183,8 @@ impl Analyzer { return Err(anyhow!("time_window is required (or provide query_string)")); }; - // ── Step 5: resolve dimensions (group_by + label_filter keys) ────── - // Parsed values are the base; explicit spec fields override / extend. - let parsed_group_by = parsed - .as_ref() - .map(|p| p.group_by_labels.as_slice()) - .unwrap_or(&[]); + // ── Step 5: resolve filters; conflicts are checked against the query ─ + // Collector retention labels remain independent deployment options. let parsed_filters: HashMap = parsed .as_ref() .map(|p| p.label_filters.clone()) @@ -229,12 +196,6 @@ impl Analyzer { m }; - let filter_keys: Vec = merged_filters.keys().cloned().collect(); - let all_group_by: Vec = dedup_dims( - &dedup_dims(parsed_group_by, &spec.group_by_labels), - &filter_keys, - ); - // ── Step 6: scalar fields ────────────────────────────────────────── let repeat_every = spec .repeat_every @@ -250,36 +211,153 @@ impl Analyzer { .transpose() .with_context(|| "invalid latency_sla")?; - let exact_required = parsed.as_ref().map(|p| p.exact_required).unwrap_or(false); - let quantiles = parsed - .as_ref() - .map(|p| p.quantiles.clone()) - .unwrap_or_default(); - - // These remaining compatibility fields do not yet feed the flat planner. - let _ = ( - &spec.shape, - &spec.data, - &spec.id, - &spec.language, - &spec.dollars, - &spec.deployment_model, + use crate::registered_workload::{declared, DeploymentOptions}; + use planner_types::workload::*; + let query = if let Some(query) = &spec.query_string { + let p = parsed.as_ref().expect("parsed above"); + anyhow::ensure!(metric_name == p.metric_name && time_window == p.time_window + && aggregations == p.aggregations && merged_filters == p.label_filters, + "explicit overrides conflict with query_string; update the query expression instead"); + query.clone() + } else { + anyhow::ensure!( + aggregations.len() == 1, + "field-only input requires one aggregation" + ); + let mut filters: Vec<_> = merged_filters + .iter() + .map(|(k, v)| format!("{k}={}", serde_json::to_string(v).expect("string"))) + .collect(); + filters.sort(); + let selector = if filters.is_empty() { + metric_name.clone() + } else { + format!("{}{{{}}}", metric_name, filters.join(",")) + }; + match aggregations[0] { + AggType::Quantile => format!( + "quantile_over_time(0.99, {selector}[{}s])", + time_window.as_secs() + ), + AggType::Cardinality => { + format!("distinct_over_time({selector}[{}s])", time_window.as_secs()) + } + AggType::Frequency => { + format!("count_over_time({selector}[{}s])", time_window.as_secs()) + } + } + }; + anyhow::ensure!( + spec.language.is_none() + || matches!(spec.language, Some(crate::types::QueryLanguage::PromQl)), + "metric registration requires PromQL" ); - - Ok(QueryWorkload { - metric_name, - label_filters: merged_filters, - group_by_labels: all_group_by, - aggregations, - time_window, - repeat_every, - accuracy, - accuracy_sla, - latency_sla, - sketch_type_override: spec.sketch_type, - exact_required, - quantiles, - }) + anyhow::ensure!( + spec.dollars.is_none(), + "dollars constraints are not supported by metric registration" + ); + let cadence = match spec.shape { + QueryShape::Periodic { every } => { + anyhow::ensure!(repeat_every.is_none_or(|r| r == every), "conflicting repetition intervals"); + Some(every) + }, + QueryShape::Streaming => return Err(anyhow!("streaming demand without a fixed cadence is not supported; specify periodic demand")), + QueryShape::OneShot => repeat_every, + }; + let requirements = QueryRequirements { + accuracy: AccuracyRequirement::Explicit(accuracy), + response_latency: latency_sla + .map(|d| LatencyRequirement::ExplicitMaxMs(d.as_secs_f64() * 1000.0)) + .unwrap_or(LatencyRequirement::Unspecified), + }; + let time_selection = TimeSelection { + scope: QueryTimeScope::RealTime, + lookback: crate::registered_workload::metric_query_range(&query)? + .map(|range| u64::try_from(range.as_millis()).map(DurationMs)) + .transpose()?, + as_of: None, + }; + let (query_batch, repeating_queries) = if let Some(cadence) = cadence { + anyhow::ensure!( + cadence.subsec_nanos().is_multiple_of(1_000_000), + "repetition interval requires whole milliseconds" + ); + let interval = u32::try_from(cadence.as_millis()) + .context("repetition interval exceeds u32 milliseconds")?; + anyhow::ensure!(interval > 0, "repetition interval must be positive"); + ( + None, + Some(vec![RepeatingEntry { + query: Query(query), + demand: RepeatedDemand::FixedInterval(RepetitionInterval(interval)), + requirements, + predictability: Predictability::Predictable { known_at: None }, + time_selection, + }]), + ) + } else { + ( + Some(vec![BatchEntry { + query: Query(query), + requirements, + predictability: Predictability::AdHoc, + invocations: 1, + execute_at: None, + time_selection, + }]), + None, + ) + }; + let wc = &spec.workload; + let rate = wc.series_count as f64 * wc.samples_per_sec_per_series; + anyhow::ensure!( + wc.samples_per_sec_per_series.is_finite() + && wc.samples_per_sec_per_series >= 0.0 + && rate.is_finite(), + "sample rate must be finite and nonnegative" + ); + let arrival = match spec.data { + DataShape::Batch => DataArrival::AtRest, + DataShape::AppendOnlyStream => DataArrival::ContinuouslyIngesting, + DataShape::Mixed => DataArrival::Mixed, + DataShape::Mutable => { + return Err(anyhow!( + "mutable data is not supported by metric registration" + )) + } + }; + let data_workload = Some(DataWorkload { + arrival, + ingestion_rate: declared(Rate(if matches!(spec.data, DataShape::Batch) { + 0.0 + } else { + rate + })), + input_cardinality: declared(wc.series_count), + distribution: declared(match wc.data_distribution { + crate::types::DataDistribution::Zipf => DataDistribution::Zipf, + crate::types::DataDistribution::Uniform => DataDistribution::Uniform, + crate::types::DataDistribution::Bursty => DataDistribution::Bursty, + }), + ingestion_volume: Evidence::default(), + }); + RegisteredWorkload::new( + QueryWorkload { + language: QueryLanguage::PromQL, + query_batch, + repeating_queries, + data_workload, + }, + DeploymentOptions { + sketch_type_override: spec.sketch_type, + query_id: spec.id, + deployment_model: spec.deployment_model, + retained_labels: dedup_dims(&spec.group_by_labels, &[]), + bytes_per_raw_sample: wc.bytes_per_raw_sample, + distinct_keys_per_window: wc.distinct_keys_per_window, + memory_budget_bytes: wc.memory_budget_bytes, + }, + ) } } @@ -301,12 +379,16 @@ pub fn parse_duration(s: &str) -> anyhow::Result { .parse() .map_err(|_| anyhow!("invalid number in duration {:?}", s))?; current_num.clear(); - match ch { - 'h' => total_secs += n * 3600, - 'm' => total_secs += n * 60, - 's' => total_secs += n, + let multiplier = match ch { + 'h' => 3600, + 'm' => 60, + 's' => 1, _ => return Err(anyhow!("unknown unit {:?} in duration {:?}", ch, s)), - } + }; + total_secs = n + .checked_mul(multiplier) + .and_then(|part| total_secs.checked_add(part)) + .ok_or_else(|| anyhow!("duration overflow in {:?}", s))?; } } if !current_num.is_empty() { @@ -394,12 +476,12 @@ mod tests { #[test] fn valid_spec() { let w = Analyzer::new().analyze(basic_spec()).unwrap(); - assert_eq!(w.metric_name, "request_latency"); - assert_eq!(w.accuracy_sla, 0.01); - assert_eq!(w.time_window, Duration::from_secs(300)); - assert_eq!(w.repeat_every, Some(Duration::from_secs(60))); - assert_eq!(w.latency_sla, Some(Duration::from_secs(600))); - assert_eq!(w.aggregations, vec![AggType::Quantile]); + assert_eq!(w.metric_name(), "request_latency"); + assert!(((1.0 - w.error_bound()) - 0.01).abs() < 1e-12); + assert_eq!(w.time_window(), Duration::from_secs(300)); + assert_eq!(w.repeat_every(), Some(Duration::from_secs(60))); + assert_eq!(w.latency_sla(), Some(Duration::from_secs(600))); + assert_eq!(w.aggregations(), vec![AggType::Quantile]); } #[test] @@ -407,22 +489,22 @@ mod tests { let mut spec = basic_spec(); spec.label_filters = [ ("service".into(), "api".into()), - ("host.name".into(), "h1".into()), + ("host_name".into(), "h1".into()), ] .into(); - spec.group_by_labels = vec!["host.name".into(), "region".into()]; + spec.group_by_labels = vec!["host_name".into(), "region".into()]; let w = Analyzer::new().analyze(spec).unwrap(); - for dim in &["host.name", "region", "service"] { + for dim in &["host_name", "region", "service"] { assert!( - w.group_by_labels.contains(&dim.to_string()), + w.group_by_labels().contains(&dim.to_string()), "missing {dim}" ); } // host.name must appear exactly once after dedup assert_eq!( - w.group_by_labels + w.group_by_labels() .iter() - .filter(|d| d.as_str() == "host.name") + .filter(|d| d.as_str() == "host_name") .count(), 1 ); @@ -432,11 +514,7 @@ mod tests { fn multiple_aggregations() { let mut spec = basic_spec(); spec.aggregations = vec!["cardinality".into(), "frequency".into()]; - let w = Analyzer::new().analyze(spec).unwrap(); - assert_eq!( - w.aggregations, - vec![AggType::Cardinality, AggType::Frequency] - ); + assert!(Analyzer::new().analyze(spec).is_err()); } #[test] @@ -550,43 +628,35 @@ mod tests { "sum by (host) (quantile_over_time(0.99, latency[5m]))", )) .unwrap(); - assert_eq!(w.metric_name, "latency"); - assert_eq!(w.aggregations, vec![AggType::Quantile]); - assert_eq!(w.time_window, Duration::from_secs(300)); - assert_eq!(w.quantiles, vec![0.99]); - assert!(w.exact_required); + assert_eq!(w.metric_name(), "latency"); + assert_eq!(w.aggregations(), vec![AggType::Quantile]); + assert_eq!(w.time_window(), Duration::from_secs(300)); + assert_eq!(w.quantiles(), vec![0.99]); + assert!(w.exact_required()); } - /// Explicit metric_name overrides the name derived from query_string. + /// A conflicting metric field cannot change only the stored projection. #[test] fn explicit_metric_name_overrides_parsed() { let mut spec = qs_only("sum by (host) (avg_over_time(cpu[5m]))"); spec.metric_name = "my_custom_metric".into(); - let w = Analyzer::new().analyze(spec).unwrap(); - assert_eq!(w.metric_name, "my_custom_metric"); - // aggregations still come from parse — `avg_over_time` is exact - // (no ASAP-tier sketch substitute for `AggIntent::Avg`), so - // `aggregations` stays empty and `exact_required` flips instead. - assert_eq!(w.aggregations, Vec::::new()); - assert!(w.exact_required); + assert!(Analyzer::new().analyze(spec).is_err()); } - /// Explicit time_window overrides the window derived from query_string. + /// A conflicting window cannot disagree with the canonical expression. #[test] fn explicit_time_window_overrides_parsed() { let mut spec = qs_only("sum by (host) (avg_over_time(cpu[5m]))"); spec.time_window = "1h".into(); - let w = Analyzer::new().analyze(spec).unwrap(); - assert_eq!(w.time_window, Duration::from_secs(3600)); + assert!(Analyzer::new().analyze(spec).is_err()); } - /// Explicit aggregations override those derived from query_string. + /// A conflicting aggregation cannot replace canonical query semantics. #[test] fn explicit_aggregations_override_parsed() { let mut spec = qs_only("sum by (host) (avg_over_time(cpu[5m]))"); // → Quantile spec.aggregations = vec!["cardinality".into()]; - let w = Analyzer::new().analyze(spec).unwrap(); - assert_eq!(w.aggregations, vec![AggType::Cardinality]); + assert!(Analyzer::new().analyze(spec).is_err()); } /// sum_over_time is a stateful exact aggregation; exact_required is set. @@ -597,8 +667,8 @@ mod tests { "sum by (service) (sum_over_time(request_bytes[1h]))", )) .unwrap(); - assert!(w.exact_required, "sum_over_time must set exact_required"); - assert_eq!(w.aggregations, vec![]); + assert!(w.exact_required(), "sum_over_time must set exact_required"); + assert_eq!(w.aggregations(), vec![]); } /// DDSketch quantile φ values are surfaced through the workload. @@ -609,7 +679,7 @@ mod tests { "sum by (host) (quantile_over_time(0.5, latency[5m]))", )) .unwrap(); - assert_eq!(w.quantiles, vec![0.5]); + assert_eq!(w.quantiles(), vec![0.5]); } /// Existing callers that supply all fields explicitly and omit @@ -617,30 +687,30 @@ mod tests { #[test] fn backward_compat_no_query_string() { let w = Analyzer::new().analyze(basic_spec()).unwrap(); - assert_eq!(w.metric_name, "request_latency"); - assert_eq!(w.aggregations, vec![AggType::Quantile]); - assert_eq!(w.time_window, Duration::from_secs(300)); - assert!(!w.exact_required); - assert!(w.quantiles.is_empty()); + assert_eq!(w.metric_name(), "request_latency"); + assert_eq!(w.aggregations(), vec![AggType::Quantile]); + assert_eq!(w.time_window(), Duration::from_secs(300)); + assert!(!w.exact_required()); + assert_eq!(w.quantiles(), vec![0.99]); } // ── design.md alignment tests ───────────────────────────────────────────── /// Typed `accuracy: Some(Epsilon(0.05))` overrides the legacy /// `accuracy_sla: 0.99` (which would translate to `Epsilon(0.01)`), - /// and the resolved value flows through to `QueryWorkload.accuracy_sla`. + /// and the resolved value flows through to `RegisteredWorkload.accuracy_sla`. #[test] fn typed_accuracy_overrides_legacy_accuracy_sla() { let mut spec = basic_spec(); spec.accuracy_sla = 0.99; // legacy: ε = 0.01 spec.accuracy = Some(AccuracyTarget::Epsilon(0.05)); let w = Analyzer::new().analyze(spec).unwrap(); - // The resolved 1.0 - 0.05 = 0.95 must reach the QueryWorkload, not + // The resolved 1.0 - 0.05 = 0.95 must reach the RegisteredWorkload, not // the legacy 0.99. assert!( - (w.accuracy_sla - 0.95).abs() < 1e-9, + ((1.0 - w.error_bound()) - 0.95).abs() < 1e-9, "got {}", - w.accuracy_sla + (1.0 - w.error_bound()) ); } @@ -683,10 +753,9 @@ mod tests { let mut spec = basic_spec(); spec.accuracy_sla = 0.2; spec.accuracy = Some(target.clone()); - let mut workload = Analyzer::new().analyze(spec).unwrap(); - assert_eq!(workload.accuracy, target); - // Mutating the deprecated reporting view cannot affect semantic binding. - workload.accuracy_sla = 0.9999; + let workload = Analyzer::new().analyze(spec).unwrap(); + assert_eq!(workload.accuracy(), target); + // Canonical requirements have no independently mutable scalar mirror. let bound = crate::physical::workload_planner::bind_workload_typed(&workload); if target == AccuracyTarget::Exact { assert!( @@ -728,7 +797,7 @@ mod tests { spec.accuracy_sla = 0.5; spec.accuracy = Some(AccuracyTarget::Exact); let w = Analyzer::new().analyze(spec).unwrap(); - assert_eq!(w.accuracy_sla, 1.0); + assert_eq!((1.0 - w.error_bound()), 1.0); } /// L1 rejects `(QueryShape::Streaming, DataShape::Batch)` per the @@ -759,14 +828,13 @@ mod tests { ); } - /// `(QueryShape::Streaming, DataShape::AppendOnlyStream)` — the - /// canonical streaming case — is accepted. + /// Continuous demand needs a supported cadence before metric registration. #[test] - fn l1_accepts_streaming_over_append_only_stream() { + fn rejects_streaming_demand_without_fixed_cadence() { let mut spec = basic_spec(); spec.shape = QueryShape::Streaming; spec.data = DataShape::AppendOnlyStream; - assert!(Analyzer::new().analyze(spec).is_ok()); + assert!(Analyzer::new().analyze(spec).is_err()); } /// JSON without any of the new fields parses correctly via serde — @@ -791,14 +859,14 @@ mod tests { assert_eq!(spec.data, DataShape::AppendOnlyStream); // And the analyzer accepts it. let w = Analyzer::new().analyze(spec).unwrap(); - assert_eq!(w.metric_name, "request_latency"); + assert_eq!(w.metric_name(), "request_latency"); // Legacy accuracy_sla=0.99 round-trips through resolution // (no typed `accuracy` supplied → translate from legacy → // Epsilon(0.01) → back to 1 - 0.01 = 0.99). assert!( - (w.accuracy_sla - 0.99).abs() < 1e-9, + ((1.0 - w.error_bound()) - 0.99).abs() < 1e-9, "got {}", - w.accuracy_sla + (1.0 - w.error_bound()) ); } @@ -816,7 +884,7 @@ mod tests { "id": "q-001", "language": "prom_ql", "accuracy": { "Epsilon": 0.02 }, - "dollars": 0.001, + "dollars": null, "deployment_model": "asaplifecycle", "shape": { "kind": "periodic", "every": { "secs": 60, "nanos": 0 } }, "data": "batch" @@ -825,7 +893,7 @@ mod tests { assert_eq!(spec.id.as_ref().unwrap().0.as_str(), "q-001"); assert_eq!(spec.language, Some(QueryLanguage::PromQl)); assert_eq!(spec.accuracy, Some(AccuracyTarget::Epsilon(0.02))); - assert_eq!(spec.dollars, Some(0.001)); + assert_eq!(spec.dollars, None); assert_eq!(spec.deployment_model.as_deref(), Some("asaplifecycle")); assert!(matches!(spec.shape, QueryShape::Periodic { .. })); assert_eq!(spec.data, DataShape::Batch); @@ -835,9 +903,9 @@ mod tests { // typed `accuracy: Epsilon(0.02)` overrode the legacy 0.5 → // resolved accuracy_sla in the workload is 1.0 - 0.02 = 0.98. assert!( - (w.accuracy_sla - 0.98).abs() < 1e-9, + ((1.0 - w.error_bound()) - 0.98).abs() < 1e-9, "got {}", - w.accuracy_sla + (1.0 - w.error_bound()) ); } } diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs index 70d6a08ba..34858c6bd 100644 --- a/control_plane/src/query_plan.rs +++ b/control_plane/src/query_plan.rs @@ -2,12 +2,15 @@ //! Serving consumes asap_types::query_plan; compilation stays in this component. mod clickhouse_exact; -pub mod logical; +pub mod residual; + pub use asap_types::query_plan::*; #[cfg(test)] use asap_types::PolicyFingerprint; use planner_types::post_asap::{SummaryExpr, SummaryFamilyType, SummaryNode}; use planner_types::pre_asap::Reduction; +#[deprecated(note = "Use query_plan::residual")] +pub use residual as logical; use std::collections::BTreeMap; #[cfg(test)] use std::collections::BTreeSet; @@ -131,7 +134,7 @@ where instant, fallback, }; - logical::finalize_residuals(&mut entry)?; + residual::finalize_residuals(&mut entry)?; Ok(entry) } @@ -229,16 +232,16 @@ where self.seen.insert(identity, id); let residual = match (&self.logical_source, &node.expr) { (Some(original), SummaryExpr::KeepPreAsap(expr)) => { - Some(logical::residual_nodes(original, expr)?) + Some(residual::residual_nodes(original, expr)?) } (Some(original), SummaryExpr::SummaryAgg { child, .. }) if matches!(child.expr, SummaryExpr::KeepPreAsap(_)) && !matches!( - crate::physical::compiler::materialization_leaf_contract(node), + crate::physical::compiler::raw_materialization_input_contract(node), Ok((_, Some(_), _)) ) => { - Some(logical::selected_residual_nodes(original, node)?) + Some(residual::selected_residual_nodes(original, node)?) } _ => None, }; @@ -311,11 +314,11 @@ where } if measures.len() == 1 => { use planner_types::pre_asap::AggIntent; let operation = match &measures[0] { - AggIntent::Sum { .. } => logical::Aggregation::Sum, - AggIntent::Count { .. } => logical::Aggregation::Count, - AggIntent::Min { .. } => logical::Aggregation::Min, - AggIntent::Max { .. } => logical::Aggregation::Max, - AggIntent::Avg { .. } => logical::Aggregation::Avg, + AggIntent::Sum { .. } => residual::Aggregation::Sum, + AggIntent::Count { .. } => residual::Aggregation::Count, + AggIntent::Min { .. } => residual::Aggregation::Min, + AggIntent::Max { .. } => residual::Aggregation::Max, + AggIntent::Avg { .. } => residual::Aggregation::Avg, _ => { return Err(QueryPlanError::Invalid( "unsupported exact value aggregation".into(), @@ -344,9 +347,9 @@ where }) .collect::, _>>()?; QueryPlanNode::Logical { - operator: logical::LogicalOperator::Aggregate { + operator: residual::ResidualQueryOperator::Aggregate { operation, - grouping: logical::Grouping { + grouping: residual::Grouping { labels, without: keys.is_without(), }, @@ -408,11 +411,11 @@ where }) .collect::, _>>()?; QueryPlanNode::Logical { - operator: logical::LogicalOperator::TopKSelection { + operator: residual::ResidualQueryOperator::TopKSelection { k: u64::try_from(*n).map_err(|_| { QueryPlanError::Invalid("TopK limit exceeds u64".into()) })?, - grouping: logical::Grouping { + grouping: residual::Grouping { labels, without: partition_by.is_without(), }, @@ -425,7 +428,7 @@ where operation: planner_types::post_asap::ValueOperation::Sort { keys, .. }, timing: planner_types::post_asap::ExecutionTiming::ReadTime, } if keys.len() == 1 => QueryPlanNode::Logical { - operator: logical::LogicalOperator::Sort { + operator: residual::ResidualQueryOperator::Sort { descending: !keys[0].ascending, }, inputs: vec![self.lower(child)?], @@ -520,7 +523,7 @@ where k: u64::try_from(*k).map_err(|_| { QueryPlanError::Invalid("CandidateTopK k exceeds u64".into()) })?, - grouping: logical::Grouping { + grouping: residual::Grouping { labels, without: grouping.is_without(), }, @@ -533,7 +536,7 @@ where operator, timing: planner_types::post_asap::ExecutionTiming::ReadTime, } if self.logical_source.is_some() => { - let operator = logical::binary_operator(operator)?; + let operator = residual::binary_operator(operator)?; QueryPlanNode::Logical { operator, inputs: vec![self.lower(lhs)?, self.lower(rhs)?], @@ -553,7 +556,7 @@ where planner_types::post_asap::ExactKind::Sum | planner_types::post_asap::ExactKind::Count ) { - let operator = logical::selected_aggregate_operator( + let operator = residual::selected_aggregate_operator( self.logical_source.as_deref().unwrap(), node, )?; @@ -568,8 +571,8 @@ where return Ok(id); } let operation = match kind { - planner_types::post_asap::ExactKind::Sum => logical::Aggregation::Sum, - planner_types::post_asap::ExactKind::Count => logical::Aggregation::Count, + planner_types::post_asap::ExactKind::Sum => residual::Aggregation::Sum, + planner_types::post_asap::ExactKind::Count => residual::Aggregation::Count, _ => { return Err(QueryPlanError::Invalid( "unsupported aggregation over selected summary values".into(), @@ -596,9 +599,9 @@ where }) .collect::, _>>()?; QueryPlanNode::Logical { - operator: logical::LogicalOperator::Aggregate { + operator: residual::ResidualQueryOperator::Aggregate { operation, - grouping: logical::Grouping { + grouping: residual::Grouping { labels, without: keys.is_without(), }, @@ -734,7 +737,7 @@ where Err(error) => { if let Some(original) = &self.logical_source { let (root, nodes) = - logical::selected_residual_nodes(original, node)?; + residual::selected_residual_nodes(original, node)?; return self.graft(id, root, nodes); } return Err(error); @@ -970,6 +973,179 @@ fn physical_grouping( Ok(PhysicalGrouping::Reduce(names)) } +#[cfg(test)] +mod catalog_binding_tests { + use super::*; + use crate::physical::summary_catalog::SummaryCatalog; + use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; + + fn fixture() -> (QueryPlan, SummaryCatalog) { + let mut config = PrecomputeMaterialization::new( + AggregationType::Sum, + String::new(), + Default::default(), + KeyByLabelNames::new(vec!["job".into()]), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + String::new(), + 10, + 10, + WindowKind::Tumbling, + String::new(), + "m".into(), + None, + None, + None, + ); + config.pane_origin_ms = Some(0); + let catalog = SummaryCatalog::from_materializations(7, 2, &[config.clone()]).unwrap(); + let entry = QueryPlanEntry { + language: crate::query_plan::QueryLanguage::PromQl, + query_id: "q".into(), + canonical_query: "sum_over_time(m[1m])".into(), + fixed_evaluation: None, + root: QueryNodeId(1), + nodes: BTreeMap::from([( + QueryNodeId(1), + QueryPlanNode::ReadMaterialization { + binding: MaterializationBinding { + full_window_slide_ms: None, + item_labels: Vec::new(), + materialization: config.policy_fingerprint().into(), + output_grouping: PhysicalGrouping::PerEntity, + window_ms: 10_000, + pane_origin_ms: Some(0), + readout_lookback_ms: Some(60_000), + }, + }, + )]), + instant: InstantExecution { + lookback_ms: 60_000, + full_history: false, + cumulative_readout: true, + }, + fallback: FallbackPolicy::ExactBackend, + }; + ( + QueryPlan { + plan_id: 7, + plan_version: 2, + clickhouse_context: None, + entries: BTreeMap::from([(entry.canonical_query.clone(), entry)]), + }, + catalog, + ) + } + fn binding(plan: &mut QueryPlan) -> &mut MaterializationBinding { + let QueryPlanNode::ReadMaterialization { binding } = plan + .entries + .values_mut() + .next() + .unwrap() + .nodes + .values_mut() + .next() + .unwrap() + else { + panic!("fixture") + }; + binding + } + + // One pane ID is compatible with a longer semantic readout window. + #[test] + fn catalog_binding_round_trip_preserves_pane_and_readout_windows() { + let (plan, catalog) = fixture(); + let wire = serde_json::to_vec(&plan).unwrap(); + let mut decoded: QueryPlan = serde_json::from_slice(&wire).unwrap(); + decoded.validate_against_catalog(&catalog).unwrap(); + assert_eq!( + decoded + .lookup("sum_over_time(m[1m])") + .unwrap() + .canonical_query, + "sum_over_time(m[1m])" + ); + assert!(String::from_utf8(wire).unwrap().contains("canonical_query")); + assert_eq!(binding(&mut decoded).window_ms, 10_000); + assert_eq!(binding(&mut decoded).readout_lookback_ms, Some(60_000)); + } + + // The catalog owns source and grouping; the binding owns only its stable ID. + #[test] + fn catalog_binding_rejects_source_grouping_and_identity_drift() { + let (plan, catalog) = fixture(); + let mut broken = plan.clone(); + binding(&mut broken).materialization = PolicyFingerprint(123).into(); + assert!(broken.validate_against_catalog(&catalog).is_err()); + let mut broken = plan.clone(); + broken.plan_version += 1; + assert!(broken.validate_against_catalog(&catalog).is_err()); + let mut broken = plan; + binding(&mut broken).window_ms = 0; + assert!(broken.validate_against_catalog(&catalog).is_err()); + } + + // Catalog descriptor corruption must fail even if the materialization exists. + #[test] + fn catalog_binding_rejects_broken_descriptor_reference() { + let (plan, mut catalog) = fixture(); + catalog.summary_descriptors.clear(); + assert!(plan.validate_against_catalog(&catalog).is_err()); + } + + #[test] + fn counter_readout_requires_counter_sds_fidelity() { + fn as_rate_plan(mut plan: QueryPlan) -> QueryPlan { + let entry = plan.entries.values_mut().next().unwrap(); + let read = entry.root; + let root = QueryNodeId(2); + entry.root = root; + entry.nodes.insert( + root, + QueryPlanNode::ExactReadout { + input: read, + readout: ExactReadout::Rate, + }, + ); + plan + } + + let (sum_plan, sum_catalog) = fixture(); + assert!(as_rate_plan(sum_plan) + .validate_against_catalog(&sum_catalog) + .unwrap_err() + .to_string() + .contains("exact counter SDS")); + + let mut counter = PrecomputeMaterialization::new( + AggregationType::Increase, + String::new(), + Default::default(), + KeyByLabelNames::new(vec!["job".into()]), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + String::new(), + 10, + 10, + WindowKind::Tumbling, + String::new(), + "m".into(), + None, + None, + None, + ); + counter.pane_origin_ms = Some(0); + let counter_catalog = + SummaryCatalog::from_materializations(7, 2, &[counter.clone()]).unwrap(); + let (mut counter_plan, _) = fixture(); + binding(&mut counter_plan).materialization = counter.policy_fingerprint().into(); + as_rate_plan(counter_plan) + .validate_against_catalog(&counter_catalog) + .unwrap(); + } +} + #[cfg(test)] mod tests { use super::*; @@ -1120,7 +1296,7 @@ mod tests { QueryPlanNode::CandidateTopK { inputs: [QueryNodeId(0), QueryNodeId(1)], k: 2, - grouping: logical::Grouping { + grouping: residual::Grouping { labels: vec![], without: false, }, @@ -1150,176 +1326,3 @@ mod tests { assert!(entry.validate(&BTreeSet::new()).is_err()); } } - -#[cfg(test)] -mod catalog_binding_tests { - use super::*; - use crate::physical::summary_catalog::SummaryCatalog; - use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; - - fn fixture() -> (QueryPlan, SummaryCatalog) { - let mut config = PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - Default::default(), - KeyByLabelNames::new(vec!["job".into()]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 10, - 10, - WindowKind::Tumbling, - String::new(), - "m".into(), - None, - None, - None, - ); - config.pane_origin_ms = Some(0); - let catalog = SummaryCatalog::from_materializations(7, 2, &[config.clone()]).unwrap(); - let entry = QueryPlanEntry { - language: crate::query_plan::QueryLanguage::PromQl, - query_id: "q".into(), - canonical_query: "sum_over_time(m[1m])".into(), - fixed_evaluation: None, - root: QueryNodeId(1), - nodes: BTreeMap::from([( - QueryNodeId(1), - QueryPlanNode::ReadMaterialization { - binding: MaterializationBinding { - full_window_slide_ms: None, - item_labels: Vec::new(), - materialization: config.policy_fingerprint().into(), - output_grouping: PhysicalGrouping::PerEntity, - window_ms: 10_000, - pane_origin_ms: Some(0), - readout_lookback_ms: Some(60_000), - }, - }, - )]), - instant: InstantExecution { - lookback_ms: 60_000, - full_history: false, - cumulative_readout: true, - }, - fallback: FallbackPolicy::ExactBackend, - }; - ( - QueryPlan { - plan_id: 7, - plan_version: 2, - clickhouse_context: None, - entries: BTreeMap::from([(entry.canonical_query.clone(), entry)]), - }, - catalog, - ) - } - fn binding(plan: &mut QueryPlan) -> &mut MaterializationBinding { - let QueryPlanNode::ReadMaterialization { binding } = plan - .entries - .values_mut() - .next() - .unwrap() - .nodes - .values_mut() - .next() - .unwrap() - else { - panic!("fixture") - }; - binding - } - - // One pane ID is compatible with a longer semantic readout window. - #[test] - fn catalog_binding_round_trip_preserves_pane_and_readout_windows() { - let (plan, catalog) = fixture(); - let wire = serde_json::to_vec(&plan).unwrap(); - let mut decoded: QueryPlan = serde_json::from_slice(&wire).unwrap(); - decoded.validate_against_catalog(&catalog).unwrap(); - assert_eq!( - decoded - .lookup("sum_over_time(m[1m])") - .unwrap() - .canonical_query, - "sum_over_time(m[1m])" - ); - assert!(String::from_utf8(wire).unwrap().contains("canonical_query")); - assert_eq!(binding(&mut decoded).window_ms, 10_000); - assert_eq!(binding(&mut decoded).readout_lookback_ms, Some(60_000)); - } - - // The catalog owns source and grouping; the binding owns only its stable ID. - #[test] - fn catalog_binding_rejects_source_grouping_and_identity_drift() { - let (plan, catalog) = fixture(); - let mut broken = plan.clone(); - binding(&mut broken).materialization = PolicyFingerprint(123).into(); - assert!(broken.validate_against_catalog(&catalog).is_err()); - let mut broken = plan.clone(); - broken.plan_version += 1; - assert!(broken.validate_against_catalog(&catalog).is_err()); - let mut broken = plan; - binding(&mut broken).window_ms = 0; - assert!(broken.validate_against_catalog(&catalog).is_err()); - } - - // Catalog descriptor corruption must fail even if the materialization exists. - #[test] - fn catalog_binding_rejects_broken_descriptor_reference() { - let (plan, mut catalog) = fixture(); - catalog.summary_descriptors.clear(); - assert!(plan.validate_against_catalog(&catalog).is_err()); - } - - #[test] - fn counter_readout_requires_counter_sds_fidelity() { - fn as_rate_plan(mut plan: QueryPlan) -> QueryPlan { - let entry = plan.entries.values_mut().next().unwrap(); - let read = entry.root; - let root = QueryNodeId(2); - entry.root = root; - entry.nodes.insert( - root, - QueryPlanNode::ExactReadout { - input: read, - readout: ExactReadout::Rate, - }, - ); - plan - } - - let (sum_plan, sum_catalog) = fixture(); - assert!(as_rate_plan(sum_plan) - .validate_against_catalog(&sum_catalog) - .unwrap_err() - .to_string() - .contains("exact counter SDS")); - - let mut counter = PrecomputeMaterialization::new( - AggregationType::Increase, - String::new(), - Default::default(), - KeyByLabelNames::new(vec!["job".into()]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 10, - 10, - WindowKind::Tumbling, - String::new(), - "m".into(), - None, - None, - None, - ); - counter.pane_origin_ms = Some(0); - let counter_catalog = - SummaryCatalog::from_materializations(7, 2, &[counter.clone()]).unwrap(); - let (mut counter_plan, _) = fixture(); - binding(&mut counter_plan).materialization = counter.policy_fingerprint().into(); - as_rate_plan(counter_plan) - .validate_against_catalog(&counter_catalog) - .unwrap(); - } -} diff --git a/control_plane/src/query_plan/clickhouse_exact.rs b/control_plane/src/query_plan/clickhouse_exact.rs index 8551ddfed..bcd31fb85 100644 --- a/control_plane/src/query_plan/clickhouse_exact.rs +++ b/control_plane/src/query_plan/clickhouse_exact.rs @@ -267,99 +267,6 @@ pub(super) fn render(expr: &QueryExpr) -> Result { } } -#[cfg(test)] -mod tests { - use super::*; - use planner_types::pre_asap::{Column, DataType, Predicate, ProjectItem}; - use std::rc::Rc; - #[test] - fn composite_cut_preserves_branch_time_and_positional_projection() { - let scan = Rc::new(QueryExpr::Scan { - source: Source::Table { - table_ref: "db.samples".into(), - }, - schema: Schema::new(vec![ - Column::new("ts", DataType::Int64, false), - Column::new("v", DataType::Float64, false), - ]), - predicates: vec![Predicate(Rc::new(QueryExpr::Compare { - left: Rc::new(QueryExpr::Column(0)), - op: CompareOpKind::Lt, - right: Rc::new(QueryExpr::Literal(ScalarValue::Int64(-100))), - }))], - }); - let project = QueryExpr::Project { - cols: vec![ProjectItem { - alias: Some("result".into()), - expr: QueryExpr::Column(1), - }], - qualifier: None, - child: scan, - }; - let sql = render(&project).unwrap(); - assert!(sql.contains("`ts` < -100")); - assert!(sql.starts_with("SELECT `v` AS `result`")); - assert!(!sql.contains("{from:")); - assert!(!sql.contains("{to:")); - } - #[test] - fn typed_list_access_renders_native_element_lookup() { - let schema = Schema::new(vec![Column::new( - "samples", - DataType::List { - element: Box::new(Column::new("item", DataType::Float64, false)), - }, - false, - )]); - let expr = QueryExpr::FunctionCall { - name: "asap_element_access".into(), - args: vec![ - QueryExpr::Column(0), - QueryExpr::Literal(ScalarValue::Int64(-1)), - ], - }; - assert_eq!( - scalar(&expr, &schema).unwrap(), - "arrayElement(`samples`, -1)" - ); - } - - #[test] - fn typed_struct_field_renders_native_lookup() { - let schema = Schema::new(vec![Column::new( - "sample", - DataType::Struct { - fields: vec![ - Column::new("ts", DataType::Int64, false), - Column::new("value", DataType::Float64, true), - ], - }, - false, - )]); - let expr = QueryExpr::FunctionCall { - name: "asap_struct_field".into(), - args: vec![ - QueryExpr::Column(0), - QueryExpr::Literal(ScalarValue::Utf8("value".into())), - ], - }; - assert_eq!( - scalar(&expr, &schema).unwrap(), - "tupleElement(`sample`, 'value')" - ); - } - - #[test] - fn unsupported_scalar_is_not_forwarded_as_arbitrary_native_code() { - let schema = Schema::new(vec![]); - let expr = QueryExpr::FunctionCall { - name: "unreviewedFunction".into(), - args: vec![], - }; - assert!(scalar(&expr, &schema).is_err()); - } -} - #[cfg(test)] mod original_tests { use super::*; @@ -470,3 +377,96 @@ mod original_tests { assert_eq!(rendered, "'a\\\\\\'b\n'"); } } + +#[cfg(test)] +mod tests { + use super::*; + use planner_types::pre_asap::{Column, DataType, Predicate, ProjectItem}; + use std::rc::Rc; + #[test] + fn composite_cut_preserves_branch_time_and_positional_projection() { + let scan = Rc::new(QueryExpr::Scan { + source: Source::Table { + table_ref: "db.samples".into(), + }, + schema: Schema::new(vec![ + Column::new("ts", DataType::Int64, false), + Column::new("v", DataType::Float64, false), + ]), + predicates: vec![Predicate(Rc::new(QueryExpr::Compare { + left: Rc::new(QueryExpr::Column(0)), + op: CompareOpKind::Lt, + right: Rc::new(QueryExpr::Literal(ScalarValue::Int64(-100))), + }))], + }); + let project = QueryExpr::Project { + cols: vec![ProjectItem { + alias: Some("result".into()), + expr: QueryExpr::Column(1), + }], + qualifier: None, + child: scan, + }; + let sql = render(&project).unwrap(); + assert!(sql.contains("`ts` < -100")); + assert!(sql.starts_with("SELECT `v` AS `result`")); + assert!(!sql.contains("{from:")); + assert!(!sql.contains("{to:")); + } + #[test] + fn typed_list_access_renders_native_element_lookup() { + let schema = Schema::new(vec![Column::new( + "samples", + DataType::List { + element: Box::new(Column::new("item", DataType::Float64, false)), + }, + false, + )]); + let expr = QueryExpr::FunctionCall { + name: "asap_element_access".into(), + args: vec![ + QueryExpr::Column(0), + QueryExpr::Literal(ScalarValue::Int64(-1)), + ], + }; + assert_eq!( + scalar(&expr, &schema).unwrap(), + "arrayElement(`samples`, -1)" + ); + } + + #[test] + fn typed_struct_field_renders_native_lookup() { + let schema = Schema::new(vec![Column::new( + "sample", + DataType::Struct { + fields: vec![ + Column::new("ts", DataType::Int64, false), + Column::new("value", DataType::Float64, true), + ], + }, + false, + )]); + let expr = QueryExpr::FunctionCall { + name: "asap_struct_field".into(), + args: vec![ + QueryExpr::Column(0), + QueryExpr::Literal(ScalarValue::Utf8("value".into())), + ], + }; + assert_eq!( + scalar(&expr, &schema).unwrap(), + "tupleElement(`sample`, 'value')" + ); + } + + #[test] + fn unsupported_scalar_is_not_forwarded_as_arbitrary_native_code() { + let schema = Schema::new(vec![]); + let expr = QueryExpr::FunctionCall { + name: "unreviewedFunction".into(), + args: vec![], + }; + assert!(scalar(&expr, &schema).is_err()); + } +} diff --git a/control_plane/src/query_plan/logical.rs b/control_plane/src/query_plan/residual.rs similarity index 91% rename from control_plane/src/query_plan/logical.rs rename to control_plane/src/query_plan/residual.rs index a2cbbb179..73b7ce8d4 100644 --- a/control_plane/src/query_plan/logical.rs +++ b/control_plane/src/query_plan/residual.rs @@ -8,7 +8,7 @@ use promql_parser::{ }; use std::collections::BTreeMap; -pub use asap_types::query_plan::logical::*; +pub use asap_types::query_plan::residual::*; /// Stable identity of a Planner-authorized materializable DAG leaf. This is a /// workload-selection key, not another physical materialization definition. @@ -54,7 +54,7 @@ impl Lower { } fn operation( &mut self, - operator: LogicalOperator, + operator: ResidualQueryOperator, inputs: Vec, ) -> Result { operator.validate(inputs.len())?; @@ -84,7 +84,7 @@ impl Lower { }) .collect(); self.operation( - LogicalOperator::Scan { + ResidualQueryOperator::Scan { metric: s.name.clone(), matchers, range_ms, @@ -101,7 +101,7 @@ impl Lower { Expr::Paren(p) => self.lower(&p.expr), Expr::Unary(u) => { let input = self.lower(&u.expr)?; - self.operation(LogicalOperator::UnaryNegate, vec![input]) + self.operation(ResidualQueryOperator::UnaryNegate, vec![input]) } Expr::VectorSelector(s) => self.scan(s, None), Expr::MatrixSelector(s) => self.scan(&s.vs, Some(millis(s.range)?)), @@ -111,7 +111,7 @@ impl Lower { } let input = self.lower(&s.expr)?; self.operation( - LogicalOperator::Subquery { + ResidualQueryOperator::Subquery { range_ms: millis(s.range)?, // Prometheus uses its configured default evaluation // interval when `[range:]` omits the resolution. The @@ -163,7 +163,7 @@ impl Lower { self.nodes = nodes_before; self.seen = seen_before; self.operation( - LogicalOperator::ExactSubquery { + ResidualQueryOperator::ExactSubquery { query: a.expr.to_string(), }, vec![], @@ -171,7 +171,7 @@ impl Lower { } }; return self.operation( - LogicalOperator::TopKSelection { + ResidualQueryOperator::TopKSelection { k: u64::try_from(k).unwrap_or(0), grouping, }, @@ -191,7 +191,7 @@ impl Lower { }; let input = self.lower(&a.expr)?; self.operation( - LogicalOperator::Aggregate { + ResidualQueryOperator::Aggregate { operation, grouping, }, @@ -200,11 +200,11 @@ impl Lower { } Expr::Call(c) => { let operator = match c.func.name { - "scalar" => LogicalOperator::VectorToScalar, - "histogram_quantile" => LogicalOperator::HistogramQuantile, - "sort" => LogicalOperator::Sort { descending: false }, - "sort_desc" => LogicalOperator::Sort { descending: true }, - name => LogicalOperator::Temporal { + "scalar" => ResidualQueryOperator::VectorToScalar, + "histogram_quantile" => ResidualQueryOperator::HistogramQuantile, + "sort" => ResidualQueryOperator::Sort { descending: false }, + "sort_desc" => ResidualQueryOperator::Sort { descending: true }, + name => ResidualQueryOperator::Temporal { operation: match name { "rate" => TemporalOperation::Rate, "increase" => TemporalOperation::Increase, @@ -251,7 +251,7 @@ impl Lower { }; let inputs = vec![self.lower(&b.lhs)?, self.lower(&b.rhs)?]; self.operation( - LogicalOperator::Binary { + ResidualQueryOperator::Binary { operation, return_bool: b.return_bool(), }, @@ -341,7 +341,7 @@ pub(super) fn residual_nodes( pub(super) fn binary_operator( operator: &planner_types::post_asap::BinaryOperator, -) -> Result { +) -> Result { if operator.vector_match.is_some() { return Err(invalid("explicit residual vector matching unsupported")); } @@ -364,174 +364,12 @@ pub(super) fn binary_operator( ))) } }; - Ok(LogicalOperator::Binary { + Ok(ResidualQueryOperator::Binary { operation, return_bool: false, }) } -#[cfg(test)] -mod tests { - use super::*; - fn instant() -> InstantExecution { - InstantExecution { - lookback_ms: 300_000, - full_history: false, - cumulative_readout: false, - } - } - - #[test] - fn complete_o11y_corpus_lowers_to_serialized_operations() { - // Every original workload occurrence must compile to an executable typed graph. - let corpus: serde_json::Value = - serde_json::from_str(include_str!("../../tests/fixtures/o11y_queries.json")).unwrap(); - for row in corpus["queries"].as_array().unwrap() { - let query = row["query"].as_str().unwrap(); - let entry = crate::query_plan::logical::compile_logical( - row["id"].as_str().unwrap().into(), - query.into(), - instant(), - FallbackPolicy::Reject, - ) - .unwrap_or_else(|error| panic!("{query}: {error}")); - let encoded = serde_json::to_string(&entry).unwrap(); - let restored: QueryPlanEntry = serde_json::from_str(&encoded).unwrap(); - restored.validate(&Default::default()).unwrap(); - assert!(!restored - .nodes - .values() - .any(|node| matches!(node, QueryPlanNode::ExactFallback { .. }))); - } - } - #[test] - fn residual_mapping_preserves_filters_and_rejects_different_sources() { - // Physical lowering must prove correspondence with the Planner-kept semantic subtree. - let query = "sum(rate(requests_total{job=\"api\"}[5m]))"; - let residual = crate::query_parser::parse_query_expr_canonical( - query, - planner_types::types::AccuracyTarget::Exact, - ) - .unwrap(); - let (_, nodes) = residual_nodes(query, &residual).unwrap(); - assert!(nodes.values().any(|node| matches!(node, QueryPlanNode::Logical { operator: LogicalOperator::Scan { matchers, .. }, .. } if matchers.iter().any(|m| m.name == "job" && m.value == "api")))); - assert!(residual_nodes("sum(rate(other_total[5m]))", &residual).is_err()); - } - #[test] - fn repeated_subexpressions_share_node_identity() { - // Serialized edges must retain CSE rather than duplicating raw work. - let entry = crate::query_plan::logical::compile_logical( - "q".into(), - "sum(up) / sum(up)".into(), - instant(), - FallbackPolicy::Reject, - ) - .unwrap(); - let QueryPlanNode::Logical { inputs, .. } = &entry.nodes[&entry.root] else { - panic!("binary expected") - }; - assert_eq!(inputs[0], inputs[1]); - } - #[test] - fn malformed_operator_arity_is_rejected_at_installation() { - // A serialized graph cannot bypass the operation's input contract. - assert!(LogicalOperator::HistogramQuantile.validate(1).is_err()); - assert!(LogicalOperator::Subquery { - range_ms: 60_000, - step_ms: 0, - offset_ms: 0 - } - .validate(1) - .is_err()); - } - - #[test] - fn real_topk_queries_lower_to_value_selection() { - for (query, k) in [ - ( - "topk(2, sum by (job) (rate(backend_process_cpu_seconds_total[1h])))", - 2, - ), - ( - "topk(2, sum by (job) (backend_process_resident_memory_bytes))", - 2, - ), - ("topk(2, max_over_time(backend_retry_backlog_depth[6h]))", 2), - ( - "topk(1, sum by (job) (increase(backend_http_5xx_total[6h])) / sum by (job) (increase(backend_http_requests_total[6h])))", - 1, - ), - ( - "topk(3, avg_over_time((sum by (job) (backend_process_resident_memory_bytes))[6h:]))", - 3, - ), - ] { - let entry = crate::query_plan::logical::compile_logical( - "topk".into(), - query.into(), - instant(), - FallbackPolicy::Reject, - ) - .unwrap_or_else(|error| panic!("{query}: {error}")); - assert!(matches!( - entry.nodes[&entry.root], - QueryPlanNode::Logical { - operator: LogicalOperator::TopKSelection { k: actual, .. }, - .. - } if actual == k - )); - } - } - - #[test] - fn topk_keeps_unsupported_child_as_exact_leaf() { - let entry = crate::query_plan::logical::compile_logical( - "topk-subquery".into(), - "topk(3, label_replace(memory_bytes, \"dst\", \"$1\", \"src\", \"(.*)\"))".into(), - instant(), - FallbackPolicy::Reject, - ) - .unwrap(); - assert!(matches!( - entry.nodes[&entry.root], - QueryPlanNode::Logical { - operator: LogicalOperator::TopKSelection { k: 3, .. }, - .. - } - )); - assert!(entry.nodes.values().any(|node| matches!( - node, - QueryPlanNode::Logical { - operator: LogicalOperator::ExactSubquery { .. }, - .. - } - ))); - } - - #[test] - fn topk_preserves_by_and_without_partitioning() { - for (query, labels, without) in [ - ("topk by (cluster) (2, m)", vec!["cluster"], false), - ("topk without (pod) (2, m)", vec!["pod"], true), - ] { - let entry = crate::query_plan::logical::compile_logical( - "topk-group".into(), - query.into(), - instant(), - FallbackPolicy::Reject, - ) - .unwrap(); - assert!(matches!( - &entry.nodes[&entry.root], - QueryPlanNode::Logical { - operator: LogicalOperator::TopKSelection { grouping, .. }, - .. - } if grouping.labels == labels && grouping.without == without - )); - } - } -} - /// Prove a physical-native substitute represents exactly the selected summary leaf. /// A second Planner invocation is an equality witness, not a replacement selection. pub(crate) fn selected_residual_nodes( @@ -604,11 +442,11 @@ pub(crate) fn selected_residual_nodes( pub(super) fn selected_aggregate_operator( original: &str, selected: &planner_types::post_asap::SummaryNode, -) -> Result { +) -> Result { let (root, nodes) = selected_residual_nodes(original, selected)?; match nodes.get(&root) { Some(QueryPlanNode::Logical { - operator: operator @ LogicalOperator::Aggregate { .. }, + operator: operator @ ResidualQueryOperator::Aggregate { .. }, .. }) => Ok(operator.clone()), _ => Err(invalid( @@ -644,7 +482,7 @@ mod hybrid_tests { FallbackPolicy::Reject, |node, _| { let (_, _, spatial_filter) = - crate::physical::compiler::materialization_leaf_contract(node) + crate::physical::compiler::raw_materialization_input_contract(node) .map_err(QueryPlanError::Invalid)?; Ok(MaterializationBinding { full_window_slide_ms: None, @@ -666,21 +504,21 @@ mod hybrid_tests { assert!(!entry.nodes.values().any(|node| matches!( node, QueryPlanNode::Logical { - operator: LogicalOperator::ExactSubquery { .. }, + operator: ResidualQueryOperator::ExactSubquery { .. }, .. } ))); assert!(!entry.nodes.values().any(|node| matches!( node, QueryPlanNode::Logical { - operator: LogicalOperator::Scan { .. }, + operator: ResidualQueryOperator::Scan { .. }, .. } ))); assert!(matches!( entry.nodes[&entry.root], QueryPlanNode::Logical { - operator: LogicalOperator::Binary { .. }, + operator: ResidualQueryOperator::Binary { .. }, .. } )); @@ -714,7 +552,7 @@ mod hybrid_tests { #[cfg(test)] mod planner_workload_tests { use super::*; - use crate::physical::compiler::{BackendLocalPlanningSnapshot, PhysicalCompiler}; + use crate::physical::compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}; fn lookback(expr: &Expr) -> u64 { match expr { @@ -729,7 +567,7 @@ mod planner_workload_tests { } } - fn compile_one(query: &str) -> crate::physical::compiler::PhysicalPlan { + fn compile_one(query: &str) -> crate::physical::compiler::CompiledPhysicalPlan { let mut fixture: serde_json::Value = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) @@ -740,12 +578,12 @@ mod planner_workload_tests { let window = lookback(&parser::parse(query).unwrap()); entry["time_selection"]["lookback"] = (if window == 0 { 300_000 } else { window }).into(); fixture["query_workload"]["repeating_queries"] = vec![entry].into(); - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_value(fixture).unwrap(); + let snapshot: BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); let (request, environment) = snapshot - .planning_request() + .into_physical_compilation_request() .unwrap_or_else(|error| panic!("{query}: {error}")); - PhysicalCompiler - .compile(request, environment) + PhysicalPlanCompiler + .compile_promql(request, environment) .unwrap_or_else(|error| panic!("{query}: {error}")) } @@ -764,7 +602,7 @@ mod planner_workload_tests { matches!( entry.nodes[&entry.root], QueryPlanNode::Logical { - operator: LogicalOperator::TopKSelection { .. }, + operator: ResidualQueryOperator::TopKSelection { .. }, .. } ), @@ -792,7 +630,7 @@ mod planner_workload_tests { assert!(matches!( entry.nodes[&entry.root], QueryPlanNode::Logical { - operator: LogicalOperator::TopKSelection { .. }, + operator: ResidualQueryOperator::TopKSelection { .. }, .. } )); @@ -830,10 +668,12 @@ mod planner_workload_tests { entries.push(entry); } fixture["query_workload"]["repeating_queries"] = entries.into(); - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_value(fixture).unwrap(); - let (request, environment) = snapshot.planning_request().unwrap(); - assert!(request.hybrid_execution); - let plan = PhysicalCompiler.compile(request, environment).unwrap(); + let snapshot: BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); + let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); + assert!(request.allow_mixed_summary_and_exact_execution); + let plan = PhysicalPlanCompiler + .compile_promql(request, environment) + .unwrap(); assert_eq!(plan.query_plan.entries.len(), 24); assert!(plan.query_plan.entries.values().all(|entry| !entry .nodes @@ -854,7 +694,7 @@ mod planner_workload_tests { let operator = selected_aggregate_operator(query, &selected).unwrap(); assert!(matches!( operator, - LogicalOperator::Aggregate { + ResidualQueryOperator::Aggregate { operation: Aggregation::Max, .. } @@ -884,7 +724,7 @@ mod planner_workload_tests { assert!(matches!( nodes[&root], QueryPlanNode::Logical { - operator: LogicalOperator::Aggregate { + operator: ResidualQueryOperator::Aggregate { operation: Aggregation::Min, .. }, @@ -915,7 +755,7 @@ pub(crate) fn selected_range_max_materialization( let (root, nodes) = selected_residual_nodes(original, node)?; let Some(QueryPlanNode::Logical { operator: - LogicalOperator::Temporal { + ResidualQueryOperator::Temporal { operation: TemporalOperation::Max, }, inputs, @@ -928,7 +768,7 @@ pub(crate) fn selected_range_max_materialization( } let Some(QueryPlanNode::Logical { operator: - LogicalOperator::Scan { + ResidualQueryOperator::Scan { metric: Some(metric), matchers, range_ms: Some(range_ms), @@ -1029,7 +869,7 @@ fn counter_contract( nodes: &BTreeMap, ) -> Option { let QueryPlanNode::Logical { - operator: LogicalOperator::Temporal { operation }, + operator: ResidualQueryOperator::Temporal { operation }, inputs, } = nodes.get(&root)? else { @@ -1044,7 +884,7 @@ fn counter_contract( } let QueryPlanNode::Logical { operator: - LogicalOperator::Scan { + ResidualQueryOperator::Scan { metric: Some(metric), matchers, range_ms: Some(range_ms), @@ -1104,7 +944,7 @@ pub fn finalize_residuals(entry: &mut QueryPlanEntry) -> Result<(), QueryPlanErr assign_retention(entry) } -pub fn materialization_candidate_keys( +pub fn eligible_materialization_keys( original: &str, selected: &std::rc::Rc, ) -> Result, QueryPlanError> { @@ -1170,7 +1010,7 @@ fn expression_shape( .get(&id) .ok_or_else(|| invalid("missing expression node"))?; if let QueryPlanNode::Logical { - operator: LogicalOperator::ExactSubquery { query }, + operator: ResidualQueryOperator::ExactSubquery { query }, .. } = node { @@ -1247,7 +1087,7 @@ pub fn externalize_residuals(entry: &mut QueryPlanEntry) -> Result<(), QueryPlan ) || matches!( node, QueryPlanNode::Logical { - operator: LogicalOperator::TopKSelection { .. }, + operator: ResidualQueryOperator::TopKSelection { .. }, .. } ); @@ -1255,9 +1095,9 @@ pub fn externalize_residuals(entry: &mut QueryPlanEntry) -> Result<(), QueryPlan if let QueryPlanNode::Logical { operator, .. } = node { exact = matches!( operator, - LogicalOperator::Scan { .. } - | LogicalOperator::ExactSubquery { .. } - | LogicalOperator::CandidateExactSubquery { .. } + ResidualQueryOperator::Scan { .. } + | ResidualQueryOperator::ExactSubquery { .. } + | ResidualQueryOperator::CandidateExactSubquery { .. } ); } for child in node.inputs() { @@ -1272,8 +1112,8 @@ pub fn externalize_residuals(entry: &mut QueryPlanEntry) -> Result<(), QueryPlan if matches!( entry.nodes.get(&id), Some(QueryPlanNode::Logical { - operator: LogicalOperator::ExactSubquery { .. } - | LogicalOperator::CandidateExactSubquery { .. }, + operator: ResidualQueryOperator::ExactSubquery { .. } + | ResidualQueryOperator::CandidateExactSubquery { .. }, .. }) ) { @@ -1286,7 +1126,7 @@ pub fn externalize_residuals(entry: &mut QueryPlanEntry) -> Result<(), QueryPlan entry.nodes.insert( id, QueryPlanNode::Logical { - operator: LogicalOperator::ExactSubquery { + operator: ResidualQueryOperator::ExactSubquery { query: query.clone(), }, inputs: vec![], @@ -1303,7 +1143,7 @@ pub fn externalize_residuals(entry: &mut QueryPlanEntry) -> Result<(), QueryPlan matches!( node, QueryPlanNode::Logical { - operator: LogicalOperator::Scan { .. }, + operator: ResidualQueryOperator::Scan { .. }, .. } ) @@ -1329,7 +1169,7 @@ fn assign_retention(entry: &mut QueryPlanEntry) -> Result<(), QueryPlanError> { .ok_or_else(|| invalid("missing index ancestor"))?; let mut child_depth = depth; if let QueryPlanNode::Logical { operator, .. } = node { - if let LogicalOperator::Subquery { + if let ResidualQueryOperator::Subquery { range_ms, offset_ms, .. @@ -1392,10 +1232,177 @@ mod remote_boundary_regressions { .unwrap(); let selected = crate::planner_selection::select_summary_default(&parsed).unwrap(); assert_eq!( - materialization_candidate_keys(query, &selected) + eligible_materialization_keys(query, &selected) .unwrap() .len(), 2 ); } } + +#[deprecated(note = "Use eligible_materialization_keys")] +pub use eligible_materialization_keys as materialization_candidate_keys; + +#[cfg(test)] +mod tests { + use super::*; + fn instant() -> InstantExecution { + InstantExecution { + lookback_ms: 300_000, + full_history: false, + cumulative_readout: false, + } + } + + #[test] + fn complete_o11y_corpus_lowers_to_serialized_operations() { + // Every original workload occurrence must compile to an executable typed graph. + let corpus: serde_json::Value = + serde_json::from_str(include_str!("../../tests/fixtures/o11y_queries.json")).unwrap(); + for row in corpus["queries"].as_array().unwrap() { + let query = row["query"].as_str().unwrap(); + let entry = crate::query_plan::residual::compile_logical( + row["id"].as_str().unwrap().into(), + query.into(), + instant(), + FallbackPolicy::Reject, + ) + .unwrap_or_else(|error| panic!("{query}: {error}")); + let encoded = serde_json::to_string(&entry).unwrap(); + let restored: QueryPlanEntry = serde_json::from_str(&encoded).unwrap(); + restored.validate(&Default::default()).unwrap(); + assert!(!restored + .nodes + .values() + .any(|node| matches!(node, QueryPlanNode::ExactFallback { .. }))); + } + } + #[test] + fn residual_mapping_preserves_filters_and_rejects_different_sources() { + // Physical lowering must prove correspondence with the Planner-kept semantic subtree. + let query = "sum(rate(requests_total{job=\"api\"}[5m]))"; + let residual = crate::query_parser::parse_query_expr_canonical( + query, + planner_types::types::AccuracyTarget::Exact, + ) + .unwrap(); + let (_, nodes) = residual_nodes(query, &residual).unwrap(); + assert!(nodes.values().any(|node| matches!(node, QueryPlanNode::Logical { operator: ResidualQueryOperator::Scan { matchers, .. }, .. } if matchers.iter().any(|m| m.name == "job" && m.value == "api")))); + assert!(residual_nodes("sum(rate(other_total[5m]))", &residual).is_err()); + } + #[test] + fn repeated_subexpressions_share_node_identity() { + // Serialized edges must retain CSE rather than duplicating raw work. + let entry = crate::query_plan::residual::compile_logical( + "q".into(), + "sum(up) / sum(up)".into(), + instant(), + FallbackPolicy::Reject, + ) + .unwrap(); + let QueryPlanNode::Logical { inputs, .. } = &entry.nodes[&entry.root] else { + panic!("binary expected") + }; + assert_eq!(inputs[0], inputs[1]); + } + #[test] + fn malformed_operator_arity_is_rejected_at_installation() { + // A serialized graph cannot bypass the operation's input contract. + assert!(ResidualQueryOperator::HistogramQuantile + .validate(1) + .is_err()); + assert!(ResidualQueryOperator::Subquery { + range_ms: 60_000, + step_ms: 0, + offset_ms: 0 + } + .validate(1) + .is_err()); + } + + #[test] + fn real_topk_queries_lower_to_value_selection() { + for (query, k) in [ + ( + "topk(2, sum by (job) (rate(backend_process_cpu_seconds_total[1h])))", + 2, + ), + ( + "topk(2, sum by (job) (backend_process_resident_memory_bytes))", + 2, + ), + ("topk(2, max_over_time(backend_retry_backlog_depth[6h]))", 2), + ( + "topk(1, sum by (job) (increase(backend_http_5xx_total[6h])) / sum by (job) (increase(backend_http_requests_total[6h])))", + 1, + ), + ( + "topk(3, avg_over_time((sum by (job) (backend_process_resident_memory_bytes))[6h:]))", + 3, + ), + ] { + let entry = crate::query_plan::residual::compile_logical( + "topk".into(), + query.into(), + instant(), + FallbackPolicy::Reject, + ) + .unwrap_or_else(|error| panic!("{query}: {error}")); + assert!(matches!( + entry.nodes[&entry.root], + QueryPlanNode::Logical { + operator: ResidualQueryOperator::TopKSelection { k: actual, .. }, + .. + } if actual == k + )); + } + } + + #[test] + fn topk_keeps_unsupported_child_as_exact_leaf() { + let entry = crate::query_plan::residual::compile_logical( + "topk-subquery".into(), + "topk(3, label_replace(memory_bytes, \"dst\", \"$1\", \"src\", \"(.*)\"))".into(), + instant(), + FallbackPolicy::Reject, + ) + .unwrap(); + assert!(matches!( + entry.nodes[&entry.root], + QueryPlanNode::Logical { + operator: ResidualQueryOperator::TopKSelection { k: 3, .. }, + .. + } + )); + assert!(entry.nodes.values().any(|node| matches!( + node, + QueryPlanNode::Logical { + operator: ResidualQueryOperator::ExactSubquery { .. }, + .. + } + ))); + } + + #[test] + fn topk_preserves_by_and_without_partitioning() { + for (query, labels, without) in [ + ("topk by (cluster) (2, m)", vec!["cluster"], false), + ("topk without (pod) (2, m)", vec!["pod"], true), + ] { + let entry = crate::query_plan::residual::compile_logical( + "topk-group".into(), + query.into(), + instant(), + FallbackPolicy::Reject, + ) + .unwrap(); + assert!(matches!( + &entry.nodes[&entry.root], + QueryPlanNode::Logical { + operator: ResidualQueryOperator::TopKSelection { grouping, .. }, + .. + } if grouping.labels == labels && grouping.without == without + )); + } + } +} diff --git a/control_plane/src/registered_workload.rs b/control_plane/src/registered_workload.rs new file mode 100644 index 000000000..4b46ff3a4 --- /dev/null +++ b/control_plane/src/registered_workload.rs @@ -0,0 +1,569 @@ +//! Registration state: canonical Planner workload plus collector deployment options. +use std::{collections::HashMap, time::Duration}; + +use anyhow::{anyhow, ensure}; +use planner_types::workload::*; + +use crate::{ + query_parser::{self, ParsedQuery}, + types::AccuracyTarget, + types::{AggType, SketchType, WorkloadCharacteristics}, +}; + +/// Facts about the collector deployment, not query semantics or data arrival. +#[derive(Debug, Clone, Default)] +pub struct DeploymentOptions { + pub sketch_type_override: Option, + pub query_id: Option, + pub deployment_model: Option, + pub retained_labels: Vec, + pub bytes_per_raw_sample: u32, + pub distinct_keys_per_window: Option, + pub memory_budget_bytes: Option, +} + +/// The registry owns one canonical query; stage-specific metadata is derived on demand. +#[derive(Debug, Clone)] +pub struct RegisteredWorkload { + workload: QueryWorkload, + pub deployment: DeploymentOptions, +} + +impl RegisteredWorkload { + pub fn new(workload: QueryWorkload, deployment: DeploymentOptions) -> anyhow::Result { + workload + .validate() + .map_err(|e| anyhow!("invalid workload: {e:?}"))?; + ensure!( + workload.language == QueryLanguage::PromQL, + "metric registration requires PromQL" + ); + ensure!( + workload.entries().count() == 1, + "metric registration requires exactly one query entry" + ); + let registered = Self { + workload, + deployment, + }; + let entry = registered.entry(); + query_parser::parse_query_expr_canonical(&entry.query.0, registered.accuracy())?; + let range = metric_query_range(&entry.query.0)?; + ensure!(matches!(entry.recurrence, QueryRecurrence::OneTime { invocations: 1, execute_at: None } | QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(_))), "metric registration supports one invocation or a fixed interval without an evaluation phase"); + ensure!( + entry.time_selection.as_of.is_none(), + "metric registration does not support as_of" + ); + ensure!( + matches!( + entry.time_selection.scope, + QueryTimeScope::RealTime | QueryTimeScope::Unknown + ), + "metric registration requires real-time selection" + ); + ensure!( + entry + .time_selection + .lookback + .is_none_or(|d| range.is_some_and(|r| u128::from(d.0) == r.as_millis())), + "lookback conflicts with the query range" + ); + if let LatencyRequirement::ExplicitMaxMs(ms) = entry.requirements.response_latency { + ensure!( + Duration::try_from_secs_f64(ms / 1000.0).is_ok(), + "latency exceeds supported duration" + ); + } + Ok(registered) + } + + pub fn workload(&self) -> &QueryWorkload { + &self.workload + } + pub fn entry(&self) -> QueryWorkloadEntry { + self.workload + .entries() + .next() + .expect("validated single query") + } + pub fn parsed(&self) -> ParsedQuery { + use planner_types::pre_asap::{AggIntent, QueryExpr}; + let query = self.entry().query.0; + let expr = query_parser::parse_query_expr_canonical(&query, self.accuracy()) + .expect("validated canonical query"); + let mut parsed = query_parser::qe_to_parsed_query(&expr); + // Temporal count is the collector's per-item frequency operation. + // Classify the canonical tree, so formatting cannot change this choice. + if matches!(&expr, QueryExpr::Aggregate { measures, child, .. } + if matches!(measures.as_slice(), [AggIntent::Count { .. }]) + && matches!(child.as_ref(), QueryExpr::TimeRange { .. })) + { + parsed.aggregations = vec![AggType::Frequency]; + parsed.exact_required = false; + } + parsed + } + pub fn metric_name(&self) -> String { + self.parsed().metric_name + } + pub fn label_filters(&self) -> HashMap { + self.parsed().label_filters + } + pub fn group_by_labels(&self) -> Vec { + let parsed = self.parsed(); + let mut labels = parsed.group_by_labels; + for label in &self.deployment.retained_labels { + if !labels.contains(label) { + labels.push(label.clone()); + } + } + let mut filter_labels: Vec<_> = parsed.label_filters.into_keys().collect(); + filter_labels.sort(); + for label in filter_labels { + if !labels.contains(&label) { + labels.push(label); + } + } + labels + } + pub fn aggregations(&self) -> Vec { + self.parsed().aggregations + } + pub fn time_window(&self) -> Duration { + self.parsed().time_window + } + pub fn quantiles(&self) -> Vec { + self.parsed().quantiles + } + pub fn exact_required(&self) -> bool { + self.parsed().exact_required + } + pub fn accuracy(&self) -> AccuracyTarget { + self.entry().requirements.accuracy.target() + } + pub fn error_bound(&self) -> f64 { + match self.accuracy() { + AccuracyTarget::Exact => 0.0, + AccuracyTarget::Epsilon(e) | AccuracyTarget::EpsilonDelta { epsilon: e, .. } => e, + } + } + pub fn repeat_every(&self) -> Option { + match self.entry().recurrence { + QueryRecurrence::Repeated( + RepeatedDemand::FixedInterval(i) + | RepeatedDemand::FixedIntervalAt { interval: i, .. }, + ) => Some(Duration::from_millis(u64::from(i.0))), + _ => None, + } + } + pub fn latency_sla(&self) -> Option { + match self.entry().requirements.response_latency { + LatencyRequirement::ExplicitMaxMs(ms) => Some(Duration::from_secs_f64(ms / 1000.0)), + LatencyRequirement::Unspecified => None, + } + } + /// Cost formulas require fresh evidence. Missing facts remain unavailable. + pub fn characteristics_at(&self, now_ms: u64) -> Option { + if self.deployment.bytes_per_raw_sample == 0 { + return None; + } + let data = self.workload.data_workload.as_ref()?; + let series = *data.input_cardinality.value_at(now_ms)?; + let rate = data.ingestion_rate.value_at(now_ms)?.0; + if series == 0 && rate > 0.0 { + return None; + } + let distribution = data.distribution.value_at(now_ms)?; + Some(WorkloadCharacteristics { + series_count: series, + samples_per_sec_per_series: if series == 0 { + 0.0 + } else { + rate / series as f64 + }, + bytes_per_raw_sample: self.deployment.bytes_per_raw_sample, + distinct_keys_per_window: self.deployment.distinct_keys_per_window, + memory_budget_bytes: self.deployment.memory_budget_bytes, + data_distribution: match distribution { + DataDistribution::Zipf => crate::types::DataDistribution::Zipf, + DataDistribution::Uniform => crate::types::DataDistribution::Uniform, + DataDistribution::Bursty => crate::types::DataDistribution::Bursty, + }, + }) + } +} + +pub(crate) fn metric_query_range(query: &str) -> anyhow::Result> { + use promql_parser::{ + label::MatchOp, + parser::{Expr, LabelModifier}, + util::{walk_expr, ExprVisitor}, + }; + struct Validator { + selectors: usize, + range: Option, + } + impl ExprVisitor for Validator { + type Error = anyhow::Error; + fn pre_visit(&mut self, expr: &Expr) -> anyhow::Result { + let selector = match expr { + Expr::VectorSelector(v) => Some(v), + Expr::MatrixSelector(m) => { + self.range = Some(m.range); + Some(&m.vs) + } + Expr::Subquery(_) => { + anyhow::bail!("metric registration does not support subqueries") + } + Expr::Aggregate(a) if matches!(a.modifier, Some(LabelModifier::Exclude(_))) => { + anyhow::bail!("metric registration does not support without grouping") + } + _ => None, + }; + if let Some(v) = selector { + self.selectors += 1; + ensure!( + v.offset.is_none() && v.at.is_none(), + "metric registration does not support offset or @ modifiers" + ); + ensure!( + v.matchers.or_matchers.is_empty() + && v.matchers + .matchers + .iter() + .all(|m| matches!(m.op, MatchOp::Equal)), + "metric registration supports equality label filters only" + ); + } + Ok(true) + } + } + let expr = promql_parser::parser::parse(query).map_err(|e| anyhow!(e))?; + let mut validator = Validator { + selectors: 0, + range: None, + }; + walk_expr(&mut validator, &expr)?; + ensure!(validator.selectors == 1, "metric registration requires exactly one selector; use the full query compilation API for multi-source expressions"); + Ok(validator.range) +} + +pub(crate) fn declared(value: T) -> Evidence { + Evidence { + value: Some(value), + source: EvidenceSource::Declared, + observed_at_ms: None, + valid_for_ms: None, + } +} + +#[cfg(test)] +pub(crate) mod fixtures { + use super::*; + /// Test inputs are rendered into canonical expressions before entering a planner. + pub struct WorkloadFixture { + pub metric_name: String, + pub label_filters: HashMap, + pub group_by_labels: Vec, + pub aggregations: Vec, + pub time_window: Duration, + pub repeat_every: Option, + pub accuracy: AccuracyTarget, + pub latency_sla: Option, + pub sketch_type_override: Option, + pub exact_required: bool, + pub quantiles: Vec, + } + impl WorkloadFixture { + pub fn build(self) -> RegisteredWorkload { + let filters = self + .label_filters + .iter() + .map(|(k, v)| format!("{k}={}", serde_json::to_string(v).unwrap())) + .collect::>() + .join(","); + let selector = format!( + "{}{{{filters}}}[{}s]", + self.metric_name, + self.time_window.as_secs() + ); + let query = if self.exact_required { + format!("sum_over_time({selector})") + } else { + self.aggregations + .iter() + .map(|agg| match agg { + AggType::Quantile => format!( + "quantile_over_time({}, {selector})", + self.quantiles.first().copied().unwrap_or(0.99) + ), + AggType::Cardinality => format!("distinct_over_time({selector})"), + AggType::Frequency => format!("count_over_time({selector})"), + }) + .collect::>() + .join(" + ") + }; + let requirements = QueryRequirements { + accuracy: AccuracyRequirement::Explicit(self.accuracy), + response_latency: self + .latency_sla + .map(|d| LatencyRequirement::ExplicitMaxMs(d.as_secs_f64() * 1000.0)) + .unwrap_or(LatencyRequirement::Unspecified), + }; + let time_selection = TimeSelection { + scope: QueryTimeScope::RealTime, + lookback: Some(DurationMs(self.time_window.as_millis() as u64)), + as_of: None, + }; + let (query_batch, repeating_queries) = match self.repeat_every { + Some(d) => ( + None, + Some(vec![RepeatingEntry { + query: Query(query), + requirements, + predictability: Predictability::Unknown, + time_selection, + demand: RepeatedDemand::FixedInterval(RepetitionInterval( + d.as_millis().try_into().unwrap(), + )), + }]), + ), + None => ( + Some(vec![BatchEntry { + query: Query(query), + requirements, + predictability: Predictability::Unknown, + time_selection, + invocations: 1, + execute_at: None, + }]), + None, + ), + }; + RegisteredWorkload::new( + QueryWorkload { + language: QueryLanguage::PromQL, + query_batch, + repeating_queries, + data_workload: None, + }, + DeploymentOptions { + sketch_type_override: self.sketch_type_override, + retained_labels: self.group_by_labels, + ..Default::default() + }, + ) + .unwrap() + } + } + impl RegisteredWorkload { + pub fn set_repeat_every(&mut self, cadence: Option) { + let entry = self.entry(); + self.workload.query_batch = None; + self.workload.repeating_queries = None; + match cadence { + Some(d) => { + self.workload.repeating_queries = Some(vec![RepeatingEntry { + query: entry.query, + requirements: entry.requirements, + predictability: entry.predictability, + time_selection: entry.time_selection, + demand: RepeatedDemand::FixedInterval(RepetitionInterval( + d.as_millis().try_into().unwrap(), + )), + }]) + } + None => { + self.workload.query_batch = Some(vec![BatchEntry { + query: entry.query, + requirements: entry.requirements, + predictability: entry.predictability, + time_selection: entry.time_selection, + invocations: 1, + execute_at: None, + }]) + } + } + } + pub fn set_label_filters(&mut self, filters: HashMap) { + let rendered = filters + .iter() + .map(|(k, v)| format!("{k}={}", serde_json::to_string(v).unwrap())) + .collect::>() + .join(","); + if let Some(batch) = &mut self.workload.query_batch { + batch[0].query.0 = batch[0].query.0.replace("{}", &format!("{{{rendered}}}")); + } + if let Some(repeating) = &mut self.workload.repeating_queries { + repeating[0].query.0 = repeating[0] + .query + .0 + .replace("{}", &format!("{{{rendered}}}")); + } + } + pub fn set_accuracy(&mut self, accuracy: AccuracyTarget) { + if let Some(batch) = &mut self.workload.query_batch { + batch[0].requirements.accuracy = AccuracyRequirement::Explicit(accuracy.clone()); + } + if let Some(repeating) = &mut self.workload.repeating_queries { + repeating[0].requirements.accuracy = AccuracyRequirement::Explicit(accuracy); + } + } + pub fn set_latency_sla(&mut self, latency: Option) { + let latency = latency + .map(|d| LatencyRequirement::ExplicitMaxMs(d.as_secs_f64() * 1000.0)) + .unwrap_or(LatencyRequirement::Unspecified); + if let Some(batch) = &mut self.workload.query_batch { + batch[0].requirements.response_latency = latency; + } + if let Some(repeating) = &mut self.workload.repeating_queries { + repeating[0].requirements.response_latency = latency; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + pipeline::{Analyzer, QuerySpec}, + store::workload::WorkloadStore, + workload::AggRole, + }; + + fn spec() -> QuerySpec { + serde_json::from_value(serde_json::json!({ + "query_string": "quantile_over_time(0.9, latency[5m])", + "accuracy_sla": 0.5, + "accuracy": {"EpsilonDelta": {"epsilon": 0.02, "delta": 0.001}}, + "repeat_every": "10s", "latency_sla": "2s" + })) + .unwrap() + } + + /// Registration and retrieval preserve the canonical query, requirements and data evidence. + #[test] + fn canonical_workload_survives_registry_roundtrip() { + let mut input = spec(); + input.workload.series_count = 10; + input.workload.samples_per_sec_per_series = 5.0; + input.workload.distinct_keys_per_window = Some(7); + input.sketch_type = Some(SketchType::KLL); + let registered = Analyzer::new().analyze(input).unwrap(); + let canonical = registered.workload().clone(); + let data = canonical.data_workload.as_ref().unwrap(); + assert_eq!(data.ingestion_rate.value, Some(Rate(50.0))); + assert_eq!(data.input_cardinality.value, Some(10)); + assert_eq!( + canonical.entries().next().unwrap().time_selection.lookback, + Some(DurationMs(300_000)) + ); + let store = WorkloadStore::new(); + store.set("latency", AggRole::Quantile, registered); + let retrieved = store.get("latency", AggRole::Quantile).unwrap(); + assert_eq!(retrieved.workload(), &canonical); + assert_eq!( + retrieved.accuracy(), + AccuracyTarget::EpsilonDelta { + epsilon: 0.02, + delta: 0.001 + } + ); + assert_eq!(retrieved.repeat_every(), Some(Duration::from_secs(10))); + assert_eq!(retrieved.latency_sla(), Some(Duration::from_secs(2))); + assert_eq!( + retrieved.deployment.sketch_type_override, + Some(SketchType::KLL) + ); + let cost = retrieved.characteristics_at(0).unwrap(); + assert_eq!(cost.samples_per_sec_per_series, 5.0); + assert_eq!(cost.distinct_keys_per_window, Some(7)); + } + + /// Missing, stale or inconsistent evidence cannot become a fabricated rate estimate. + #[test] + fn unavailable_data_evidence_stays_unavailable() { + let original = Analyzer::new().analyze(spec()).unwrap(); + let mut canonical = original.workload().clone(); + let data = canonical.data_workload.as_mut().unwrap(); + data.ingestion_rate = Evidence { + value: Some(Rate(50.0)), + source: EvidenceSource::Observed, + observed_at_ms: Some(10), + valid_for_ms: Some(5), + }; + let registered = + RegisteredWorkload::new(canonical.clone(), original.deployment.clone()).unwrap(); + assert!(registered.characteristics_at(15).is_some()); + assert!(registered.characteristics_at(16).is_none()); + canonical.data_workload.as_mut().unwrap().ingestion_rate = Evidence::default(); + let unknown = + RegisteredWorkload::new(canonical.clone(), original.deployment.clone()).unwrap(); + assert!(unknown.characteristics_at(0).is_none()); + let data = canonical.data_workload.as_mut().unwrap(); + data.ingestion_rate = declared(Rate(50.0)); + data.input_cardinality = declared(0); + let inconsistent = RegisteredWorkload::new(canonical, original.deployment).unwrap(); + assert!(inconsistent.characteristics_at(0).is_none()); + } + + /// Unsupported stage projections fail explicitly, before any deployment is emitted. + #[test] + fn rejects_lossy_metric_projections() { + for query in [ + "quantile_over_time(0.9, latency{job!=\"api\"}[5m])", + "quantile_over_time(0.9, latency{job=~\"api.*\"}[5m])", + "sum(a) / sum(b)", + "sum_over_time(latency[5m] offset 1h)", + "sum without(instance)(latency)", + "avg_over_time(latency[10m:1m])", + ] { + let mut input = spec(); + input.query_string = Some(query.into()); + assert!( + Analyzer::new().analyze(input).is_err(), + "must reject {query}" + ); + } + } + + /// Cadence and data arrival are independent, and millisecond cadence conversion is checked. + #[test] + fn recurrence_is_separate_from_arrival() { + let mut input = spec(); + input.repeat_every = None; + input.data = crate::types::DataShape::Batch; + let once = Analyzer::new().analyze(input.clone()).unwrap(); + assert!(matches!( + once.entry().recurrence, + QueryRecurrence::OneTime { .. } + )); + assert_eq!( + once.workload().data_workload.as_ref().unwrap().arrival, + DataArrival::AtRest + ); + input.repeat_every = Some("10s".into()); + let repeating = Analyzer::new().analyze(input.clone()).unwrap(); + assert_eq!(repeating.repeat_every(), Some(Duration::from_secs(10))); + assert_eq!( + repeating.workload().data_workload.as_ref().unwrap().arrival, + DataArrival::AtRest + ); + for cadence in ["0s", "4294968s"] { + input.repeat_every = Some(cadence.into()); + assert!(Analyzer::new().analyze(input.clone()).is_err()); + } + } + + /// Public canonical inputs cannot silently discard known time-selection facts. + #[test] + fn rejects_conflicting_canonical_time_selection() { + let original = Analyzer::new().analyze(spec()).unwrap(); + let mut canonical = original.workload().clone(); + canonical.repeating_queries.as_mut().unwrap()[0] + .time_selection + .lookback = Some(DurationMs(600_000)); + assert!(RegisteredWorkload::new(canonical, original.deployment).is_err()); + } +} diff --git a/control_plane/src/replan.rs b/control_plane/src/replan.rs index f2f313ee1..9e89e9cfb 100644 --- a/control_plane/src/replan.rs +++ b/control_plane/src/replan.rs @@ -33,7 +33,7 @@ use crate::physical::plan_cache::CachedDeploymentPlanner; use crate::physical::stage_split; use crate::physical::workload_planner as rules; use crate::store::{PlanStore, WorkloadStore}; -use crate::types::QueryWorkload; +use crate::types::RegisteredWorkload; use crate::workload::AggRole; fn short_hash(s: &str) -> String { @@ -227,12 +227,12 @@ impl Replanner { role: AggRole, agent_id: &str, ) -> Option { - let (workload, _wc) = self.workload_store.get(metric, role)?; + let workload = self.workload_store.get(metric, role)?; self.try_emit_typed_edge_yaml_for_workload(&workload, agent_id) } /// Same as [`try_emit_typed_edge_yaml`] but takes the - /// `QueryWorkload` directly. Used by `replan_metric` which already + /// `RegisteredWorkload` directly. Used by `replan_metric` which already /// has the workload in scope. /// /// `agent_id` is threaded into the emitted opamp `X-Agent-ID` header @@ -240,7 +240,7 @@ impl Replanner { /// pass `"$AGENT_ID"` and rely on the agent container's env. fn try_emit_typed_edge_yaml_for_workload( &self, - workload: &QueryWorkload, + workload: &RegisteredWorkload, agent_id: &str, ) -> Option { let deployment_expr = rules::bind_workload_typed(workload)?; @@ -426,7 +426,7 @@ impl Replanner { return false; } let mut any = false; - for (role, _, _) in pairs { + for (role, _) in pairs { if self.replan_metric_role(metric, role).await { any = true; } @@ -437,7 +437,7 @@ impl Replanner { /// Re-plans a single `(metric, role)` pair and pushes updated configs. /// Returns `true` on success, `false` if the pair is unknown. pub async fn replan_metric_role(&self, metric: &str, role: AggRole) -> bool { - let Some((workload, wc)) = self.workload_store.get(metric, role) else { + let Some(workload) = self.workload_store.get(metric, role) else { warn!(metric, role = %role, "replan requested but workload not found in store"); return false; }; @@ -448,7 +448,8 @@ impl Replanner { // previously established baseline — the whole point of a re-plan is to // re-optimise with current EMA data. self.planner.reset(metric); - let plan = self.planner.plan(&workload, Some(&wc)); + + let plan = self.planner.plan(&workload); self.plan_store.set(metric, role, plan.clone()); // Push agent config only to agents registered for this specific @@ -582,7 +583,7 @@ impl Replanner { /// return `No result for query`. fn build_backend_stage_config( &self, - workload: &QueryWorkload, + workload: &RegisteredWorkload, role: AggRole, ) -> Option { // ── Typed sketch path (Quantile / Cardinality / TopK / Frequency) ── @@ -597,7 +598,7 @@ impl Replanner { // through `extract_edge_facts` fails, and always leaves // `grouping` empty (`QueryExpr::Aggregate.by` is positional // `ColumnId`s with no label-name resolution today). The - // `QueryWorkload` carries both unambiguously, and every + // `RegisteredWorkload` carries both unambiguously, and every // aggregation under one workload shares them. // Per-metric item_label (the high-card dimension a CMS/CountSketch // hashes): threaded into the policy params so the data-plane ingest @@ -611,12 +612,12 @@ impl Replanner { .unwrap_or_default(); for agg in &mut be.aggregations { if agg.metric_name.is_empty() { - agg.metric_name = workload.metric_name.clone(); + agg.metric_name = workload.metric_name().clone(); } if agg.window_secs == 0 { - agg.window_secs = workload.time_window.as_secs(); + agg.window_secs = workload.time_window().as_secs(); } - agg.grouping = workload.group_by_labels.clone(); + agg.grouping = workload.group_by_labels().clone(); agg.item_label = item_labels.get(&agg.metric_name).cloned(); } return Some(be); @@ -666,17 +667,17 @@ impl Replanner { AggregationInput, BackendAggregation, BackendStageConfig, }; use planner_types::post_asap::SummaryFamilyType; - let window_secs = workload.time_window.as_secs().max(1); + let window_secs = workload.time_window().as_secs().max(1); Some(BackendStageConfig { aggregations: vec![BackendAggregation { item_label: None, heap_update_mode: None, - aggregation_id: format!("exact-{}-{}", workload.metric_name, role), - metric_name: workload.metric_name.clone(), + aggregation_id: format!("exact-{}-{}", workload.metric_name(), role), + metric_name: workload.metric_name().clone(), family: SummaryFamilyType::ExactAggregate(exact_kind, exact_params), window_secs, spatial_filter: String::new(), - grouping: workload.group_by_labels.clone(), + grouping: workload.group_by_labels().clone(), // ExactAgg consumes raw values at the backend (the agent // ships counter samples; the backend's // SumAccumulator integrates them). @@ -886,21 +887,22 @@ mod tests { )) } - fn test_workload(metric: &str) -> (QueryWorkload, WorkloadCharacteristics) { - let wl = QueryWorkload { + fn test_workload(metric: &str) -> (RegisteredWorkload, WorkloadCharacteristics) { + let wl = crate::registered_workload::fixtures::WorkloadFixture { metric_name: metric.into(), label_filters: HashMap::new(), group_by_labels: vec![], aggregations: vec![AggType::Quantile], time_window: Duration::from_secs(300), repeat_every: None, - accuracy_sla: 0.01, + accuracy: crate::types::AccuracyTarget::Epsilon(0.01), latency_sla: None, sketch_type_override: None, exact_required: false, quantiles: vec![], - }; + } + .build(); (wl, WorkloadCharacteristics::default()) } @@ -944,8 +946,8 @@ mod tests { #[tokio::test] async fn replan_known_metric_updates_plan_store() { let r = make_replanner(); - let (wl, wc) = test_workload("latency"); - r.workload_store.set("latency", AggRole::Quantile, wl, wc); + let (wl, _wc) = test_workload("latency"); + r.workload_store.set("latency", AggRole::Quantile, wl); r.plan_store.set("latency", AggRole::Quantile, make_plan()); let ok = r.replan_metric("latency").await; @@ -958,8 +960,8 @@ mod tests { #[tokio::test] async fn replan_expired_replans_only_expired() { let r = make_replanner(); - let (wl, wc) = test_workload("old"); - r.workload_store.set("old", AggRole::Quantile, wl, wc); + let (wl, _wc) = test_workload("old"); + r.workload_store.set("old", AggRole::Quantile, wl); // Insert an already-expired plan. let mut expired_plan = make_plan(); @@ -967,8 +969,8 @@ mod tests { r.plan_store.set("old", AggRole::Quantile, expired_plan); // Insert a still-active plan for "active". - let (awl, awc) = test_workload("active"); - r.workload_store.set("active", AggRole::Quantile, awl, awc); + let (awl, _awc) = test_workload("active"); + r.workload_store.set("active", AggRole::Quantile, awl); r.plan_store.set("active", AggRole::Quantile, make_plan()); r.replan_expired().await; @@ -981,8 +983,8 @@ mod tests { #[tokio::test] async fn register_then_violation_replans_correct_metric() { let r = make_replanner(); - let (wl, wc) = test_workload("req_rate"); - r.workload_store.set("req_rate", AggRole::Quantile, wl, wc); + let (wl, _wc) = test_workload("req_rate"); + r.workload_store.set("req_rate", AggRole::Quantile, wl); r.plan_store.set("req_rate", AggRole::Quantile, make_plan()); r.register_agent("agent-1", "req_rate", AggRole::Quantile) @@ -1011,15 +1013,13 @@ mod tests { #[tokio::test] async fn replan_multi_role_metric_updates_both_plans() { let r = make_replanner(); - let (wl_q, wc_q) = test_workload("http_requests_total"); - let mut wl_s = wl_q.clone(); - wl_s.aggregations = vec![AggType::Quantile]; // analyzer-shaped (test fixture) - let wc_s = wc_q.clone(); + let (wl_q, _wc_q) = test_workload("http_requests_total"); + let wl_s = wl_q.clone(); r.workload_store - .set("http_requests_total", AggRole::Quantile, wl_q, wc_q); + .set("http_requests_total", AggRole::Quantile, wl_q); r.workload_store - .set("http_requests_total", AggRole::Sum, wl_s, wc_s); + .set("http_requests_total", AggRole::Sum, wl_s); r.plan_store .set("http_requests_total", AggRole::Quantile, make_plan()); r.plan_store @@ -1047,10 +1047,9 @@ mod tests { #[tokio::test] async fn replan_metric_role_only_touches_target_role() { let r = make_replanner(); - let (wl, wc) = test_workload("m"); - r.workload_store - .set("m", AggRole::Quantile, wl.clone(), wc.clone()); - r.workload_store.set("m", AggRole::Sum, wl, wc); + let (wl, _wc) = test_workload("m"); + r.workload_store.set("m", AggRole::Quantile, wl.clone()); + r.workload_store.set("m", AggRole::Sum, wl); // Make the Sum-role plan expired and Quantile plan fresh. let mut sum_plan = make_plan(); @@ -1077,10 +1076,9 @@ mod tests { #[tokio::test] async fn agent_serving_multiple_roles_triggers_per_role_replan() { let r = make_replanner(); - let (wl, wc) = test_workload("m"); - r.workload_store - .set("m", AggRole::Quantile, wl.clone(), wc.clone()); - r.workload_store.set("m", AggRole::Sum, wl, wc); + let (wl, _wc) = test_workload("m"); + r.workload_store.set("m", AggRole::Quantile, wl.clone()); + r.workload_store.set("m", AggRole::Sum, wl); r.plan_store.set("m", AggRole::Quantile, make_plan()); r.plan_store.set("m", AggRole::Sum, make_plan()); @@ -1131,8 +1129,8 @@ mod tests { let _env = EnvVarGuard::set("USE_TYPED_STAGE_SPLIT", "1"); let r = make_replanner(); - let (wl, wc) = test_workload("latency"); - r.workload_store.set("latency", AggRole::Quantile, wl, wc); + let (wl, _wc) = test_workload("latency"); + r.workload_store.set("latency", AggRole::Quantile, wl); r.plan_store.set("latency", AggRole::Quantile, make_plan()); let yaml = r @@ -1185,8 +1183,8 @@ mod tests { let _env = EnvVarGuard::unset("USE_TYPED_STAGE_SPLIT"); let r = make_replanner(); - let (wl, wc) = test_workload("latency"); - r.workload_store.set("latency", AggRole::Quantile, wl, wc); + let (wl, _wc) = test_workload("latency"); + r.workload_store.set("latency", AggRole::Quantile, wl); r.plan_store.set("latency", AggRole::Quantile, make_plan()); // Drive the legacy emitter directly — same code diff --git a/control_plane/src/store/workload.rs b/control_plane/src/store/workload.rs index 81fc0d15d..cae4278fc 100644 --- a/control_plane/src/store/workload.rs +++ b/control_plane/src/store/workload.rs @@ -1,4 +1,4 @@ -//! Persists the `QueryWorkload` + `WorkloadCharacteristics` associated with +//! Persists the canonical workload and deployment options associated with //! each planned `(metric, AggRole)` pair so that the re-planner can re-run //! `plan()` without needing the original `QuerySpec` HTTP payload. //! @@ -11,14 +11,14 @@ use std::collections::HashMap; use std::sync::RwLock; -use crate::types::{QueryWorkload, WorkloadCharacteristics}; +use crate::types::RegisteredWorkload; use crate::workload::AggRole; /// Composite key `(metric_name, role)` for the store. pub type WorkloadKey = (String, AggRole); pub struct WorkloadStore { - inner: RwLock>, + inner: RwLock>, } impl Default for WorkloadStore { @@ -41,26 +41,16 @@ impl WorkloadStore { /// only happen when metric AND role coincide — re-registering the /// same `(metric, role)` is the legitimate update path (controller /// HTTP `POST /api/v1/plan` re-issuing the same shape). - pub fn set( - &self, - metric: impl Into, - role: AggRole, - wl: QueryWorkload, - wc: WorkloadCharacteristics, - ) { + pub fn set(&self, metric: impl Into, role: AggRole, wl: RegisteredWorkload) { self.inner .write() .unwrap() - .insert((metric.into(), role), (wl, wc)); + .insert((metric.into(), role), wl); } - /// Returns a clone of `(workload, characteristics)` for the + /// Returns a clone of the registration for the /// `(metric, role)` pair if known. - pub fn get( - &self, - metric: &str, - role: AggRole, - ) -> Option<(QueryWorkload, WorkloadCharacteristics)> { + pub fn get(&self, metric: &str, role: AggRole) -> Option { self.inner .read() .unwrap() @@ -68,19 +58,16 @@ impl WorkloadStore { .cloned() } - /// Returns every `(workload, characteristics)` pair registered for + /// Returns every `(role, registration)` pair registered for /// `metric`, across all roles. Empty vec when nothing is registered /// for the metric. Order is unspecified — sort if determinism matters. - pub fn get_all_for_metric( - &self, - metric: &str, - ) -> Vec<(AggRole, QueryWorkload, WorkloadCharacteristics)> { + pub fn get_all_for_metric(&self, metric: &str) -> Vec<(AggRole, RegisteredWorkload)> { self.inner .read() .unwrap() .iter() .filter(|((m, _), _)| m == metric) - .map(|((_, role), (wl, wc))| (*role, wl.clone(), wc.clone())) + .map(|((_, role), wl)| (*role, wl.clone())) .collect() } @@ -106,34 +93,30 @@ mod tests { use std::collections::HashMap; use std::time::Duration; - fn wl(name: &str) -> QueryWorkload { - QueryWorkload { + fn wl(name: &str) -> RegisteredWorkload { + crate::registered_workload::fixtures::WorkloadFixture { metric_name: name.into(), label_filters: HashMap::new(), group_by_labels: vec![], aggregations: vec![AggType::Quantile], time_window: Duration::from_secs(300), repeat_every: None, - accuracy_sla: 0.01, + accuracy: crate::types::AccuracyTarget::Epsilon(0.01), latency_sla: None, sketch_type_override: None, exact_required: false, quantiles: vec![], } + .build() } #[test] fn set_and_get() { let s = WorkloadStore::new(); - s.set( - "latency", - AggRole::Quantile, - wl("latency"), - WorkloadCharacteristics::default(), - ); - let (got, _) = s.get("latency", AggRole::Quantile).unwrap(); - assert_eq!(got.metric_name, "latency"); + s.set("latency", AggRole::Quantile, wl("latency")); + let got = s.get("latency", AggRole::Quantile).unwrap(); + assert_eq!(got.metric_name(), "latency"); } #[test] @@ -145,34 +128,19 @@ mod tests { #[test] fn unknown_role_for_known_metric_returns_none() { let s = WorkloadStore::new(); - s.set( - "m", - AggRole::Quantile, - wl("m"), - WorkloadCharacteristics::default(), - ); + s.set("m", AggRole::Quantile, wl("m")); assert!(s.get("m", AggRole::Sum).is_none()); } #[test] fn overwrite_same_role_replaces() { let s = WorkloadStore::new(); - s.set( - "m", - AggRole::Quantile, - wl("m"), - WorkloadCharacteristics::default(), - ); + s.set("m", AggRole::Quantile, wl("m")); let mut updated = wl("m"); - updated.accuracy_sla = 0.05; - s.set( - "m", - AggRole::Quantile, - updated, - WorkloadCharacteristics::default(), - ); - let (got, _) = s.get("m", AggRole::Quantile).unwrap(); - assert_eq!(got.accuracy_sla, 0.05); + updated.set_accuracy(crate::types::AccuracyTarget::Epsilon(0.05)); + s.set("m", AggRole::Quantile, updated); + let got = s.get("m", AggRole::Quantile).unwrap(); + assert_eq!(got.error_bound(), 0.05); } #[test] @@ -181,51 +149,33 @@ mod tests { // and each entry persists independently of the others. let s = WorkloadStore::new(); let mut wl_q = wl("http_requests_total"); - wl_q.accuracy_sla = 0.01; + wl_q.set_accuracy(crate::types::AccuracyTarget::Epsilon(0.01)); let mut wl_s = wl("http_requests_total"); - wl_s.accuracy_sla = 0.02; + wl_s.set_accuracy(crate::types::AccuracyTarget::Epsilon(0.02)); let mut wl_c = wl("http_requests_total"); - wl_c.accuracy_sla = 0.03; - s.set( - "http_requests_total", - AggRole::Quantile, - wl_q, - WorkloadCharacteristics::default(), - ); - s.set( - "http_requests_total", - AggRole::Sum, - wl_s, - WorkloadCharacteristics::default(), - ); - s.set( - "http_requests_total", - AggRole::Count, - wl_c, - WorkloadCharacteristics::default(), - ); + wl_c.set_accuracy(crate::types::AccuracyTarget::Epsilon(0.03)); + s.set("http_requests_total", AggRole::Quantile, wl_q); + s.set("http_requests_total", AggRole::Sum, wl_s); + s.set("http_requests_total", AggRole::Count, wl_c); // All three persist (the pre-B2 store would have collapsed // them onto one key, only the last survives). assert_eq!( s.get("http_requests_total", AggRole::Quantile) .unwrap() - .0 - .accuracy_sla, + .error_bound(), 0.01 ); assert_eq!( s.get("http_requests_total", AggRole::Sum) .unwrap() - .0 - .accuracy_sla, + .error_bound(), 0.02 ); assert_eq!( s.get("http_requests_total", AggRole::Count) .unwrap() - .0 - .accuracy_sla, + .error_bound(), 0.03 ); // `get_all_for_metric` surfaces all three. @@ -236,18 +186,8 @@ mod tests { #[test] fn remove_clears_only_target_role() { let s = WorkloadStore::new(); - s.set( - "m", - AggRole::Quantile, - wl("m"), - WorkloadCharacteristics::default(), - ); - s.set( - "m", - AggRole::Sum, - wl("m"), - WorkloadCharacteristics::default(), - ); + s.set("m", AggRole::Quantile, wl("m")); + s.set("m", AggRole::Sum, wl("m")); s.remove("m", AggRole::Quantile); assert!(s.get("m", AggRole::Quantile).is_none()); assert!(s.get("m", AggRole::Sum).is_some()); @@ -311,18 +251,13 @@ mod tests { let store = WorkloadStore::new(); for entry in &entries { let role = derive_agg_role(entry); - store.set( - &entry.metric_name, - role, - wl(&entry.metric_name), - WorkloadCharacteristics::default(), - ); + store.set(&entry.metric_name, role, wl(&entry.metric_name)); } // Both distinct roles persist after the loop (vs pre-B2: only // the last `set` survives because the key was metric only). let all = store.get_all_for_metric("http_requests_total"); - let roles: std::collections::HashSet<_> = all.iter().map(|(r, _, _)| *r).collect(); + let roles: std::collections::HashSet<_> = all.iter().map(|(r, _)| *r).collect(); assert!( roles.contains(&crate::workload::AggRole::Sum), "Sum-role plan must survive after the pre-pop loop; got {roles:?}" @@ -336,18 +271,8 @@ mod tests { #[test] fn keys_returns_all_pairs() { let s = WorkloadStore::new(); - s.set( - "a", - AggRole::Quantile, - wl("a"), - WorkloadCharacteristics::default(), - ); - s.set( - "b", - AggRole::Sum, - wl("b"), - WorkloadCharacteristics::default(), - ); + s.set("a", AggRole::Quantile, wl("a")); + s.set("b", AggRole::Sum, wl("b")); let mut keys = s.keys(); keys.sort(); assert_eq!( diff --git a/control_plane/src/types.rs b/control_plane/src/types.rs index 38b92b2a4..772e9bc2a 100644 --- a/control_plane/src/types.rs +++ b/control_plane/src/types.rs @@ -1,31 +1,15 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; use std::time::Duration; // ── Workload characteristics ─────────────────────────────────────────────────── -/// Hint about the statistical distribution of keys in the data stream. -/// Affects fill-rate estimation and therefore delta compression projections. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum DataDistribution { - /// Zipf-distributed keys (s ≈ 1.1). A small number of keys dominate, - /// so only a fraction of sketch cells are touched per window. This is - /// the typical production case. - #[default] - Zipf, - /// All keys are equally probable. Every window fills the sketch more - /// uniformly; delta compression benefit is lower. - Uniform, - /// Traffic arrives in bursts with a concentrated key set. Effective - /// fill rate is lower on average but spikes can reach Uniform levels. - Bursty, -} +pub use planner_types::workload::DataDistribution; /// Observable characteristics of the incoming data stream. /// -/// Callers supply these alongside a [`QueryWorkload`] so the planner can +/// Compatibility input and transient cost projection of canonical data facts. +/// The registry stores Planner DataWorkload; cost routines use this view to /// compare raw vs. sketch-full vs. sketch-delta transmission costs and /// estimate the CPU / memory overhead at the SDK or agent collector. /// @@ -251,41 +235,7 @@ impl std::fmt::Display for ProcessorMode { // ── Core types ──────────────────────────────────────────────────────────────── -#[derive(Debug, Clone)] -pub struct QueryWorkload { - pub metric_name: String, - pub label_filters: HashMap, - pub group_by_labels: Vec, - pub aggregations: Vec, - pub time_window: Duration, - pub repeat_every: Option, - /// Authoritative query requirement; never reconstructed from the legacy view. - pub accuracy: crate::types::AccuracyTarget, - /// Deprecated confidence-style view retained for compatibility reporting only. - pub accuracy_sla: f64, - pub latency_sla: Option, - /// When set, the planner must use this sketch type instead of running - /// the cost model. Allows pinning for collectors that support a subset. - pub sketch_type_override: Option, - /// When true, sketches offer no benefit and the plan must use raw - /// pass-through (SP-2–SP-4 collapse to raw-preservation). - /// Set for stateful per-sample queries (RSI, MACD, stochastic, SUM). - pub exact_required: bool, - /// Quantile φ targets implied by the query (e.g. [0.5] for TWAP, - /// [0.0, 1.0] for price range). Empty for non-quantile workloads. - pub quantiles: Vec, -} - -impl QueryWorkload { - /// Scalar sizing input for legacy cost formulas, not a confidence guarantee. - pub fn error_bound(&self) -> f64 { - match self.accuracy { - crate::types::AccuracyTarget::Exact => 0.0, - crate::types::AccuracyTarget::Epsilon(epsilon) - | crate::types::AccuracyTarget::EpsilonDelta { epsilon, .. } => epsilon, - } - } -} +pub use crate::registered_workload::RegisteredWorkload; // ── Sketch defaults (YAML-configurable) ────────────────────────────────────── diff --git a/control_plane/src/workload.rs b/control_plane/src/workload.rs index d851d32ba..95663d339 100644 --- a/control_plane/src/workload.rs +++ b/control_plane/src/workload.rs @@ -237,7 +237,7 @@ pub struct WorkloadEntry { /// Optional explicit sketch family override. When set, the planner pins /// this family for the metric (modulo `(sketch, statistic)` validity /// by ASAPPlanner's legal candidate enumeration). Threaded - /// into `QueryWorkload::sketch_type_override` by the registry pre-pop + /// into `RegisteredWorkload::sketch_type_override` by the registry pre-pop /// path so the typed L4 binding (`bind_workload_typed`) honours it. /// /// MVP-§46 contract entries 5–8 in `deploy/configs/mvp-workload.yaml` @@ -264,12 +264,12 @@ pub struct WorkloadEntry { /// http_requests_total_latency_ms[30s])` carries no `by (...)` /// clause, so the PromQL parser surfaces an EMPTY group_by_labels. /// Without a declarative field the analyzer ends up with an empty - /// `QueryWorkload.group_by_labels` → an empty `keep_keys` list → + /// `RegisteredWorkload.group_by_labels` → an empty `keep_keys` list → /// the agent strips ALL attrs and mints a single sid per metric /// (instead of one per `(metric, zone)`), defeating the streaming- /// config contract. /// - /// Threaded into `QueryWorkload::group_by_labels` by the registry + /// Threaded into `RegisteredWorkload::group_by_labels` by the registry /// pre-pop loop in `main`, so it merges with any `by (...)` keys /// the PromQL parser surfaces. Empty / missing ⇒ same behaviour as /// pre-B3 (no allowlist injected). @@ -363,7 +363,7 @@ pub struct WorkloadEntry { /// to reach the planner with *different* flush periods — the startup path /// hardcoded `None`. Threaded into `QuerySpec::repeat_every` by the /// registry pre-pop loop in `main`, which the analyzer parses into - /// [`crate::types::QueryWorkload::repeat_every`]. + /// [`crate::types::RegisteredWorkload::repeat_every`]. /// /// `None` / missing ⇒ unchanged behaviour (the cost model falls back to /// its window-derived flush rate). @@ -383,7 +383,11 @@ pub fn query_spec_for_entry(entry: &WorkloadEntry) -> crate::pipeline::QuerySpec metric_name: entry.metric_name.clone(), label_filters: Default::default(), group_by_labels: entry.grouping_labels.clone(), - aggregations: vec!["quantile".into()], + aggregations: if entry.query_string.is_some() { + vec![] + } else { + vec!["quantile".into()] + }, time_window: if entry.query_string.is_some() { String::new() } else { @@ -636,13 +640,13 @@ mod tests { let analyzer = crate::pipeline::Analyzer::new(); let declared = analyzer.analyze(query_spec_for_entry(&entries[0])).unwrap(); assert_eq!( - declared.repeat_every, + declared.repeat_every(), Some(std::time::Duration::from_secs(30)) ); // An entry that declares no cadence keeps the historical `None`, so the // cost model falls back to its window-derived flush rate. let undeclared = analyzer.analyze(query_spec_for_entry(&entries[1])).unwrap(); - assert_eq!(undeclared.repeat_every, None); + assert_eq!(undeclared.repeat_every(), None); } /// A cadence the duration parser cannot read is a declaration error, not a diff --git a/crates/asap_types/src/aggregation_config.rs b/crates/asap_types/src/aggregation_config.rs index 1809eddba..92dfd41dc 100644 --- a/crates/asap_types/src/aggregation_config.rs +++ b/crates/asap_types/src/aggregation_config.rs @@ -187,9 +187,9 @@ pub struct PrecomputeMaterialization { #[derive(Debug, Clone)] pub struct AggregationIdInfo { /// `PolicyFingerprint::as_u64()` of the key aggregation's config. - pub aggregation_id_for_key: u64, + pub key_policy_fingerprint: u64, /// `PolicyFingerprint::as_u64()` of the value aggregation's config. - pub aggregation_id_for_value: u64, + pub value_policy_fingerprint: u64, pub aggregation_type_for_key: AggregationType, pub aggregation_type_for_value: AggregationType, } @@ -197,7 +197,7 @@ pub struct AggregationIdInfo { impl AggregationIdInfo {} /// Compatibility name for legacy streaming-config and precompute call sites. -/// New PhysicalPlan code should use [`PrecomputeMaterialization`]. +/// New CompiledPhysicalPlan code should use [`PrecomputeMaterialization`]. pub type AggregationConfig = PrecomputeMaterialization; impl PrecomputeMaterialization { diff --git a/crates/asap_types/src/executable_plan.rs b/crates/asap_types/src/executable_plan.rs index fe59854ae..04647cd44 100644 --- a/crates/asap_types/src/executable_plan.rs +++ b/crates/asap_types/src/executable_plan.rs @@ -281,7 +281,7 @@ mod tests { use super::*; // Shared installation metadata must be safe to retain in cross-thread - // ActivePhysicalPlan snapshots without importing the compiler crate. + // RuntimePhysicalPlan snapshots without importing the compiler crate. #[test] fn installed_contract_is_send_sync_and_preserves_wire_identity() { fn send_sync() {} diff --git a/crates/asap_types/src/grouping_projection.rs b/crates/asap_types/src/grouping_projection.rs index 49cd1cdd9..23c0ae721 100644 --- a/crates/asap_types/src/grouping_projection.rs +++ b/crates/asap_types/src/grouping_projection.rs @@ -143,57 +143,6 @@ impl<'de> Deserialize<'de> for GroupingProjection { } } -#[cfg(test)] -mod tests { - use super::*; - /// Legacy label lists become one non-null string column per name. - #[test] - fn legacy_and_typed_groups_share_one_projection() { - let legacy: GroupingProjection = - serde_json::from_value(serde_json::json!({"labels":["job"]})).unwrap(); - assert_eq!( - legacy.columns(), - &[Column::new("job", DataType::Utf8, false)] - ); - let typed: GroupingProjection = serde_json::from_value( - serde_json::json!([{"name":"job","dtype":"utf8","nullable":false}]), - ) - .unwrap(); - assert_eq!(typed, legacy); - assert_eq!(typed.serialize_to_json(), serde_json::json!(["job"])); - assert_eq!( - serde_json::to_value(&typed).unwrap(), - serde_json::json!(["job"]) - ); - #[derive(Serialize)] - struct LegacyConfig { - #[serde(serialize_with = "serialize_config_grouping")] - grouping_labels: GroupingProjection, - } - assert_eq!( - serde_json::to_value(LegacyConfig { - grouping_labels: typed - }) - .unwrap(), - serde_json::json!({"grouping_labels":{"labels":["job"]}}) - ); - } - /// Numeric grouping types must survive wire transport rather than become labels. - #[test] - fn typed_group_retains_type_and_rejects_duplicate_columns() { - let typed = GroupingProjection::new(vec![Column::new("tenant", DataType::Int64, true)]); - let decoded: GroupingProjection = - serde_json::from_value(typed.serialize_to_json()).unwrap(); - assert_eq!(decoded, typed); - assert!(!decoded.is_legacy_labels()); - let duplicate = GroupingProjection::new(vec![ - Column::new("tenant", DataType::Int64, true), - Column::new("tenant", DataType::Utf8, false), - ]); - assert!(duplicate.validate().is_err()); - } -} - #[cfg(test)] mod identity_tests { use super::*; @@ -453,3 +402,54 @@ mod population_key_tests { } } } + +#[cfg(test)] +mod tests { + use super::*; + /// Legacy label lists become one non-null string column per name. + #[test] + fn legacy_and_typed_groups_share_one_projection() { + let legacy: GroupingProjection = + serde_json::from_value(serde_json::json!({"labels":["job"]})).unwrap(); + assert_eq!( + legacy.columns(), + &[Column::new("job", DataType::Utf8, false)] + ); + let typed: GroupingProjection = serde_json::from_value( + serde_json::json!([{"name":"job","dtype":"utf8","nullable":false}]), + ) + .unwrap(); + assert_eq!(typed, legacy); + assert_eq!(typed.serialize_to_json(), serde_json::json!(["job"])); + assert_eq!( + serde_json::to_value(&typed).unwrap(), + serde_json::json!(["job"]) + ); + #[derive(Serialize)] + struct LegacyConfig { + #[serde(serialize_with = "serialize_config_grouping")] + grouping_labels: GroupingProjection, + } + assert_eq!( + serde_json::to_value(LegacyConfig { + grouping_labels: typed + }) + .unwrap(), + serde_json::json!({"grouping_labels":{"labels":["job"]}}) + ); + } + /// Numeric grouping types must survive wire transport rather than become labels. + #[test] + fn typed_group_retains_type_and_rejects_duplicate_columns() { + let typed = GroupingProjection::new(vec![Column::new("tenant", DataType::Int64, true)]); + let decoded: GroupingProjection = + serde_json::from_value(typed.serialize_to_json()).unwrap(); + assert_eq!(decoded, typed); + assert!(!decoded.is_legacy_labels()); + let duplicate = GroupingProjection::new(vec![ + Column::new("tenant", DataType::Int64, true), + Column::new("tenant", DataType::Utf8, false), + ]); + assert!(duplicate.validate().is_err()); + } +} diff --git a/crates/asap_types/src/policy_registry.rs b/crates/asap_types/src/policy_registry.rs index af8ca349a..4b4f9e95c 100644 --- a/crates/asap_types/src/policy_registry.rs +++ b/crates/asap_types/src/policy_registry.rs @@ -22,7 +22,7 @@ //! //! Two `AggregationConfig`s that produce the same `PolicyFingerprint` //! ARE the same policy. The registry treats this as a *deduplication* -//! invariant — if two distinct entries in the source `aggregation_configs` +//! invariant — if two distinct entries in the source `materializations_by_policy_fingerprint` //! map produce the same fingerprint, the later one wins (last-write //! semantics). In practice the source should never contain duplicates; //! if it does, that's a control-plane bug worth surfacing in telemetry diff --git a/crates/asap_types/src/producer_plan.rs b/crates/asap_types/src/producer_plan.rs index 57cfa7e3a..0d9e58457 100644 --- a/crates/asap_types/src/producer_plan.rs +++ b/crates/asap_types/src/producer_plan.rs @@ -17,7 +17,8 @@ pub struct CollectorMaterialization { pub group_by: Vec, pub window_secs: u64, pub abstract_window_framework: SummaryWindowFramework, - pub window_implementation_id: String, + #[serde(rename = "window_implementation_id", alias = "window_realization_id")] + pub window_realization_id: String, pub slide_secs: u64, #[serde( default, @@ -161,7 +162,7 @@ pub struct AdaptiveU64Bounds { /// Guardrails for telemetry-driven runtime adaptation. This is an /// authorization contract, not an instruction to mutate the active plan. -/// Every accepted change becomes a staged successor PhysicalPlan. +/// Every accepted change becomes a staged successor CompiledPhysicalPlan. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] #[serde(deny_unknown_fields)] pub struct RuntimeAdaptationPolicy { diff --git a/crates/asap_types/src/query_plan.rs b/crates/asap_types/src/query_plan.rs index a1aaa9a44..6da01d180 100644 --- a/crates/asap_types/src/query_plan.rs +++ b/crates/asap_types/src/query_plan.rs @@ -5,7 +5,10 @@ //! node IDs. Serving executes this graph without reconstructing Planner IR or //! searching for compatible materializations. -pub mod logical; +pub mod residual; + +#[deprecated(note = "Use query_plan::residual")] +pub use residual as logical; use std::collections::{BTreeMap, BTreeSet}; @@ -553,7 +556,7 @@ pub enum QueryPlanNode { output_schema: planner_types::post_asap::SummarySchema, }, Logical { - operator: logical::LogicalOperator, + operator: residual::ResidualQueryOperator, inputs: Vec, }, Scalar { @@ -587,7 +590,7 @@ pub enum QueryPlanNode { CandidateTopK { inputs: [QueryNodeId; 2], k: u64, - grouping: logical::Grouping, + grouping: residual::Grouping, completeness: CandidateCompleteness, }, /// An exact subtree evaluated outside ASAP. Its results enter the query DAG diff --git a/crates/asap_types/src/query_plan/logical.rs b/crates/asap_types/src/query_plan/residual.rs similarity index 94% rename from crates/asap_types/src/query_plan/logical.rs rename to crates/asap_types/src/query_plan/residual.rs index 5c3f6e138..8d76d645c 100644 --- a/crates/asap_types/src/query_plan/logical.rs +++ b/crates/asap_types/src/query_plan/residual.rs @@ -8,7 +8,7 @@ fn invalid(message: impl Into) -> QueryPlanError { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum LogicalOperator { +pub enum ResidualQueryOperator { /// A maximal exact scalar/vector subtree evaluated by Prometheus. ExactSubquery { query: String, @@ -114,7 +114,7 @@ pub enum TemporalOperation { Count, } -impl LogicalOperator { +impl ResidualQueryOperator { pub fn validate(&self, inputs: usize) -> Result<(), QueryPlanError> { let expected = match self { Self::Scan { .. } | Self::ExactSubquery { .. } => 0, @@ -153,3 +153,7 @@ impl LogicalOperator { Ok(()) } } + +// Compatibility imports; new callers use the domain names above. +#[deprecated(note = "Use ResidualQueryOperator")] +pub use ResidualQueryOperator as LogicalOperator; diff --git a/crates/asap_types/src/routing_index.rs b/crates/asap_types/src/routing_index.rs index 86bd497cf..4e0dd5153 100644 --- a/crates/asap_types/src/routing_index.rs +++ b/crates/asap_types/src/routing_index.rs @@ -27,7 +27,7 @@ //! "cheap, but call at swap time not per query, if it shows up in //! profiles" note `StreamingConfig::policy_registry`'s own doc comment //! already flags — is a further, larger change (it means threading a -//! cached derived value through `HotReloadStreamingConfig`'s swap path) +//! cached derived value through `StreamingConfigHandle`'s swap path) //! and is not done by this type on its own. use std::collections::{BTreeSet, HashMap}; diff --git a/crates/asap_types/src/sds.rs b/crates/asap_types/src/sds.rs index a44fff11d..f24f27798 100644 --- a/crates/asap_types/src/sds.rs +++ b/crates/asap_types/src/sds.rs @@ -1072,6 +1072,32 @@ fn data_descriptor_id( DataDescriptorId(key) } +#[cfg(test)] +mod partition_identity_tests { + use super::*; + #[test] + fn entity_and_global_population_have_distinct_identity() { + let legacy = DataDescriptor::new_typed( + DataSourceIdentity::TimeSeries { metric: "m".into() }, + ValueProjectionIdentity::SampleValue, + "", + Vec::::new(), + "v1", + ); + let entity = legacy + .clone() + .with_partitioning(Some(PopulationPartitioning::PerEntity)); + let grouped = legacy + .clone() + .with_partitioning(Some(PopulationPartitioning::Grouped)); + assert_ne!(entity.id, grouped.id); + assert_ne!(entity.id, legacy.id); + assert_eq!(legacy.clone().with_partitioning(None).id, legacy.id); + entity.validate().unwrap(); + grouped.validate().unwrap(); + } +} + #[cfg(test)] mod tests { use super::*; @@ -1622,29 +1648,3 @@ mod tests { ]))); } } - -#[cfg(test)] -mod partition_identity_tests { - use super::*; - #[test] - fn entity_and_global_population_have_distinct_identity() { - let legacy = DataDescriptor::new_typed( - DataSourceIdentity::TimeSeries { metric: "m".into() }, - ValueProjectionIdentity::SampleValue, - "", - Vec::::new(), - "v1", - ); - let entity = legacy - .clone() - .with_partitioning(Some(PopulationPartitioning::PerEntity)); - let grouped = legacy - .clone() - .with_partitioning(Some(PopulationPartitioning::Grouped)); - assert_ne!(entity.id, grouped.id); - assert_ne!(entity.id, legacy.id); - assert_eq!(legacy.clone().with_partitioning(None).id, legacy.id); - entity.validate().unwrap(); - grouped.validate().unwrap(); - } -} diff --git a/data_plane/benches/sketch_db.rs b/data_plane/benches/sketch_db.rs index 136257868..01162475e 100644 --- a/data_plane/benches/sketch_db.rs +++ b/data_plane/benches/sketch_db.rs @@ -15,7 +15,7 @@ //! `iter_batched` (criterion-0.5.1 `src/bencher.rs:264-272`) consumes its //! input by value, so the per-iteration `Drop` of the setup `SketchStore` //! runs inside the timed region. The `SketchStore` Drop scales linearly -//! with `num_sids` (each `SketchInstanceMetadata` owns a `String`, +//! with `num_sids` (each `SummarySeriesMetadata` owns a `String`, //! `BTreeSet`, `SketchConfig`, `Option` — roughly ~100 ns //! to drop apiece). At 10k sids that's ~1 ms of pure drop work per //! "append" — i.e. ~1000× the actual `append_sample` cost. See @@ -38,7 +38,7 @@ use data_plane::storage_engines::sketch_db::data::{ AccuracyBound, AggKind, AggregationType, Capability, SketchAlgorithm, SketchConfig, SketchEncoding, }; -use data_plane::storage_engines::sketch_db::index::{SketchInstanceMetadata, SketchStore}; +use data_plane::storage_engines::sketch_db::index::{SketchStore, SummarySeriesMetadata}; use data_plane::storage_engines::SketchSampleState; // ── Payload builders ──────────────────────────────────────────────────────── @@ -109,8 +109,8 @@ fn sketch_meta( sid: u64, algorithm: SketchAlgorithm, config: SketchConfig, -) -> SketchInstanceMetadata { - SketchInstanceMetadata { +) -> SummarySeriesMetadata { + SummarySeriesMetadata { sid, metric_name: "bench_metric".into(), group_by_keys: BTreeSet::new(), @@ -131,8 +131,8 @@ fn sketch_meta( } } -fn precompute_meta(sid: u64, metric: &str, agg_type: AggregationType) -> SketchInstanceMetadata { - SketchInstanceMetadata { +fn precompute_meta(sid: u64, metric: &str, agg_type: AggregationType) -> SummarySeriesMetadata { + SummarySeriesMetadata { sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), @@ -424,7 +424,7 @@ fn matching_streaming_config(metric: &str) -> data_plane::storage_engines::types } /// `reconcile_from_streaming_config` ran on EVERY ingest batch and, in -/// the pre-optimization code, deep-cloned every `SketchInstanceMetadata` +/// the pre-optimization code, deep-cloned every `SummarySeriesMetadata` /// in the catalog (`snapshot_instances()`) — the dominant ingest-path /// CPU cost in live `perf` profiling (BTreeMap/String clone + malloc /// churn). This bench measures one un-gated reconcile against a diff --git a/data_plane/examples/audit_clickhouse_fallback.rs b/data_plane/examples/audit_clickhouse_fallback.rs index cd04319f0..81781851b 100644 --- a/data_plane/examples/audit_clickhouse_fallback.rs +++ b/data_plane/examples/audit_clickhouse_fallback.rs @@ -7,14 +7,14 @@ use control_plane::{ query_plan::{ClickHousePlanningContext, QueryPlan}, }; use data_plane::{ - drivers::query::servers::http::{build_active_physical_plan, PhysicalPlanInstallRequest}, + drivers::query::servers::http::{validate_and_build_runtime_plan, PhysicalPlanInstallRequest}, query_engines::asap_clickhouse_query_engine::{ accelerator::CatalogClickHouseAccelerator, ClickHouseAccelerationOutcome, ClickHouseAccelerator, ClickHouseHttpFallback, ClickHouseHttpServer, }, storage_engines::{ sketch_db::index::SketchStore, - types::{BackendStorageRouting, HotReloadActivePhysicalPlan}, + types::{ActivePhysicalPlanHandle, BackendStorageRouting}, }, }; use planner_types::{ @@ -74,7 +74,7 @@ async fn main() { let mut precompute_plan = PrecomputePlan::build_backend_local(envelope.clone(), vec![]).unwrap(); precompute_plan.summary_catalog = Some(reference.clone()); - let mut transmission_plan = control_plane::physical::compiler::compile_transmission_plan( + let mut transmission_plan = control_plane::physical::compiler::build_transmission_plan( envelope, &precompute_plan, &BTreeMap::new(), @@ -91,7 +91,7 @@ async fn main() { }), entries: BTreeMap::new(), }; - let active = build_active_physical_plan( + let active = validate_and_build_runtime_plan( PhysicalPlanInstallRequest { summary_catalog: catalog, collector_plans: vec![], @@ -112,7 +112,7 @@ async fn main() { )); let accelerator = Arc::new(CatalogClickHouseAccelerator::with_active_physical_plan( Arc::new(SketchStore::new()), - HotReloadActivePhysicalPlan::new(active), + ActivePhysicalPlanHandle::new(active), )); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); diff --git a/data_plane/examples/sketch_db_diag.rs b/data_plane/examples/sketch_db_diag.rs index fdf27f6bb..325d72b0c 100644 --- a/data_plane/examples/sketch_db_diag.rs +++ b/data_plane/examples/sketch_db_diag.rs @@ -4,7 +4,7 @@ //! Criterion's `iter_batched` consumes the input by value and the input's //! `Drop` runs *inside* the timed region (see criterion-0.5.1 //! `src/bencher.rs:264-272`). Our `setup` builds a `SketchStore` with N -//! `SketchInstanceMetadata` entries registered; dropping that store at N=10_000 +//! `SummarySeriesMetadata` entries registered; dropping that store at N=10_000 //! pays for 10k Drops of `metric_name: String`, `BTreeSet`, `SketchConfig`, //! `AccuracyBound`, etc. That drop, not `append_sample`, is what makes the //! 10k case look ~90× slower than the 100-sid case. @@ -27,7 +27,7 @@ use data_plane::storage_engines::sketch_db::data::{ AccuracyBound, AggKind, Capability, SketchAlgorithm, SketchConfig, SketchEncoding, SketchSampleState, }; -use data_plane::storage_engines::sketch_db::index::{SketchInstanceMetadata, SketchStore}; +use data_plane::storage_engines::sketch_db::index::{SketchStore, SummarySeriesMetadata}; fn ddsketch_payload() -> Vec { let mut sk = DdSketch::new(0.01); @@ -46,11 +46,11 @@ fn ddsketch_payload() -> Vec { .encode_to_vec() } -fn dd_meta(sid: u64) -> SketchInstanceMetadata { +fn dd_meta(sid: u64) -> SummarySeriesMetadata { let cfg = SketchConfig::DDSketch { relative_accuracy: 0.01, }; - SketchInstanceMetadata { + SummarySeriesMetadata { sid, metric_name: "bench_metric".into(), group_by_keys: BTreeSet::new(), diff --git a/data_plane/src/bin/monitor_coordinator_harness.rs b/data_plane/src/bin/monitor_coordinator_harness.rs index 5ae094be2..3f993ed2a 100644 --- a/data_plane/src/bin/monitor_coordinator_harness.rs +++ b/data_plane/src/bin/monitor_coordinator_harness.rs @@ -18,7 +18,7 @@ //! carry the same key or registrations are rejected as unconfigured. Empty //! (default) = ungrouped. -use data_plane::monitor::{MonitorConfig, MonitorCoordinator, MonitorServiceImpl}; +use data_plane::update_sampling::{MonitorConfig, MonitorCoordinator, MonitorServiceImpl}; /// The edge_id the Go e2edriver connects as (asap-precompute-go/monitor/ /// grpcclient/cmd/e2edriver/main.go: `monitor.NewEngine("e2e-edge", ...)`). diff --git a/data_plane/src/drivers/ingest/otel.rs b/data_plane/src/drivers/ingest/otel.rs index c3ebfaffc..8de15bfd4 100644 --- a/data_plane/src/drivers/ingest/otel.rs +++ b/data_plane/src/drivers/ingest/otel.rs @@ -629,10 +629,10 @@ fn resolve_bucket_sid_for_agg_config( &agg_kind_canonical, |sid| { ingest_state - .sketch_index + .summary_store .validate_routed_catalog_generation(captured_generation)?; let activation = ingest_state - .sketch_index + .summary_store .authorize_series_reactivation(sid, config.policy_fingerprint().into())?; if activation .as_deref() @@ -656,21 +656,21 @@ async fn route_otlp_to_precompute( // Snapshot the latest agg_configs from the hot-reload handle so // new aggregations are visible without restart. - let physical_plan_snapshot = ingest_state.physical_plan_snapshot(); - let catalog_generation = physical_plan_snapshot + let active_physical_plan_snapshot = ingest_state.active_physical_plan_snapshot(); + let catalog_generation = active_physical_plan_snapshot .as_ref() .and_then(|plan| plan.precompute_plan.summary_catalog.clone()) .map(Arc::new); - let snap = physical_plan_snapshot + let snap = active_physical_plan_snapshot .as_ref() - .map(|plan| plan.runtime_config.clone()) + .map(|plan| plan.streaming_config.clone()) .unwrap_or_else(|| ingest_state.config_snapshot()); - let agg_configs = snap.get_all_aggregation_configs(); + let agg_configs = snap.materializations(); // Reconcile sid lifecycle using the current streaming-config snapshot. // The store enforces the write barrier for retired and expired instances. // A config-Arc identity check skips catalog scans while the config is unchanged. let _ = crate::storage_engines::sketch_db::lifecycle::reconcile_if_config_changed( - ingest_state.sketch_index.as_ref(), + ingest_state.summary_store.as_ref(), &snap, crate::storage_engines::sketch_db::DEFAULT_RETIREMENT_RETENTION, ); @@ -911,16 +911,16 @@ async fn route_modified_otlp_sketches_to_precompute( // Load the generation exactly once. Deriving both the runtime config and // transmission contract from this Arc prevents an activation between two // independent ArcSwap loads from producing a torn ingest view. - let physical_plan_snapshot = ingest_state.physical_plan_snapshot(); - let snap = physical_plan_snapshot + let active_physical_plan_snapshot = ingest_state.active_physical_plan_snapshot(); + let snap = active_physical_plan_snapshot .as_ref() - .map(|plan| plan.runtime_config.clone()) + .map(|plan| plan.streaming_config.clone()) .unwrap_or_else(|| ingest_state.config_snapshot()); - let catalog_generation = physical_plan_snapshot + let catalog_generation = active_physical_plan_snapshot .as_ref() .and_then(|plan| plan.precompute_plan.summary_catalog.clone()) .map(Arc::new); - let active_physical_plan = physical_plan_snapshot.filter(|plan| plan.plan_id() != 0); + let active_physical_plan = active_physical_plan_snapshot.filter(|plan| plan.plan_id() != 0); let lineage_batch_guard = active_physical_plan .as_ref() .map(|_| ingest_state.observability.frame_lineage.lock_batch()); @@ -931,10 +931,10 @@ async fn route_modified_otlp_sketches_to_precompute( // the batch was accepted, while a contract error applies none of it. preflight_summary_frames(request, ingest_state, active)?; } - let agg_configs = snap.get_all_aggregation_configs(); + let agg_configs = snap.materializations(); // Reconcile sid lifecycle only when the configuration snapshot changes. let _ = crate::storage_engines::sketch_db::lifecycle::reconcile_if_config_changed( - ingest_state.sketch_index.as_ref(), + ingest_state.summary_store.as_ref(), &snap, crate::storage_engines::sketch_db::DEFAULT_RETIREMENT_RETENTION, ); @@ -1102,7 +1102,7 @@ async fn route_modified_otlp_sketches_to_precompute( // Canonicalize the metric name once per metric: strip the // agent-side sketch-family suffix (`_kll`, `_hll`, …) so // the whole ingest pipeline — sid resolution, series-key - // snapshot cache, `SketchInstanceMetadata.metric_name`, + // snapshot cache, `SummarySeriesMetadata.metric_name`, // `derive_sketch_policy_fp`, and the legacy precompute // router match below — keys on the RAW metric name that // the controller's streaming-config and the query @@ -1221,7 +1221,7 @@ async fn route_modified_otlp_sketches_to_precompute( // because `fp=""` is a perfectly good mint/lookup key. let resolved_sid: Option = if dp.series_id != 0 && attrs_pairs.is_empty() { let sid = dp.series_id; - if ingest_state.sketch_index.instance(sid).is_some() { + if ingest_state.summary_store.instance(sid).is_some() { Some(sid) } else { unknown_sids.push(sid); @@ -1271,12 +1271,12 @@ async fn route_modified_otlp_sketches_to_precompute( &agg_kind_canonical, |sid| { ingest_state - .sketch_index + .summary_store .validate_routed_catalog_generation( catalog_generation.as_deref(), )?; let activation = ingest_state - .sketch_index + .summary_store .authorize_series_reactivation(sid, definition)?; if activation.as_deref().is_some_and(|generation| { Some(generation) != catalog_generation.as_deref() @@ -1330,7 +1330,7 @@ async fn route_modified_otlp_sketches_to_precompute( if let Some(frame) = frame_identity.as_ref() { let observed_policy = ingest_state - .sketch_index + .summary_store .instance(sid) .map(|metadata| metadata.policy_fp) .unwrap_or_else(|| { @@ -1349,7 +1349,7 @@ async fn route_modified_otlp_sketches_to_precompute( } } - // register a `SketchInstanceMetadata` on + // register a `SummarySeriesMetadata` on // first sight of `sid` and append this DP's sketch // state to the per-sid columnar storage. The instance // is keyed by sid, so subsequent DPs on the same sid @@ -1359,11 +1359,11 @@ async fn route_modified_otlp_sketches_to_precompute( // and its key set IS the group-by KEY set. { use crate::storage_engines::sketch_db::index::{ - AccuracyBound, Capability, SketchAlgorithm, SketchInstanceMetadata, + AccuracyBound, Capability, SketchAlgorithm, SummarySeriesMetadata, }; use std::collections::BTreeSet; - if ingest_state.sketch_index.instance(sid).is_none() { + if ingest_state.summary_store.instance(sid).is_none() { let algorithm = sketch_algorithm_for(&dp); let cap = match algorithm { SketchAlgorithm::DDSketch | SketchAlgorithm::Kll => { @@ -1426,7 +1426,7 @@ async fn route_modified_otlp_sketches_to_precompute( let item_label_for_sid: Option = { snap.get_aggregation_config(policy_fp.as_u64()) .or_else(|| { - snap.get_all_aggregation_configs() + snap.materializations() .values() .find(|c| c.metric == canonical_name) }) @@ -1435,7 +1435,7 @@ async fn route_modified_otlp_sketches_to_precompute( .filter(|s| !s.is_empty()) .map(|s| s.to_string()) }; - ingest_state.sketch_index.register(SketchInstanceMetadata { + ingest_state.summary_store.register(SummarySeriesMetadata { sid, metric_name: canonical_name.clone(), group_by_keys, @@ -1453,9 +1453,9 @@ async fn route_modified_otlp_sketches_to_precompute( policy_fp, }); if let Some(label) = &item_label_for_sid { - ingest_state.sketch_index.set_item_label(sid, label); + ingest_state.summary_store.set_item_label(sid, label); } - } else if let Some(existing) = ingest_state.sketch_index.instance(sid) { + } else if let Some(existing) = ingest_state.summary_store.instance(sid) { // P1-4 (a) — one-way capability UPGRADE. The sid // was first registered from a non-heap frame // (PROTO, a delta, or a heap-LESS MSGPACK), so it @@ -1495,7 +1495,7 @@ async fn route_modified_otlp_sketches_to_precompute( // in place (same sid → same policy/metric // index slots), so this is an atomic swap to // the stronger capability. - ingest_state.sketch_index.register(upgraded); + ingest_state.summary_store.register(upgraded); debug!( "OTLP sketch sid {} upgraded {:?} -> FrequencyTopk({:?}) \ on heap-bearing frame (metric={}, encoding={})", @@ -1706,7 +1706,7 @@ async fn route_modified_otlp_sketches_to_precompute( == asap_types::producer_plan::SummaryFrameKind::Full { ingest_state - .sketch_index + .summary_store .clear_summary_lineage_incomplete(sid, frame); } } @@ -1726,7 +1726,7 @@ async fn route_modified_otlp_sketches_to_precompute( } Err(error) => { ingest_state - .sketch_index + .summary_store .mark_summary_lineage_incomplete(sid, frame); warn!( plan_id = frame.plan_id, @@ -1758,7 +1758,7 @@ async fn route_modified_otlp_sketches_to_precompute( ); let encoding = encoding_to_handle(dp.encoding).unwrap_or(SketchEncoding::ProtoFull); - if !ingest_state.sketch_index.append_sample( + if !ingest_state.summary_store.append_sample( sid, label_values, window, @@ -2058,7 +2058,7 @@ fn derive_sketch_policy_fp( /// keyed on the *bare* metric `request_size_bytes`) and the query /// analyzer (`control_plane::asap_tier_analysis`, which lifts the bare /// metric name out of the PromQL selector) speak the bare name. With -/// the suffix left on, `SketchInstanceMetadata.metric_name` is the +/// the suffix left on, `SummarySeriesMetadata.metric_name` is the /// suffixed form, so `SketchIndex::instances_matching(bare, …)` and /// `find_matching_policies` / `find_policy_by_content` (all of which /// compare `metric_name` for equality) never match — every warm sketch @@ -2221,7 +2221,7 @@ struct ModifiedOtlpSketchDp { /// the SketchStore's columnar storage keys on. start_time_unix_nano: u64, /// sketch-instance configuration lifted off the parent - /// container. Drives `SketchInstanceMetadata.sketch_config` and the + /// container. Drives `SummarySeriesMetadata.sketch_config` and the /// derived `AccuracyBound`. container_config: crate::storage_engines::sketch_db::index::SketchConfig, } @@ -2233,7 +2233,7 @@ struct ModifiedOtlpSketchDp { fn preflight_summary_frames( request: &ExportMetricsServiceRequest, ingest_state: &IngestState, - active: &crate::storage_engines::types::ActivePhysicalPlan, + active: &crate::storage_engines::types::RuntimePhysicalPlan, ) -> Result<(), String> { use asap_otel_proto::tonic::metrics::v1::metric::Data; @@ -2241,7 +2241,7 @@ fn preflight_summary_frames( metric_name: &str, mut dp: ModifiedOtlpSketchDp, ingest_state: &IngestState, - active: &crate::storage_engines::types::ActivePhysicalPlan, + active: &crate::storage_engines::types::RuntimePhysicalPlan, ) -> Result { let canonical_name = canonical_sketch_metric_name(metric_name, dp.algorithm.clone()); let frame = @@ -2336,7 +2336,7 @@ fn preflight_summary_frames( // schema. Either route must agree with the declared materialization. let observed = if dp.series_id != 0 && dp.attrs.is_empty() { ingest_state - .sketch_index + .summary_store .instance(dp.series_id) .map(|metadata| metadata.policy_fp) .ok_or_else(|| { @@ -3232,7 +3232,7 @@ mod canonical_metric_name_tests { //! strip of the agent's `_` suffix (ASAPCollector //! `asapedgeprocessor` sets `MetricSuffix: "_" + family`). Without //! it, warm-tier sketch queries against the raw metric name - //! capability-miss because `SketchInstanceMetadata.metric_name` and + //! capability-miss because `SummarySeriesMetadata.metric_name` and //! the controller's streaming-config policy `metric` never line up. use super::*; @@ -3625,7 +3625,7 @@ mod sid_resolution_tests { use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::precompute_engine::series_router::SeriesRouter; use crate::storage_engines::sketch_db::index::SketchStore; - use crate::storage_engines::types::{HotReloadStreamingConfig, StreamingConfig}; + use crate::storage_engines::types::{StreamingConfig, StreamingConfigHandle}; use asap_otel_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; use asap_otel_proto::tonic::common::v1::{any_value::Value as AnyVal, AnyValue, KeyValue}; use asap_otel_proto::tonic::metrics::v1::{ @@ -3639,7 +3639,7 @@ mod sid_resolution_tests { let (tx, mut rx) = mpsc::channel(1024); let router = SeriesRouter::new(vec![tx]); let streaming = StreamingConfig::new(std::collections::HashMap::new()); - let hot_reload = HotReloadStreamingConfig::new(streaming.clone()); + let hot_reload = StreamingConfigHandle::new(streaming.clone()); let state = Arc::new(IngestState { router, samples_ingested: std::sync::atomic::AtomicU64::new(0), @@ -3648,7 +3648,7 @@ mod sid_resolution_tests { pass_raw_samples: false, sketch_snapshots: dashmap::DashMap::new(), series_resolver: Arc::new(SeriesIdResolver::new()), - sketch_index: Arc::new(SketchStore::new()), + summary_store: Arc::new(SketchStore::new()), observability: crate::precompute_engine::ingest_handler::IngestObservability::default(), }); let drain = tokio::spawn(async move { while rx.recv().await.is_some() {} }); @@ -3724,11 +3724,11 @@ mod sid_resolution_tests { "resolver mints a non-zero sid (zero is reserved on the wire)" ); assert_eq!( - state.sketch_index.instance_count(), + state.summary_store.instance_count(), 1, "SketchStore registered one instance under the resolver-minted sid" ); - assert!(state.sketch_index.instance(assigned.series_id).is_some()); + assert!(state.summary_store.instance(assigned.series_id).is_some()); drop(state); let _ = drain.await; @@ -3760,7 +3760,7 @@ mod sid_resolution_tests { outcome.series_assignments.is_empty(), "no assignment when attrs are missing" ); - assert_eq!(state.sketch_index.instance_count(), 0); + assert_eq!(state.summary_store.instance_count(), 0); drop(state); let _ = drain.await; @@ -3789,7 +3789,7 @@ mod sid_resolution_tests { .expect("seed ingest succeeds"); let assigned_sid = seed_outcome.series_assignments[0].series_id; assert!( - state.sketch_index.instance(assigned_sid).is_some(), + state.summary_store.instance(assigned_sid).is_some(), "seed registers the resolver-minted sid" ); @@ -3829,7 +3829,7 @@ mod sid_resolution_tests { ); // The assigned sid stays registered — the second DP routed to // it via the resolver's cache hit. - assert!(state.sketch_index.instance(assigned_sid).is_some()); + assert!(state.summary_store.instance(assigned_sid).is_some()); drop(state); let _ = drain.await; @@ -4070,7 +4070,7 @@ mod sid_resolution_tests { "no fresh assignment when sender already had a valid binding" ); // Both DPs landed against the same sid — no proliferation. - assert_eq!(state.sketch_index.instance_count(), 1); + assert_eq!(state.summary_store.instance_count(), 1); drop(state); let _ = drain.await; @@ -4410,7 +4410,7 @@ mod sid_resolution_tests { .await .expect("ingest succeeds"); let sid = out1.series_assignments[0].series_id; - let meta1 = state.sketch_index.instance(sid).expect("sid registered"); + let meta1 = state.summary_store.instance(sid).expect("sid registered"); assert_eq!( meta1.capability, Some(Capability::FrequencyEstimate(Some(SketchAlgorithm::Cms))), @@ -4441,7 +4441,7 @@ mod sid_resolution_tests { .expect("ingest succeeds"); let meta2 = state - .sketch_index + .summary_store .instance(sid) .expect("sid still registered"); assert_eq!( @@ -4453,7 +4453,7 @@ mod sid_resolution_tests { ); // Still one instance — the upgrade is an in-place overwrite, not a // new sid. - assert_eq!(state.sketch_index.instance_count(), 1); + assert_eq!(state.summary_store.instance_count(), 1); // ── Frame 3: a later heap-LESS frame must NOT downgrade. ── let plain2 = asap_sketchlib::CountMinSketch::new(ROWS as usize, COLS as usize); @@ -4471,7 +4471,7 @@ mod sid_resolution_tests { .await .expect("ingest succeeds"); let meta3 = state - .sketch_index + .summary_store .instance(sid) .expect("sid still registered"); assert_eq!( @@ -4532,11 +4532,11 @@ mod sid_resolution_tests { "no unknown sids — the DP was ingestable, not dropped" ); assert_eq!( - state.sketch_index.instance_count(), + state.summary_store.instance_count(), 1, "SketchStore registered the global-aggregation instance" ); - assert!(state.sketch_index.instance(assigned.series_id).is_some()); + assert!(state.summary_store.instance(assigned.series_id).is_some()); drop(state); let _ = drain.await; @@ -4557,7 +4557,7 @@ mod sid_bucketing_tests { use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::precompute_engine::series_router::{SeriesRouter, WorkerMessage}; use crate::storage_engines::sketch_db::index::SketchStore; - use crate::storage_engines::types::{HotReloadStreamingConfig, StreamingConfig}; + use crate::storage_engines::types::{StreamingConfig, StreamingConfigHandle}; use asap_otel_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; use asap_otel_proto::tonic::common::v1::{any_value::Value as AnyVal, AnyValue, KeyValue}; use asap_otel_proto::tonic::metrics::v1::{ @@ -4677,7 +4677,7 @@ mod sid_bucketing_tests { let mut configs = HashMap::new(); configs.insert(cfg.policy_fp_u64(), cfg.clone()); let streaming = StreamingConfig::new(configs); - let hot_reload = HotReloadStreamingConfig::new(streaming); + let hot_reload = StreamingConfigHandle::new(streaming); let resolver = Arc::new(SeriesIdResolver::new()); let state = Arc::new(IngestState { @@ -4688,7 +4688,7 @@ mod sid_bucketing_tests { pass_raw_samples: false, sketch_snapshots: dashmap::DashMap::new(), series_resolver: resolver.clone(), - sketch_index: Arc::new(SketchStore::new()), + summary_store: Arc::new(SketchStore::new()), observability: crate::precompute_engine::ingest_handler::IngestObservability::default(), }); diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index bd6da65f6..7de8bd913 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -204,7 +204,7 @@ impl PrometheusRemoteWriteReceiver { let generation = self .inner .ingest - .physical_plan_snapshot() + .active_physical_plan_snapshot() .and_then(|plan| plan.precompute_plan.summary_catalog.clone()) .ok_or("finite completion requires a catalog generation")?; self.inner.ingest.router.drain().await?; @@ -213,7 +213,7 @@ impl PrometheusRemoteWriteReceiver { if self .inner .ingest - .sketch_index + .summary_store .seal_finite_summary_input(&generation)? { break; @@ -228,13 +228,13 @@ impl PrometheusRemoteWriteReceiver { let plan = self .inner .ingest - .physical_plan_snapshot() + .active_physical_plan_snapshot() .ok_or("finite maintenance requires an installed physical plan")?; if plan.precompute_plan.summary_catalog.as_ref() != Some(&generation) { return Err("finite maintenance generation changed during drain".into()); } crate::precompute_engine::maintenance_runtime::execute_finite_maintenance( - &self.inner.ingest.sketch_index, + &self.inner.ingest.summary_store, &self.inner.ingest.series_resolver, &plan.precompute_plan, )?; @@ -291,7 +291,7 @@ impl PrometheusRemoteWriteReceiver { let physical_plan = self .inner .ingest - .physical_plan_snapshot() + .active_physical_plan_snapshot() .ok_or(RemoteWriteError::InactivePhysicalPlan)?; if physical_plan.precompute_plan.envelope.plan_id == 0 || !matches!( @@ -452,7 +452,7 @@ impl PrometheusRemoteWriteReceiver { let revision = self .inner .ingest - .sketch_index + .summary_store .admit_summary_updates(&generation, coordinates)?; Ok(Some(Arc::new( crate::storage_engines::types::SummaryInputRevision { @@ -660,7 +660,7 @@ fn canonicalize_labels( fn route_messages( samples: &[CanonicalSample], ingest: &Arc, - physical_plan: &crate::storage_engines::types::ActivePhysicalPlan, + physical_plan: &crate::storage_engines::types::RuntimePhysicalPlan, ) -> Result, RemoteWriteError> { type Bucket = ( u64, @@ -668,14 +668,14 @@ fn route_messages( Arc, ); type RoutedSample = (String, i64, f64); - let snapshot = physical_plan.runtime_config.clone(); + let snapshot = physical_plan.streaming_config.clone(); let _ = crate::storage_engines::sketch_db::lifecycle::reconcile_if_config_changed( - ingest.sketch_index.as_ref(), + ingest.summary_store.as_ref(), &snapshot, crate::storage_engines::sketch_db::DEFAULT_RETIREMENT_RETENTION, ); let configs = snapshot - .get_all_aggregation_configs() + .materializations() .values() .filter(|config| config.derived_input.is_none()) .filter_map(|config| { @@ -767,11 +767,11 @@ fn route_messages( let sid = ingest .series_resolver .resolve_with_reactivation(&config.metric, attrs_fp, &materialization_kind, |sid| { - ingest.sketch_index.validate_routed_catalog_generation( + ingest.summary_store.validate_routed_catalog_generation( physical_plan.precompute_plan.summary_catalog.as_ref(), )?; let activation = ingest - .sketch_index + .summary_store .authorize_series_reactivation(sid, policy_fp.into())?; if let Some(generation) = &activation { if physical_plan.precompute_plan.summary_catalog.as_ref() @@ -928,8 +928,8 @@ mod tests { use crate::precompute_engine::ingest_handler::IngestObservability; use crate::precompute_engine::series_router::SeriesRouter; use crate::storage_engines::types::{ - ActivePhysicalPlan, BackendStorageRouting, HotReloadActivePhysicalPlan, - HotReloadStreamingConfig, StreamingConfig, + ActivePhysicalPlanHandle, BackendStorageRouting, RuntimePhysicalPlan, StreamingConfig, + StreamingConfigHandle, }; use tokio::sync::mpsc; @@ -939,7 +939,7 @@ mod tests { .unwrap() } - fn physical_config(streaming: StreamingConfig) -> HotReloadStreamingConfig { + fn physical_config(streaming: StreamingConfig) -> StreamingConfigHandle { use asap_types::producer_plan::{FrameIdentityContract, SequenceScope, TransmissionPlan}; use control_plane::physical::compiler::{ IngestContract, IngestProtocol, PlanEnvelope, PrecomputePlan, TimestampUnit, @@ -956,7 +956,7 @@ mod tests { capability_snapshot_id: "test".into(), }; let configs = streaming - .aggregation_configs + .materializations_by_policy_fingerprint .values() .cloned() .collect::>(); @@ -971,7 +971,7 @@ mod tests { plan_version: reference.plan_version, snapshot_sha256: reference.snapshot_sha256, }; - let active = ActivePhysicalPlan { + let active = RuntimePhysicalPlan { envelope: envelope.clone(), summary_catalog: Some(catalog), precompute_plan: PrecomputePlan { @@ -988,7 +988,11 @@ mod tests { schemas: Vec::new(), producers: Vec::new(), executable_dags: Default::default(), - materializations: streaming.aggregation_configs.values().cloned().collect(), + materializations: streaming + .materializations_by_policy_fingerprint + .values() + .cloned() + .collect(), }, transmission_plan: TransmissionPlan { summary_catalog: None, @@ -1001,11 +1005,11 @@ mod tests { }, rules: Vec::new(), }, - runtime_config: Arc::new(streaming), + streaming_config: Arc::new(streaming), query_plan: Arc::new(asap_types::query_plan::QueryPlan::empty()), storage_routing: Arc::new(BackendStorageRouting::empty()), }; - HotReloadStreamingConfig::from_active(HotReloadActivePhysicalPlan::new(active)) + StreamingConfigHandle::from_active_physical_plan(ActivePhysicalPlanHandle::new(active)) } fn receiver(config: PrometheusRemoteWriteConfig) -> PrometheusRemoteWriteReceiver { @@ -1018,7 +1022,7 @@ mod tests { pass_raw_samples: false, sketch_snapshots: dashmap::DashMap::new(), series_resolver: Arc::new(super::super::SeriesIdResolver::new()), - sketch_index: Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()), + summary_store: Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()), observability: IngestObservability::default(), }); PrometheusRemoteWriteReceiver::new(config, ingest) @@ -1064,14 +1068,14 @@ mod tests { pass_raw_samples: false, sketch_snapshots: dashmap::DashMap::new(), series_resolver: Arc::new(super::super::SeriesIdResolver::new()), - sketch_index: Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()), + summary_store: Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()), observability: IngestObservability::default(), }); ingest - .sketch_index + .summary_store .install_summary_catalog( ingest - .physical_plan_snapshot() + .active_physical_plan_snapshot() .unwrap() .summary_catalog .as_ref() @@ -1088,7 +1092,7 @@ mod tests { #[test] fn canonical_routing_preserves_group_labels_without_series_text_roundtrip() { let (base, _worker) = configured_receiver(); - let snapshot = base.inner.ingest.physical_plan_snapshot().unwrap(); + let snapshot = base.inner.ingest.active_physical_plan_snapshot().unwrap(); let mut config = snapshot.precompute_plan.materializations[0].clone(); config.population_key_encoding = asap_types::PopulationKeyEncoding::CanonicalLabelsV1; config.partitioning = Some(asap_types::sds::PopulationPartitioning::Grouped); @@ -1096,7 +1100,7 @@ mod tests { config.policy_fp_u64(), config.clone(), )]))); - let physical = hot.physical_plan_snapshot().unwrap(); + let physical = hot.active_physical_plan_snapshot().unwrap(); // A direct routing fixture; public installation remains intentionally gated. let (sender, _worker) = mpsc::channel(8); let ingest = Arc::new(IngestState { @@ -1107,11 +1111,11 @@ mod tests { pass_raw_samples: false, sketch_snapshots: dashmap::DashMap::new(), series_resolver: Arc::new(super::super::SeriesIdResolver::new()), - sketch_index: Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()), + summary_store: Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()), observability: IngestObservability::default(), }); ingest - .sketch_index + .summary_store .install_summary_catalog(physical.summary_catalog.as_ref().unwrap().clone()) .unwrap(); let value = "a;z=\"x,y\\q"; @@ -1212,7 +1216,7 @@ mod tests { (pooled_kll_fp.0, pooled_kll), ])); let hot_reload = physical_config(streaming); - let physical_plan = hot_reload.physical_plan_snapshot().unwrap(); + let physical_plan = hot_reload.active_physical_plan_snapshot().unwrap(); let (sender, _worker) = mpsc::channel(8); let ingest = Arc::new(IngestState { router: SeriesRouter::new(vec![sender]), @@ -1222,11 +1226,11 @@ mod tests { pass_raw_samples: false, sketch_snapshots: dashmap::DashMap::new(), series_resolver: Arc::new(super::super::SeriesIdResolver::new()), - sketch_index: Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()), + summary_store: Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()), observability: IngestObservability::default(), }); ingest - .sketch_index + .summary_store .install_summary_catalog(physical_plan.summary_catalog.as_ref().unwrap().clone()) .unwrap(); let request = WriteRequest { @@ -1371,11 +1375,11 @@ mod tests { router: SeriesRouter::new(vec![sender]), samples_ingested: AtomicU64::new(0), samples_blocked_by_schema_barrier: AtomicU64::new(0), - hot_reload_config: HotReloadStreamingConfig::new(StreamingConfig::default()), + hot_reload_config: StreamingConfigHandle::new(StreamingConfig::default()), pass_raw_samples: false, sketch_snapshots: dashmap::DashMap::new(), series_resolver: Arc::new(super::super::SeriesIdResolver::new()), - sketch_index: Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()), + summary_store: Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()), observability: IngestObservability::default(), }); let receiver = PrometheusRemoteWriteReceiver::new(Default::default(), ingest); @@ -1573,7 +1577,7 @@ mod tests { let second = queued.recv().await.unwrap(); let ingest = &receiver.inner.ingest; let sink = Arc::new(SketchStoreSink::new( - ingest.sketch_index.clone(), + ingest.summary_store.clone(), ingest.hot_reload_config.clone(), ingest.series_resolver.clone(), )); @@ -1607,7 +1611,7 @@ mod tests { let policy = *ingest .hot_reload_config .snapshot() - .aggregation_configs + .materializations_by_policy_fingerprint .keys() .next() .unwrap(); @@ -1621,7 +1625,7 @@ mod tests { readout_lookback_ms: None, }; let context = QueryExecutionContext { - index: &ingest.sketch_index, + index: &ingest.summary_store, t0_ms: 0, t1_ms: 60_000, is_cumulative: true, diff --git a/data_plane/src/drivers/ingest/series_resolver.rs b/data_plane/src/drivers/ingest/series_resolver.rs index 938815ae2..61942fb27 100644 --- a/data_plane/src/drivers/ingest/series_resolver.rs +++ b/data_plane/src/drivers/ingest/series_resolver.rs @@ -849,104 +849,6 @@ pub fn canonical_attrs_fingerprint(attrs: &[(&str, &str)]) -> String { buf } -#[cfg(test)] -mod tests { - use super::*; - - /// Stand-in canonical `AggKind` string. Tests don't care about the - /// specific encoding — the resolver only uses the value for key - /// equality. Production callers compute this via - /// `AggKind::canonical_string()`. - const TEST_AGG: &str = "sketch:DDSketch:D:0.01"; - - #[test] - fn versioned_population_routing_separates_delimiter_collisions() { - use asap_types::PopulationKeyEncoding::{CanonicalLabelsV1, LegacyDelimited}; - let a = [("a", "b;c=d")]; - let b = [("a", "b"), ("c", "d")]; - assert_eq!( - population_attrs_fingerprint(LegacyDelimited, &a).unwrap(), - "a=b;c=d;" - ); - assert_eq!( - population_attrs_fingerprint(LegacyDelimited, &a), - population_attrs_fingerprint(LegacyDelimited, &b) - ); - let ka = population_attrs_fingerprint(CanonicalLabelsV1, &a).unwrap(); - let kb = population_attrs_fingerprint(CanonicalLabelsV1, &b).unwrap(); - assert_ne!(ka, kb); - assert_eq!( - kb, - population_attrs_fingerprint(CanonicalLabelsV1, &[("c", "d"), ("a", "b")]).unwrap() - ); - let resolver = SeriesIdResolver::new(); - assert_ne!( - resolver.resolve("m", &ka, TEST_AGG), - resolver.resolve("m", &kb, TEST_AGG) - ); - assert!( - population_attrs_fingerprint(CanonicalLabelsV1, &[("a", "1"), ("a", "2")]).is_err() - ); - } - - #[test] - fn idempotent_same_input_same_sid() { - let r = SeriesIdResolver::new(); - let sid1 = r.resolve("http_requests_total", "zone=z0;", TEST_AGG); - let sid2 = r.resolve("http_requests_total", "zone=z0;", TEST_AGG); - assert_eq!(sid1, sid2, "same input must produce same sid"); - } - - #[test] - fn distinct_inputs_distinct_sids() { - let r = SeriesIdResolver::new(); - let s_z0 = r.resolve("metric_a", "zone=z0;", TEST_AGG); - let s_z1 = r.resolve("metric_a", "zone=z1;", TEST_AGG); - assert_ne!(s_z0, s_z1); - } - - #[test] - fn distinct_metrics_same_attrs_distinct_sids() { - let r = SeriesIdResolver::new(); - let s_a = r.resolve("metric_a", "zone=z0;", TEST_AGG); - let s_b = r.resolve("metric_b", "zone=z0;", TEST_AGG); - assert_ne!(s_a, s_b); - } - - #[test] - fn distinct_agg_kinds_same_series_distinct_sids() { - // Two aggregations over the same (metric, attrs) tuple — e.g. - // a DDSketch and a Sum on `http_latency_ms{zone=z0}` — get - // SEPARATE sids. This is the core property of Interpretation B: - // sid identity is `(metric, attrs, agg_kind)`. - let r = SeriesIdResolver::new(); - let s_dd = r.resolve("http_latency_ms", "zone=z0;", "sketch:DDSketch:D:0.01"); - let s_sum = r.resolve("http_latency_ms", "zone=z0;", "precompute:Sum:"); - assert_ne!( - s_dd, s_sum, - "different agg_kinds over the same series must mint distinct sids", - ); - } - - #[test] - fn fingerprint_sorts_keys() { - let f1 = canonical_attrs_fingerprint(&[("zone", "z0"), ("rack", "r00")]); - let f2 = canonical_attrs_fingerprint(&[("rack", "r00"), ("zone", "z0")]); - assert_eq!(f1, f2, "fingerprint must be order-independent"); - assert_eq!(f1, "rack=r00;zone=z0;"); - } - - #[test] - fn lookup_returns_existing_without_mint() { - let r = SeriesIdResolver::new(); - let sid = r.resolve("m", "k=v;", TEST_AGG); - assert_eq!(r.lookup("m", "k=v;", TEST_AGG), Some(sid)); - assert_eq!(r.lookup("m", "k=v2;", TEST_AGG), None); - // Same (metric, attrs) but different agg_kind is a miss. - assert_eq!(r.lookup("m", "k=v;", "precompute:Sum:"), None); - } -} - #[cfg(test)] mod persistence_tests { use super::*; @@ -1290,3 +1192,101 @@ mod persistence_tests { assert_ne!(sid, sid2); } } + +#[cfg(test)] +mod tests { + use super::*; + + /// Stand-in canonical `AggKind` string. Tests don't care about the + /// specific encoding — the resolver only uses the value for key + /// equality. Production callers compute this via + /// `AggKind::canonical_string()`. + const TEST_AGG: &str = "sketch:DDSketch:D:0.01"; + + #[test] + fn versioned_population_routing_separates_delimiter_collisions() { + use asap_types::PopulationKeyEncoding::{CanonicalLabelsV1, LegacyDelimited}; + let a = [("a", "b;c=d")]; + let b = [("a", "b"), ("c", "d")]; + assert_eq!( + population_attrs_fingerprint(LegacyDelimited, &a).unwrap(), + "a=b;c=d;" + ); + assert_eq!( + population_attrs_fingerprint(LegacyDelimited, &a), + population_attrs_fingerprint(LegacyDelimited, &b) + ); + let ka = population_attrs_fingerprint(CanonicalLabelsV1, &a).unwrap(); + let kb = population_attrs_fingerprint(CanonicalLabelsV1, &b).unwrap(); + assert_ne!(ka, kb); + assert_eq!( + kb, + population_attrs_fingerprint(CanonicalLabelsV1, &[("c", "d"), ("a", "b")]).unwrap() + ); + let resolver = SeriesIdResolver::new(); + assert_ne!( + resolver.resolve("m", &ka, TEST_AGG), + resolver.resolve("m", &kb, TEST_AGG) + ); + assert!( + population_attrs_fingerprint(CanonicalLabelsV1, &[("a", "1"), ("a", "2")]).is_err() + ); + } + + #[test] + fn idempotent_same_input_same_sid() { + let r = SeriesIdResolver::new(); + let sid1 = r.resolve("http_requests_total", "zone=z0;", TEST_AGG); + let sid2 = r.resolve("http_requests_total", "zone=z0;", TEST_AGG); + assert_eq!(sid1, sid2, "same input must produce same sid"); + } + + #[test] + fn distinct_inputs_distinct_sids() { + let r = SeriesIdResolver::new(); + let s_z0 = r.resolve("metric_a", "zone=z0;", TEST_AGG); + let s_z1 = r.resolve("metric_a", "zone=z1;", TEST_AGG); + assert_ne!(s_z0, s_z1); + } + + #[test] + fn distinct_metrics_same_attrs_distinct_sids() { + let r = SeriesIdResolver::new(); + let s_a = r.resolve("metric_a", "zone=z0;", TEST_AGG); + let s_b = r.resolve("metric_b", "zone=z0;", TEST_AGG); + assert_ne!(s_a, s_b); + } + + #[test] + fn distinct_agg_kinds_same_series_distinct_sids() { + // Two aggregations over the same (metric, attrs) tuple — e.g. + // a DDSketch and a Sum on `http_latency_ms{zone=z0}` — get + // SEPARATE sids. This is the core property of Interpretation B: + // sid identity is `(metric, attrs, agg_kind)`. + let r = SeriesIdResolver::new(); + let s_dd = r.resolve("http_latency_ms", "zone=z0;", "sketch:DDSketch:D:0.01"); + let s_sum = r.resolve("http_latency_ms", "zone=z0;", "precompute:Sum:"); + assert_ne!( + s_dd, s_sum, + "different agg_kinds over the same series must mint distinct sids", + ); + } + + #[test] + fn fingerprint_sorts_keys() { + let f1 = canonical_attrs_fingerprint(&[("zone", "z0"), ("rack", "r00")]); + let f2 = canonical_attrs_fingerprint(&[("rack", "r00"), ("zone", "z0")]); + assert_eq!(f1, f2, "fingerprint must be order-independent"); + assert_eq!(f1, "rack=r00;zone=z0;"); + } + + #[test] + fn lookup_returns_existing_without_mint() { + let r = SeriesIdResolver::new(); + let sid = r.resolve("m", "k=v;", TEST_AGG); + assert_eq!(r.lookup("m", "k=v;", TEST_AGG), Some(sid)); + assert_eq!(r.lookup("m", "k=v2;", TEST_AGG), None); + // Same (metric, attrs) but different agg_kind is a miss. + assert_eq!(r.lookup("m", "k=v;", "precompute:Sum:"), None); + } +} diff --git a/data_plane/src/drivers/query/adapters/prometheus_http.rs b/data_plane/src/drivers/query/adapters/prometheus_http.rs index fb1a296bb..1ede26e6b 100644 --- a/data_plane/src/drivers/query/adapters/prometheus_http.rs +++ b/data_plane/src/drivers/query/adapters/prometheus_http.rs @@ -366,12 +366,12 @@ impl HttpProtocolAdapter for PrometheusHttpAdapter { async fn handle_runtime_info( &self, - sketch_index: Arc, + summary_store: Arc, ) -> Result, StatusCode> { debug!("Handling runtime info request in Prometheus adapter"); // Read first-seen timestamps from per-sid metadata for runtime diagnostics. - let earliest_timestamps = sketch_index.earliest_timestamps_per_series_id(); + let earliest_timestamps = summary_store.earliest_timestamps_per_series_id(); // Get runtime info from fallback if available let mut runtime_data = if let Some(fallback) = &self.config.fallback { diff --git a/data_plane/src/drivers/query/adapters/traits.rs b/data_plane/src/drivers/query/adapters/traits.rs index e4ac075de..90f2d6aa8 100644 --- a/data_plane/src/drivers/query/adapters/traits.rs +++ b/data_plane/src/drivers/query/adapters/traits.rs @@ -150,16 +150,16 @@ pub trait HttpProtocolAdapter: Send + Sync { /// optionally forward to fallback backend for additional info. async fn handle_runtime_info( &self, - sketch_index: std::sync::Arc, + summary_store: std::sync::Arc, ) -> Result, StatusCode>; async fn handle_runtime_info_with_headers( &self, - sketch_index: std::sync::Arc, + summary_store: std::sync::Arc, headers: HashMap, ) -> Result, StatusCode> { // Adapters may override this to forward request headers. let _ = headers; - self.handle_runtime_info(sketch_index).await + self.handle_runtime_info(summary_store).await } } diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index bec471888..4b7a1aaa4 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -146,10 +146,10 @@ pub struct HttpServer { /// map. See `docs/design-gorilla-s3-cold-engine.md` §8. query_router: Arc, /// Sketch storage for runtime diagnostics. - sketch_index: Arc, + summary_store: Arc, /// Hot-reloadable `StreamingConfig` source. `None` when hot-reload /// is not wired up by the caller (unit tests, legacy binaries). - hot_reload_config: Option, + hot_reload_config: Option, /// Per-metric storage-backend routing table consulted by the HTTP /// instant-query handler at request time. When `Some(..)` and the /// query parses, the handler extracts the metric name from the @@ -191,7 +191,7 @@ pub struct HttpServer { /// Serializes multi-document physical-plan publication so two control /// plane generations cannot interleave their plan projections and catalog. physical_plan_lock: Arc>, - active_physical_plan: Option, + active_physical_plan: Option, physical_plan_lifecycle: Option, remote_write: Option, } @@ -203,10 +203,10 @@ struct AppState { /// See [`HttpServer::query_router`]. query_router: Arc, /// Attach sketch storage for HTTP runtime diagnostics. - sketch_index: Arc, + summary_store: Arc, adapter: Arc, fallback: Option>, - hot_reload_config: Option, + hot_reload_config: Option, /// See [`HttpServer::backend_storage_routing`]. backend_storage_routing: Option, /// Backfill registry (sketch DB §10). See `HttpServer::backfill`. @@ -216,7 +216,7 @@ struct AppState { /// See [`HttpServer::probe_cache`]. probe_cache: Option>, physical_plan_lock: Arc>, - active_physical_plan: Option, + active_physical_plan: Option, physical_plan_lifecycle: Option, remote_write: Option, } @@ -225,7 +225,7 @@ impl HttpServer { pub fn new( config: HttpServerConfig, query_engine: Arc, - sketch_index: Arc, + summary_store: Arc, ) -> Self { // Bootstrap the capability router with `ASAPQueryEngine` // registered under its canonical query-engine id. @@ -237,7 +237,7 @@ impl HttpServer { adapter_override: None, query_engine, query_router, - sketch_index, + summary_store, hot_reload_config: None, backend_storage_routing: None, backfill: None, @@ -304,13 +304,13 @@ impl HttpServer { self } - /// Attach a `HotReloadStreamingConfig` handle so the + /// Attach a `StreamingConfigHandle` handle so the /// `GET/POST /api/v1/streaming-config` endpoints can read and /// swap the currently active config. Without this handle the /// endpoints return `503 Service Unavailable`. pub fn with_hot_reload_config( mut self, - handle: crate::storage_engines::types::HotReloadStreamingConfig, + handle: crate::storage_engines::types::StreamingConfigHandle, ) -> Self { self.hot_reload_config = Some(handle); self @@ -318,7 +318,7 @@ impl HttpServer { pub fn with_active_physical_plan( mut self, - handle: crate::storage_engines::types::HotReloadActivePhysicalPlan, + handle: crate::storage_engines::types::ActivePhysicalPlanHandle, ) -> Self { self.physical_plan_lifecycle = Some( crate::storage_engines::types::PhysicalPlanLifecycle::new(handle.clone()), @@ -430,7 +430,7 @@ impl HttpServer { config: self.config.clone(), query_engine: self.query_engine, query_router: self.query_router, - sketch_index: self.sketch_index, + summary_store: self.summary_store, adapter: adapter.clone(), fallback: self.config.adapter_config.fallback.clone(), hot_reload_config: self.hot_reload_config.clone(), @@ -558,7 +558,7 @@ impl HttpServer { config: self.config.clone(), query_engine: self.query_engine.clone(), query_router: self.query_router.clone(), - sketch_index: self.sketch_index.clone(), + summary_store: self.summary_store.clone(), adapter: adapter.clone(), fallback: self.config.adapter_config.fallback.clone(), hot_reload_config: self.hot_reload_config.clone(), @@ -854,13 +854,13 @@ async fn process_query_request( /// single-target metrics keep their original semantics — every shape /// resolves to the one configured backend. fn resolve_metric_storage(state: &AppState, query: &str, tenant: &str) -> StorageBackend { - // A non-bootstrap atomic PhysicalPlan owns routing. Every request first + // A non-bootstrap atomic CompiledPhysicalPlan owns routing. Every request first // enters the ASAP engine, where QueryPlan lookup either executes its // compiler-bound DAG or returns an explicit fallback reason. Consulting // the legacy shape/SID candidate heuristics here would bypass QueryPlan // (and can also discard the request's explicit evaluation timestamp). if state.active_physical_plan.as_ref().is_some_and(|active| { - let snapshot = active.snapshot(); + let snapshot = active.active_snapshot(); snapshot.query_plan.plan_id != 0 && !snapshot.query_plan.entries.is_empty() }) { debug!( @@ -906,7 +906,7 @@ fn resolve_metric_storage(state: &AppState, query: &str, tenant: &str) -> Storag crate::storage_engines::types::QueryOperatorShape::RatePostHoc | crate::storage_engines::types::QueryOperatorShape::Topk ) - && metric_has_exact_agg_sum_sid(&state.sketch_index, &metric_name) + && metric_has_exact_agg_sum_sid(&state.summary_store, &metric_name) { debug!( "resolve_metric_storage: overriding {:?} → SketchStore \ @@ -948,8 +948,8 @@ fn resolve_metric_storage(state: &AppState, query: &str, tenant: &str) -> Storag shape, crate::storage_engines::types::QueryOperatorShape::Topk ) - && !metric_has_frequency_topk_sid(&state.sketch_index, &metric_name) - && !metric_has_exact_agg_sum_sid(&state.sketch_index, &metric_name) + && !metric_has_frequency_topk_sid(&state.summary_store, &metric_name) + && !metric_has_exact_agg_sum_sid(&state.summary_store, &metric_name) { debug!( "resolve_metric_storage: overriding SketchStore → \ @@ -986,7 +986,7 @@ fn resolve_metric_storage(state: &AppState, query: &str, tenant: &str) -> Storag crate::storage_engines::types::QueryOperatorShape::Sum ) && query_is_sum_over_time(&expr) - && metric_has_exact_agg_sum_sid(&state.sketch_index, &metric_name) + && metric_has_exact_agg_sum_sid(&state.summary_store, &metric_name) { debug!( "resolve_metric_storage: overriding SketchStore → \ @@ -2110,7 +2110,7 @@ async fn handle_runtime_info( // Delegate to adapter for protocol-specific handling state .adapter - .handle_runtime_info_with_headers(state.sketch_index.clone(), forwarding_headers) + .handle_runtime_info_with_headers(state.summary_store.clone(), forwarding_headers) .await } @@ -2615,7 +2615,7 @@ mod tests { use crate::precompute_engine::ingest_handler::{IngestObservability, IngestState}; use crate::precompute_engine::series_router::SeriesRouter; use crate::query_engines::ASAPQueryEngine; - use crate::storage_engines::types::{HotReloadStreamingConfig, StreamingConfig}; + use crate::storage_engines::types::{StreamingConfig, StreamingConfigHandle}; use prost::Message; use reqwest::Client; use std::sync::atomic::AtomicU64; @@ -2715,12 +2715,12 @@ mod tests { asap_types::summary_catalog::SummaryCatalog::from_materializations(7, 1, &[]).unwrap(), ); let generation = catalog.reference().unwrap(); - let sketch_index = Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); - sketch_index + let summary_store = Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); + summary_store .install_summary_catalog(Arc::clone(&catalog)) .unwrap(); - let active = crate::storage_engines::types::HotReloadActivePhysicalPlan::new( - crate::storage_engines::types::ActivePhysicalPlan { + let active = crate::storage_engines::types::ActivePhysicalPlanHandle::new( + crate::storage_engines::types::RuntimePhysicalPlan { envelope: envelope.clone(), summary_catalog: Some(Arc::clone(&catalog)), precompute_plan: PrecomputePlan { @@ -2750,7 +2750,7 @@ mod tests { }, rules: Vec::new(), }, - runtime_config: streaming_config.clone(), + streaming_config: streaming_config.clone(), query_plan: Arc::new(asap_types::query_plan::QueryPlan { plan_id: 7, plan_version: 1, @@ -2762,7 +2762,7 @@ mod tests { ), }, ); - let hot_reload = HotReloadStreamingConfig::from_active(active.clone()); + let hot_reload = StreamingConfigHandle::from_active_physical_plan(active.clone()); let (sender, _worker) = mpsc::channel(8); let ingest = Arc::new(IngestState { router: SeriesRouter::new(vec![sender]), @@ -2772,7 +2772,7 @@ mod tests { pass_raw_samples: false, sketch_snapshots: dashmap::DashMap::new(), series_resolver: Arc::new(crate::drivers::ingest::SeriesIdResolver::new()), - sketch_index: Arc::clone(&sketch_index), + summary_store: Arc::clone(&summary_store), observability: IngestObservability::default(), }); let receiver = @@ -2785,7 +2785,7 @@ mod tests { adapter_config, }, Arc::new(ASAPQueryEngine::new(15_000)), - sketch_index, + summary_store, ) .with_active_physical_plan(active) .with_remote_write(receiver.clone()); @@ -2851,9 +2851,7 @@ mod tests { ); } - async fn setup_test_server_with_hot_reload( - hot_reload: Option, - ) -> u16 { + async fn setup_test_server_with_hot_reload(hot_reload: Option) -> u16 { let adapter_config = AdapterConfig::prometheus_promql( "http://127.0.0.1:9999".to_string(), // Unused for this test false, // forward_unsupported_queries @@ -2984,7 +2982,7 @@ mod tests { /// the GET snapshot emission. #[tokio::test] async fn test_streaming_config_hot_reload_round_trip() { - let hot_reload = HotReloadStreamingConfig::new(StreamingConfig::default()); + let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); let server_port = setup_test_server_with_hot_reload(Some(hot_reload.clone())).await; let client = Client::new(); @@ -3070,13 +3068,13 @@ aggregations: let after_body: serde_json::Value = after.json().await.unwrap(); assert_eq!(after_body["aggregation_count"], 2); - // The underlying HotReloadStreamingConfig handle (cloned into + // The underlying StreamingConfigHandle handle (cloned into // the server at setup) also reflects the swap — proving that // downstream consumers that re-snapshot would see the new // state. PR 5: the map is keyed on fingerprints, so just // assert the entry count. let direct_snap = hot_reload.snapshot(); - assert_eq!(direct_snap.aggregation_configs.len(), 2); + assert_eq!(direct_snap.materializations_by_policy_fingerprint.len(), 2); } #[tokio::test] @@ -3107,7 +3105,7 @@ aggregations: #[tokio::test] async fn test_streaming_config_hot_reload_rejects_bad_yaml() { - let hot_reload = HotReloadStreamingConfig::new(StreamingConfig::default()); + let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); let server_port = setup_test_server_with_hot_reload(Some(hot_reload)).await; let client = Client::new(); @@ -3132,8 +3130,8 @@ aggregations: /// legacy `SchemaRegistry` is gone, so there is no longer a /// `schemas` parameter — every reconcile decision is sid-level. async fn setup_test_server_with_hot_reload_and_sketch_index( - hot_reload: HotReloadStreamingConfig, - sketch_index: Arc, + hot_reload: StreamingConfigHandle, + summary_store: Arc, ) -> u16 { let adapter_config = AdapterConfig::prometheus_promql("http://127.0.0.1:9999".to_string(), false); @@ -3145,7 +3143,7 @@ aggregations: let streaming_config = Arc::new(StreamingConfig::default()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let server = - HttpServer::new(config, query_engine, sketch_index).with_hot_reload_config(hot_reload); + HttpServer::new(config, query_engine, summary_store).with_hot_reload_config(hot_reload); server .start_test_server() .await @@ -3163,10 +3161,10 @@ aggregations: group_by: &[&str], ) { use crate::storage_engines::sketch_db::data::AggKind; - use crate::storage_engines::sketch_db::index::SketchInstanceMetadata; + use crate::storage_engines::sketch_db::index::SummarySeriesMetadata; use std::collections::BTreeSet; let group_by_keys: BTreeSet = group_by.iter().map(|s| s.to_string()).collect(); - store.register(SketchInstanceMetadata { + store.register(SummarySeriesMetadata { sid, metric_name: metric.to_string(), group_by_keys, @@ -3196,16 +3194,16 @@ aggregations: use crate::storage_engines::sketch_db::index::SketchStore; use crate::storage_engines::sketch_db::AggStatus; - let hot_reload = HotReloadStreamingConfig::new(StreamingConfig::default()); - let sketch_index = Arc::new(SketchStore::new()); + let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); + let summary_store = Arc::new(SketchStore::new()); // Pre-register two Active sids whose signatures match the // first config below; only sid 1 will survive the second // swap. - register_precompute_sid(&sketch_index, 1, "cpu_usage", &["host"]); - register_precompute_sid(&sketch_index, 2, "mem_usage", &["host"]); + register_precompute_sid(&summary_store, 1, "cpu_usage", &["host"]); + register_precompute_sid(&summary_store, 2, "mem_usage", &["host"]); let server_port = setup_test_server_with_hot_reload_and_sketch_index( hot_reload.clone(), - sketch_index.clone(), + summary_store.clone(), ) .await; let client = Client::new(); @@ -3262,11 +3260,11 @@ aggregations: "no sid should retire when every signature still appears in the new config; got {retired_ids:?}", ); assert_eq!( - sketch_index.instance(1).unwrap().status(), + summary_store.instance(1).unwrap().status(), AggStatus::Active ); assert_eq!( - sketch_index.instance(2).unwrap().status(), + summary_store.instance(2).unwrap().status(), AggStatus::Active ); @@ -3305,11 +3303,11 @@ aggregations: .collect::>(); assert_eq!(retired, vec![2u64]); assert_eq!( - sketch_index.instance(1).unwrap().status(), + summary_store.instance(1).unwrap().status(), AggStatus::Active ); assert_eq!( - sketch_index.instance(2).unwrap().status(), + summary_store.instance(2).unwrap().status(), AggStatus::Retired ); } @@ -3327,7 +3325,7 @@ aggregations: // `PolicyFingerprint::from_config`. The `agg_ids_added` u64 // in the HTTP response is the fingerprint's `as_u64()` form, // NOT the literal `42` the YAML once spelled out. - let hot_reload = HotReloadStreamingConfig::new(StreamingConfig::default()); + let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); let server_port = setup_test_server_with_hot_reload(Some(hot_reload)).await; let client = Client::new(); @@ -3378,13 +3376,13 @@ aggregations: // handler force-retires it. use crate::storage_engines::sketch_db::index::SketchStore; - let hot_reload = HotReloadStreamingConfig::new(StreamingConfig::default()); - let sketch_index = Arc::new(SketchStore::new()); - register_precompute_sid(&sketch_index, 1, "m1", &[]); - register_precompute_sid(&sketch_index, 2, "m2", &[]); + let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); + let summary_store = Arc::new(SketchStore::new()); + register_precompute_sid(&summary_store, 1, "m1", &[]); + register_precompute_sid(&summary_store, 2, "m2", &[]); let server_port = setup_test_server_with_hot_reload_and_sketch_index( hot_reload.clone(), - sketch_index.clone(), + summary_store.clone(), ) .await; let client = Client::new(); @@ -3474,7 +3472,7 @@ aggregations: // attached (every `HttpServer` carries one). With no // registered sids the endpoint reports an empty array, not // a 503. - let hot_reload = HotReloadStreamingConfig::new(StreamingConfig::default()); + let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); let server_port = setup_test_server_with_hot_reload(Some(hot_reload)).await; let client = Client::new(); @@ -3500,13 +3498,13 @@ aggregations: use crate::storage_engines::sketch_db::index::SketchStore; use crate::storage_engines::sketch_db::AggStatus; - let hot_reload = HotReloadStreamingConfig::new(StreamingConfig::default()); - let sketch_index = Arc::new(SketchStore::new()); - register_precompute_sid(&sketch_index, 11, "cpu", &["host"]); - register_precompute_sid(&sketch_index, 22, "mem", &["host"]); + let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); + let summary_store = Arc::new(SketchStore::new()); + register_precompute_sid(&summary_store, 11, "cpu", &["host"]); + register_precompute_sid(&summary_store, 22, "mem", &["host"]); let server_port = setup_test_server_with_hot_reload_and_sketch_index( hot_reload.clone(), - sketch_index.clone(), + summary_store.clone(), ) .await; let client = Client::new(); @@ -3525,7 +3523,7 @@ aggregations: assert_eq!(body["schema"]["sid"], 11); assert_eq!(body["schema"]["status"], "retired"); assert_eq!( - sketch_index.instance(11).unwrap().status(), + summary_store.instance(11).unwrap().status(), AggStatus::Retired ); @@ -3543,7 +3541,7 @@ aggregations: assert_eq!(body["schema"]["sid"], 22); assert_eq!(body["schema"]["status"], "expired"); assert_eq!( - sketch_index.instance(22).unwrap().status(), + summary_store.instance(22).unwrap().status(), AggStatus::Expired ); @@ -3565,11 +3563,11 @@ aggregations: async fn test_get_timeline_missing_param_returns_400() { use crate::storage_engines::sketch_db::index::SketchStore; - let hot_reload = HotReloadStreamingConfig::new(StreamingConfig::default()); - let sketch_index = Arc::new(SketchStore::new()); + let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); + let summary_store = Arc::new(SketchStore::new()); let server_port = setup_test_server_with_hot_reload_and_sketch_index( hot_reload.clone(), - sketch_index.clone(), + summary_store.clone(), ) .await; let client = Client::new(); @@ -3611,7 +3609,7 @@ aggregations: // reads from the sid catalog (always attached) instead of the // optional `SchemaRegistry`. Empty catalog → empty segments, // not a 503. - let hot_reload = HotReloadStreamingConfig::new(StreamingConfig::default()); + let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); let server_port = setup_test_server_with_hot_reload(Some(hot_reload)).await; let client = Client::new(); let resp = client @@ -3704,14 +3702,14 @@ aggregations: agg_map.insert(fp, cfg); } let streaming_config = Arc::new(StreamingConfig::new(agg_map)); - let hot_reload = HotReloadStreamingConfig::from_arc(streaming_config.clone()); + let hot_reload = StreamingConfigHandle::from_arc(streaming_config.clone()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); - let sketch_index = Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); + let summary_store = Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); for marker in active_agg_ids { let fp = marker_to_fp[marker]; - register_precompute_sid(&sketch_index, fp, &format!("metric_{marker}"), &[]); + register_precompute_sid(&summary_store, fp, &format!("metric_{marker}"), &[]); } - let server = HttpServer::new(config, query_engine, sketch_index) + let server = HttpServer::new(config, query_engine, summary_store) .with_backfill_registry(registry) .with_hot_reload_config(hot_reload); let port = server @@ -3863,7 +3861,7 @@ aggregations: async fn test_backfill_endpoints_503_without_registry() { // Build a server with NO backfill registry attached — every // backfill endpoint should 503. - let hot_reload = HotReloadStreamingConfig::new(StreamingConfig::default()); + let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); let server_port = setup_test_server_with_hot_reload(Some(hot_reload)).await; let client = Client::new(); @@ -4055,7 +4053,7 @@ aggregations: /// `QueryEngine`s. The hot-reload `StreamingConfig` is pinned at /// `metric_storage_backend` so query dispatch follows the /// requested capability axis. Returns the bound port + the - /// `HotReloadStreamingConfig` handle so tests can swap the + /// `StreamingConfigHandle` handle so tests can swap the /// `storage_backend` mid-flight if they need to. async fn setup_test_server_with_router( metric_storage_backend: StorageBackend, @@ -4073,7 +4071,7 @@ aggregations: let streaming_cfg = StreamingConfig::with_storage_backend(Default::default(), metric_storage_backend); let streaming_arc = Arc::new(streaming_cfg); - let hot_reload = HotReloadStreamingConfig::from_arc(streaming_arc.clone()); + let hot_reload = StreamingConfigHandle::from_arc(streaming_arc.clone()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let mut server = HttpServer::new( config, @@ -4117,7 +4115,7 @@ aggregations: // decisions must come from the per-metric routing table. let streaming_cfg = StreamingConfig::default(); let streaming_arc = Arc::new(streaming_cfg); - let hot_reload = HotReloadStreamingConfig::from_arc(streaming_arc.clone()); + let hot_reload = StreamingConfigHandle::from_arc(streaming_arc.clone()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let mut server = HttpServer::new( config, @@ -5027,7 +5025,7 @@ aggregations: crate::query_engines::routing::HotReloadBackendStorageRouting, ) { use crate::query_engines::routing::HotReloadBackendStorageRouting; - use crate::storage_engines::types::{HotReloadStreamingConfig, StreamingConfig}; + use crate::storage_engines::types::{StreamingConfig, StreamingConfigHandle}; let adapter_config = AdapterConfig::prometheus_promql("http://127.0.0.1:9999".to_string(), false); @@ -5038,7 +5036,7 @@ aggregations: }; let streaming_cfg = StreamingConfig::default(); let streaming_arc = Arc::new(streaming_cfg); - let hot_reload = HotReloadStreamingConfig::from_arc(streaming_arc.clone()); + let hot_reload = StreamingConfigHandle::from_arc(streaming_arc.clone()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let routing_handle = HotReloadBackendStorageRouting::empty(); let server = HttpServer::new( @@ -5575,7 +5573,7 @@ aggregations: let streaming_cfg = StreamingConfig::with_storage_backend(Default::default(), metric_storage_backend); let streaming_arc = Arc::new(streaming_cfg); - let hot_reload = HotReloadStreamingConfig::from_arc(streaming_arc.clone()); + let hot_reload = StreamingConfigHandle::from_arc(streaming_arc.clone()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let mut server = HttpServer::new( config, @@ -5880,7 +5878,7 @@ async fn handle_health(State(state): State) -> axum::response::Respons let Some(active) = state .active_physical_plan .as_ref() - .map(|handle| handle.snapshot()) + .map(|handle| handle.active_snapshot()) else { return (StatusCode::SERVICE_UNAVAILABLE, "no active PhysicalPlan").into_response(); }; @@ -5932,11 +5930,11 @@ async fn handle_store_metrics(State(state): State) -> axum::response:: use axum::response::IntoResponse; // Per-sid first-seen timestamps are available without I/O. - let timestamps = state.sketch_index.earliest_timestamps_per_series_id(); + let timestamps = state.summary_store.earliest_timestamps_per_series_id(); let body = serde_json::json!({ "status": "success", "sid_count": timestamps.len(), - "approx_resident_bytes": state.sketch_index.approx_resident_bytes(), + "approx_resident_bytes": state.summary_store.approx_resident_bytes(), "earliest_timestamps_per_series_id": timestamps}); (StatusCode::OK, axum::Json(body)).into_response() } @@ -5958,8 +5956,8 @@ async fn handle_get_streaming_config(State(state): State) -> axum::res let snap = handle.snapshot(); let body = serde_json::json!({ "status": "success", - "aggregation_count": snap.aggregation_configs.len(), - "aggregation_ids": snap.aggregation_configs.keys().copied().collect::>(), + "aggregation_count": snap.materializations_by_policy_fingerprint.len(), + "aggregation_ids": snap.materializations_by_policy_fingerprint.keys().copied().collect::>(), "streaming_config": &*snap}); (StatusCode::OK, axum::Json(body)).into_response() } @@ -6008,9 +6006,17 @@ async fn handle_post_streaming_config( } }; - let new_ids: HashSet = new_config.aggregation_configs.keys().copied().collect(); + let new_ids: HashSet = new_config + .materializations_by_policy_fingerprint + .keys() + .copied() + .collect(); let old_arc = handle.swap(new_config); - let old_ids: HashSet = old_arc.aggregation_configs.keys().copied().collect(); + let old_ids: HashSet = old_arc + .materializations_by_policy_fingerprint + .keys() + .copied() + .collect(); let added: Vec = new_ids.difference(&old_ids).copied().collect(); let removed: Vec = old_ids.difference(&new_ids).copied().collect(); @@ -6019,7 +6025,7 @@ async fn handle_post_streaming_config( "streaming-config hot-reload removed agg_ids {:?} — any in-flight \ precompute worker groups for these ids will continue with their \ construction-time config until they close naturally (phase 1 \ - limitation; see HotReloadStreamingConfig module doc)", + limitation; see StreamingConfigHandle module doc)", removed ); } @@ -6034,7 +6040,7 @@ async fn handle_post_streaming_config( // `SketchStore::ingest_precompute_for_agg_config`). let snap = handle.snapshot(); let sid_summary = crate::storage_engines::sketch_db::lifecycle::reconcile_from_streaming_config( - state.sketch_index.as_ref(), + state.summary_store.as_ref(), snap.as_ref(), crate::storage_engines::sketch_db::DEFAULT_RETIREMENT_RETENTION, ); @@ -6052,10 +6058,10 @@ pub use asap_types::plan_publication::PhysicalPlanInstallRequest; /// Decode and cross-validate every backend view before it can become visible. /// Used by both startup artifact loading and the staged HTTP install path. -pub fn build_active_physical_plan( +pub fn validate_and_build_runtime_plan( request: PhysicalPlanInstallRequest, default_routing: Arc, -) -> Result { +) -> Result { use std::collections::BTreeSet; request .precompute_plan @@ -6111,10 +6117,10 @@ pub fn build_active_physical_plan( { return Err("physical subplans have different plan identity/version".into()); } - let runtime_config = + let streaming_config = crate::storage_engines::types::StreamingConfig::new(runtime_materializations); - let typed_fps: BTreeSet<_> = runtime_config - .aggregation_configs + let typed_fps: BTreeSet<_> = streaming_config + .materializations_by_policy_fingerprint .keys() .copied() .map(asap_types::PolicyFingerprint) @@ -6130,12 +6136,12 @@ pub fn build_active_physical_plan( ), None => default_routing, }; - Ok(crate::storage_engines::types::ActivePhysicalPlan { + Ok(crate::storage_engines::types::RuntimePhysicalPlan { envelope: envelope.clone(), summary_catalog: Some(Arc::new(request.summary_catalog)), precompute_plan: request.precompute_plan, transmission_plan: request.transmission_plan, - runtime_config: Arc::new(runtime_config), + streaming_config: Arc::new(streaming_config), query_plan: Arc::new(request.query_plan), storage_routing, }) @@ -6168,7 +6174,7 @@ async fn handle_post_physical_plan( ) .into_response(); }; - let current = active_handle.snapshot(); + let current = active_handle.active_snapshot(); if current.transmission_plan.envelope.plan_id != 0 { if let Err(error) = current.transmission_plan.authorize_successor( &request.transmission_plan, @@ -6186,7 +6192,7 @@ async fn handle_post_physical_plan( } } let _guard = state.physical_plan_lock.lock().await; - let active = match build_active_physical_plan(request, current.storage_routing.clone()) { + let active = match validate_and_build_runtime_plan(request, current.storage_routing.clone()) { Ok(active) => active, Err(error) => { return ( @@ -6273,7 +6279,7 @@ async fn handle_activate_physical_plan( .into_response(); }; let _guard = state.physical_plan_lock.lock().await; - let store = Arc::clone(&state.sketch_index); + let store = Arc::clone(&state.summary_store); let remote_write = state.remote_write.clone(); let old = match lifecycle.activate_with_prepare( request.plan_id, @@ -6305,7 +6311,7 @@ async fn handle_activate_physical_plan( .into_response() } }; - let activated = active_handle.snapshot(); + let activated = active_handle.active_snapshot(); let clickhouse_plan_count = activated .query_plan .entries @@ -6319,16 +6325,16 @@ async fn handle_activate_physical_plan( tokio::spawn(async move { loop { if Arc::strong_count(&old) == 1 { - lifecycle.retire_drained(draining_id, draining_version); + lifecycle.mark_drained_plan_retired(draining_id, draining_version); break; } tokio::time::sleep(Duration::from_millis(10)).await; } }); } - let snap = activated.runtime_config.clone(); + let snap = activated.streaming_config.clone(); let retired = crate::storage_engines::sketch_db::lifecycle::reconcile_from_streaming_config( - state.sketch_index.as_ref(), + state.summary_store.as_ref(), snap.as_ref(), crate::storage_engines::sketch_db::DEFAULT_RETIREMENT_RETENTION, ); @@ -6348,7 +6354,7 @@ async fn handle_summary_inventory(State(state): State) -> axum::respon let Some(active) = state .active_physical_plan .as_ref() - .map(|handle| handle.snapshot()) + .map(|handle| handle.active_snapshot()) else { return ( StatusCode::SERVICE_UNAVAILABLE, @@ -6377,7 +6383,7 @@ async fn handle_summary_inventory(State(state): State) -> axum::respon }) .collect(); let reporter = std::env::var("HOSTNAME").unwrap_or_else(|_| "asapquery-backend".into()); - match state.sketch_index.observed_summary_inventory( + match state.summary_store.observed_summary_inventory( &reporter, &reporter, &producers, @@ -6606,12 +6612,12 @@ async fn handle_get_schemas( }; let mut entries: Vec = state - .sketch_index + .summary_store .snapshot_instances() .iter() .filter(|m| allowed.contains(&m.status())) .map(|metadata| { - let descriptors = state.sketch_index.descriptors_for_series_id(metadata.sid); + let descriptors = state.summary_store.descriptors_for_series_id(metadata.sid); sid_instance_to_json(metadata, descriptors.as_ref()) }) .collect(); @@ -6635,7 +6641,7 @@ fn status_str(s: crate::storage_engines::sketch_db::AggStatus) -> &'static str { /// Encode sid metadata, including identity, lifecycle timestamps, and status. fn sid_instance_to_json( - m: &crate::storage_engines::sketch_db::index::SketchInstanceMetadata, + m: &crate::storage_engines::sketch_db::index::SummarySeriesMetadata, descriptors: Option<&( std::sync::Arc, std::sync::Arc, @@ -6666,12 +6672,12 @@ async fn handle_post_schema_retire( axum::extract::Path(sid): axum::extract::Path, ) -> axum::response::Response { use axum::response::IntoResponse; - match state.sketch_index.force_retire( + match state.summary_store.force_retire( sid, crate::storage_engines::sketch_db::DEFAULT_RETIREMENT_RETENTION, ) { Some(meta) => { - let descriptors = state.sketch_index.descriptors_for_series_id(sid); + let descriptors = state.summary_store.descriptors_for_series_id(sid); let body = serde_json::json!({ "status": "success", "schema": sid_instance_to_json(&meta, descriptors.as_ref())}); @@ -6695,9 +6701,9 @@ async fn handle_post_schema_expire( axum::extract::Path(sid): axum::extract::Path, ) -> axum::response::Response { use axum::response::IntoResponse; - match state.sketch_index.force_expire(sid) { + match state.summary_store.force_expire(sid) { Some(meta) => { - let descriptors = state.sketch_index.descriptors_for_series_id(sid); + let descriptors = state.summary_store.descriptors_for_series_id(sid); let body = serde_json::json!({ "status": "success", "schema": sid_instance_to_json(&meta, descriptors.as_ref())}); @@ -6776,7 +6782,7 @@ async fn handle_get_timeline( // content-derived signature id (xxh64 of metric + agg_kind + // group_by_keys), stable across restarts. let segments = crate::storage_engines::sketch_db::query::timeline::timeline_for_metric( - &state.sketch_index, + &state.summary_store, metric, start_ms, end_ms, @@ -6920,7 +6926,7 @@ async fn handle_post_backfill_job( // "ingest started at the unix epoch" — backfill can then // cover up to wall-clock-now. let earliest = state - .sketch_index + .summary_store .snapshot_instances() .into_iter() .filter(|m| m.metric_name == agg_cfg.metric) @@ -7134,17 +7140,17 @@ mod logical_provenance_tests { #[cfg(test)] mod catalog_install_tests { - use super::{build_active_physical_plan, PhysicalPlanInstallRequest}; + use super::{validate_and_build_runtime_plan, PhysicalPlanInstallRequest}; use std::sync::Arc; fn request() -> PhysicalPlanInstallRequest { - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" )) .unwrap(); let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); PhysicalPlanInstallRequest { summary_catalog: plan.summary_catalog, @@ -7158,8 +7164,8 @@ mod catalog_install_tests { } fn install( request: PhysicalPlanInstallRequest, - ) -> Result { - build_active_physical_plan( + ) -> Result { + validate_and_build_runtime_plan( request, Arc::new(crate::storage_engines::types::BackendStorageRouting::empty()), ) @@ -7168,8 +7174,8 @@ mod catalog_install_tests { #[test] fn invalid_clickhouse_entry_cannot_change_active_generation() { let active = install(request()).expect("baseline plan installs"); - let handle = crate::storage_engines::types::HotReloadActivePhysicalPlan::new(active); - let before = handle.snapshot(); + let handle = crate::storage_engines::types::ActivePhysicalPlanHandle::new(active); + let before = handle.active_snapshot(); let mut candidate = request(); candidate.query_plan.clickhouse_context = Some(asap_types::query_plan::ClickHousePlanningContext { @@ -7207,7 +7213,7 @@ mod catalog_install_tests { let error = install(candidate).expect_err("invalid SQL binding must fail staging"); assert!(error.contains("pane origin"), "{error}"); - let after = handle.snapshot(); + let after = handle.active_snapshot(); assert_eq!(after.plan_id(), before.plan_id()); assert_eq!(after.plan_version(), before.plan_version()); } @@ -7242,18 +7248,18 @@ mod catalog_install_tests { // A same-version snapshot replacement fails before the active generation changes. #[test] fn catalog_install_rejects_drift_without_replacing_active_snapshot() { - let active = crate::storage_engines::types::HotReloadActivePhysicalPlan::new( + let active = crate::storage_engines::types::ActivePhysicalPlanHandle::new( install(request()).unwrap(), ); - let before = active.snapshot(); + let before = active.active_snapshot(); let mut changed = request(); changed.summary_catalog.plan_version += 1; assert!(install(changed).is_err()); - assert!(Arc::ptr_eq(&before, &active.snapshot())); + assert!(Arc::ptr_eq(&before, &active.active_snapshot())); let mut changed = request(); changed.summary_catalog.summary_descriptors.clear(); assert!(install(changed).is_err()); - assert!(Arc::ptr_eq(&before, &active.snapshot())); + assert!(Arc::ptr_eq(&before, &active.active_snapshot())); } // Physical pane width cannot be replaced by the semantic lookback at install. @@ -7300,3 +7306,6 @@ mod catalog_install_tests { assert!(error.contains("pane origin"), "{error}"); } } + +#[deprecated(note = "Use validate_and_build_runtime_plan")] +pub use validate_and_build_runtime_plan as build_active_physical_plan; diff --git a/data_plane/src/lib.rs b/data_plane/src/lib.rs index 9feffede1..913e5c149 100644 --- a/data_plane/src/lib.rs +++ b/data_plane/src/lib.rs @@ -25,13 +25,14 @@ )] pub mod drivers; -pub mod monitor; +pub mod update_sampling; + +#[deprecated(note = "Use update_sampling")] +pub use update_sampling as monitor; pub mod precompute_engine; pub mod query_engines; pub mod storage_engines; -#[cfg(test)] -pub mod tests; pub mod utils; // Re-export commonly used types to avoid glob import conflicts @@ -61,3 +62,6 @@ pub use precompute_engine::PrecomputeEngine; pub use utils::read_streaming_config; pub type Result = std::result::Result>; + +#[cfg(test)] +pub mod tests; diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index da59d7f17..8b7d2ff2c 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -51,7 +51,7 @@ struct Args { /// Versioned canonical QueryWorkload + DataWorkload and backend-local /// implementation evidence. The ASAPQuery profile invokes the pinned - /// Planner and PhysicalCompiler at startup when this is supplied. + /// Planner and PhysicalPlanCompiler at startup when this is supplied. #[arg(long)] planning_snapshot: Option, @@ -524,7 +524,7 @@ async fn main() -> Result<()> { let startup_artifact = if let Some(path) = args.planning_snapshot.as_ref() { let bytes = fs::read(path)?; - let mut snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let mut snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_slice(&bytes).map_err(|error| { format!( "failed to decode planning snapshot {}: {error}", @@ -534,12 +534,14 @@ async fn main() -> Result<()> { let runtime_memory_budget = u64::try_from(args.persistence_memory_limit_mb) .unwrap_or(u64::MAX) .saturating_mul(1024 * 1024); - snapshot.implementation.max_retained_summary_bytes = snapshot - .implementation - .max_retained_summary_bytes + snapshot + .physical_inputs + .retained_summary_memory_budget_bytes = snapshot + .physical_inputs + .retained_summary_memory_budget_bytes .min(runtime_memory_budget); let plan = snapshot - .compile() + .compile_promql() .map_err(|error| format!("startup planning failed for {}: {error}", path.display()))?; if let Some(comparison) = &plan.cost_comparison { info!( @@ -570,7 +572,7 @@ async fn main() -> Result<()> { None }; let startup_physical_plan = if let Some(artifact) = startup_artifact { - let active = data_plane::drivers::query::servers::http::build_active_physical_plan( + let active = data_plane::drivers::query::servers::http::validate_and_build_runtime_plan( artifact, Arc::new(data_plane::storage_engines::types::BackendStorageRouting::empty()), ) @@ -601,7 +603,7 @@ async fn main() -> Result<()> { None }; let streaming_config = match startup_physical_plan.as_ref() { - Some(active) => active.runtime_config.clone(), + Some(active) => active.streaming_config.clone(), None => Arc::new(read_streaming_config( args.streaming_config .as_deref() @@ -610,7 +612,7 @@ async fn main() -> Result<()> { }; info!( "Loaded streaming config with {} entries", - streaming_config.get_all_aggregation_configs().len() + streaming_config.materializations().len() ); info!("Streaming config: {:?}", streaming_config); @@ -648,12 +650,12 @@ async fn main() -> Result<()> { } else { Arc::new(data_plane::drivers::ingest::series_resolver::SeriesIdResolver::new()) }; - let sketch_index = Arc::new(data_plane::storage_engines::sketch_db::index::SketchStore::new()); + let summary_store = Arc::new(data_plane::storage_engines::sketch_db::index::SketchStore::new()); if let Some(catalog) = startup_physical_plan .as_ref() .and_then(|plan| plan.summary_catalog.as_ref()) { - sketch_index + summary_store .install_summary_catalog(Arc::clone(catalog)) .map_err(std::io::Error::other)?; } @@ -701,7 +703,7 @@ async fn main() -> Result<()> { index_persistence_dir ); Some( - sketch_index + summary_store .start_persistence(cfg) .expect("SketchStore::start_persistence failed"), ) @@ -733,7 +735,7 @@ async fn main() -> Result<()> { schemas: Vec::new(), producers: Vec::new(), materializations: streaming_config - .aggregation_configs + .materializations_by_policy_fingerprint .values() .cloned() .collect(), @@ -752,12 +754,12 @@ async fn main() -> Result<()> { rules: Vec::new(), }; let initial_active_plan = startup_physical_plan.unwrap_or_else(|| { - data_plane::storage_engines::types::ActivePhysicalPlan { + data_plane::storage_engines::types::RuntimePhysicalPlan { envelope: initial_precompute_plan.envelope.clone(), summary_catalog: None, precompute_plan: initial_precompute_plan, transmission_plan: initial_transmission_plan, - runtime_config: streaming_config.clone(), + streaming_config: streaming_config.clone(), query_plan: Arc::new(asap_types::query_plan::QueryPlan::empty()), storage_routing: Arc::new( data_plane::storage_engines::types::BackendStorageRouting::empty(), @@ -765,14 +767,14 @@ async fn main() -> Result<()> { } }); let active_physical_plan = - data_plane::storage_engines::types::HotReloadActivePhysicalPlan::new(initial_active_plan); + data_plane::storage_engines::types::ActivePhysicalPlanHandle::new(initial_active_plan); let hot_reload_config = - data_plane::storage_engines::types::HotReloadStreamingConfig::from_active( + data_plane::storage_engines::types::StreamingConfigHandle::from_active_physical_plan( active_physical_plan.clone(), ); // Query execution reads generation-consistent runtime configuration from - // the ActivePhysicalPlan installed below. + // the RuntimePhysicalPlan installed below. let engine = { let mut engine = ASAPQueryEngine::new(args.prometheus_scrape_interval) // Phase 5 wire-in (refactor 2026-05): hand the ASAP-tier @@ -780,7 +782,7 @@ async fn main() -> Result<()> { // drives the Phase 6 archive failover via // EngineError::CapabilityMiss when the ASAP tier is empty // / ghost / unknown. - .with_sketch_index(sketch_index.clone()) + .with_sketch_index(summary_store.clone()) .with_active_physical_plan(active_physical_plan.clone()) .with_exact_subquery_endpoint(args.prometheus_server.clone()) .with_metricsql_exact_subquery_endpoint(args.victoriametrics_url.clone()); @@ -834,7 +836,7 @@ async fn main() -> Result<()> { // below is for the eviction service + diagnostic plumbing // until subsequent M2.3.6 sub-PRs delete those too. let output_sink = Arc::new(SketchStoreSink::new( - sketch_index.clone(), + summary_store.clone(), hot_reload_config.clone(), series_resolver.clone(), )); @@ -843,12 +845,12 @@ async fn main() -> Result<()> { hot_reload_config.clone(), output_sink, series_resolver.clone(), - sketch_index.clone(), + summary_store.clone(), ); if let Some(endpoint) = args.erp_runtime_samples_endpoint.clone() { let generation = engine .ingest_state() - .physical_plan_snapshot() + .active_physical_plan_snapshot() .and_then(|plan| plan.precompute_plan.summary_catalog.clone()) .ok_or_else(|| { std::io::Error::other("ERP observation requires an installed catalog") @@ -864,7 +866,7 @@ async fn main() -> Result<()> { info!("Starting precompute engine (ingest adapters share its bounded worker queues)"); // Log memory diagnostics for the shared sketch store. - let diag_index = sketch_index.clone(); + let diag_index = summary_store.clone(); tokio::spawn(async move { spawn_memory_diagnostics(diag_index, Some(worker_diagnostics)).await; }); @@ -885,7 +887,7 @@ async fn main() -> Result<()> { // write-idle, fully-flushed sketch sids while keeping their queryable // metadata, bounding resident registry memory under series churn. if args.idle_sid_evict_secs > 0 { - let evict_index = sketch_index.clone(); + let evict_index = summary_store.clone(); let idle_ms = args.idle_sid_evict_secs.saturating_mul(1000); // Sweep a few times per idle horizon, clamped to a sane cadence. let sweep = std::time::Duration::from_secs(args.idle_sid_evict_secs.clamp(10, 60)); @@ -959,7 +961,7 @@ async fn main() -> Result<()> { // sample_p grant. Global-threshold alerting is retired (see // data_plane::monitor module docs) — this coordinator never fires one. let monitor_handle = if args.enable_monitor_coordinator { - use data_plane::monitor::{ + use data_plane::update_sampling::{ Functional, MonitorConfig, MonitorCoordinator, MonitorServiceImpl, }; let specs: Vec = streaming_config @@ -1067,13 +1069,13 @@ async fn main() -> Result<()> { // by the control plane through their HTTP endpoints. // HTTP endpoints inspect lifecycle metadata in the shared sketch store. - let mut server = HttpServer::new(http_config, engine, sketch_index.clone()) + let mut server = HttpServer::new(http_config, engine, summary_store.clone()) .with_active_physical_plan(active_physical_plan.clone()) .with_probe_cache(probe_cache.clone()); if args.profile == RuntimeProfile::Distributed { // Legacy partial-document endpoints remain available to distributed // deployments. The compatibility profile deliberately exposes only - // the atomic PhysicalPlan stage/activate lifecycle. + // the atomic CompiledPhysicalPlan stage/activate lifecycle. server = server.with_hot_reload_config(hot_reload_config.clone()); } @@ -1135,14 +1137,14 @@ async fn main() -> Result<()> { ); data_plane::storage_engines::types::BackendStorageRouting::empty() }; - if active_physical_plan.snapshot().plan_id() == 0 { - let current = active_physical_plan.snapshot(); - active_physical_plan.swap(data_plane::storage_engines::types::ActivePhysicalPlan { + if active_physical_plan.active_snapshot().plan_id() == 0 { + let current = active_physical_plan.active_snapshot(); + active_physical_plan.swap(data_plane::storage_engines::types::RuntimePhysicalPlan { envelope: current.envelope.clone(), summary_catalog: current.summary_catalog.clone(), precompute_plan: current.precompute_plan.clone(), transmission_plan: current.transmission_plan.clone(), - runtime_config: current.runtime_config.clone(), + streaming_config: current.streaming_config.clone(), query_plan: current.query_plan.clone(), storage_routing: Arc::new(bootstrap_routing), }); @@ -1247,7 +1249,7 @@ async fn main() -> Result<()> { data_plane::storage_engines::sketch_db::BackfillServiceConfig::default(), ) // Backfill and live ingest share the same sid resolver and sketch store. - .with_sketch_index(sketch_index.clone()) + .with_sketch_index(summary_store.clone()) .with_series_resolver(series_resolver.clone()); info!( "Spawning BackfillService drain loop (reader factory: default — Prometheus sources wired, S3/OtherSketch fail fast)" @@ -1285,7 +1287,7 @@ async fn main() -> Result<()> { data_plane::storage_engines::sketch_db::DEFAULT_RETIREMENT_RETENTION, ); let svc = data_plane::storage_engines::sketch_db::SchemaEvictionService::new( - sketch_index.clone(), + summary_store.clone(), backfill_registry.clone(), data_plane::storage_engines::sketch_db::SchemaEvictionConfig { poll_interval: std::time::Duration::from_secs(args.schema_eviction_poll_secs), @@ -1330,7 +1332,7 @@ async fn main() -> Result<()> { ); let accelerator = Arc::new( data_plane::query_engines::asap_clickhouse_query_engine::accelerator::CatalogClickHouseAccelerator::with_active_physical_plan_and_exact_backend( - sketch_index.clone(), + summary_store.clone(), active_physical_plan.clone(), fallback.clone(), ), @@ -1420,7 +1422,7 @@ fn process_resident_bytes() -> usize { /// Periodic memory diagnostics logger — runs every 30 seconds. async fn spawn_memory_diagnostics( - sketch_index: Arc, + summary_store: Arc, worker_diagnostics: Option>, ) { use data_plane::storage_engines::sketch_db::index::persistence::EpochSource; @@ -1431,8 +1433,8 @@ async fn spawn_memory_diagnostics( interval.tick().await; // Per-sid sketch-store diagnostics. - let instance_count = sketch_index.instance_count(); - let series_count = sketch_index.series_len(); + let instance_count = summary_store.instance_count(); + let series_count = summary_store.series_len(); // `approx_memory_bytes` is the flusher's EVICTABLE-payload gauge: // it counts only live sketch payloads (current_epoch + sealed), so // it correctly reads ~0 once everything has been flushed to disk. @@ -1440,8 +1442,8 @@ async fn spawn_memory_diagnostics( // per-sid registry + intern caches stay resident and are not // flushable. Report evictable payload, structural overhead, their // total store estimate, and process RSS ground truth separately. - let payload_bytes = sketch_index.approx_memory_bytes(); - let resident_bytes = sketch_index.approx_resident_bytes(); + let payload_bytes = summary_store.approx_memory_bytes(); + let resident_bytes = summary_store.approx_resident_bytes(); let structural_bytes = resident_bytes.saturating_sub(payload_bytes); let rss_bytes = process_resident_bytes(); info!( diff --git a/data_plane/src/precompute_engine/engine.rs b/data_plane/src/precompute_engine/engine.rs index a49bad0e7..fa451ca36 100644 --- a/data_plane/src/precompute_engine/engine.rs +++ b/data_plane/src/precompute_engine/engine.rs @@ -3,7 +3,7 @@ use crate::precompute_engine::ingest_handler::IngestState; use crate::precompute_engine::output_sink::OutputSink; use crate::precompute_engine::series_router::{SeriesRouter, WorkerMessage}; use crate::precompute_engine::worker::{Worker, WorkerRuntimeConfig}; -use crate::storage_engines::types::HotReloadStreamingConfig; +use crate::storage_engines::types::StreamingConfigHandle; use std::sync::atomic::{AtomicI64, AtomicUsize}; use std::sync::Arc; use tokio::sync::mpsc; @@ -20,15 +20,14 @@ pub struct PrecomputeWorkerDiagnostics { /// Creates worker threads and the series router. The ingest state /// (router + hot-reload handle) is built eagerly in `new()` so that /// ingest sources (currently OTLP) can hold a handle and push data -/// into the same worker pool. The legacy Prometheus / VictoriaMetrics -/// remote-write HTTP listener was deleted alongside the rest of the -/// remote-write ingest path — backend ingest is OTLP-only now. +/// into the same worker pool. OTLP ingestion and the backend-local Remote Write +/// profile both use the installed materialization view. pub struct PrecomputeEngine { config: PrecomputeEngineConfig, output_sink: Arc, diagnostics: Arc, ingest_state: Arc, - hot_reload_config: HotReloadStreamingConfig, + hot_reload_config: StreamingConfigHandle, /// Worker receivers, one per worker. Taken by `run()` when spawning workers. receivers: Vec>, } @@ -36,10 +35,10 @@ pub struct PrecomputeEngine { impl PrecomputeEngine { pub fn new( config: PrecomputeEngineConfig, - hot_reload_config: HotReloadStreamingConfig, + hot_reload_config: StreamingConfigHandle, output_sink: Arc, series_resolver: Arc, - sketch_index: Arc, + summary_store: Arc, ) -> Self { let worker_group_counts = (0..config.num_workers) .map(|_| Arc::new(AtomicUsize::new(0))) @@ -77,7 +76,7 @@ impl PrecomputeEngine { pass_raw_samples: config.pass_raw_samples, sketch_snapshots: dashmap::DashMap::new(), series_resolver, - sketch_index, + summary_store, observability: crate::precompute_engine::ingest_handler::IngestObservability::new(), }); @@ -104,10 +103,8 @@ impl PrecomputeEngine { } /// Start the precompute engine. This spawns worker tasks and the - /// periodic flush timer, then blocks until shutdown. The legacy - /// Prometheus / VictoriaMetrics HTTP ingest listener has been - /// removed; ingest now flows in via the OTLP receiver, which holds - /// the same `IngestState` handle returned by `ingest_state()`. + /// periodic flush timer, then blocks until shutdown. Protocol receivers + /// submit through the shared `IngestState` returned by `ingest_state()`. pub async fn run(mut self) -> Result<(), Box> { let num_workers = self.config.num_workers; let output_sink: Arc = Arc::new( diff --git a/data_plane/src/precompute_engine/ingest_handler.rs b/data_plane/src/precompute_engine/ingest_handler.rs index e264dcbf5..d975eb6a9 100644 --- a/data_plane/src/precompute_engine/ingest_handler.rs +++ b/data_plane/src/precompute_engine/ingest_handler.rs @@ -1,6 +1,6 @@ use crate::precompute_engine::series_router::SeriesRouter; use crate::precompute_engine::worker::parse_labels_from_series_key; -use crate::storage_engines::types::HotReloadStreamingConfig; +use crate::storage_engines::types::StreamingConfigHandle; use asap_types::aggregation_config::AggregationConfig; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; @@ -121,7 +121,7 @@ pub struct IngestState { /// Hot-reloadable streaming config. On each ingest batch, the /// router snapshots the latest config to derive agg_configs. /// This replaces the old frozen `Vec>`. - pub hot_reload_config: HotReloadStreamingConfig, + pub hot_reload_config: StreamingConfigHandle, /// When true, skip group-key extraction and pass raw samples through. pub pass_raw_samples: bool, /// Per-series reconstructed sketch bases, keyed by series identity. Full frames @@ -142,7 +142,7 @@ pub struct IngestState { /// every modified-OTLP first-class sketch DataPoint; queried by /// the `ASAPQueryEngine` query path (ASAP-tier hit / ghost / unknown /// classification drives the Phase 6 archive failover). - pub sketch_index: Arc, + pub summary_store: Arc, /// CQ-6 / RES-1 — per-reason silent-drop counters plus the /// `sketch_snapshots` eviction configuration. Grouped into one /// `Default`-constructible field so the counters can live on @@ -163,10 +163,17 @@ impl IngestState { self.hot_reload_config.snapshot() } + pub fn active_physical_plan_snapshot( + &self, + ) -> Option> { + self.hot_reload_config.active_physical_plan_snapshot() + } + + #[deprecated(note = "use active_physical_plan_snapshot")] pub fn physical_plan_snapshot( &self, - ) -> Option> { - self.hot_reload_config.physical_plan_snapshot() + ) -> Option> { + self.active_physical_plan_snapshot() } /// RES-1 — record that a per-series snapshot base for `window_start` @@ -331,7 +338,7 @@ mod tests { map.insert(agg_id, make_config(agg_id, metric)); let streaming = StreamingConfig::new(map); let hot_reload = - crate::storage_engines::types::HotReloadStreamingConfig::new(streaming.clone()); + crate::storage_engines::types::StreamingConfigHandle::new(streaming.clone()); let state = Arc::new(IngestState { router, @@ -343,7 +350,7 @@ mod tests { series_resolver: Arc::new( crate::drivers::ingest::series_resolver::SeriesIdResolver::new(), ), - sketch_index: Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()), + summary_store: Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()), observability: IngestObservability::default(), }); diff --git a/data_plane/src/precompute_engine/maintenance_runtime.rs b/data_plane/src/precompute_engine/maintenance_runtime.rs index 0f5149d21..b32fb8df0 100644 --- a/data_plane/src/precompute_engine/maintenance_runtime.rs +++ b/data_plane/src/precompute_engine/maintenance_runtime.rs @@ -5,7 +5,7 @@ use super::subdag_scheduler::{ execute_precompute_sink, IdempotentCommitSink, MaterializationCommitKey, PrecomputeOperatorRegistry, ScheduleError, }; -use crate::storage_engines::types::{AggregateCore, HotReloadStreamingConfig, PrecomputedOutput}; +use crate::storage_engines::types::{AggregateCore, PrecomputedOutput, StreamingConfigHandle}; use asap_types::executable_plan::{BackendExecutableBinding, BackendNodeBinding}; use planner_types::post_asap::{ExecutableDagNode, ExecutableOperatorPayload, PostAsapNodeId}; use sha2::{Digest, Sha256}; @@ -1611,12 +1611,12 @@ struct CommitRegistry(Mutex); impl CommitRegistry { fn plan_snapshot( &self, - plans: &HotReloadStreamingConfig, - ) -> Result>, String> { + plans: &StreamingConfigHandle, + ) -> Result>, String> { let mut state = self.0.lock().map_err(|_| "commit registry poisoned")?; // Read the authoritative generation while holding the registry lock, // so an old in-flight batch cannot restore an obsolete generation. - let plan = plans.physical_plan_snapshot(); + let plan = plans.active_physical_plan_snapshot(); let generation = plan .as_ref() .map(|plan| (plan.plan_id(), plan.plan_version())); @@ -1779,13 +1779,13 @@ impl IdempotentCommitSink for CommitRegistry { /// With no matching DAG, the source output is forwarded unchanged. pub struct MaintenanceDagSink { inner: Arc, - plans: HotReloadStreamingConfig, + plans: StreamingConfigHandle, commits: CommitRegistry, batch_guard: Mutex<()>, } impl MaintenanceDagSink { - pub fn new(inner: Arc, plans: HotReloadStreamingConfig) -> Self { + pub fn new(inner: Arc, plans: StreamingConfigHandle) -> Self { Self { inner, plans, @@ -1796,7 +1796,7 @@ impl MaintenanceDagSink { fn execute_one( &self, - plan: &crate::storage_engines::types::ActivePhysicalPlan, + plan: &crate::storage_engines::types::RuntimePhysicalPlan, output: PrecomputedOutput, state: Box, ) -> Result, String> { @@ -2106,6 +2106,44 @@ impl OutputSink for MaintenanceDagSink { } } +/// Materializations affected by an admitted source update, following installed +/// semantic dependencies rather than assuming source and output identities match. +pub(crate) fn affected_materializations( + plan: &asap_types::precompute_plan::PrecomputePlan, + source: asap_types::sds::SummaryDefinitionId, +) -> BTreeSet { + use asap_types::executable_plan::BackendNodeBinding; + let mut affected = BTreeSet::from([source]); + for installed in plan.executable_dags.values() { + let mut reachable = installed.binding.nodes.iter().filter_map(|(node, binding)| { + matches!(binding, BackendNodeBinding::Materialization { summary_definition } if *summary_definition == source).then_some(*node) + }).collect::>(); + let mut frontier = reachable.iter().copied().collect::>(); + while let Some(producer) = frontier.pop() { + for edge in &installed.document.edges { + let immutable = matches!(installed.binding.node(edge.consumer), + Some(BackendNodeBinding::Materialization { summary_definition }) + if plan.materializations.iter().any(|config| + config.policy_fingerprint() == summary_definition.fingerprint() + && config.derived_input.is_some())); + if edge.producer == producer && !immutable && reachable.insert(edge.consumer) { + frontier.push(edge.consumer); + } + } + } + for sink in &installed.binding.precompute_sinks { + if reachable.contains(sink) { + if let Some(BackendNodeBinding::Materialization { summary_definition }) = + installed.binding.nodes.get(sink) + { + affected.insert(*summary_definition); + } + } + } + } + affected +} + #[cfg(test)] mod tests { use super::*; @@ -2310,10 +2348,10 @@ mod tests { .unwrap(); snapshot["query_workload"]["repeating_queries"][0]["query"] = "sum(sum_over_time(m[1m]))".into(); - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_value(snapshot).unwrap(); let bundle = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let mut source_config = bundle.precompute_plan.materializations[0].clone(); // This operator fixture supplies global raw populations; its config @@ -3534,13 +3572,13 @@ mod tests { fn summary_update_rejects_multiple_output_populations_before_updating() { use planner_types::post_asap::{GroupingStrategy, SummaryUpdate}; use planner_types::pre_asap::{ColumnRef, Reduction}; - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); let mut config = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap() .precompute_plan .materializations[0] @@ -3595,13 +3633,13 @@ mod tests { fn dds_maintenance_rejects_nonpositive_population_before_returning_summary() { use planner_types::post_asap::{GroupingStrategy, SummaryUpdate}; use planner_types::pre_asap::{ColumnRef, Reduction}; - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); let mut config = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap() .precompute_plan .materializations[0] @@ -3758,7 +3796,7 @@ mod tests { #[test] fn downstream_failure_does_not_acknowledge_maintenance_publication() { use crate::storage_engines::types::{ - ActivePhysicalPlan, HotReloadActivePhysicalPlan, StreamingConfig, + ActivePhysicalPlanHandle, RuntimePhysicalPlan, StreamingConfig, }; use asap_types::executable_plan::{InstalledPostAsapDag, OwnedPostAsapDag}; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -3791,10 +3829,10 @@ mod tests { .unwrap(); snapshot["query_workload"]["repeating_queries"][0]["query"] = "sum(sum_over_time(m[1m]))".into(); - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_value(snapshot).unwrap(); let mut bundle = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let target_config = &bundle.precompute_plan.materializations[0]; let long_step = target_config.window_size.max( @@ -3842,12 +3880,12 @@ mod tests { binding, }, )]); - let active = ActivePhysicalPlan { + let active = RuntimePhysicalPlan { envelope: bundle.precompute_plan.envelope.clone(), summary_catalog: Some(Arc::new(bundle.summary_catalog)), precompute_plan: bundle.precompute_plan, transmission_plan: bundle.transmission_plan, - runtime_config: Arc::new(StreamingConfig::new(Default::default())), + streaming_config: Arc::new(StreamingConfig::new(Default::default())), query_plan: Arc::new(bundle.query_plan), storage_routing: Arc::new(Default::default()), }; @@ -3862,7 +3900,7 @@ mod tests { }); let sink = MaintenanceDagSink::new( downstream.clone(), - HotReloadStreamingConfig::from_active(HotReloadActivePhysicalPlan::new( + StreamingConfigHandle::from_active_physical_plan(ActivePhysicalPlanHandle::new( active.clone(), )), ); @@ -4071,41 +4109,3 @@ mod tests { assert!(commits.get(&key).unwrap().is_none()); } } - -/// Materializations affected by an admitted source update, following installed -/// semantic dependencies rather than assuming source and output identities match. -pub(crate) fn affected_materializations( - plan: &asap_types::precompute_plan::PrecomputePlan, - source: asap_types::sds::SummaryDefinitionId, -) -> BTreeSet { - use asap_types::executable_plan::BackendNodeBinding; - let mut affected = BTreeSet::from([source]); - for installed in plan.executable_dags.values() { - let mut reachable = installed.binding.nodes.iter().filter_map(|(node, binding)| { - matches!(binding, BackendNodeBinding::Materialization { summary_definition } if *summary_definition == source).then_some(*node) - }).collect::>(); - let mut frontier = reachable.iter().copied().collect::>(); - while let Some(producer) = frontier.pop() { - for edge in &installed.document.edges { - let immutable = matches!(installed.binding.node(edge.consumer), - Some(BackendNodeBinding::Materialization { summary_definition }) - if plan.materializations.iter().any(|config| - config.policy_fingerprint() == summary_definition.fingerprint() - && config.derived_input.is_some())); - if edge.producer == producer && !immutable && reachable.insert(edge.consumer) { - frontier.push(edge.consumer); - } - } - } - for sink in &installed.binding.precompute_sinks { - if reachable.contains(sink) { - if let Some(BackendNodeBinding::Materialization { summary_definition }) = - installed.binding.nodes.get(sink) - { - affected.insert(*summary_definition); - } - } - } - } - affected -} diff --git a/data_plane/src/precompute_engine/multisource_coordinator.rs b/data_plane/src/precompute_engine/multisource_coordinator.rs index 0b305fa81..2e3dcc5f7 100644 --- a/data_plane/src/precompute_engine/multisource_coordinator.rs +++ b/data_plane/src/precompute_engine/multisource_coordinator.rs @@ -617,10 +617,10 @@ mod tests { query["demand"]["fixed_interval_at"]["evaluation_phase"] = 0.into(); query["time_selection"]["lookback"] = 60000.into(); wire["query_workload"]["repeating_queries"] = serde_json::json!([query]); - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_value(wire).unwrap(); let mut plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap() .precompute_plan; let target = plan diff --git a/data_plane/src/precompute_engine/output_sink.rs b/data_plane/src/precompute_engine/output_sink.rs index e11870417..a710e57a9 100644 --- a/data_plane/src/precompute_engine/output_sink.rs +++ b/data_plane/src/precompute_engine/output_sink.rs @@ -1,7 +1,7 @@ use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::precompute_engine::ingest_handler::IngestObservability; use crate::storage_engines::sketch_db::index::SketchStore; -use crate::storage_engines::types::hot_reload_config::HotReloadStreamingConfig; +use crate::storage_engines::types::hot_reload_config::StreamingConfigHandle; use crate::storage_engines::types::{AggregateCore, PrecomputedOutput}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; @@ -51,8 +51,8 @@ fn consume_in_order(items: Vec, mut persist: impl FnMut(&T) -> bool) -> us /// its fingerprint. Consume accumulators in order so catch-up batches release /// each pane as soon as it is serialized. pub struct SketchStoreSink { - sketch_index: Arc, - hot_reload: HotReloadStreamingConfig, + summary_store: Arc, + hot_reload: StreamingConfigHandle, /// Single shared resolver across the ingest + precompute paths. Under /// the registry-allocated sid model (PR-1..3), this is the canonical /// mint authority — precompute sids share the same `next_sid` counter @@ -72,12 +72,12 @@ pub struct SketchStoreSink { impl SketchStoreSink { pub fn new( - sketch_index: Arc, - hot_reload: HotReloadStreamingConfig, + summary_store: Arc, + hot_reload: StreamingConfigHandle, series_resolver: Arc, ) -> Self { Self { - sketch_index, + summary_store, hot_reload, series_resolver, observability: None, @@ -155,7 +155,7 @@ impl SketchStoreSink { crate::precompute_engine::metrics::record_materialized_outputs(1) }); } - self.sketch_index + self.summary_store .validate_routed_catalog_generation(output.catalog_generation.as_deref()) .ok()?; writer @@ -163,11 +163,11 @@ impl SketchStoreSink { |metric, fp, ak| { resolver .resolve_with_reactivation(metric, fp, ak, |sid| { - self.sketch_index.validate_routed_catalog_generation( + self.summary_store.validate_routed_catalog_generation( output.catalog_generation.as_deref(), )?; let activation = - self.sketch_index.authorize_series_reactivation( + self.summary_store.authorize_series_reactivation( sid, output.policy_fp.into(), )?; @@ -212,7 +212,7 @@ impl SketchStoreSink { time_range: asap_types::sds::HalfOpenTimeRange { start_ms, end_ms }, group_values, }; - if let Err(error) = self.sketch_index.publish_admitted_summary_update( + if let Err(error) = self.summary_store.publish_admitted_summary_update( &revision.generation, &coordinate, revision.first_revision, @@ -230,7 +230,7 @@ impl SketchStoreSink { } true } else { - self.sketch_index + self.summary_store .publish_unadmitted_summary_update(persist) .is_some() } @@ -426,11 +426,11 @@ mod tests { let mut configs = HashMap::new(); configs.insert(agg_id, cfg); let streaming = StreamingConfig::new(configs); - let hot_reload = HotReloadStreamingConfig::new(streaming.clone()); + let hot_reload = StreamingConfigHandle::new(streaming.clone()); - let sketch_index = Arc::new(SketchStore::new()); + let summary_store = Arc::new(SketchStore::new()); let sink = SketchStoreSink::new( - sketch_index.clone(), + summary_store.clone(), hot_reload, Arc::new(SeriesIdResolver::new()), ); @@ -443,16 +443,16 @@ mod tests { sink.emit_batch(vec![(output, acc)]).expect("emit ok"); assert_eq!( - sketch_index.instance_count(), + summary_store.instance_count(), 1, "SketchStore should have one precompute instance" ); - let instances = sketch_index + let instances = summary_store .list_by_status(crate::storage_engines::sketch_db::lifecycle::AggStatus::Active); assert_eq!(instances.len(), 1); let meta = instances[0].clone(); let sid = meta.sid; - assert_eq!(sketch_index.classify(sid), SeriesLookup::Hit); + assert_eq!(summary_store.classify(sid), SeriesLookup::Hit); assert!( matches!( meta.agg_kind, @@ -494,10 +494,7 @@ mod tests { Arc::new(SeriesIdResolver::open(temporary.path().join("resolver.wal")).unwrap()); let sink = SketchStoreSink::new( store.clone(), - HotReloadStreamingConfig::new(StreamingConfig::new(HashMap::from([( - fingerprint.0, - cfg, - )]))), + StreamingConfigHandle::new(StreamingConfig::new(HashMap::from([(fingerprint.0, cfg)]))), resolver, ); let original_generation = Arc::new(catalog.reference().unwrap()); @@ -577,10 +574,10 @@ mod tests { .insert("alpha".into(), serde_json::json!(0.01)); let policy_fp = cfg.policy_fp_u64(); let hot_reload = - HotReloadStreamingConfig::new(StreamingConfig::new(HashMap::from([(policy_fp, cfg)]))); - let sketch_index = Arc::new(SketchStore::new()); + StreamingConfigHandle::new(StreamingConfig::new(HashMap::from([(policy_fp, cfg)]))); + let summary_store = Arc::new(SketchStore::new()); let sink = SketchStoreSink::new( - sketch_index.clone(), + summary_store.clone(), hot_reload, Arc::new(SeriesIdResolver::new()), ); @@ -593,7 +590,7 @@ mod tests { )]) .expect("emit sketch"); - let meta = sketch_index + let meta = summary_store .list_by_status(crate::storage_engines::sketch_db::lifecycle::AggStatus::Active) .into_iter() .next() @@ -605,8 +602,8 @@ mod tests { .. } )); - assert_eq!(sketch_index.query_range(meta.sid, 1_000, 2_000).len(), 1); - assert!(sketch_index + assert_eq!(summary_store.query_range(meta.sid, 1_000, 2_000).len(), 1); + assert!(summary_store .query_exact_agg_range(meta.sid, 1_000, 2_000) .is_empty()); } @@ -616,10 +613,10 @@ mod tests { // Streaming config does NOT contain agg_id=99 — the sink // reports a recoverable error rather than acknowledging a lost write. let streaming = StreamingConfig::new(HashMap::new()); - let hot_reload = HotReloadStreamingConfig::new(streaming.clone()); - let sketch_index = Arc::new(SketchStore::new()); + let hot_reload = StreamingConfigHandle::new(streaming.clone()); + let summary_store = Arc::new(SketchStore::new()); let sink = SketchStoreSink::new( - sketch_index.clone(), + summary_store.clone(), hot_reload, Arc::new(SeriesIdResolver::new()), ); @@ -628,7 +625,7 @@ mod tests { let acc: Box = Box::new(SumAccumulator::with_sum(1.0)); sink.emit_batch(vec![(output, acc)]) .expect_err("unpersisted output must not be acknowledged"); - assert_eq!(sketch_index.instance_count(), 0); + assert_eq!(summary_store.instance_count(), 0); } /// CQ-6 — a registry-miss (policy_fp not in the running streaming @@ -637,11 +634,11 @@ mod tests { #[test] fn sink_increments_policy_miss_counter_on_registry_miss() { let streaming = StreamingConfig::new(HashMap::new()); - let hot_reload = HotReloadStreamingConfig::new(streaming.clone()); - let sketch_index = Arc::new(SketchStore::new()); + let hot_reload = StreamingConfigHandle::new(streaming.clone()); + let summary_store = Arc::new(SketchStore::new()); let obs = Arc::new(IngestObservability::new()); let sink = SketchStoreSink::new( - sketch_index.clone(), + summary_store.clone(), hot_reload, Arc::new(SeriesIdResolver::new()), ) @@ -654,7 +651,7 @@ mod tests { .expect_err("unpersisted output must not be acknowledged"); assert_eq!( - sketch_index.instance_count(), + summary_store.instance_count(), 0, "no write on registry miss" ); diff --git a/data_plane/src/precompute_engine/worker.rs b/data_plane/src/precompute_engine/worker.rs index bdead2c92..2c48006f4 100644 --- a/data_plane/src/precompute_engine/worker.rs +++ b/data_plane/src/precompute_engine/worker.rs @@ -9,7 +9,7 @@ use crate::precompute_engine::output_sink::OutputSink; use crate::precompute_engine::series_router::WorkerMessage; use crate::precompute_engine::window_manager::WindowManager; use crate::storage_engines::types::{ - AggregateCore, HotReloadStreamingConfig, KeyByLabelValues, PrecomputedOutput, + AggregateCore, KeyByLabelValues, PrecomputedOutput, StreamingConfigHandle, }; use asap_types::aggregation_config::AggregationConfig; use asap_types::PolicyFingerprint; @@ -162,7 +162,7 @@ pub struct Worker { /// Hot-reload handle — workers read config directly from ArcSwap /// instead of holding a local copy. All components see the same /// config at the same time. - hot_reload: HotReloadStreamingConfig, + hot_reload: StreamingConfigHandle, /// Allowed lateness in ms. allowed_lateness_ms: i64, /// When true, skip aggregation and pass raw samples through. @@ -199,7 +199,7 @@ impl Worker { id: usize, receiver: mpsc::Receiver, output_sink: Arc, - hot_reload: HotReloadStreamingConfig, + hot_reload: StreamingConfigHandle, runtime_config: WorkerRuntimeConfig, group_count: Arc, worker_watermark: Arc, @@ -415,7 +415,7 @@ impl Worker { /// hot-reload snapshot the first time we see this sid; `group_key` is /// remembered on the `GroupState` for emit-time label rendering. /// - /// Reads config directly from the `HotReloadStreamingConfig` + /// Reads config directly from the `StreamingConfigHandle` /// ArcSwap handle, so new policies from a config swap are visible /// immediately — no message passing, no delay. /// Returns None if `policy_fp` has no matching config (e.g. arrived @@ -1899,15 +1899,15 @@ mod tests { ) } - /// Build a fresh `HotReloadStreamingConfig` from a map of agg_id + /// Build a fresh `StreamingConfigHandle` from a map of agg_id /// → AggregationConfig. Worker::new takes this handle instead of /// the old `HashMap>`. Tests use this /// helper instead of constructing the handle inline at every /// callsite. fn make_hot_reload( configs: HashMap, - ) -> crate::storage_engines::types::HotReloadStreamingConfig { - crate::storage_engines::types::HotReloadStreamingConfig::new( + ) -> crate::storage_engines::types::StreamingConfigHandle { + crate::storage_engines::types::StreamingConfigHandle::new( crate::storage_engines::types::StreamingConfig::new(configs), ) } @@ -2704,13 +2704,13 @@ aggregations: // PR 5: the streaming-config key is the policy fingerprint. let agg_id = *streaming_config - .get_all_aggregation_configs() + .materializations() .keys() .next() .expect("one agg"); assert!(streaming_config.contains(agg_id)); - let agg_configs = streaming_config.get_all_aggregation_configs().clone(); + let agg_configs = streaming_config.materializations().clone(); let sink = Arc::new(CapturingOutputSink::new()); let mut worker = make_worker(agg_configs, sink.clone(), false, 0, LateDataPolicy::Drop); diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs index 2ca7c7b70..bc4d4741d 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs @@ -23,7 +23,7 @@ use crate::storage_engines::sketch_db::index::SketchStore; pub struct CatalogClickHouseAccelerator { pub store: Arc, - active_physical_plan: Option, + active_physical_plan: Option, exact_backend: Option>, } @@ -38,7 +38,7 @@ impl CatalogClickHouseAccelerator { pub fn with_active_physical_plan( store: Arc, - active: crate::storage_engines::types::HotReloadActivePhysicalPlan, + active: crate::storage_engines::types::ActivePhysicalPlanHandle, ) -> Self { let mut accelerator = Self::empty(store); accelerator.active_physical_plan = Some(active); @@ -51,7 +51,7 @@ impl CatalogClickHouseAccelerator { /// nodes unexecutable. pub fn with_active_physical_plan_and_exact_backend( store: Arc, - active: crate::storage_engines::types::HotReloadActivePhysicalPlan, + active: crate::storage_engines::types::ActivePhysicalPlanHandle, exact_backend: Arc, ) -> Self { Self::with_active_physical_plan(store, active).with_exact_backend(exact_backend) @@ -168,7 +168,11 @@ fn requested_format(request: &ClickHouseQueryRequest) -> Result ClickHouseAccelerationOutcome { - let Some(physical) = self.active_physical_plan.as_ref().map(|h| h.snapshot()) else { + let Some(physical) = self + .active_physical_plan + .as_ref() + .map(|h| h.active_snapshot()) + else { return ClickHouseAccelerationOutcome::Fallback( ClickHouseAccelerationFallback::CatalogMiss, ); @@ -228,7 +232,7 @@ impl CatalogClickHouseAccelerator { async fn execute_bound( &self, request: &ClickHouseQueryRequest, - physical: &crate::storage_engines::types::ActivePhysicalPlan, + physical: &crate::storage_engines::types::RuntimePhysicalPlan, entry: &asap_types::query_plan::QueryPlanEntry, runtime_range: Option<(u64, u64)>, ) -> ClickHouseAccelerationOutcome { @@ -359,7 +363,7 @@ mod tests { use crate::{ precompute_engine::operators::SumAccumulator, - storage_engines::sketch_db::index::{AggKind, Capability, SketchInstanceMetadata}, + storage_engines::sketch_db::index::{AggKind, Capability, SummarySeriesMetadata}, }; use asap_types::query_plan::{ ClickHousePlanningContext, ExactReadout, ExternalExactOutput, ExternalExactRequest, @@ -733,7 +737,7 @@ mod tests { .install_summary_catalog(Arc::new(sds.clone())) .unwrap(); if seed { - store.register(SketchInstanceMetadata { + store.register(SummarySeriesMetadata { sid: 7, metric_name: "requests".into(), group_by_keys: BTreeSet::new(), @@ -779,14 +783,14 @@ mod tests { ) .unwrap(); precompute.summary_catalog = Some(sds.reference().unwrap()); - let mut transmission = control_plane::physical::compiler::compile_transmission_plan( + let mut transmission = control_plane::physical::compiler::build_transmission_plan( envelope.clone(), &precompute, &BTreeMap::new(), ) .unwrap(); transmission.summary_catalog = Some(sds.reference().unwrap()); - let active = crate::drivers::query::servers::http::build_active_physical_plan( + let active = crate::drivers::query::servers::http::validate_and_build_runtime_plan( crate::drivers::query::servers::http::PhysicalPlanInstallRequest { summary_catalog: sds, collector_plans: vec![], @@ -801,7 +805,7 @@ mod tests { .unwrap(); let accelerator = CatalogClickHouseAccelerator::with_active_physical_plan( store, - crate::storage_engines::types::HotReloadActivePhysicalPlan::new(active), + crate::storage_engines::types::ActivePhysicalPlanHandle::new(active), ); let request = ClickHouseQueryRequest { method: Method::GET, @@ -883,7 +887,7 @@ mod tests { let (accelerator, mut request) = fixture_with_sql(1_000, Arc::new(SketchStore::new()), true, true).await; let active = accelerator.active_physical_plan.as_ref().unwrap(); - let mut snapshot = active.snapshot().as_ref().clone(); + let mut snapshot = active.active_snapshot().as_ref().clone(); let context = snapshot.query_plan.clickhouse_context.as_ref().unwrap(); let second_sql = "SELECT sum(value) FROM requests WHERE timestamp >= 1000 AND timestamp < 2000"; @@ -950,7 +954,7 @@ mod tests { let (accelerator, mut request) = fixture_with_sql(1_000, Arc::new(SketchStore::new()), true, true).await; let active = accelerator.active_physical_plan.as_ref().unwrap(); - let mut snapshot = active.snapshot().as_ref().clone(); + let mut snapshot = active.active_snapshot().as_ref().clone(); let context = snapshot.query_plan.clickhouse_context.as_ref().unwrap(); let (fixed, _, _) = control_plane::clickhouse::bind_clickhouse_sql( &request.sql, @@ -1040,7 +1044,7 @@ mod tests { cfg.value_projection = Some(asap_types::sds::ValueProjectionIdentity::Column { name: "value".into(), }); - let hot = crate::storage_engines::types::HotReloadStreamingConfig::from_arc(Arc::new( + let hot = crate::storage_engines::types::StreamingConfigHandle::from_arc(Arc::new( crate::storage_engines::types::StreamingConfig::new(HashMap::from([( cfg.policy_fp_u64(), cfg.clone(), @@ -1141,7 +1145,7 @@ mod tests { .active_physical_plan .as_ref() .unwrap() - .snapshot(); + .active_snapshot(); let entry = mixed_summary_external_entry( physical.query_plan.entries.values().next().unwrap().clone(), ); @@ -1187,7 +1191,7 @@ mod tests { .active_physical_plan .as_ref() .unwrap() - .snapshot(); + .active_snapshot(); let mut query_plan = physical.query_plan.as_ref().clone(); let key = query_plan.entries.keys().next().unwrap().clone(); let entry = mixed_summary_external_entry(query_plan.entries[&key].clone()); @@ -1200,7 +1204,7 @@ mod tests { )); let accelerator = CatalogClickHouseAccelerator::with_active_physical_plan_and_exact_backend( accelerator.store.clone(), - crate::storage_engines::types::HotReloadActivePhysicalPlan::new(active), + crate::storage_engines::types::ActivePhysicalPlanHandle::new(active), exact_backend.clone(), ); let ClickHouseAccelerationOutcome::Accelerated(response) = diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/clickhouse_result_adapter.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/clickhouse_result_adapter.rs index cdc46d262..4541d3d28 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/clickhouse_result_adapter.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/clickhouse_result_adapter.rs @@ -308,113 +308,6 @@ pub fn raw_response(response: ClickHouseRawResponse) -> Response { output } -#[cfg(test)] -mod tests { - use super::*; - use arrow::{ - array::{Int64Array, StringArray}, - datatypes::{DataType, Field, Schema}, - }; - use std::sync::Arc; - - #[test] - fn nullable_fields_remain_explicit_in_json_and_tsv() { - let batch = RecordBatch::try_new( - Arc::new(Schema::new(vec![ - Field::new("value", DataType::Float64, true), - Field::new("label", DataType::Utf8, true), - ])), - vec![ - Arc::new(Float64Array::from(vec![Some(1.25), None, Some(2.5)])), - Arc::new(StringArray::from(vec![Some(""), None, Some("\\N")])), - ], - ) - .unwrap(); - let result = ClickHouseQueryResult { - batches: vec![batch], - }; - let json: serde_json::Value = - serde_json::from_slice(&result.encode(ClickHouseFormat::Json).unwrap()).unwrap(); - assert_eq!( - json["data"], - serde_json::json!([{ "value":1.25,"label":"" }, { "value":null,"label":null }, {"value":2.5,"label":"\\N"}]) - ); - let lines = result.encode(ClickHouseFormat::JsonEachRow).unwrap(); - let null_row: serde_json::Value = - serde_json::from_slice(lines.split(|byte| *byte == b'\n').nth(1).unwrap()).unwrap(); - assert_eq!(null_row, serde_json::json!({"value":null,"label":null})); - assert_eq!( - result.encode(ClickHouseFormat::TabSeparated).unwrap(), - b"1.25\t\n\\N\t\\N\n2.5\t\\\\N\n" - ); - } - - #[test] - fn empty_map_bottom_type_uses_clickhouse_nothing() { - let entries = DataType::Struct( - vec![ - Field::new("key", DataType::Null, false), - Field::new("value", DataType::Null, false), - ] - .into(), - ); - let dtype = DataType::Map(Arc::new(Field::new("entries", entries, false)), false); - assert_eq!(clickhouse_type(&dtype, false), "Map(Nothing, Nothing)"); - assert_eq!(clickhouse_type(&DataType::Null, true), "Nullable(Nothing)"); - } - - #[test] - fn map_timestamp_transport_is_not_assumed_to_match_native_formatting() { - let entries = DataType::Struct( - vec![ - Field::new("key", DataType::Utf8, false), - Field::new( - "value", - DataType::Timestamp(arrow::datatypes::TimeUnit::Millisecond, None), - false, - ), - ] - .into(), - ); - let dtype = DataType::Map(Arc::new(Field::new("entries", entries, false)), false); - let batch = RecordBatch::try_new( - Arc::new(Schema::new(vec![Field::new("m", dtype.clone(), false)])), - vec![arrow::array::new_empty_array(&dtype)], - ) - .unwrap(); - let result = ClickHouseQueryResult { - batches: vec![batch], - }; - assert!(result.encode(ClickHouseFormat::Json).is_err()); - assert!(result.encode(ClickHouseFormat::TabSeparated).is_err()); - } - - #[test] - fn encodes_table_without_using_promql_query_result() { - let schema = Arc::new(Schema::new(vec![ - Field::new("zone", DataType::Utf8, false), - Field::new("count", DataType::Int64, false), - ])); - let batch = RecordBatch::try_new( - schema, - vec![ - Arc::new(StringArray::from(vec!["a\tb"])), - Arc::new(Int64Array::from(vec![7])), - ], - ) - .unwrap(); - let result = ClickHouseQueryResult { - batches: vec![batch], - }; - assert_eq!( - result.encode(ClickHouseFormat::TabSeparated).unwrap(), - b"a\\tb\t7\n" - ); - assert!(result.encode(ClickHouseFormat::JsonEachRow).is_err()); - assert!(result.encode(ClickHouseFormat::Json).is_err()); - } -} - struct JsonArrowRows<'a>(&'a [RecordBatch]); impl Serialize for JsonArrowRows<'_> { fn serialize(&self, serializer: S) -> Result { @@ -586,3 +479,110 @@ fn map_literal(array: &dyn Array, row: usize) -> Result) -> QueryExpr { + QueryExpr::FunctionCall { + name: name.into(), + args, + } + } + fn text(value: &str) -> QueryExpr { + QueryExpr::Literal(ScalarValue::Utf8(value.into())) + } + + #[test] + fn map_access_uses_declared_default_and_first_duplicate() { + let dtype = DataType::Map { + key: Box::new(DataType::Utf8), + value: Box::new(DataType::Int64), + value_nullable: false, + }; + let schema = Schema::new(vec![Column::new("m", dtype, false)]); + let access = function("asap_map_access", vec![QueryExpr::Column(0), text("a")]); + assert_eq!( + eval(&access, &[Cell::Map(vec![])], &schema).unwrap(), + Cell::Int64(0) + ); + assert_eq!( + eval( + &access, + &[Cell::Map(vec![ + (Cell::Utf8("a".into()), Cell::Int64(7)), + (Cell::Utf8("a".into()), Cell::Int64(9)) + ])], + &schema + ) + .unwrap(), + Cell::Int64(7) + ); + let nullable = Schema::new(vec![Column::new( + "m", + DataType::Map { + key: Box::new(DataType::Utf8), + value: Box::new(DataType::Int64), + value_nullable: true, + }, + false, + )]); + assert_eq!( + eval(&access, &[Cell::Map(vec![])], &nullable).unwrap(), + Cell::Null + ); + let null_key = function( + "asap_map_access", + vec![QueryExpr::Column(0), QueryExpr::Literal(ScalarValue::Null)], + ); + assert_eq!( + eval(&null_key, &[Cell::Map(vec![])], &schema).unwrap(), + Cell::Null + ); + } + + #[test] + fn map_concat_preserves_duplicates_and_empty_map() { + let map = |value| { + function( + "map", + vec![text("a"), QueryExpr::Literal(ScalarValue::Int64(value))], + ) + }; + let concat = function("mapConcat", vec![function("map", vec![]), map(7), map(9)]); + let schema = Schema::new(vec![]); + assert_eq!( + eval(&concat, &[], &schema).unwrap(), + Cell::Map(vec![ + (Cell::Utf8("a".into()), Cell::Int64(7)), + (Cell::Utf8("a".into()), Cell::Int64(9)) + ]) + ); + let mixed = function( + "map", + vec![ + text("a"), + QueryExpr::Literal(ScalarValue::Int64(1)), + text("b"), + QueryExpr::Literal(ScalarValue::Float64(2.5)), + ], + ); + assert!(eval(&mixed, &[], &schema).is_err()); + } + + #[test] + fn sorting_nested_nan_fails_before_comparator_can_treat_it_as_equal() { + let dtype = DataType::Map { + key: Box::new(DataType::Utf8), + value: Box::new(DataType::Float64), + value_nullable: false, + }; + let input = ClickHouseRelation { + rows: vec![vec![Cell::Map(vec![( + Cell::Utf8("a".into()), + Cell::Float64(f64::NAN), + )])]], + fields: vec![("m".into(), dtype.clone(), false)], + coverage: None, + }; + let schema = SummarySchema { + fields: vec![planner_types::post_asap::SummaryField { + name: "m".into(), + dtype: SummaryFamilyType::Plain(dtype), + nullable: false, + }], + time_index: None, + }; + let operation = ValueOperation::Sort { + keys: vec![SortKey { + expr: QueryExpr::Column(0), + ascending: true, + nulls_first: false, + }], + partition_by: planner_types::pre_asap::GroupKeys::none(), + }; + assert!(ClickHouseRelationalAdapter + .apply_operation(&operation, &schema, input) + .is_err()); + } + + #[test] + fn mixed_comparison_preserves_integer_precision_and_boundaries() { + assert_eq!( + integer_float_cmp(9_007_199_254_740_993, 9_007_199_254_740_992.0), + Some(Ordering::Greater) + ); + assert_eq!( + integer_float_cmp(i64::MAX, 9_223_372_036_854_775_808.0), + Some(Ordering::Less) + ); + assert_eq!( + integer_float_cmp(i64::MIN, -9_223_372_036_854_775_808.0), + Some(Ordering::Equal) + ); + assert_eq!(integer_float_cmp(-1, -1.5), Some(Ordering::Greater)); + assert_eq!(integer_float_cmp(1, 1.5), Some(Ordering::Less)); + assert_eq!(integer_float_cmp(0, f64::INFINITY), Some(Ordering::Less)); + assert_eq!( + integer_float_cmp(0, f64::NEG_INFINITY), + Some(Ordering::Greater) + ); + assert_eq!(integer_float_cmp(0, f64::NAN), None); + } + + #[test] + fn integer_modulo_never_rounds_through_float() { + assert_eq!( + arithmetic( + &ArithmeticOpKind::Mod, + Cell::Int64(9_007_199_254_740_993), + Cell::Int64(2) + ) + .unwrap(), + Cell::Int64(1) + ); + assert_eq!( + arithmetic(&ArithmeticOpKind::Mod, Cell::Int64(-7), Cell::Int64(3)).unwrap(), + Cell::Int64(-1) + ); + assert!(arithmetic(&ArithmeticOpKind::Mod, Cell::Int64(7), Cell::Int64(0)).is_err()); + assert!(arithmetic( + &ArithmeticOpKind::Mod, + Cell::Int64(i64::MIN), + Cell::Int64(-1) + ) + .is_err()); + } +} + #[cfg(test)] mod tests { use super::*; @@ -1515,180 +1692,3 @@ mod tests { ); } } - -#[cfg(test)] -mod scalar_contract_tests { - use super::*; - use planner_types::pre_asap::{Column, Schema}; - - fn function(name: &str, args: Vec) -> QueryExpr { - QueryExpr::FunctionCall { - name: name.into(), - args, - } - } - fn text(value: &str) -> QueryExpr { - QueryExpr::Literal(ScalarValue::Utf8(value.into())) - } - - #[test] - fn map_access_uses_declared_default_and_first_duplicate() { - let dtype = DataType::Map { - key: Box::new(DataType::Utf8), - value: Box::new(DataType::Int64), - value_nullable: false, - }; - let schema = Schema::new(vec![Column::new("m", dtype, false)]); - let access = function("asap_map_access", vec![QueryExpr::Column(0), text("a")]); - assert_eq!( - eval(&access, &[Cell::Map(vec![])], &schema).unwrap(), - Cell::Int64(0) - ); - assert_eq!( - eval( - &access, - &[Cell::Map(vec![ - (Cell::Utf8("a".into()), Cell::Int64(7)), - (Cell::Utf8("a".into()), Cell::Int64(9)) - ])], - &schema - ) - .unwrap(), - Cell::Int64(7) - ); - let nullable = Schema::new(vec![Column::new( - "m", - DataType::Map { - key: Box::new(DataType::Utf8), - value: Box::new(DataType::Int64), - value_nullable: true, - }, - false, - )]); - assert_eq!( - eval(&access, &[Cell::Map(vec![])], &nullable).unwrap(), - Cell::Null - ); - let null_key = function( - "asap_map_access", - vec![QueryExpr::Column(0), QueryExpr::Literal(ScalarValue::Null)], - ); - assert_eq!( - eval(&null_key, &[Cell::Map(vec![])], &schema).unwrap(), - Cell::Null - ); - } - - #[test] - fn map_concat_preserves_duplicates_and_empty_map() { - let map = |value| { - function( - "map", - vec![text("a"), QueryExpr::Literal(ScalarValue::Int64(value))], - ) - }; - let concat = function("mapConcat", vec![function("map", vec![]), map(7), map(9)]); - let schema = Schema::new(vec![]); - assert_eq!( - eval(&concat, &[], &schema).unwrap(), - Cell::Map(vec![ - (Cell::Utf8("a".into()), Cell::Int64(7)), - (Cell::Utf8("a".into()), Cell::Int64(9)) - ]) - ); - let mixed = function( - "map", - vec![ - text("a"), - QueryExpr::Literal(ScalarValue::Int64(1)), - text("b"), - QueryExpr::Literal(ScalarValue::Float64(2.5)), - ], - ); - assert!(eval(&mixed, &[], &schema).is_err()); - } - - #[test] - fn sorting_nested_nan_fails_before_comparator_can_treat_it_as_equal() { - let dtype = DataType::Map { - key: Box::new(DataType::Utf8), - value: Box::new(DataType::Float64), - value_nullable: false, - }; - let input = ClickHouseRelation { - rows: vec![vec![Cell::Map(vec![( - Cell::Utf8("a".into()), - Cell::Float64(f64::NAN), - )])]], - fields: vec![("m".into(), dtype.clone(), false)], - coverage: None, - }; - let schema = SummarySchema { - fields: vec![planner_types::post_asap::SummaryField { - name: "m".into(), - dtype: SummaryFamilyType::Plain(dtype), - nullable: false, - }], - time_index: None, - }; - let operation = ValueOperation::Sort { - keys: vec![SortKey { - expr: QueryExpr::Column(0), - ascending: true, - nulls_first: false, - }], - partition_by: planner_types::pre_asap::GroupKeys::none(), - }; - assert!(ClickHouseRelationalAdapter - .apply_operation(&operation, &schema, input) - .is_err()); - } - - #[test] - fn mixed_comparison_preserves_integer_precision_and_boundaries() { - assert_eq!( - integer_float_cmp(9_007_199_254_740_993, 9_007_199_254_740_992.0), - Some(Ordering::Greater) - ); - assert_eq!( - integer_float_cmp(i64::MAX, 9_223_372_036_854_775_808.0), - Some(Ordering::Less) - ); - assert_eq!( - integer_float_cmp(i64::MIN, -9_223_372_036_854_775_808.0), - Some(Ordering::Equal) - ); - assert_eq!(integer_float_cmp(-1, -1.5), Some(Ordering::Greater)); - assert_eq!(integer_float_cmp(1, 1.5), Some(Ordering::Less)); - assert_eq!(integer_float_cmp(0, f64::INFINITY), Some(Ordering::Less)); - assert_eq!( - integer_float_cmp(0, f64::NEG_INFINITY), - Some(Ordering::Greater) - ); - assert_eq!(integer_float_cmp(0, f64::NAN), None); - } - - #[test] - fn integer_modulo_never_rounds_through_float() { - assert_eq!( - arithmetic( - &ArithmeticOpKind::Mod, - Cell::Int64(9_007_199_254_740_993), - Cell::Int64(2) - ) - .unwrap(), - Cell::Int64(1) - ); - assert_eq!( - arithmetic(&ArithmeticOpKind::Mod, Cell::Int64(-7), Cell::Int64(3)).unwrap(), - Cell::Int64(-1) - ); - assert!(arithmetic(&ArithmeticOpKind::Mod, Cell::Int64(7), Cell::Int64(0)).is_err()); - assert!(arithmetic( - &ArithmeticOpKind::Mod, - Cell::Int64(i64::MIN), - Cell::Int64(-1) - ) - .is_err()); - } -} diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/server.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/server.rs index aedbac3c6..967d314c2 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/server.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/server.rs @@ -124,6 +124,107 @@ impl ClickHouseHttpServer { } } +fn request( + method: Method, + params: HashMap, + headers: HeaderMap, + body: Bytes, +) -> Result { + let parameters: BTreeMap<_, _> = params.into_iter().collect(); + let sql = parameters + .get("query") + .cloned() + .or_else(|| String::from_utf8(body.to_vec()).ok()) + .unwrap_or_default(); + if sql.trim().is_empty() { + return Err((StatusCode::BAD_REQUEST, "missing query").into_response()); + } + Ok(ClickHouseQueryRequest { + method, + sql, + body, + parameters, + headers, + }) +} + +async fn query_get( + State(state): State, + Query(params): Query>, + headers: HeaderMap, +) -> Response { + let request = match request(Method::GET, params, headers, Bytes::new()) { + Ok(v) => v, + Err(e) => return e, + }; + execute_or_fallback(&state, &request).await +} + +async fn query_post( + State(state): State, + Query(params): Query>, + headers: HeaderMap, + body: Bytes, +) -> Response { + let request = match request(Method::POST, params, headers, body) { + Ok(v) => v, + Err(e) => return e, + }; + execute_or_fallback(&state, &request).await +} + +async fn execute_or_fallback(state: &ServerState, request: &ClickHouseQueryRequest) -> Response { + match state.accelerator.execute(request).await { + ClickHouseAccelerationOutcome::Accelerated(response) => { + let mut response = raw_response(response); + response + .headers_mut() + .entry("x-asap-execution") + .or_insert(axum::http::HeaderValue::from_static("warm")); + response + .headers_mut() + .entry("x-asap-execution-detail") + .or_insert(axum::http::HeaderValue::from_static("asap")); + response + } + ClickHouseAccelerationOutcome::Fallback(reason) => { + tracing::info!( + failure_stage = reason.stage(), + failure_reason = reason.reason_code(), + failure_detail = ?reason, + "ClickHouse acceleration routed to exact fallback" + ); + let stage = reason.stage(); + let reason = reason.reason_code(); + return match state.fallback.execute(request).await { + Ok(v) => { + let mut response = raw_response(v); + for (name, value) in [ + ("x-asap-execution", "exact_fallback"), + ("x-asap-execution-detail", reason), + ("x-asap-failure-stage", stage), + ("x-asap-failure-reason", reason), + ] { + response.headers_mut().insert( + axum::http::HeaderName::from_static(name), + axum::http::HeaderValue::from_static(value), + ); + } + response + } + Err(e) => (StatusCode::BAD_GATEWAY, e.to_string()).into_response(), + }; + } + } +} + +async fn ping(State(state): State) -> Response { + match state.fallback.ping().await { + Ok(v) => raw_response(v), + Err(e) => (StatusCode::BAD_GATEWAY, e.to_string()).into_response(), + } +} + #[cfg(test)] mod tests { use super::*; @@ -307,104 +408,3 @@ mod tests { } } } - -fn request( - method: Method, - params: HashMap, - headers: HeaderMap, - body: Bytes, -) -> Result { - let parameters: BTreeMap<_, _> = params.into_iter().collect(); - let sql = parameters - .get("query") - .cloned() - .or_else(|| String::from_utf8(body.to_vec()).ok()) - .unwrap_or_default(); - if sql.trim().is_empty() { - return Err((StatusCode::BAD_REQUEST, "missing query").into_response()); - } - Ok(ClickHouseQueryRequest { - method, - sql, - body, - parameters, - headers, - }) -} - -async fn query_get( - State(state): State, - Query(params): Query>, - headers: HeaderMap, -) -> Response { - let request = match request(Method::GET, params, headers, Bytes::new()) { - Ok(v) => v, - Err(e) => return e, - }; - execute_or_fallback(&state, &request).await -} - -async fn query_post( - State(state): State, - Query(params): Query>, - headers: HeaderMap, - body: Bytes, -) -> Response { - let request = match request(Method::POST, params, headers, body) { - Ok(v) => v, - Err(e) => return e, - }; - execute_or_fallback(&state, &request).await -} - -async fn execute_or_fallback(state: &ServerState, request: &ClickHouseQueryRequest) -> Response { - match state.accelerator.execute(request).await { - ClickHouseAccelerationOutcome::Accelerated(response) => { - let mut response = raw_response(response); - response - .headers_mut() - .entry("x-asap-execution") - .or_insert(axum::http::HeaderValue::from_static("warm")); - response - .headers_mut() - .entry("x-asap-execution-detail") - .or_insert(axum::http::HeaderValue::from_static("asap")); - response - } - ClickHouseAccelerationOutcome::Fallback(reason) => { - tracing::info!( - failure_stage = reason.stage(), - failure_reason = reason.reason_code(), - failure_detail = ?reason, - "ClickHouse acceleration routed to exact fallback" - ); - let stage = reason.stage(); - let reason = reason.reason_code(); - return match state.fallback.execute(request).await { - Ok(v) => { - let mut response = raw_response(v); - for (name, value) in [ - ("x-asap-execution", "exact_fallback"), - ("x-asap-execution-detail", reason), - ("x-asap-failure-stage", stage), - ("x-asap-failure-reason", reason), - ] { - response.headers_mut().insert( - axum::http::HeaderName::from_static(name), - axum::http::HeaderValue::from_static(value), - ); - } - response - } - Err(e) => (StatusCode::BAD_GATEWAY, e.to_string()).into_response(), - }; - } - } -} - -async fn ping(State(state): State) -> Response { - match state.fallback.ping().await { - Ok(v) => raw_response(v), - Err(e) => (StatusCode::BAD_GATEWAY, e.to_string()).into_response(), - } -} diff --git a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs index 3009c68cc..394453757 100644 --- a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs +++ b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs @@ -184,18 +184,18 @@ mod tests { use super::*; use asap_types::summary_catalog::SummaryCatalog; use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; - use control_plane::physical::compiler::BackendLocalPlanningSnapshot; + use control_plane::physical::compiler::BackendLocalPlanningInput; - fn fixture() -> control_plane::physical::compiler::PhysicalPlan { + fn fixture() -> control_plane::physical::compiler::CompiledPhysicalPlan { let mut value: serde_json::Value = serde_json::from_str(include_str!( "../../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); value["query_workload"]["repeating_queries"][0]["query"] = "sum(sum_over_time(m[1m]))".into(); - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_value(value).unwrap(); + let snapshot: BackendLocalPlanningInput = serde_json::from_value(value).unwrap(); crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap() } diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index 5a4f8d96d..52070e1b2 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -89,7 +89,7 @@ pub struct ASAPQueryEngine { /// EngineRouter's archive failover. When `None`, the /// engine behaves as it did before Phase 5 wire-in (every query /// goes through `handle_query`'s legacy path). - sketch_index: Option>, + summary_store: Option>, /// Phase-5 hybrid-stitch hook — set by `with_archive_engine` from /// `main.rs`'s engine builder. When the ASAP-tier reducer reports a /// `ASAPTierResult.coverage` narrower than the requested @@ -104,7 +104,7 @@ pub struct ASAPQueryEngine { Option>, /// Generation-consistent physical snapshot used by the production query /// path. The QueryPlan and SummaryCatalog must come from the same snapshot. - active_physical_plan: Option, + active_physical_plan: Option, exact_subquery_endpoint: Option, metricsql_exact_subquery_endpoint: Option, exact_subquery_client: reqwest::Client, @@ -117,7 +117,7 @@ impl ASAPQueryEngine { now_ms: u64, ) -> Result { - let physical = self.physical_plan_snapshot().ok_or_else(|| { + let physical = self.active_physical_plan_snapshot().ok_or_else(|| { crate::query_engines::EngineError::capability_miss( "query_plan", "no active physical plan", @@ -129,7 +129,9 @@ impl ASAPQueryEngine { .map_err(|error| { crate::query_engines::EngineError::capability_miss("query_plan", error.to_string()) })?; - let leaves = self.prepare_logical(&physical, planned, &[now_ms]).await?; + let leaves = self + .prepare_query_inputs(&physical, planned, &[now_ms]) + .await?; let (mut result, mut stats) = self.execute_logical_entry(&physical, planned, &leaves, now_ms)?; stats.remote_evaluations = leaves.values().map(|leaf| leaf.remote_evaluations).sum(); @@ -146,7 +148,7 @@ impl ASAPQueryEngine { step_ms: u64, ) -> Result { - let physical = self.physical_plan_snapshot().ok_or_else(|| { + let physical = self.active_physical_plan_snapshot().ok_or_else(|| { crate::query_engines::EngineError::capability_miss( "query_plan", "no active physical plan", @@ -163,12 +165,12 @@ impl ASAPQueryEngine { } /// Construct the query executor. Runtime configuration is read only from - /// the generation-consistent `ActivePhysicalPlan` installed separately. + /// the generation-consistent `RuntimePhysicalPlan` installed separately. pub fn new(prometheus_scrape_interval: u64) -> Self { Self { prometheus_scrape_interval, control_plane_client: None, - sketch_index: None, + summary_store: None, archive_engine: None, active_physical_plan: None, exact_subquery_endpoint: None, @@ -189,9 +191,9 @@ impl ASAPQueryEngine { self.metricsql_exact_subquery_endpoint = Some(endpoint); self } - async fn prepare_logical( + async fn prepare_query_inputs( &self, - physical: &crate::storage_engines::types::ActivePhysicalPlan, + physical: &crate::storage_engines::types::RuntimePhysicalPlan, entry: &asap_types::query_plan::QueryPlanEntry, times: &[u64], ) -> Result { @@ -256,7 +258,7 @@ impl ASAPQueryEngine { fn execute_logical_entry( &self, - physical: &crate::storage_engines::types::ActivePhysicalPlan, + physical: &crate::storage_engines::types::RuntimePhysicalPlan, entry: &asap_types::query_plan::QueryPlanEntry, leaves: &super::logical_dag::PreparedLeaves, at: u64, @@ -269,7 +271,7 @@ impl ASAPQueryEngine { > { use crate::query_engines::EngineError; let revision = self - .sketch_index + .summary_store .as_ref() .map(|index| index.summary_update_revision()); let result = @@ -301,7 +303,7 @@ impl ASAPQueryEngine { subtree.instant.full_history = false; subtree.instant.cumulative_readout = true; let requirement = readiness_requirement(&subtree); - let index = self.sketch_index.as_ref().ok_or_else(|| { + let index = self.summary_store.as_ref().ok_or_else(|| { EngineError::capability_miss( "installed_logical_dag", "summary store unavailable", @@ -382,7 +384,7 @@ impl ASAPQueryEngine { )) }); let current = self - .sketch_index + .summary_store .as_ref() .map(|index| index.summary_update_revision()); if match (revision, current) { @@ -400,7 +402,7 @@ impl ASAPQueryEngine { async fn execute_logical_range( &self, - physical: &crate::storage_engines::types::ActivePhysicalPlan, + physical: &crate::storage_engines::types::RuntimePhysicalPlan, entry: &asap_types::query_plan::QueryPlanEntry, start: u64, end: u64, @@ -420,7 +422,7 @@ impl ASAPQueryEngine { let times: Vec = (0..=(end - start) / step) .map(|n| start + n * step) .collect(); - let leaves = self.prepare_logical(physical, entry, ×).await?; + let leaves = self.prepare_query_inputs(physical, entry, ×).await?; let mut series = std::collections::BTreeMap::, RangeVectorElement>::new(); let mut total = super::logical_dag::ExecutionStats::default(); @@ -475,18 +477,18 @@ impl ASAPQueryEngine { pub fn with_active_physical_plan( mut self, - handle: crate::storage_engines::types::HotReloadActivePhysicalPlan, + handle: crate::storage_engines::types::ActivePhysicalPlanHandle, ) -> Self { self.active_physical_plan = Some(handle); self } - fn physical_plan_snapshot( + fn active_physical_plan_snapshot( &self, - ) -> Option> { + ) -> Option> { self.active_physical_plan .as_ref() - .map(|handle| handle.snapshot()) + .map(|handle| handle.active_snapshot()) .filter(|plan| plan.plan_id() != 0) } @@ -511,7 +513,7 @@ impl ASAPQueryEngine { mut self, index: Arc, ) -> Self { - self.sketch_index = Some(index); + self.summary_store = Some(index); self } @@ -628,7 +630,7 @@ impl ASAPQueryEngine { step_ms: u64, ) -> Result { - if let Some(physical) = self.physical_plan_snapshot() { + if let Some(physical) = self.active_physical_plan_snapshot() { if let Ok(entry) = physical.query_plan.lookup(query) { if entry.nodes.values().any(|node| { matches!(node, asap_types::query_plan::QueryPlanNode::Logical { .. }) @@ -639,14 +641,14 @@ impl ASAPQueryEngine { } } } - let Some(idx) = self.sketch_index.as_ref() else { + let Some(idx) = self.summary_store.as_ref() else { return Err(crate::query_engines::EngineError::capability_miss( crate::storage_engines::types::StorageBackend::SketchStore.data_source_id(), format!("ASAPQueryEngine: no sketch index for `{query}` — failing over"), )); }; - let physical_plan = self.physical_plan_snapshot(); + let physical_plan = self.active_physical_plan_snapshot(); let mut readiness = None; let planned = match physical_plan.as_ref() { Some(physical_plan) => match physical_plan.query_plan.lookup(query) { @@ -972,10 +974,10 @@ impl crate::query_engines::routing::query_engine_routing::QueryEngine for ASAPQu now_ms: u64, ) -> Result { - if let Some(physical) = self.physical_plan_snapshot() { + if let Some(physical) = self.active_physical_plan_snapshot() { if let Ok(entry) = physical.query_plan.lookup(query) { let leaves = self - .prepare_logical(&physical, entry, &[now_ms]) + .prepare_query_inputs(&physical, entry, &[now_ms]) .await .map_err(|error| { tracing::warn!(query, error = %error, "installed query DAG preparation failed"); @@ -998,8 +1000,8 @@ impl crate::query_engines::routing::query_engine_routing::QueryEngine for ASAPQu // SummaryCatalog/materialization resolver → SID lookup → DAG executor. // A typed resolver/executor error becomes CapabilityMiss, which lets // EngineRouter continue to the archive backend. - if let Some(idx) = self.sketch_index.as_ref() { - let physical_plan = self.physical_plan_snapshot(); + if let Some(idx) = self.summary_store.as_ref() { + let physical_plan = self.active_physical_plan_snapshot(); let mut readiness = None; let planned = match physical_plan.as_ref() { Some(physical_plan) => match physical_plan.query_plan.lookup(query) { @@ -1457,11 +1459,11 @@ mod aux_pushdown_tests { fn make_engine() -> ASAPQueryEngine { use crate::storage_engines::types::{ - CleanupPolicy, HotReloadStreamingConfig, StreamingConfig, + CleanupPolicy, StreamingConfig, StreamingConfigHandle, }; let sc = Arc::new(StreamingConfig::new(HashMap::new())); - let hr = HotReloadStreamingConfig::from_arc(sc.clone()); + let hr = StreamingConfigHandle::from_arc(sc.clone()); let _ = sc; ASAPQueryEngine::new(60) } @@ -1586,10 +1588,10 @@ mod asap_tier_classify_tests { use crate::query_engines::routing::query_engine_routing::QueryEngine as _; use crate::query_engines::EngineError; use crate::storage_engines::sketch_db::index::{ - AccuracyBound, Capability, SketchAlgorithm, SketchConfig, SketchInstanceMetadata, - SketchSampleState, SketchStore, + AccuracyBound, Capability, SketchAlgorithm, SketchConfig, SketchSampleState, SketchStore, + SummarySeriesMetadata, }; - use crate::storage_engines::types::{CleanupPolicy, HotReloadStreamingConfig}; + use crate::storage_engines::types::{CleanupPolicy, StreamingConfigHandle}; use std::collections::{BTreeMap, BTreeSet}; /// `sum by (zone) (http_requests_total)` end-to-end via the @@ -1620,7 +1622,7 @@ mod asap_tier_classify_tests { for (i, zone) in zones.iter().enumerate() { let sid = 9000 + i as u64; - idx.register(SketchInstanceMetadata { + idx.register(SummarySeriesMetadata { sid, metric_name: "http_requests_total".to_string(), group_by_keys: ["zone".to_string()].into_iter().collect(), @@ -1704,14 +1706,14 @@ mod asap_tier_classify_tests { assert_eq!(by_zone.get("z3").copied(), Some(400.0)); } - // Build a now-anchored KLL `SketchInstanceMetadata` + sample so the + // Build a now-anchored KLL `SummarySeriesMetadata` + sample so the // engine's instant/range default lookbacks reach it. Mirrors the // live MVP workload: the agent emits a bare-named KLL sketch // (`http_requests_total_latency_ms`) into the SketchStore. - fn kll_meta(sid: u64, metric: &str) -> SketchInstanceMetadata { + fn kll_meta(sid: u64, metric: &str) -> SummarySeriesMetadata { // Latest ASAPPlanner sizes an epsilon=0.01 KLL at k=269. let cfg = SketchConfig::Kll { k: 269 }; - SketchInstanceMetadata { + SummarySeriesMetadata { sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), @@ -1746,10 +1748,10 @@ mod asap_tier_classify_tests { env.encode_to_vec() } - fn hll_meta(sid: u64, metric: &str) -> SketchInstanceMetadata { + fn hll_meta(sid: u64, metric: &str) -> SummarySeriesMetadata { // Latest ASAPPlanner requires p=14 for a 1% HLL error target. let cfg = SketchConfig::Hll { precision: 14 }; - SketchInstanceMetadata { + SummarySeriesMetadata { sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), @@ -2266,7 +2268,7 @@ mod asap_tier_classify_tests { for (i, (zone, per_window)) in [("z0", 600.0_f64), ("z1", 900.0)].iter().enumerate() { let sid = 14_000 + i as u64; - idx.register(SketchInstanceMetadata { + idx.register(SummarySeriesMetadata { sid, metric_name: "http_requests_total".to_string(), group_by_keys: ["zone".to_string()].into_iter().collect(), @@ -2362,7 +2364,7 @@ mod asap_tier_classify_tests { ) { // Matches ControlPlaneCostModel's epsilon=0.01 CMS sizing. let cfg = SketchConfig::CountMin { rows: 5, cols: 512 }; - idx.register(SketchInstanceMetadata { + idx.register(SummarySeriesMetadata { sid, metric_name: metric.to_string(), group_by_keys: group_by @@ -2476,9 +2478,9 @@ mod outer_agg_integration_tests { use crate::query_engines::EngineError; use crate::storage_engines::sketch_db::index::{ AccuracyBound, Capability, SketchAlgorithm, SketchConfig, SketchEncoding, - SketchInstanceMetadata, SketchSampleState, SketchStore, + SketchSampleState, SketchStore, SummarySeriesMetadata, }; - use crate::storage_engines::types::HotReloadStreamingConfig; + use crate::storage_engines::types::StreamingConfigHandle; use asap_sketchlib::DdSketch; use asap_sketchlib::MessagePackCodec; use std::collections::{BTreeMap, BTreeSet}; @@ -2494,11 +2496,11 @@ mod outer_agg_integration_tests { sk.to_msgpack().expect("ddsketch msgpack serialization") } - fn dd_meta_for(sid: u64, metric: &str, group_by: &[&str]) -> SketchInstanceMetadata { + fn dd_meta_for(sid: u64, metric: &str, group_by: &[&str]) -> SummarySeriesMetadata { let cfg = SketchConfig::DDSketch { relative_accuracy: 0.01, }; - SketchInstanceMetadata { + SummarySeriesMetadata { sid, metric_name: metric.to_string(), group_by_keys: group_by @@ -2708,9 +2710,9 @@ mod range_stitch_tests { use crate::query_engines::EngineError; use crate::storage_engines::sketch_db::index::{ AccuracyBound, Capability, SketchAlgorithm, SketchConfig, SketchEncoding, - SketchInstanceMetadata, SketchSampleState, SketchStore, + SketchSampleState, SketchStore, SummarySeriesMetadata, }; - use crate::storage_engines::types::{HotReloadStreamingConfig, KeyByLabelValues}; + use crate::storage_engines::types::{KeyByLabelValues, StreamingConfigHandle}; use async_trait::async_trait; use std::collections::{BTreeMap, BTreeSet}; @@ -2771,9 +2773,9 @@ mod range_stitch_tests { /// A CountMin FrequencyEstimate sid — `count_over_time` over it emits one /// PER-WINDOW sample (not a single cumulative scalar), which is what the /// range stitch needs so warm contributes one value per covered window. - fn cms_meta(sid: u64, metric: &str) -> SketchInstanceMetadata { + fn cms_meta(sid: u64, metric: &str) -> SummarySeriesMetadata { let cfg = SketchConfig::CountMin { rows: 5, cols: 512 }; - SketchInstanceMetadata { + SummarySeriesMetadata { sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), @@ -2852,13 +2854,13 @@ mod range_stitch_tests { FallbackPolicy, InstantExecution, QueryLanguage, QueryNodeId, QueryPlanEntry, QueryPlanNode, }; - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" )) .unwrap(); let mut plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let identity = asap_types::query_plan::canonical_promql("1 + 2").unwrap(); plan.query_plan.entries.insert( @@ -2888,7 +2890,7 @@ mod range_stitch_tests { fallback: FallbackPolicy::ExactBackend, }, ); - let mut active = crate::drivers::query::servers::http::build_active_physical_plan( + let mut active = crate::drivers::query::servers::http::validate_and_build_runtime_plan( crate::drivers::query::servers::http::PhysicalPlanInstallRequest { summary_catalog: plan.summary_catalog, collector_plans: plan.collector_plans, @@ -2902,8 +2904,8 @@ mod range_stitch_tests { ) .unwrap(); active.envelope.expiry_unix_ms = None; - let active = crate::storage_engines::types::HotReloadActivePhysicalPlan::new(active); - let hot = HotReloadStreamingConfig::from_active(active.clone()); + let active = crate::storage_engines::types::ActivePhysicalPlanHandle::new(active); + let hot = StreamingConfigHandle::from_active_physical_plan(active.clone()); let engine = ASAPQueryEngine::new(15).with_active_physical_plan(active); let error = engine .execute_metricsql_at(&identity, 1_000) diff --git a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs index c0ae6e712..1630051e2 100644 --- a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs +++ b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs @@ -2,8 +2,8 @@ use super::logical_dag::{PreparedLeaf, PreparedLeaves, Value}; use crate::query_engines::EngineError; use asap_types::query_plan::{ - logical::LogicalOperator, ExternalExactInput, ExternalExactRequest, QueryLanguage, QueryNodeId, - QueryPlanEntry, QueryPlanNode, + logical::ResidualQueryOperator, ExternalExactInput, ExternalExactRequest, QueryLanguage, + QueryNodeId, QueryPlanEntry, QueryPlanNode, }; use std::collections::{BTreeMap, BTreeSet, HashMap}; @@ -17,7 +17,7 @@ fn miss(message: impl Into) -> EngineError { /// Traverse only the installed graph, including epoch-aligned nested subquery grids. #[derive(Debug, Clone)] enum ExactLeaf { - Legacy(LogicalOperator), + Legacy(ResidualQueryOperator), External(ExternalExactRequest), } @@ -47,14 +47,14 @@ fn leaves( .ok_or_else(|| miss("missing installed node"))?; match node { QueryPlanNode::Logical { operator, inputs } => match operator { - LogicalOperator::Scan { .. } => { + ResidualQueryOperator::Scan { .. } => { return Err(miss("local raw Scan is forbidden in deployed plans")) } - LogicalOperator::ExactSubquery { .. } - | LogicalOperator::CandidateExactSubquery { .. } => { + ResidualQueryOperator::ExactSubquery { .. } + | ResidualQueryOperator::CandidateExactSubquery { .. } => { result.insert((id, at), ExactLeaf::Legacy(operator.clone())); } - LogicalOperator::Subquery { + ResidualQueryOperator::Subquery { range_ms, step_ms, offset_ms, @@ -116,7 +116,7 @@ pub(super) fn external_dependencies( ); } else if matches!( leaf, - ExactLeaf::Legacy(LogicalOperator::CandidateExactSubquery { .. }) + ExactLeaf::Legacy(ResidualQueryOperator::CandidateExactSubquery { .. }) ) { let input = *entry.nodes[&id] .inputs() @@ -298,10 +298,13 @@ pub(super) async fn prepare_external( for ((id, at), leaf) in leaves(entry, times)? { u64::try_from(at).map_err(|_| miss("subquery predates epoch"))?; let (language, query, candidate_input) = match &leaf { - ExactLeaf::Legacy(LogicalOperator::ExactSubquery { query }) => { + ExactLeaf::Legacy(ResidualQueryOperator::ExactSubquery { query }) => { (QueryLanguage::PromQl, query.clone(), None) } - ExactLeaf::Legacy(LogicalOperator::CandidateExactSubquery { query, item_label }) => ( + ExactLeaf::Legacy(ResidualQueryOperator::CandidateExactSubquery { + query, + item_label, + }) => ( QueryLanguage::PromQl, query.clone(), Some((entry.nodes[&id].inputs()[0], item_label.as_str())), @@ -751,7 +754,7 @@ mod tests { #[tokio::test] async fn exact_leaf_calls_prometheus_and_combines_with_prepared_summary() { // A successful exact branch remains an intermediate, not a whole-root fallback. - use asap_types::query_plan::logical::BinaryOperation; + use asap_types::query_plan::residual::BinaryOperation; use std::sync::{ atomic::{AtomicUsize, Ordering}, Arc, @@ -771,7 +774,7 @@ mod tests { ( QueryNodeId(0), QueryPlanNode::Logical { - operator: LogicalOperator::Binary { + operator: ResidualQueryOperator::Binary { operation: BinaryOperation::Div, return_bool: false, }, @@ -785,7 +788,7 @@ mod tests { ( QueryNodeId(2), QueryPlanNode::Logical { - operator: LogicalOperator::ExactSubquery { query: "b".into() }, + operator: ResidualQueryOperator::ExactSubquery { query: "b".into() }, inputs: vec![], }, ), @@ -827,7 +830,7 @@ mod tests { repeated.nodes.insert( QueryNodeId(1), QueryPlanNode::Logical { - operator: LogicalOperator::ExactSubquery { query: "b".into() }, + operator: ResidualQueryOperator::ExactSubquery { query: "b".into() }, inputs: vec![], }, ); @@ -863,7 +866,7 @@ mod tests { use crate::query_engines::query_result::{InstantVectorElement, QueryResult}; use crate::storage_engines::sketch_db::{ data::AggKind, - index::{Capability, SketchInstanceMetadata}, + index::{Capability, SummarySeriesMetadata}, }; use crate::storage_engines::types::{KeyByLabelValues, Measurement}; use asap_types::query_plan::{ @@ -877,7 +880,7 @@ mod tests { const AT: u64 = 300_000; const MATERIALIZATION: asap_types::PolicyFingerprint = asap_types::PolicyFingerprint(9001); let store = crate::storage_engines::sketch_db::index::SketchStore::new(); - store.register(SketchInstanceMetadata { + store.register(SummarySeriesMetadata { sid: 41, metric_name: "http_requests_total".into(), group_by_keys: std::collections::BTreeSet::from(["job".into()]), @@ -909,7 +912,7 @@ mod tests { ( QueryNodeId(0), QueryPlanNode::Logical { - operator: LogicalOperator::Binary { + operator: ResidualQueryOperator::Binary { operation: BinaryOperation::Div, return_bool: false, }, @@ -919,7 +922,7 @@ mod tests { ( QueryNodeId(1), QueryPlanNode::Logical { - operator: LogicalOperator::ExactSubquery { + operator: ResidualQueryOperator::ExactSubquery { query: exact_query.into(), }, inputs: vec![], @@ -1035,7 +1038,7 @@ mod tests { let entry = entry(BTreeMap::from([( QueryNodeId(0), QueryPlanNode::Logical { - operator: LogicalOperator::Scan { + operator: ResidualQueryOperator::Scan { metric: Some("m".into()), matchers: vec![], range_ms: None, diff --git a/data_plane/src/query_engines/asap_query_engine/live_serve.rs b/data_plane/src/query_engines/asap_query_engine/live_serve.rs index 15690399a..577409c86 100644 --- a/data_plane/src/query_engines/asap_query_engine/live_serve.rs +++ b/data_plane/src/query_engines/asap_query_engine/live_serve.rs @@ -131,7 +131,7 @@ mod tests { use std::collections::BTreeMap; use crate::storage_engines::sketch_db::data::AggKind; - use crate::storage_engines::sketch_db::index::{Capability, SketchInstanceMetadata}; + use crate::storage_engines::sketch_db::index::{Capability, SummarySeriesMetadata}; #[test] fn flag_off_spellings_disable_live_serve() { @@ -149,7 +149,7 @@ mod tests { fn formal_range_plan_returns_exact_requested_steps() { let idx = SketchStore::new(); let policy = asap_types::PolicyFingerprint(901); - idx.register(SketchInstanceMetadata { + idx.register(SummarySeriesMetadata { sid: 9, metric_name: "bytes".into(), group_by_keys: std::collections::BTreeSet::new(), diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs index 93c124072..201966b01 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs @@ -4,8 +4,8 @@ use crate::query_engines::{ EngineError, }; use crate::storage_engines::types::KeyByLabelValues; -use asap_types::query_plan::logical::{ - Aggregation, BinaryOperation, Grouping, LogicalOperator, TemporalOperation, +use asap_types::query_plan::residual::{ + Aggregation, BinaryOperation, Grouping, ResidualQueryOperator, TemporalOperation, }; use asap_types::query_plan::{CandidateCompleteness, QueryNodeId, QueryPlanEntry, QueryPlanNode}; use std::collections::{BTreeMap, BTreeSet}; @@ -185,9 +185,9 @@ impl Result> Evaluator<' QueryPlanNode::Logical { operator, inputs } => { if matches!( operator, - LogicalOperator::Scan { .. } - | LogicalOperator::ExactSubquery { .. } - | LogicalOperator::CandidateExactSubquery { .. } + ResidualQueryOperator::Scan { .. } + | ResidualQueryOperator::ExactSubquery { .. } + | ResidualQueryOperator::CandidateExactSubquery { .. } ) { return Err(miss( "installed Prometheus leaf was not prepared; backend raw execution is forbidden", @@ -224,7 +224,7 @@ impl Result> Evaluator<' } fn logical( &mut self, - operator: LogicalOperator, + operator: ResidualQueryOperator, inputs: &[QueryNodeId], at: i64, ) -> Result { @@ -235,14 +235,14 @@ impl Result> Evaluator<' .ok_or_else(|| miss("missing logical input")) }; match operator { - LogicalOperator::ExactSubquery { .. } - | LogicalOperator::CandidateExactSubquery { .. } => { + ResidualQueryOperator::ExactSubquery { .. } + | ResidualQueryOperator::CandidateExactSubquery { .. } => { Err(miss("Prometheus exact leaf was not prepared")) } - LogicalOperator::Scan { .. } => { + ResidualQueryOperator::Scan { .. } => { Err(miss("local raw Scan is forbidden in deployed plans")) } - LogicalOperator::UnaryNegate => match self.eval(input(0)?, at)? { + ResidualQueryOperator::UnaryNegate => match self.eval(input(0)?, at)? { Value::Scalar(value) => Ok(Value::Scalar(-value)), Value::Vector(values) => Ok(Value::Vector( values @@ -252,7 +252,7 @@ impl Result> Evaluator<' )), _ => Err(miss("cannot negate range vector")), }, - LogicalOperator::VectorToScalar => { + ResidualQueryOperator::VectorToScalar => { let values = vector(self.eval(input(0)?, at)?)?; Ok(Value::Scalar(if values.len() == 1 { values[0].1 @@ -260,18 +260,18 @@ impl Result> Evaluator<' f64::NAN })) } - LogicalOperator::Aggregate { + ResidualQueryOperator::Aggregate { operation, grouping, } => { let values = vector(self.eval(input(0)?, at)?)?; Ok(Value::Vector(aggregate(operation, &grouping, values))) } - LogicalOperator::TopKSelection { k, grouping } => { + ResidualQueryOperator::TopKSelection { k, grouping } => { let values = vector(self.eval(input(0)?, at)?)?; Ok(Value::Vector(topk_selection(k, &grouping, values))) } - LogicalOperator::Binary { + ResidualQueryOperator::Binary { operation, return_bool, } => { @@ -279,7 +279,7 @@ impl Result> Evaluator<' let right = self.eval(input(1)?, at)?; binary(operation, return_bool, left, right) } - LogicalOperator::Temporal { operation } => { + ResidualQueryOperator::Temporal { operation } => { let Value::Matrix(values, start, end) = self.eval(input(0)?, at)? else { return Err(miss("temporal operator requires range vector")); }; @@ -340,7 +340,7 @@ impl Result> Evaluator<' .collect(), )) } - LogicalOperator::Sort { descending } => { + ResidualQueryOperator::Sort { descending } => { let mut values = vector(self.eval(input(0)?, at)?)?; values.sort_by(|a, b| { if a.1.is_nan() && b.1.is_nan() { @@ -357,7 +357,7 @@ impl Result> Evaluator<' }); Ok(Value::Vector(values)) } - LogicalOperator::HistogramQuantile => { + ResidualQueryOperator::HistogramQuantile => { let Value::Scalar(quantile) = self.eval(input(0)?, at)? else { return Err(miss("quantile requires scalar")); }; @@ -374,7 +374,7 @@ impl Result> Evaluator<' .collect(), )) } - LogicalOperator::Subquery { + ResidualQueryOperator::Subquery { range_ms, step_ms, offset_ms, @@ -804,7 +804,7 @@ mod topk_tests { #[test] fn installed_topk_combines_with_prometheus_exact_child() { - let mut entry = control_plane::query_plan::logical::compile_logical( + let mut entry = control_plane::query_plan::residual::compile_logical( "hybrid-topk".into(), "topk(2, m)".into(), InstantExecution { @@ -815,7 +815,7 @@ mod topk_tests { FallbackPolicy::ExactBackend, ) .unwrap(); - control_plane::query_plan::logical::finalize_residuals(&mut entry).unwrap(); + control_plane::query_plan::residual::finalize_residuals(&mut entry).unwrap(); let leaf = entry .nodes .iter() @@ -823,7 +823,7 @@ mod topk_tests { matches!( node, QueryPlanNode::Logical { - operator: LogicalOperator::ExactSubquery { .. }, + operator: ResidualQueryOperator::ExactSubquery { .. }, .. } ) @@ -890,7 +890,7 @@ mod topk_tests { ( QueryNodeId(0), QueryPlanNode::Logical { - operator: LogicalOperator::ExactSubquery { + operator: ResidualQueryOperator::ExactSubquery { query: "m[1s]".into(), }, inputs: vec![], @@ -899,7 +899,7 @@ mod topk_tests { ( QueryNodeId(1), QueryPlanNode::Logical { - operator: LogicalOperator::Temporal { operation }, + operator: ResidualQueryOperator::Temporal { operation }, inputs: vec![QueryNodeId(0)], }, ), @@ -974,7 +974,7 @@ mod topk_tests { ( root, QueryPlanNode::Logical { - operator: LogicalOperator::TopKSelection { + operator: ResidualQueryOperator::TopKSelection { k: 2, grouping: Grouping { labels: vec![], diff --git a/data_plane/src/query_engines/asap_query_engine/mod.rs b/data_plane/src/query_engines/asap_query_engine/mod.rs index 3fc00fb45..b9db4172f 100644 --- a/data_plane/src/query_engines/asap_query_engine/mod.rs +++ b/data_plane/src/query_engines/asap_query_engine/mod.rs @@ -13,10 +13,10 @@ pub mod summary_executor; pub use crate::storage_engines::sketch_db::query as asap_tier; +pub use engine::ASAPQueryEngine; + #[cfg(test)] pub mod tests; -pub use engine::ASAPQueryEngine; - #[cfg(test)] mod test_plan; diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index 356c9e0bc..c2ddc7eed 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -742,7 +742,7 @@ mod tests { } use crate::storage_engines::sketch_db::data::{AggKind, SketchConfig}; use crate::storage_engines::sketch_db::index::{ - AccuracyBound, Capability, SketchAlgorithm, SketchInstanceMetadata, SketchSampleState, + AccuracyBound, Capability, SketchAlgorithm, SketchSampleState, SummarySeriesMetadata, }; fn register_hll(idx: &SketchStore, sid: u64, service: &str, items: &[&str]) { @@ -755,7 +755,7 @@ mod tests { let cfg = SketchConfig::Hll { precision: 14 }; let mut group_by_keys = std::collections::BTreeSet::new(); group_by_keys.insert("service".to_string()); - idx.register(SketchInstanceMetadata { + idx.register(SummarySeriesMetadata { sid, metric_name: "unique_users".to_string(), group_by_keys, @@ -797,7 +797,7 @@ mod tests { let cfg = SketchConfig::DDSketch { relative_accuracy: 0.01, }; - idx.register(SketchInstanceMetadata { + idx.register(SummarySeriesMetadata { sid: 1, metric_name: "latency_ms".to_string(), group_by_keys: std::collections::BTreeSet::new(), @@ -1023,7 +1023,7 @@ mod tests { fn exact_agg_outcome_reports_window_end_coverage() { let idx = SketchStore::new(); idx.register( - crate::storage_engines::sketch_db::index::SketchInstanceMetadata { + crate::storage_engines::sketch_db::index::SummarySeriesMetadata { sid: 1, metric_name: "bytes_total".to_string(), group_by_keys: std::collections::BTreeSet::new(), @@ -1071,7 +1071,7 @@ mod tests { #[test] fn compiled_window_schedules_execute_exact_ranges() { use crate::precompute_engine::window_manager::WindowManager; - use control_plane::physical::compiler::{BackendLocalPlanningSnapshot, PhysicalCompiler}; + use control_plane::physical::compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}; for evaluation_secs in [20, 45, 60, 120, 90] { for phase_ms in [0, 5_000] { for full in [false, true] { @@ -1088,16 +1088,18 @@ mod tests { entry["demand"]["fixed_interval_at"] = serde_json::json!({ "interval": evaluation_secs * 1_000, "evaluation_phase": phase_ms }); - let snapshot: BackendLocalPlanningSnapshot = + let snapshot: BackendLocalPlanningInput = serde_json::from_value(snapshot).unwrap(); - let (mut request, env) = snapshot.planning_request().unwrap(); - request.queries[0].window_implementations.retain(|c| { - matches!( - c.layout, - asap_types::WindowMaterializationLayout::FullWindow - ) == full - }); - let plan = PhysicalCompiler.compile(request, env).unwrap(); + let (mut request, env) = snapshot.into_physical_compilation_request().unwrap(); + request.queries[0] + .window_realization_candidates + .retain(|c| { + matches!( + c.layout, + asap_types::WindowMaterializationLayout::FullWindow + ) == full + }); + let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); let config = &plan.precompute_plan.materializations[0]; let manager = WindowManager::with_layout( config.window_size, @@ -1116,7 +1118,7 @@ mod tests { } } let idx = SketchStore::new(); - idx.register(SketchInstanceMetadata { + idx.register(SummarySeriesMetadata { sid: 7, metric_name: "a".into(), group_by_keys: Default::default(), @@ -1168,7 +1170,7 @@ mod tests { // Compile the two readouts, store one pane series, and execute the actual ratio. #[test] fn compiled_shared_sum_panes_preserve_each_lookback() { - use control_plane::physical::compiler::{BackendLocalPlanningSnapshot, PhysicalCompiler}; + use control_plane::physical::compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}; let mut snapshot: serde_json::Value = serde_json::from_str(include_str!( "../../../../docs/examples/asapquery-planning-snapshot.json" )) @@ -1177,14 +1179,14 @@ mod tests { entry["query"] = serde_json::json!("sum_over_time(a[1m]) / sum_over_time(a[10m])"); entry["requirements"]["accuracy"]["explicit"] = serde_json::json!("Exact"); entry["demand"]["fixed_interval_at"]["interval"] = serde_json::json!(60_000); - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_value(snapshot).unwrap(); - let (request, env) = snapshot.planning_request().unwrap(); - let plan = PhysicalCompiler.compile(request, env).unwrap(); + let snapshot: BackendLocalPlanningInput = serde_json::from_value(snapshot).unwrap(); + let (request, env) = snapshot.into_physical_compilation_request().unwrap(); + let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); let config = &plan.precompute_plan.materializations[0]; let policy = config.policy_fingerprint(); let idx = SketchStore::new(); - idx.register(SketchInstanceMetadata { + idx.register(SummarySeriesMetadata { sid: 7, metric_name: "a".into(), group_by_keys: Default::default(), @@ -1258,7 +1260,7 @@ mod tests { fn repeated_multi_pane_reads_exclude_expired_state_and_reject_gaps() { let idx = SketchStore::new(); let policy = asap_types::PolicyFingerprint(777); - idx.register(SketchInstanceMetadata { + idx.register(SummarySeriesMetadata { sid: 7, metric_name: "requests_total".into(), group_by_keys: std::collections::BTreeSet::new(), @@ -1357,7 +1359,7 @@ mod tests { fn exact_query_plan_rate_uses_reset_aware_readout() { let idx = SketchStore::new(); let policy = asap_types::PolicyFingerprint(777); - idx.register(SketchInstanceMetadata { + idx.register(SummarySeriesMetadata { sid: 7, metric_name: "requests_total".into(), group_by_keys: std::collections::BTreeSet::new(), diff --git a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs index 954ca155d..27a0fb67e 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs @@ -1390,7 +1390,7 @@ mod tests { use super::*; use crate::query_engines::asap_query_engine::summary_exec::{execute, ExecOutcome}; use crate::storage_engines::sketch_db::index::{ - AccuracyBound, Capability, SketchInstanceMetadata, SketchSampleState, SketchStore, + AccuracyBound, Capability, SketchSampleState, SketchStore, SummarySeriesMetadata, }; use planner_types::post_asap::{SummaryField, SummarySchema}; use planner_types::pre_asap::{Column, DataType, Schema}; @@ -1536,9 +1536,9 @@ mod tests { }) } - fn kll_meta(sid: u64, metric: &str, group_by: &[&str]) -> SketchInstanceMetadata { + fn kll_meta(sid: u64, metric: &str, group_by: &[&str]) -> SummarySeriesMetadata { let cfg = SketchConfig::Kll { k: 200 }; - SketchInstanceMetadata { + SummarySeriesMetadata { sid, metric_name: metric.to_string(), group_by_keys: group_by @@ -1559,9 +1559,9 @@ mod tests { } } - fn hll_meta(sid: u64, metric: &str) -> SketchInstanceMetadata { + fn hll_meta(sid: u64, metric: &str) -> SummarySeriesMetadata { let cfg = SketchConfig::Hll { precision: 10 }; - SketchInstanceMetadata { + SummarySeriesMetadata { sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), @@ -1605,9 +1605,9 @@ mod tests { sk.to_msgpack().expect("encode HLL msgpack") } - fn cms_meta(sid: u64, metric: &str) -> SketchInstanceMetadata { + fn cms_meta(sid: u64, metric: &str) -> SummarySeriesMetadata { let cfg = SketchConfig::CountMin { rows: 4, cols: 256 }; - SketchInstanceMetadata { + SummarySeriesMetadata { sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), @@ -1678,9 +1678,9 @@ mod tests { }) } - fn cms_with_heap_meta(sid: u64, metric: &str) -> SketchInstanceMetadata { + fn cms_with_heap_meta(sid: u64, metric: &str) -> SummarySeriesMetadata { let cfg = SketchConfig::CountMin { rows: 4, cols: 256 }; - SketchInstanceMetadata { + SummarySeriesMetadata { sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), @@ -1749,8 +1749,8 @@ mod tests { // ── ExactAgg (Sum) fixtures ──────────────────────────────────────── - fn sum_exact_agg_meta(sid: u64, metric: &str, group_by: &[&str]) -> SketchInstanceMetadata { - SketchInstanceMetadata { + fn sum_exact_agg_meta(sid: u64, metric: &str, group_by: &[&str]) -> SummarySeriesMetadata { + SummarySeriesMetadata { sid, metric_name: metric.to_string(), group_by_keys: group_by diff --git a/data_plane/src/query_engines/asap_query_engine/test_plan.rs b/data_plane/src/query_engines/asap_query_engine/test_plan.rs index 9f90196d2..8b9f5b57a 100644 --- a/data_plane/src/query_engines/asap_query_engine/test_plan.rs +++ b/data_plane/src/query_engines/asap_query_engine/test_plan.rs @@ -3,10 +3,10 @@ use super::engine::ASAPQueryEngine; use crate::drivers::query::servers::http::{ - build_active_physical_plan, PhysicalPlanInstallRequest, + validate_and_build_runtime_plan, PhysicalPlanInstallRequest, }; use crate::storage_engines::sketch_db::index::SketchStore; -use crate::storage_engines::types::{BackendStorageRouting, HotReloadActivePhysicalPlan}; +use crate::storage_engines::types::{ActivePhysicalPlanHandle, BackendStorageRouting}; use asap_types::precompute_plan::{PlanEnvelope, PrecomputePlan, BACKEND_COMPAT}; use asap_types::query_plan::*; use asap_types::PrecomputeMaterialization; @@ -83,7 +83,7 @@ pub(super) fn install( index: &SketchStore, configs: &[(PrecomputeMaterialization, Vec)], entries: Vec, -) -> HotReloadActivePhysicalPlan { +) -> ActivePhysicalPlanHandle { let envelope = PlanEnvelope { plan_id: 1, plan_version: 1, @@ -101,7 +101,7 @@ pub(super) fn install( let mut precompute = PrecomputePlan::build(envelope.clone(), materializations, &["fixture".into()]).unwrap(); precompute.summary_catalog = Some(catalog.reference().unwrap()); - let mut transmission = control_plane::physical::compiler::compile_transmission_plan( + let mut transmission = control_plane::physical::compiler::build_transmission_plan( envelope, &precompute, &BTreeMap::new(), @@ -118,7 +118,7 @@ pub(super) fn install( index.register(metadata); } } - let active = build_active_physical_plan( + let active = validate_and_build_runtime_plan( PhysicalPlanInstallRequest { summary_catalog: catalog, collector_plans: vec![], @@ -139,7 +139,7 @@ pub(super) fn install( Arc::new(BackendStorageRouting::empty()), ) .unwrap(); - HotReloadActivePhysicalPlan::new(active) + ActivePhysicalPlanHandle::new(active) } pub(super) fn engine( diff --git a/data_plane/src/query_engines/routing/backend_storage_routing.rs b/data_plane/src/query_engines/routing/backend_storage_routing.rs index 3ee0c2d9f..97923f885 100644 --- a/data_plane/src/query_engines/routing/backend_storage_routing.rs +++ b/data_plane/src/query_engines/routing/backend_storage_routing.rs @@ -802,7 +802,7 @@ pub fn routing_table_hash(table: &BackendStorageRouting) -> String { // --------------------------------------------------------------------------- /// Per-tenant atomic-swap wrapper around `BackendStorageRouting`, -/// mirroring [`crate::storage_engines::types::HotReloadStreamingConfig`]. Lets the +/// mirroring [`crate::storage_engines::types::StreamingConfigHandle`]. Lets the /// `POST /api/v1/storage_routing` HTTP handler swap one tenant's table /// at runtime without restarting the backend or touching any other /// tenant's table. Cloneable; clones share the underlying `ArcSwap` so @@ -851,7 +851,7 @@ pub struct HotReloadBackendStorageRouting { /// once and pick the tenant's `Arc`. inner: std::sync::Arc>>>, - active: Option, + active: Option, } impl HotReloadBackendStorageRouting { @@ -892,8 +892,8 @@ impl HotReloadBackendStorageRouting { } } - pub fn from_active(active: crate::storage_engines::types::HotReloadActivePhysicalPlan) -> Self { - let initial = active.snapshot().storage_routing.clone(); + pub fn from_active(active: crate::storage_engines::types::ActivePhysicalPlanHandle) -> Self { + let initial = active.active_snapshot().storage_routing.clone(); let mut map = HashMap::new(); map.insert(initial.tenant().to_string(), initial); Self { @@ -919,7 +919,7 @@ impl HotReloadBackendStorageRouting { /// caller's lifetime; concurrent swaps don't invalidate it. pub fn snapshot_for_tenant(&self, tenant: &str) -> std::sync::Arc { if let Some(active) = &self.active { - let routing = active.snapshot().storage_routing.clone(); + let routing = active.active_snapshot().storage_routing.clone(); if routing.tenant() == tenant || tenant == DEFAULT_TENANT { return routing; } diff --git a/data_plane/src/storage_engines/mod.rs b/data_plane/src/storage_engines/mod.rs index 9a0ffcb72..72e910c8d 100644 --- a/data_plane/src/storage_engines/mod.rs +++ b/data_plane/src/storage_engines/mod.rs @@ -24,7 +24,7 @@ pub mod types; pub use sketch_db::index::{ AccuracyBound, Capability, SeriesLookup, SketchAlgorithm, SketchConfig, SketchEncoding, - SketchInstanceMetadata, SketchSampleState, SketchStore, SketchTimeSeries, + SketchSampleState, SketchStore, SketchTimeSeries, SummarySeriesMetadata, }; pub use sketch_db::AggStatus; pub use traits::*; diff --git a/data_plane/src/storage_engines/sketch_db/backfill/mod.rs b/data_plane/src/storage_engines/sketch_db/backfill/mod.rs index 2a9d28bfe..0e970b821 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/mod.rs @@ -863,6 +863,29 @@ fn now_ms() -> u64 { .unwrap_or(0) } +// 2026-05 reorg: backfill-* and the two BackfillSource impls moved +// into this folder as submodules. +pub mod clickhouse_reader; +pub mod processor; +pub mod prometheus_reader; +pub mod raw_sample_reader; +pub mod service; +pub mod window_builder; +pub mod worker; + +pub use clickhouse_reader::{clickhouse_reader_factory, ClickHouseReader, ClickHouseReaderConfig}; +pub use processor::BackfillWindowProcessor; +pub use prometheus_reader::PrometheusReader; +pub use raw_sample_reader::{ + LabelFilter, MockRawSampleReader, RawSample, RawSampleReader, RawSampleReaderError, +}; +pub use service::{ + default_reader_factory, noop_reader_factory, BackfillService, BackfillServiceConfig, + BackfillServiceHandle, ReaderFactory, +}; +pub use window_builder::build_backfilled_accumulator; +pub use worker::{BackfillWorker, BackfillWorkerError, WindowProcessor}; + #[cfg(test)] mod tests { use super::*; @@ -1405,26 +1428,3 @@ mod tests { assert_eq!(r.coverage(1, (50, 10)), Coverage::Complete); } } - -// 2026-05 reorg: backfill-* and the two BackfillSource impls moved -// into this folder as submodules. -pub mod clickhouse_reader; -pub mod processor; -pub mod prometheus_reader; -pub mod raw_sample_reader; -pub mod service; -pub mod window_builder; -pub mod worker; - -pub use clickhouse_reader::{clickhouse_reader_factory, ClickHouseReader, ClickHouseReaderConfig}; -pub use processor::BackfillWindowProcessor; -pub use prometheus_reader::PrometheusReader; -pub use raw_sample_reader::{ - LabelFilter, MockRawSampleReader, RawSample, RawSampleReader, RawSampleReaderError, -}; -pub use service::{ - default_reader_factory, noop_reader_factory, BackfillService, BackfillServiceConfig, - BackfillServiceHandle, ReaderFactory, -}; -pub use window_builder::build_backfilled_accumulator; -pub use worker::{BackfillWorker, BackfillWorkerError, WindowProcessor}; diff --git a/data_plane/src/storage_engines/sketch_db/backfill/processor.rs b/data_plane/src/storage_engines/sketch_db/backfill/processor.rs index a8f8733ff..430aca740 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/processor.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/processor.rs @@ -21,7 +21,7 @@ use tracing::debug; use crate::drivers::ingest::population_attrs_fingerprint; use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::precompute_engine::worker::parse_labels_from_series_key; -use crate::storage_engines::types::{AggregateCore, HotReloadStreamingConfig, KeyByLabelValues}; +use crate::storage_engines::types::{AggregateCore, KeyByLabelValues, StreamingConfigHandle}; use asap_types::aggregation_config::AggregationConfig; use asap_types::PolicyFingerprint; @@ -126,10 +126,10 @@ pub struct BackfillWindowProcessor { /// `StreamingConfig` at each window to find the /// `AggregationConfig` for `agg_id`. The snapshot is cheap /// (Arc refcount bump) so we don't optimise further. - config: HotReloadStreamingConfig, + config: StreamingConfigHandle, /// Destination for rebuilt windows. Tests may omit it to record registry /// provenance without storing payloads. - sketch_index: Option>, + summary_store: Option>, /// Shared sid mint authority. Same `SeriesIdResolver` the OTel /// ingest path uses, so backfilled precompute sids land in the /// same unified namespace as live precompute / sketch sids. Only @@ -148,13 +148,13 @@ pub struct BackfillWindowProcessor { impl BackfillWindowProcessor { pub fn new( - config: HotReloadStreamingConfig, + config: StreamingConfigHandle, registry: Arc, job_id: u64, ) -> Self { Self { config, - sketch_index: None, + summary_store: None, series_resolver: None, registry, job_id, @@ -166,10 +166,10 @@ impl BackfillWindowProcessor { /// so existing call sites opt in with one chained call. pub fn with_sketch_index( mut self, - sketch_index: Arc, + summary_store: Arc, ) -> Self { - self.catalog_generation = sketch_index.active_catalog_generation(); - self.sketch_index = Some(sketch_index); + self.catalog_generation = summary_store.active_catalog_generation(); + self.summary_store = Some(summary_store); self } @@ -249,7 +249,7 @@ impl WindowProcessor for BackfillWindowProcessor { r.as_ref(), &config, &sample.labels, - self.sketch_index.as_deref(), + self.summary_store.as_deref(), self.catalog_generation.as_deref(), )?, None => fallback_bucket_id(&group_key), @@ -314,7 +314,7 @@ impl WindowProcessor for BackfillWindowProcessor { // Tests without a sketch index record provenance only. Writes with an index // require the shared sid resolver; otherwise log a warning and skip them. - if let Some(idx) = self.sketch_index.as_ref() { + if let Some(idx) = self.summary_store.as_ref() { match self.series_resolver.as_ref() { Some(_resolver) => { // B7.7 — sid is pre-resolved per bucket above; hand @@ -412,7 +412,7 @@ mod tests { let cfg = sum_config(1, "latency", vec!["svc"]); let fp = cfg.policy_fp_u64(); let streaming = streaming_config_with(cfg.clone()); - let hot = HotReloadStreamingConfig::from_arc(streaming.clone()); + let hot = StreamingConfigHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); let job_id = registry.create( fp, @@ -456,7 +456,7 @@ mod tests { async fn unknown_agg_id_fails_cleanly() { let cfg = sum_config(1, "m", vec![]); let streaming = streaming_config_with(cfg); - let hot = HotReloadStreamingConfig::from_arc(streaming.clone()); + let hot = StreamingConfigHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); let job_id = registry.create( 999, @@ -479,7 +479,7 @@ mod tests { let cfg = sum_config(1, "m", vec![]); let fp = cfg.policy_fp_u64(); let streaming = streaming_config_with(cfg); - let hot = HotReloadStreamingConfig::from_arc(streaming.clone()); + let hot = StreamingConfigHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); let job_id = registry.create( fp, @@ -500,7 +500,7 @@ mod tests { let cfg = sum_config(1, "latency", vec!["svc"]); let fp = cfg.policy_fp_u64(); let streaming = streaming_config_with(cfg); - let hot = HotReloadStreamingConfig::from_arc(streaming.clone()); + let hot = StreamingConfigHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); let job_id = registry.create( fp, @@ -775,7 +775,7 @@ mod tests { /// /// Drives `process_window` end-to-end with samples spanning two /// distinct `svc` values × two samples each. Asserts: - /// - exactly two `SketchInstanceMetadata` entries land in the + /// - exactly two `SummarySeriesMetadata` entries land in the /// `SketchStore` (one per distinct sid bucket) /// - their sids equal what the shared `SeriesIdResolver` would /// mint for the same `(metric, grouping-values, agg_kind)` @@ -792,16 +792,16 @@ mod tests { let cfg = sum_config(1, "latency", vec!["svc"]); let fp = cfg.policy_fp_u64(); let streaming = streaming_config_with(cfg.clone()); - let hot = HotReloadStreamingConfig::from_arc(streaming.clone()); + let hot = StreamingConfigHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); - let sketch_index = Arc::new(SketchStore::new()); + let summary_store = Arc::new(SketchStore::new()); let catalog = asap_types::summary_catalog::SummaryCatalog::from_materializations( 1, 1, &[cfg.clone()], ) .unwrap(); - sketch_index + summary_store .install_summary_catalog(Arc::new(catalog)) .unwrap(); let resolver = Arc::new(SeriesIdResolver::new()); @@ -814,7 +814,7 @@ mod tests { ); let processor = BackfillWindowProcessor::new(hot, registry.clone(), job_id) - .with_sketch_index(sketch_index.clone()) + .with_sketch_index(summary_store.clone()) .with_series_resolver(resolver.clone()); // Two distinct svc values × two samples each. Same window @@ -849,7 +849,7 @@ mod tests { // Two distinct sids landed in the index. assert_eq!( - sketch_index.instance_count(), + summary_store.instance_count(), 2, "one sid per distinct svc bucket" ); @@ -863,8 +863,8 @@ mod tests { let sid_b = resolve_backfill_bucket_sid(&resolver, &cfg, "latency{svc=\"b\"}", None, None).unwrap(); assert_ne!(sid_a, sid_b, "distinct svc values mint distinct sids"); - assert_eq!(sketch_index.classify(sid_a), SeriesLookup::Hit); - assert_eq!(sketch_index.classify(sid_b), SeriesLookup::Hit); + assert_eq!(summary_store.classify(sid_a), SeriesLookup::Hit); + assert_eq!(summary_store.classify(sid_b), SeriesLookup::Hit); // Provenance was recorded once per window (not once per // bucket) — same shape as the pre-rekey path. diff --git a/data_plane/src/storage_engines/sketch_db/backfill/prometheus_reader.rs b/data_plane/src/storage_engines/sketch_db/backfill/prometheus_reader.rs index 035af3977..8bee0b53a 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/prometheus_reader.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/prometheus_reader.rs @@ -316,79 +316,6 @@ impl<'de> Deserialize<'de> for MatrixPoint { } } -#[cfg(test)] -mod tests { - use super::*; - use std::collections::HashMap; - - #[test] - fn build_promql_bare_metric_when_no_filters() { - let f = LabelFilter::for_metric("latency"); - assert_eq!(build_promql(&f), "latency"); - } - - #[test] - fn build_promql_stable_ordering() { - let f = LabelFilter::for_metric("m") - .with_label("b", "2") - .with_label("a", "1"); - // Sorted by key regardless of insertion order. - assert_eq!(build_promql(&f), r#"m{a="1",b="2"}"#); - } - - #[test] - fn build_promql_escapes_quotes_and_backslashes() { - let f = LabelFilter::for_metric("m").with_label("k", r#"val"with\slash"#); - assert_eq!(build_promql(&f), r#"m{k="val\"with\\slash"}"#); - } - - #[test] - fn render_series_key_uses_name_and_sorts_labels() { - let mut metric = HashMap::new(); - metric.insert("__name__".to_string(), "latency".to_string()); - metric.insert("svc".to_string(), "a".to_string()); - metric.insert("env".to_string(), "prod".to_string()); - assert_eq!( - render_series_key("fallback", &metric), - r#"latency{env="prod",svc="a"}"# - ); - } - - #[test] - fn render_series_key_handles_bare_metric() { - let mut metric = HashMap::new(); - metric.insert("__name__".to_string(), "lone".to_string()); - assert_eq!(render_series_key("fallback", &metric), "lone"); - } - - #[test] - fn render_series_key_falls_back_to_default_when_no_name() { - let metric: HashMap = HashMap::new(); - assert_eq!(render_series_key("fallback", &metric), "fallback"); - } - - #[test] - fn fractional_seconds_round_trip() { - assert_eq!(ms_to_fractional_seconds(0), "0.000"); - assert_eq!(ms_to_fractional_seconds(1_234_567), "1234.567"); - assert_eq!(fractional_seconds_to_ms(1.5), 1500); - assert_eq!(fractional_seconds_to_ms(0.0), 0); - } - - #[tokio::test] - async fn read_samples_inverted_range_returns_invalid_range() { - let r = PrometheusReader::new("http://unused"); - let err = r - .read_samples(100, 50, &LabelFilter::for_metric("m")) - .await - .unwrap_err(); - match err { - RawSampleReaderError::InvalidRange { .. } => {} - other => panic!("expected InvalidRange, got {other}"), - } - } -} - // ── Integration tests against a mock Prometheus HTTP server ───────── #[cfg(test)] @@ -670,3 +597,76 @@ mod integration_tests { handle.abort(); } } + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + #[test] + fn build_promql_bare_metric_when_no_filters() { + let f = LabelFilter::for_metric("latency"); + assert_eq!(build_promql(&f), "latency"); + } + + #[test] + fn build_promql_stable_ordering() { + let f = LabelFilter::for_metric("m") + .with_label("b", "2") + .with_label("a", "1"); + // Sorted by key regardless of insertion order. + assert_eq!(build_promql(&f), r#"m{a="1",b="2"}"#); + } + + #[test] + fn build_promql_escapes_quotes_and_backslashes() { + let f = LabelFilter::for_metric("m").with_label("k", r#"val"with\slash"#); + assert_eq!(build_promql(&f), r#"m{k="val\"with\\slash"}"#); + } + + #[test] + fn render_series_key_uses_name_and_sorts_labels() { + let mut metric = HashMap::new(); + metric.insert("__name__".to_string(), "latency".to_string()); + metric.insert("svc".to_string(), "a".to_string()); + metric.insert("env".to_string(), "prod".to_string()); + assert_eq!( + render_series_key("fallback", &metric), + r#"latency{env="prod",svc="a"}"# + ); + } + + #[test] + fn render_series_key_handles_bare_metric() { + let mut metric = HashMap::new(); + metric.insert("__name__".to_string(), "lone".to_string()); + assert_eq!(render_series_key("fallback", &metric), "lone"); + } + + #[test] + fn render_series_key_falls_back_to_default_when_no_name() { + let metric: HashMap = HashMap::new(); + assert_eq!(render_series_key("fallback", &metric), "fallback"); + } + + #[test] + fn fractional_seconds_round_trip() { + assert_eq!(ms_to_fractional_seconds(0), "0.000"); + assert_eq!(ms_to_fractional_seconds(1_234_567), "1234.567"); + assert_eq!(fractional_seconds_to_ms(1.5), 1500); + assert_eq!(fractional_seconds_to_ms(0.0), 0); + } + + #[tokio::test] + async fn read_samples_inverted_range_returns_invalid_range() { + let r = PrometheusReader::new("http://unused"); + let err = r + .read_samples(100, 50, &LabelFilter::for_metric("m")) + .await + .unwrap_err(); + match err { + RawSampleReaderError::InvalidRange { .. } => {} + other => panic!("expected InvalidRange, got {other}"), + } + } +} diff --git a/data_plane/src/storage_engines/sketch_db/backfill/service.rs b/data_plane/src/storage_engines/sketch_db/backfill/service.rs index af75b380a..c0b35971a 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/service.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/service.rs @@ -58,7 +58,7 @@ use crate::storage_engines::sketch_db::backfill::worker::BackfillWorker; use crate::storage_engines::sketch_db::backfill::{ BackfillRegistry, BackfillSource, BackfillStatus, }; -use crate::storage_engines::types::HotReloadStreamingConfig; +use crate::storage_engines::types::StreamingConfigHandle; /// Given a `BackfillSource`, return a reader that can read raw /// samples from it. Used by the service to pick a concrete reader @@ -99,11 +99,11 @@ impl Default for BackfillServiceConfig { pub struct BackfillService { registry: Arc, /// Destination for rebuilt windows. - sketch_index: Option>, + summary_store: Option>, /// Shared sid mint authority — wired alongside `sketch_index` so /// backfilled precompute sids share the namespace with live ingest. series_resolver: Option>, - config_source: HotReloadStreamingConfig, + config_source: StreamingConfigHandle, reader_factory: ReaderFactory, service_config: BackfillServiceConfig, } @@ -111,13 +111,13 @@ pub struct BackfillService { impl BackfillService { pub fn new( registry: Arc, - config_source: HotReloadStreamingConfig, + config_source: StreamingConfigHandle, reader_factory: ReaderFactory, service_config: BackfillServiceConfig, ) -> Self { Self { registry, - sketch_index: None, + summary_store: None, series_resolver: None, config_source, reader_factory, @@ -129,9 +129,9 @@ impl BackfillService { /// there. Builder-style; safe to omit (legacy tests). pub fn with_sketch_index( mut self, - sketch_index: Arc, + summary_store: Arc, ) -> Self { - self.sketch_index = Some(sketch_index); + self.summary_store = Some(summary_store); self } @@ -245,7 +245,7 @@ impl BackfillService { self.registry.clone(), job.job_id, ); - if let Some(idx) = self.sketch_index.as_ref() { + if let Some(idx) = self.summary_store.as_ref() { processor = processor.with_sketch_index(idx.clone()); } if let Some(resolver) = self.series_resolver.as_ref() { @@ -405,7 +405,7 @@ mod tests { let mut cfg = sum_config(1, "latency"); cfg.table_name = Some("expected_table".into()); let agg_fp = cfg.policy_fp_u64(); - let hot = HotReloadStreamingConfig::from_arc(streaming_with(cfg)); + let hot = StreamingConfigHandle::from_arc(streaming_with(cfg)); let registry = Arc::new(BackfillRegistry::new()); let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); let called = calls.clone(); @@ -441,7 +441,7 @@ mod tests { let cfg = sum_config(1, "latency"); let agg_fp = cfg.policy_fp_u64(); let streaming = streaming_with(cfg); - let hot = HotReloadStreamingConfig::from_arc(streaming.clone()); + let hot = StreamingConfigHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); // Factory returns a fresh mock reader per call — seeded with a @@ -488,7 +488,7 @@ mod tests { let cfg = sum_config(1, "latency"); let agg_fp = cfg.policy_fp_u64(); let streaming = streaming_with(cfg); - let hot = HotReloadStreamingConfig::from_arc(streaming.clone()); + let hot = StreamingConfigHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); let service = BackfillService::new( @@ -522,7 +522,7 @@ mod tests { let cfg = sum_config(1, "latency"); let agg_fp = cfg.policy_fp_u64(); let streaming = streaming_with(cfg); - let hot = HotReloadStreamingConfig::from_arc(streaming.clone()); + let hot = StreamingConfigHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); // Factory records the order in which it's invoked. @@ -584,7 +584,7 @@ mod tests { async fn service_shutdown_stops_the_loop() { let cfg = sum_config(1, "m"); let streaming = streaming_with(cfg); - let hot = HotReloadStreamingConfig::from_arc(streaming.clone()); + let hot = StreamingConfigHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); let service = BackfillService::new( diff --git a/data_plane/src/storage_engines/sketch_db/data/mod.rs b/data_plane/src/storage_engines/sketch_db/data/mod.rs index 01c297274..e8f3d873f 100644 --- a/data_plane/src/storage_engines/sketch_db/data/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/data/mod.rs @@ -63,7 +63,7 @@ pub use asap_types::AggregationType; /// Sketch-instance configuration carried per-Metric on the OTLP wire /// (Phase 2 lifted these from per-DP up to the parent sketch container). /// Backend reads the relevant variant at ingest time and stores it in -/// `SketchInstanceMetadata.agg_kind`. +/// `SummarySeriesMetadata.agg_kind`. #[derive(Debug, Clone)] pub enum SketchConfig { UnivMon { diff --git a/data_plane/src/storage_engines/sketch_db/index/epoch_columnar.rs b/data_plane/src/storage_engines/sketch_db/index/epoch_columnar.rs index 4cb979fe6..544b2acbb 100644 --- a/data_plane/src/storage_engines/sketch_db/index/epoch_columnar.rs +++ b/data_plane/src/storage_engines/sketch_db/index/epoch_columnar.rs @@ -795,7 +795,7 @@ pub struct SidStoreData { /// `0` means "never written" / freshly (re)hydrated. Drives idle-sid /// eviction: a sid with no writes for the idle threshold whose state /// is fully durable on disk can have this whole `SidStoreData` dropped - /// from memory while its queryable `SketchInstanceMetadata` is kept + /// from memory while its queryable `SummarySeriesMetadata` is kept /// (the series stays answerable from the disk tier and rehydrates on /// the next write). Updated under the per-sid write lock the append /// path already holds, so it costs nothing extra on the hot path. diff --git a/data_plane/src/storage_engines/sketch_db/index/maintenance.rs b/data_plane/src/storage_engines/sketch_db/index/maintenance.rs index 20cedea52..65c5907bc 100644 --- a/data_plane/src/storage_engines/sketch_db/index/maintenance.rs +++ b/data_plane/src/storage_engines/sketch_db/index/maintenance.rs @@ -737,13 +737,13 @@ mod tests { fn complete_population_keeps_every_sid_and_rejects_missing_live_binding() { // Multiple SIDs are inventory entries, never an implicit singleton; // removing a live binding cannot hide its retained durable population. - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let mut first = plan.precompute_plan.materializations[0].clone(); first.aggregation_type = asap_types::AggregationType::Sum; @@ -848,13 +848,13 @@ mod tests { fn cohort_requires_every_durable_source_in_one_catalog_generation() { // Neither a missing second window nor a new catalog may yield a // partially acquired cohort, even when the first source is complete. - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let mut first = plan.precompute_plan.materializations[0].clone(); first.aggregation_type = asap_types::AggregationType::Sum; @@ -1084,10 +1084,10 @@ mod tests { entry["demand"]["fixed_interval_at"]["evaluation_phase"] = 0.into(); entry["time_selection"]["lookback"] = 60_000.into(); fixture["query_workload"]["repeating_queries"] = serde_json::json!([entry]); - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let source = plan .precompute_plan @@ -1400,13 +1400,13 @@ mod tests { fn committed_recovery_restores_live_seal_and_rejects_additive_derived_writes() { // A durable sidecar may survive an I/O error before the caller updates // its live seal. Recovery must repair admission, not only return a hit. - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let mut source_config = plan.precompute_plan.materializations[0].clone(); source_config.aggregation_type = asap_types::AggregationType::Sum; diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index 0ee018b7f..6d9089126 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -1,7 +1,7 @@ //! Sketch index — Phase 5 of the controller-into-backend refactor (2026-05). //! //! Two-level index: -//! - `instances`: sid → SketchInstanceMetadata (one entry per logical +//! - `instances`: sid → SummarySeriesMetadata (one entry per logical //! sketch instance — its metric name, group-by KEY set, capability, //! sketch_type, sketch_config, accuracy bound). //! - `series`: sid → per-sid storage (`SidStoreData`) carrying the @@ -193,7 +193,7 @@ fn build_attrs_fp_and_label_map( /// Lifecycle status is derived from retirement and expiry timestamps plus the /// wall clock. Only active instances accept writes. #[derive(Debug, Clone)] -pub struct SketchInstanceMetadata { +pub struct SummarySeriesMetadata { pub sid: u64, pub metric_name: String, /// The group-by KEY set — `dp.attributes.keys()` after the agent's @@ -237,7 +237,7 @@ pub struct SketchInstanceMetadata { pub policy_fp: PolicyFingerprint, } -impl SketchInstanceMetadata { +impl SummarySeriesMetadata { /// Compute the current `AggStatus` against the wall clock. /// Mirrors `AggSchema::status` — purely a function of timestamps. pub fn status(&self) -> AggStatus { @@ -621,7 +621,7 @@ pub struct SketchStore { /// per-attribute-set CMS (only the bucket total is meaningful). /// Kept as a decoupled side-table so recording item_label does not /// change sid identity (`AggKind` canonical string) or churn the many - /// `SketchInstanceMetadata` / `AggKind::Sketch` literals. + /// `SummarySeriesMetadata` / `AggKind::Sketch` literals. item_labels: RwLock>, /// sid → per-sid columnar storage. Empty `SidStoreData` (or absent /// key) for ghost sids — query path detects this and falls through @@ -811,14 +811,14 @@ impl SketchStore { /// absent from `policy_to_series_ids` / `metric_to_series_ids` (the pre-fix race /// where the two indexes were written under separate sequential /// locks). See the index-field doc comments for the full invariant. - pub fn register(&self, meta: SketchInstanceMetadata) { + pub fn register(&self, meta: SummarySeriesMetadata) { let mut instances = self.instances.write().unwrap(); self.register_with_instances(meta, &mut instances); } fn register_with_instances( &self, - meta: SketchInstanceMetadata, + meta: SummarySeriesMetadata, instances: &mut HashMap, ) -> bool { let sid = meta.sid; @@ -1136,7 +1136,7 @@ impl SketchStore { /// Look up the metadata for a sid (cloned because callers usually /// release the index lock before working with it). - pub fn instance(&self, sid: u64) -> Option> { + pub fn instance(&self, sid: u64) -> Option> { self.instances .read() .unwrap() @@ -1349,7 +1349,7 @@ impl SketchStore { /// (which would deadlock) and should stay allocation-light — extract /// the small data you need (a `Capability` clone, a `bool`) and act /// after this returns. - pub fn with_instance R>( + pub fn with_instance R>( &self, sid: u64, f: F, @@ -1359,7 +1359,7 @@ impl SketchStore { } /// Append a window's sketch state under `sid`. Caller is responsible - /// for ensuring the corresponding `SketchInstanceMetadata` was + /// for ensuring the corresponding `SummarySeriesMetadata` was /// registered (or the sketch arrives orphan and the caller chooses /// to drop / reject / register-on-the-fly). /// @@ -2436,7 +2436,7 @@ impl SketchStore { /// `SidStoreData` (epoch columns + intern-table label cache + the /// `series` slot) for every sketch sid that has gone write-idle past /// `idle_threshold_ms` AND whose state is fully durable on disk, while - /// KEEPING its [`SketchInstanceMetadata`] in `instances`. + /// KEEPING its [`SummarySeriesMetadata`] in `instances`. /// /// Why keep the metadata: the query path's disk union /// ([`Self::query_range`] → `union_disk_parts_into`) needs @@ -2519,7 +2519,7 @@ impl SketchStore { // 1. SeriesId bindings, compatibility metadata, and shared SDS descriptors. if let Ok(insts) = self.instances.read() { for m in insts.values() { - total += std::mem::size_of::(); + total += std::mem::size_of::(); total += m.metric_name.len(); for k in &m.group_by_keys { total += k.len() + std::mem::size_of::(); @@ -2559,7 +2559,7 @@ impl SketchStore { /// Snapshot shared metadata handles without holding the registry lock /// across user code. This is O(N) pointer cloning and does not copy /// descriptor strings, label sets, or aggregation configuration. - pub fn snapshot_instances(&self) -> Vec> { + pub fn snapshot_instances(&self) -> Vec> { match self.instances.read() { Ok(map) => map .values() @@ -2613,7 +2613,7 @@ impl SketchStore { /// allocation-light. Callers that need to mutate or call user code /// should collect the cheap data they need (e.g. `Vec` of /// sids) here, then act after this returns. - pub fn for_each_instance(&self, mut f: F) { + pub fn for_each_instance(&self, mut f: F) { if let Ok(map) = self.instances.read() { for (sid, meta) in map.iter() { f(*sid, meta); @@ -2624,7 +2624,7 @@ impl SketchStore { /// Iterate (clones) all instance metadata matching `status`. /// Used by the eviction service to enumerate `Expired` sids /// without holding a long read lock. - pub fn list_by_status(&self, status: AggStatus) -> Vec> { + pub fn list_by_status(&self, status: AggStatus) -> Vec> { let map = match self.instances.read() { Ok(m) => m, Err(_) => return Vec::new(), @@ -2695,7 +2695,7 @@ impl SketchStore { &self, sid: u64, retention: Duration, - ) -> Option> { + ) -> Option> { let _mutation = self.begin_state_mutation(); let mut map = self.instances.write().ok()?; let instance = map.get_mut(&sid)?; @@ -2714,7 +2714,7 @@ impl SketchStore { /// state, or `None` if the sid is unknown. Intended for /// operator / debug-endpoint use so eviction can be observed in /// e2e tests without waiting out retirement retention. - pub fn force_expire(&self, sid: u64) -> Option> { + pub fn force_expire(&self, sid: u64) -> Option> { let _mutation = self.begin_state_mutation(); let mut map = self.instances.write().ok()?; let instance = map.get_mut(&sid)?; @@ -2823,7 +2823,7 @@ impl SketchStore { /// `series` DashMap is touched after the index guards are released /// (it is independently keyed and not part of the metadata-index /// invariant). - pub fn remove_instance(&self, sid: u64) -> Option> { + pub fn remove_instance(&self, sid: u64) -> Option> { let _mutation = self.begin_state_mutation(); let removed = { // Fixed lock order: instances → policy_to_series_ids → metric_to_series_ids. @@ -3011,7 +3011,7 @@ impl SketchStore { // state was reachable only through the legacy precompute // query path; capability-matching couldn't see it. if !self.register_with_instances( - SketchInstanceMetadata { + SummarySeriesMetadata { sid, metric_name: agg_cfg.metric.clone(), group_by_keys, @@ -3357,7 +3357,7 @@ impl SketchStore { }; let capability = rec.capability(); let accuracy = rec.accuracy(); - self.register(SketchInstanceMetadata { + self.register(SummarySeriesMetadata { sid: rec.sid, metric_name: rec.metric_name, group_by_keys: rec.group_by_keys.into_iter().collect(), @@ -3585,6 +3585,22 @@ impl crate::storage_engines::sketch_db::index::persistence::EpochSource for Sket } } +// Compatibility imports; new callers use the domain names above. +#[deprecated(note = "Use SummarySeriesMetadata")] +pub use SummarySeriesMetadata as SketchInstanceMetadata; + +// 2026-05 reorg: generic epoch-partitioned columnar storage lives +// alongside the store that uses it. +mod admission; +mod maintenance; +pub(crate) use maintenance::{CompleteRawMaintenanceCohort, FrozenExactWindows}; +pub mod epoch_columnar; + +// `persistence` moved up to `sketch_db::persistence`. Re-exported here +// so legacy `crate::storage_engines::sketch_db::index::persistence::*` +// paths continue working without consumer changes. +pub use crate::storage_engines::sketch_db::persistence; + #[cfg(test)] mod tests { use super::*; @@ -3612,18 +3628,18 @@ mod tests { assert_eq!(gapped.query(0, 90_000), None); } - fn meta(sid: u64) -> SketchInstanceMetadata { + fn meta(sid: u64) -> SummarySeriesMetadata { meta_with_policy(sid, asap_types::PolicyFingerprint::UNSET) } fn meta_with_policy( sid: u64, policy_fp: asap_types::PolicyFingerprint, - ) -> SketchInstanceMetadata { + ) -> SummarySeriesMetadata { let cfg = SketchConfig::DDSketch { relative_accuracy: 0.01, }; - SketchInstanceMetadata { + SummarySeriesMetadata { sid, metric_name: "m".into(), group_by_keys: BTreeSet::new(), @@ -3692,13 +3708,13 @@ mod tests { #[test] fn observed_inventory_uses_installed_catalog_and_real_store_entries() { - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" )) .unwrap(); let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); let store = SketchStore::new(); @@ -3767,13 +3783,13 @@ mod tests { #[test] fn registered_series_without_payload_is_not_a_summary_instance() { - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" )) .unwrap(); let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); let store = SketchStore::new(); @@ -4552,7 +4568,7 @@ mod tests { /// Metadata with a chosen metric name + group-by key set, so the /// secondary-index tests can register several metrics/keys. - fn meta_metric_keys(sid: u64, metric: &str, keys: &[&str]) -> SketchInstanceMetadata { + fn meta_metric_keys(sid: u64, metric: &str, keys: &[&str]) -> SummarySeriesMetadata { let mut m = meta(sid); m.metric_name = metric.to_string(); m.group_by_keys = keys.iter().map(|k| k.to_string()).collect(); @@ -4671,7 +4687,7 @@ mod tests { /// Metadata with a single group-by key `host`, so the disk read-back /// path can rebuild the `{host: }` label map from the stored /// values vector. - fn meta_with_host_key(sid: u64) -> SketchInstanceMetadata { + fn meta_with_host_key(sid: u64) -> SummarySeriesMetadata { let mut m = meta(sid); m.group_by_keys = ["host".to_string()].into_iter().collect(); m @@ -4927,13 +4943,13 @@ mod tests { use crate::storage_engines::sketch_db::index::persistence::metadata::{ SidMetaRecord, SidMetadataStore, }; - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" )) .unwrap(); let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); let metadata = meta_with_policy(507, fingerprint); @@ -4970,13 +4986,13 @@ mod tests { #[test] fn catalog_reactivation_uses_new_physical_series_without_old_disk_payload() { use crate::drivers::ingest::series_resolver::SeriesIdResolver; - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" )) .unwrap(); let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); let definition = fingerprint.into(); @@ -5071,13 +5087,13 @@ mod tests { fn completed_windows_reject_late_updates_after_restart() { // Completion is a storage admission rule, including legacy producers, // and survives restart without allowing a correction into consumed state. - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" )) .unwrap(); let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); let directory = tempfile::tempdir().unwrap(); @@ -5159,13 +5175,13 @@ mod tests { fn finite_completion_flushes_payload_before_persisting_immutability() { // With neither memory pressure nor a hot-tier deadline, completion must // explicitly flush its payload before persisting a non-replayable window. - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" )) .unwrap(); let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); let directory = tempfile::tempdir().unwrap(); @@ -5269,13 +5285,13 @@ mod tests { #[test] fn durable_lifecycle_is_not_resurrected_by_restart_or_a_stale_flush() { - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" )) .unwrap(); let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); let directory = tempfile::tempdir().unwrap(); @@ -5353,13 +5369,13 @@ mod tests { #[test] fn observed_inventory_includes_durable_instances_after_restart() { - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" )) .unwrap(); let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); let definition_id = SummaryDefinitionId::from(fingerprint); @@ -5433,9 +5449,9 @@ mod tests { // they FAIL ("No result"); with the metadata-sidecar fix they pass // because recovery re-registers the disk-resident sids. - fn meta_kll_host(sid: u64) -> SketchInstanceMetadata { + fn meta_kll_host(sid: u64) -> SummarySeriesMetadata { let cfg = SketchConfig::Kll { k: 200 }; - SketchInstanceMetadata { + SummarySeriesMetadata { sid, metric_name: "http_latency".into(), group_by_keys: ["host".to_string()].into_iter().collect(), @@ -6033,15 +6049,3 @@ mod tests { assert_eq!(idx.series.len(), 2); } } - -// 2026-05 reorg: generic epoch-partitioned columnar storage lives -// alongside the store that uses it. -mod admission; -mod maintenance; -pub(crate) use maintenance::{CompleteRawMaintenanceCohort, FrozenExactWindows}; -pub mod epoch_columnar; - -// `persistence` moved up to `sketch_db::persistence`. Re-exported here -// so legacy `crate::storage_engines::sketch_db::index::persistence::*` -// paths continue working without consumer changes. -pub use crate::storage_engines::sketch_db::persistence; diff --git a/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs b/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs index 5a59e5ed6..d7de8c2bd 100644 --- a/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs +++ b/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs @@ -42,20 +42,20 @@ impl Default for SchemaEvictionConfig { /// The backfill registry is retained but is not consulted for cancellation. pub struct SchemaEvictionService { backfill: Arc, - sketch_index: Arc, + summary_store: Arc, retirement_retention: Duration, config: SchemaEvictionConfig, } impl SchemaEvictionService { pub fn new( - sketch_index: Arc, + summary_store: Arc, backfill: Arc, config: SchemaEvictionConfig, ) -> Self { Self { backfill, - sketch_index, + summary_store, retirement_retention: DEFAULT_RETIREMENT_RETENTION, config, } @@ -99,7 +99,7 @@ impl SchemaEvictionService { /// the service deterministically without spinning up a tokio /// runtime + polling loop. pub fn run_once(&self) { - let expired = self.sketch_index.list_by_status(AggStatus::Expired); + let expired = self.summary_store.list_by_status(AggStatus::Expired); if expired.is_empty() { return; } @@ -128,7 +128,7 @@ impl SchemaEvictionService { ); continue; } - let removed = self.sketch_index.remove_instance(sid).is_some(); + let removed = self.summary_store.remove_instance(sid).is_some(); info!( sid, %metric, @@ -246,7 +246,7 @@ mod tests { } fn write_one( - sketch_index: &SketchStore, + summary_store: &SketchStore, streaming_config: &StreamingConfig, agg_id: u64, ts: u64, @@ -270,7 +270,7 @@ mod tests { static RESOLVER: Arc = Arc::new(SeriesIdResolver::new()); } let resolver = RESOLVER.with(|r| r.clone()); - sketch_index + summary_store .ingest_precompute_for_agg_config( |m, fp, ak| resolver.resolve(m, fp, ak), agg_cfg, @@ -285,21 +285,21 @@ mod tests { /// sids stay `Active`. fn fixture_with_expired_metric_1() -> (Arc, Arc) { let (initial, id_to_fp) = make_streaming_config(&[1, 2]); - let sketch_index = Arc::new(SketchStore::new()); - let sid_a = write_one(&sketch_index, &initial, id_to_fp[&1], 100); - let _ = write_one(&sketch_index, &initial, id_to_fp[&1], 200); - let _ = write_one(&sketch_index, &initial, id_to_fp[&2], 300); + let summary_store = Arc::new(SketchStore::new()); + let sid_a = write_one(&summary_store, &initial, id_to_fp[&1], 100); + let _ = write_one(&summary_store, &initial, id_to_fp[&1], 200); + let _ = write_one(&summary_store, &initial, id_to_fp[&2], 300); // Both metric_1 writes share the same agg-signature → one // sid; mark it Expired directly. metric_2's sid stays Active. - sketch_index.force_expire(sid_a); + summary_store.force_expire(sid_a); - (Arc::new(BackfillRegistry::new()), sketch_index) + (Arc::new(BackfillRegistry::new()), summary_store) } #[tokio::test(flavor = "current_thread")] async fn run_once_drops_expired_sid() { - let (backfill, sketch_index) = fixture_with_expired_metric_1(); - let before_metric_2: usize = sketch_index + let (backfill, summary_store) = fixture_with_expired_metric_1(); + let before_metric_2: usize = summary_store .list_by_status(AggStatus::Active) .into_iter() .filter(|m| m.metric_name == "metric_2") @@ -307,7 +307,7 @@ mod tests { assert!(before_metric_2 >= 1, "fixture seeded metric_2 sid"); let svc = SchemaEvictionService::new( - sketch_index.clone(), + summary_store.clone(), backfill, SchemaEvictionConfig { poll_interval: Duration::from_secs(60), @@ -318,16 +318,16 @@ mod tests { // metric_1's expired sid is gone; metric_2's active sid // remains. - let metric_1_remaining = sketch_index + let metric_1_remaining = summary_store .list_by_status(AggStatus::Active) .into_iter() - .chain(sketch_index.list_by_status(AggStatus::Retired)) - .chain(sketch_index.list_by_status(AggStatus::Expired)) + .chain(summary_store.list_by_status(AggStatus::Retired)) + .chain(summary_store.list_by_status(AggStatus::Expired)) .filter(|m| m.metric_name == "metric_1") .count(); assert_eq!(metric_1_remaining, 0, "expired sid must be removed"); assert!( - sketch_index + summary_store .list_by_status(AggStatus::Active) .iter() .any(|m| m.metric_name == "metric_2"), @@ -339,19 +339,19 @@ mod tests { async fn run_once_is_noop_without_expired_sids() { let (initial, id_to_fp) = make_streaming_config(&[1]); let backfill = Arc::new(BackfillRegistry::new()); - let sketch_index = Arc::new(SketchStore::new()); - let _ = write_one(&sketch_index, &initial, id_to_fp[&1], 100); - let before = sketch_index.instance_count(); + let summary_store = Arc::new(SketchStore::new()); + let _ = write_one(&summary_store, &initial, id_to_fp[&1], 100); + let before = summary_store.instance_count(); let svc = SchemaEvictionService::new( - sketch_index.clone(), + summary_store.clone(), backfill, SchemaEvictionConfig::default(), ); svc.run_once(); assert_eq!( - sketch_index.instance_count(), + summary_store.instance_count(), before, "no expired sids — SketchStore untouched" ); @@ -359,10 +359,10 @@ mod tests { #[tokio::test(flavor = "current_thread")] async fn dry_run_logs_but_does_not_drop() { - let (backfill, sketch_index) = fixture_with_expired_metric_1(); - let before = sketch_index.instance_count(); + let (backfill, summary_store) = fixture_with_expired_metric_1(); + let before = summary_store.instance_count(); let svc = SchemaEvictionService::new( - sketch_index.clone(), + summary_store.clone(), backfill, SchemaEvictionConfig { poll_interval: Duration::from_secs(60), @@ -372,7 +372,7 @@ mod tests { svc.run_once(); // Dry-run: every sid stays. - assert_eq!(sketch_index.instance_count(), before); + assert_eq!(summary_store.instance_count(), before); } #[test] diff --git a/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs b/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs index 944d6e892..f9a834934 100644 --- a/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs +++ b/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs @@ -71,7 +71,7 @@ pub fn reconcile_from_streaming_config( // First pass: find orphaned Active sids without cloning the // catalog. `reconcile_from_streaming_config` runs on every ingest // batch, and the old `snapshot_instances()` here deep-cloned every - // `SketchInstanceMetadata` (String + BTreeSet + AggKind) + // `SummarySeriesMetadata` (String + BTreeSet + AggKind) // on each call — the dominant ingest-path CPU cost in profiling // (BTreeMap/String clone + malloc churn). We only need to read each // instance's signature under the read lock; collect just the cheap @@ -178,7 +178,7 @@ fn signature_from_agg_config(cfg: &AggregationConfig) -> Vec { fn build_live_signature_set(config: &StreamingConfig) -> HashSet> { config - .get_all_aggregation_configs() + .materializations() .values() .map(signature_from_agg_config) .collect() @@ -270,7 +270,7 @@ mod tests { use asap_types::KeyByLabelNames; use crate::storage_engines::sketch_db::data::AggKind; - use crate::storage_engines::sketch_db::index::{SketchInstanceMetadata, SketchStore}; + use crate::storage_engines::sketch_db::index::{SketchStore, SummarySeriesMetadata}; fn agg_config( metric: &str, @@ -301,9 +301,9 @@ mod tests { metric: &str, agg_type: AggregationType, group_by: Vec<&str>, - ) -> SketchInstanceMetadata { + ) -> SummarySeriesMetadata { let group_by_keys: BTreeSet = group_by.into_iter().map(|s| s.to_string()).collect(); - SketchInstanceMetadata { + SummarySeriesMetadata { sid, metric_name: metric.to_string(), group_by_keys, @@ -407,9 +407,9 @@ mod tests { kind: crate::storage_engines::sketch_db::data::SketchAlgorithm, config: crate::storage_engines::sketch_db::data::SketchConfig, group_by: Vec<&str>, - ) -> SketchInstanceMetadata { + ) -> SummarySeriesMetadata { let group_by_keys: BTreeSet = group_by.into_iter().map(|s| s.to_string()).collect(); - SketchInstanceMetadata { + SummarySeriesMetadata { sid, metric_name: metric.to_string(), group_by_keys, diff --git a/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs b/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs index 388bc05b2..552883c1c 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs @@ -6,7 +6,7 @@ //! `agg_id`), the label *values* (`KeyByLabelValues`), the //! `sketch_type_name`, the encoding tag, the time bounds, and the //! opaque sketch bytes. They do NOT carry the pieces of -//! [`SketchInstanceMetadata`](crate::storage_engines::sketch_db::index::SketchInstanceMetadata) +//! [`SummarySeriesMetadata`](crate::storage_engines::sketch_db::index::SummarySeriesMetadata) //! that the QUERY path needs to find and serve a series: //! //! * `metric_name` — the analyzer's diff --git a/data_plane/src/storage_engines/sketch_db/persistence/source.rs b/data_plane/src/storage_engines/sketch_db/persistence/source.rs index 734cfaf6a..661b6a927 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/source.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/source.rs @@ -132,7 +132,7 @@ pub trait EpochSource: Send + Sync { fn approx_memory_bytes(&self) -> usize; /// Persistable instance metadata for `sid` — the pieces of the store's - /// `SketchInstanceMetadata` the QUERY path needs but the on-disk part + /// `SummarySeriesMetadata` the QUERY path needs but the on-disk part /// format does NOT carry (metric name, group-by KEYS, structured /// `AggKind`). Called by the flusher right before it makes a part /// durable so recovery can re-register the sid as a queryable instance diff --git a/data_plane/src/storage_engines/sketch_db/query/timeline.rs b/data_plane/src/storage_engines/sketch_db/query/timeline.rs index c06c84522..92a64b0bf 100644 --- a/data_plane/src/storage_engines/sketch_db/query/timeline.rs +++ b/data_plane/src/storage_engines/sketch_db/query/timeline.rs @@ -13,7 +13,7 @@ use std::collections::{BTreeMap, BTreeSet}; use xxhash_rust::xxh64::xxh64; use crate::storage_engines::sketch_db::data::{AggKind, SketchAlgorithm, SketchConfig}; -use crate::storage_engines::sketch_db::index::{SketchInstanceMetadata, SketchStore}; +use crate::storage_engines::sketch_db::index::{SketchStore, SummarySeriesMetadata}; use crate::storage_engines::sketch_db::lifecycle::AggStatus; /// A single `(agg_id, clipped_range)` segment returned by @@ -129,7 +129,7 @@ struct SignatureKey { encoded: Vec, } -fn signature_key(meta: &SketchInstanceMetadata) -> SignatureKey { +fn signature_key(meta: &SummarySeriesMetadata) -> SignatureKey { let mut buf: Vec = Vec::new(); buf.extend_from_slice(meta.metric_name.as_bytes()); buf.push(0); @@ -168,7 +168,7 @@ impl Default for AggSignatureGroup { } impl AggSignatureGroup { - fn fold_in(&mut self, meta: &SketchInstanceMetadata) { + fn fold_in(&mut self, meta: &SummarySeriesMetadata) { // Stable signature id: xxh64 over the same canonical encoding // `signature_key` uses. Computed lazily on first fold-in; // every sid in the group produces the same hash. @@ -297,8 +297,8 @@ mod tests { first_seen: i64, retired: Option, expires: Option, - ) -> SketchInstanceMetadata { - SketchInstanceMetadata { + ) -> SummarySeriesMetadata { + SummarySeriesMetadata { sid, metric_name: metric.into(), group_by_keys: BTreeSet::new(), diff --git a/data_plane/src/storage_engines/sketch_db/sds.rs b/data_plane/src/storage_engines/sketch_db/sds.rs index 38a3d98de..0848da10a 100644 --- a/data_plane/src/storage_engines/sketch_db/sds.rs +++ b/data_plane/src/storage_engines/sketch_db/sds.rs @@ -8,7 +8,7 @@ use std::collections::HashMap; use std::sync::{Arc, RwLock, Weak}; use super::data::{AggKind, SketchConfig}; -use super::index::SketchInstanceMetadata; +use super::index::SummarySeriesMetadata; #[cfg(test)] use crate::storage_engines::types::AggregationType; pub use asap_types::sds::{ @@ -81,7 +81,7 @@ fn legacy_summary(kind: &AggKind) -> SummaryDescriptor { /// copied into every pane row. #[derive(Debug, Clone)] pub struct SdsBinding { - pub metadata: Arc, + pub metadata: Arc, pub summary_descriptor: Arc, pub data_descriptor: Arc, /// Immutable provenance of this physical series lifetime, shared with @@ -90,7 +90,7 @@ pub struct SdsBinding { } impl std::ops::Deref for SdsBinding { - type Target = SketchInstanceMetadata; + type Target = SummarySeriesMetadata; fn deref(&self) -> &Self::Target { &self.metadata @@ -148,7 +148,7 @@ impl SummaryDescriptorRegistry { self.authoritative_catalog.read().unwrap().clone() } - pub fn bind(&self, metadata: SketchInstanceMetadata) -> Result { + pub fn bind(&self, metadata: SummarySeriesMetadata) -> Result { let authoritative = self.authoritative_snapshot(); let configured = if let Some((catalog, _)) = authoritative.as_ref() { if metadata.policy_fp.is_unset() { @@ -301,8 +301,8 @@ mod tests { filter: &str, agg_type: AggregationType, policy: u64, - ) -> SketchInstanceMetadata { - SketchInstanceMetadata { + ) -> SummarySeriesMetadata { + SummarySeriesMetadata { sid, metric_name: metric.into(), group_by_keys: BTreeSet::from(["job".into()]), diff --git a/data_plane/src/storage_engines/types/hot_reload_config.rs b/data_plane/src/storage_engines/types/hot_reload_config.rs index 1bcc28334..c679d0bcd 100644 --- a/data_plane/src/storage_engines/types/hot_reload_config.rs +++ b/data_plane/src/storage_engines/types/hot_reload_config.rs @@ -7,7 +7,7 @@ //! //! ## How the pieces see the swap //! -//! Runtime bootstrap readers share clones of the same `HotReloadStreamingConfig` +//! Runtime bootstrap readers share clones of the same `StreamingConfigHandle` //! handle (internally `Arc>`), so they //! observe the swap at the same instant: //! @@ -23,7 +23,7 @@ //! `GroupState` for them. //! //! Query execution obtains its generation-consistent runtime configuration, -//! query plan, and catalog from `ActivePhysicalPlan` instead of this handle. +//! query plan, and catalog from `RuntimePhysicalPlan` instead of this handle. //! //! ## Config-upgrade contract for the control plane //! @@ -83,9 +83,9 @@ use arc_swap::ArcSwap; use crate::storage_engines::types::StreamingConfig; /// One immutable, generation-consistent runtime snapshot. Every execution -/// subsystem must project its view from the same `Arc`. +/// subsystem must project its view from the same `Arc`. #[derive(Debug, Clone)] -pub struct ActivePhysicalPlan { +pub struct RuntimePhysicalPlan { /// Authoritative generation and lifecycle identity shared by every plan /// projection in this immutable snapshot. pub envelope: asap_types::precompute_plan::PlanEnvelope, @@ -93,12 +93,12 @@ pub struct ActivePhysicalPlan { pub summary_catalog: Option>, pub precompute_plan: asap_types::precompute_plan::PrecomputePlan, pub transmission_plan: asap_types::producer_plan::TransmissionPlan, - pub runtime_config: Arc, + pub streaming_config: Arc, pub query_plan: Arc, pub storage_routing: Arc, } -impl ActivePhysicalPlan { +impl RuntimePhysicalPlan { pub fn plan_id(&self) -> u64 { self.envelope.plan_id } @@ -123,8 +123,8 @@ impl ActivePhysicalPlan { } #[derive(Clone)] -pub struct HotReloadActivePhysicalPlan { - inner: Arc>, +pub struct ActivePhysicalPlanHandle { + inner: Arc>, readiness: Arc>, } @@ -153,7 +153,7 @@ struct MaterializationReadinessState { } impl MaterializationReadinessState { - fn for_plan(plan: &ActivePhysicalPlan) -> Self { + fn for_plan(plan: &RuntimePhysicalPlan) -> Self { let plan_id = plan.plan_id(); let plan_version = plan.plan_version(); let statuses = plan @@ -265,18 +265,18 @@ pub enum PhysicalPlanLifecycleError { #[derive(Clone)] pub struct PhysicalPlanLifecycle { - active: HotReloadActivePhysicalPlan, + active: ActivePhysicalPlanHandle, state: Arc>, } struct PhysicalPlanLifecycleState { - staged: BTreeMap<(u64, u64), ActivePhysicalPlan>, + staged: BTreeMap<(u64, u64), RuntimePhysicalPlan>, statuses: BTreeMap<(u64, u64), PhysicalPlanStatus>, } impl PhysicalPlanLifecycle { - pub fn new(active: HotReloadActivePhysicalPlan) -> Self { - let snapshot = active.snapshot(); + pub fn new(active: ActivePhysicalPlanHandle) -> Self { + let snapshot = active.active_snapshot(); let mut statuses = BTreeMap::new(); if snapshot.plan_id() != 0 { statuses.insert( @@ -295,7 +295,7 @@ impl PhysicalPlanLifecycle { pub fn stage( &self, - plan: ActivePhysicalPlan, + plan: RuntimePhysicalPlan, now: u64, ) -> Result<(), PhysicalPlanLifecycleError> { let key = (plan.plan_id(), plan.plan_version()); @@ -304,7 +304,7 @@ impl PhysicalPlanLifecycle { return Err(PhysicalPlanLifecycleError::Expired { expiry, now }); } } - let active = self.active.snapshot(); + let active = self.active.active_snapshot(); if active.plan_id() != 0 && key.1 <= active.plan_version() { return Err(PhysicalPlanLifecycleError::StaleVersion { plan_id: key.0, @@ -359,7 +359,7 @@ impl PhysicalPlanLifecycle { plan_id: u64, plan_version: u64, now: u64, - ) -> Result, PhysicalPlanLifecycleError> { + ) -> Result, PhysicalPlanLifecycleError> { self.activate_with_prepare(plan_id, plan_version, now, |_| Ok::<(), String>(())) } @@ -371,8 +371,8 @@ impl PhysicalPlanLifecycle { plan_id: u64, plan_version: u64, now: u64, - prepare: impl FnOnce(&ActivePhysicalPlan) -> Result<(), E>, - ) -> Result, PhysicalPlanLifecycleError> + prepare: impl FnOnce(&RuntimePhysicalPlan) -> Result<(), E>, + ) -> Result, PhysicalPlanLifecycleError> where E: std::fmt::Display, { @@ -399,7 +399,7 @@ impl PhysicalPlanLifecycle { return Err(PhysicalPlanLifecycleError::Expired { expiry, now }); } } - let current = self.active.snapshot(); + let current = self.active.active_snapshot(); if current.plan_id() != 0 && plan.plan_version() <= current.plan_version() { return Err(PhysicalPlanLifecycleError::StaleVersion { plan_id, @@ -433,7 +433,7 @@ impl PhysicalPlanLifecycle { /// Mark a superseded generation retired after all readers of the old /// immutable snapshot have drained. - pub(crate) fn retire_drained(&self, plan_id: u64, plan_version: u64) { + pub(crate) fn mark_drained_plan_retired(&self, plan_id: u64, plan_version: u64) { if let Some(status) = self .state .lock() @@ -448,7 +448,7 @@ impl PhysicalPlanLifecycle { } } -fn status_for(plan: &ActivePhysicalPlan, phase: PhysicalPlanPhase) -> PhysicalPlanStatus { +fn status_for(plan: &RuntimePhysicalPlan, phase: PhysicalPlanPhase) -> PhysicalPlanStatus { PhysicalPlanStatus { plan_id: plan.plan_id(), plan_version: plan.plan_version(), @@ -458,8 +458,8 @@ fn status_for(plan: &ActivePhysicalPlan, phase: PhysicalPlanPhase) -> PhysicalPl } } -impl HotReloadActivePhysicalPlan { - pub fn new(initial: ActivePhysicalPlan) -> Self { +impl ActivePhysicalPlanHandle { + pub fn new(initial: RuntimePhysicalPlan) -> Self { let readiness = MaterializationReadinessState::for_plan(&initial); Self { inner: Arc::new(ArcSwap::new(Arc::new(initial))), @@ -467,11 +467,11 @@ impl HotReloadActivePhysicalPlan { } } - pub fn snapshot(&self) -> Arc { + pub fn active_snapshot(&self) -> Arc { self.inner.load_full() } - pub fn swap(&self, next: ActivePhysicalPlan) -> Arc { + pub fn swap(&self, next: RuntimePhysicalPlan) -> Arc { let next_readiness = MaterializationReadinessState::for_plan(&next); let old = self.inner.swap(Arc::new(next)); *self @@ -554,10 +554,10 @@ impl HotReloadActivePhysicalPlan { } } -impl std::fmt::Debug for HotReloadActivePhysicalPlan { +impl std::fmt::Debug for ActivePhysicalPlanHandle { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let snapshot = self.snapshot(); - f.debug_struct("HotReloadActivePhysicalPlan") + let snapshot = self.active_snapshot(); + f.debug_struct("ActivePhysicalPlanHandle") .field("plan_id", &snapshot.plan_id()) .field("query_count", &snapshot.query_plan.entries.len()) .field( @@ -568,16 +568,18 @@ impl std::fmt::Debug for HotReloadActivePhysicalPlan { } } -/// Thin wrapper around `ArcSwap` with ergonomic -/// snapshot + swap helpers. Cloneable; clones share the same -/// underlying `ArcSwap` so all holders see the same swaps. +/// Streaming materialization view with two compatibility modes. Legacy mode +/// owns a swappable config; active-plan mode reads the config from the current +/// immutable runtime plan. In active-plan mode, `swap` only updates the legacy +/// backing slot and does not publish a new plan. Use plan activation to change +/// the authoritative configuration. #[derive(Clone)] -pub struct HotReloadStreamingConfig { +pub struct StreamingConfigHandle { inner: Arc>, - active: Option, + active: Option, } -impl HotReloadStreamingConfig { +impl StreamingConfigHandle { /// Construct with an initial `StreamingConfig`. Takes ownership — /// callers who need to keep their own handle should `.clone()` the /// `StreamingConfig` before calling `new`. @@ -598,8 +600,8 @@ impl HotReloadStreamingConfig { } } - pub fn from_active(active: HotReloadActivePhysicalPlan) -> Self { - let initial = active.snapshot().runtime_config.clone(); + pub fn from_active_physical_plan(active: ActivePhysicalPlanHandle) -> Self { + let initial = active.active_snapshot().streaming_config.clone(); Self { inner: Arc::new(ArcSwap::new(initial)), active: Some(active), @@ -612,12 +614,12 @@ impl HotReloadStreamingConfig { pub fn snapshot(&self) -> Arc { self.active .as_ref() - .map(|a| a.snapshot().runtime_config.clone()) + .map(|a| a.active_snapshot().streaming_config.clone()) .unwrap_or_else(|| self.inner.load_full()) } - pub fn physical_plan_snapshot(&self) -> Option> { - self.active.as_ref().map(|active| active.snapshot()) + pub fn active_physical_plan_snapshot(&self) -> Option> { + self.active.as_ref().map(|active| active.active_snapshot()) } /// Atomically replace the current config. The previous `Arc` is @@ -630,15 +632,51 @@ impl HotReloadStreamingConfig { } } -impl std::fmt::Debug for HotReloadStreamingConfig { +impl std::fmt::Debug for StreamingConfigHandle { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let snap = self.snapshot(); - f.debug_struct("HotReloadStreamingConfig") - .field("num_agg_configs", &snap.aggregation_configs.len()) + f.debug_struct("StreamingConfigHandle") + .field( + "num_agg_configs", + &snap.materializations_by_policy_fingerprint.len(), + ) .finish() } } +// Compatibility imports; new callers use the domain names above. +#[deprecated(note = "Use ActivePhysicalPlanHandle")] +pub use ActivePhysicalPlanHandle as HotReloadActivePhysicalPlan; +#[deprecated(note = "Use RuntimePhysicalPlan")] +pub use RuntimePhysicalPlan as ActivePhysicalPlan; + +#[deprecated(note = "Use StreamingConfigHandle")] +pub use StreamingConfigHandle as HotReloadStreamingConfig; + +impl StreamingConfigHandle { + #[deprecated(note = "Use from_active_physical_plan")] + pub fn from_active(active: ActivePhysicalPlanHandle) -> Self { + Self::from_active_physical_plan(active) + } + #[deprecated(note = "Use active_physical_plan_snapshot")] + pub fn physical_plan_snapshot(&self) -> Option> { + self.active_physical_plan_snapshot() + } +} +impl PhysicalPlanLifecycle { + #[deprecated(note = "Use mark_drained_plan_retired")] + pub fn retire_drained(&self, plan_id: u64, plan_version: u64) { + self.mark_drained_plan_retired(plan_id, plan_version) + } +} + +impl ActivePhysicalPlanHandle { + #[deprecated(note = "Use active_snapshot")] + pub fn snapshot(&self) -> Arc { + self.active_snapshot() + } +} + #[cfg(test)] mod tests { use super::*; @@ -654,7 +692,7 @@ mod tests { plan_version: u64, activation_unix_ms: u64, expiry_unix_ms: Option, - ) -> ActivePhysicalPlan { + ) -> RuntimePhysicalPlan { let envelope = asap_types::precompute_plan::PlanEnvelope { plan_id, plan_version, @@ -665,7 +703,7 @@ mod tests { planner_revision: control_plane::physical::compiler::PLANNER_REVISION.into(), capability_snapshot_id: "test".into(), }; - ActivePhysicalPlan { + RuntimePhysicalPlan { envelope: envelope.clone(), summary_catalog: None, precompute_plan: asap_types::precompute_plan::PrecomputePlan { @@ -705,7 +743,7 @@ mod tests { }, rules: Vec::new(), }, - runtime_config: Arc::new(StreamingConfig::new(HashMap::new())), + streaming_config: Arc::new(StreamingConfig::new(HashMap::new())), query_plan: Arc::new(asap_types::query_plan::QueryPlan { plan_id, plan_version, @@ -756,46 +794,56 @@ mod tests { #[test] fn snapshot_reflects_initial_config() { let (cfg, id_to_fp) = cfg_with_ids(&[1, 2, 3]); - let hr = HotReloadStreamingConfig::new(cfg); + let hr = StreamingConfigHandle::new(cfg); let snap = hr.snapshot(); - assert_eq!(snap.aggregation_configs.len(), 3); - assert!(snap.aggregation_configs.contains_key(&id_to_fp[&2])); + assert_eq!(snap.materializations_by_policy_fingerprint.len(), 3); + assert!(snap + .materializations_by_policy_fingerprint + .contains_key(&id_to_fp[&2])); } #[test] fn swap_replaces_config_atomically() { let (cfg1, id_to_fp1) = cfg_with_ids(&[1, 2]); let (cfg2, id_to_fp2) = cfg_with_ids(&[3, 4, 5]); - let hr = HotReloadStreamingConfig::new(cfg1); + let hr = StreamingConfigHandle::new(cfg1); let old = hr.swap(cfg2); // Old snapshot still reflects pre-swap contents. - assert_eq!(old.aggregation_configs.len(), 2); - assert!(old.aggregation_configs.contains_key(&id_to_fp1[&1])); + assert_eq!(old.materializations_by_policy_fingerprint.len(), 2); + assert!(old + .materializations_by_policy_fingerprint + .contains_key(&id_to_fp1[&1])); // New snapshot reflects post-swap contents. let new_snap = hr.snapshot(); - assert_eq!(new_snap.aggregation_configs.len(), 3); - assert!(new_snap.aggregation_configs.contains_key(&id_to_fp2[&5])); - assert!(!new_snap.aggregation_configs.contains_key(&id_to_fp1[&1])); + assert_eq!(new_snap.materializations_by_policy_fingerprint.len(), 3); + assert!(new_snap + .materializations_by_policy_fingerprint + .contains_key(&id_to_fp2[&5])); + assert!(!new_snap + .materializations_by_policy_fingerprint + .contains_key(&id_to_fp1[&1])); } #[test] fn clones_share_underlying_swap() { let (cfg1, _) = cfg_with_ids(&[1]); let (cfg2, id_to_fp2) = cfg_with_ids(&[2, 3]); - let hr = HotReloadStreamingConfig::new(cfg1); + let hr = StreamingConfigHandle::new(cfg1); let hr_clone = hr.clone(); hr.swap(cfg2); // The clone sees the swap because both handles share the // same ArcSwap inside. let snap = hr_clone.snapshot(); - assert_eq!(snap.aggregation_configs.len(), 2); - assert!(snap.aggregation_configs.contains_key(&id_to_fp2[&3])); + assert_eq!(snap.materializations_by_policy_fingerprint.len(), 2); + assert!(snap + .materializations_by_policy_fingerprint + .contains_key(&id_to_fp2[&3])); } #[test] fn concurrent_readers_see_consistent_snapshot() { let (cfg1, _) = cfg_with_ids(&[1, 2]); - let hr = HotReloadStreamingConfig::new(cfg1); + let hr = StreamingConfigHandle::new(cfg1); let hr_writer = hr.clone(); let writer = thread::spawn(move || { for i in 0..50 { @@ -810,7 +858,7 @@ mod tests { // Under race, the snapshot must be internally // consistent — either 2 entries (original) or 3 // (post-swap). Never a torn state. - let n = snap.aggregation_configs.len(); + let n = snap.materializations_by_policy_fingerprint.len(); assert!(n == 2 || n == 3, "torn snapshot: {n} entries"); } }); @@ -820,33 +868,33 @@ mod tests { #[test] fn physical_plan_stages_activates_drains_and_retires() { - let active = HotReloadActivePhysicalPlan::new(physical_plan(7, 1, 100, None)); + let active = ActivePhysicalPlanHandle::new(physical_plan(7, 1, 100, None)); let lifecycle = PhysicalPlanLifecycle::new(active.clone()); lifecycle .stage(physical_plan(7, 2, 200, Some(500)), 150) .unwrap(); - assert_eq!(active.snapshot().plan_version(), 1); + assert_eq!(active.active_snapshot().plan_version(), 1); assert!(matches!( lifecycle.activate(7, 2, 199), Err(PhysicalPlanLifecycleError::ActivationNotReached { .. }) )); let old = lifecycle.activate(7, 2, 200).unwrap(); assert_eq!(old.plan_version(), 1); - assert_eq!(active.snapshot().plan_version(), 2); + assert_eq!(active.active_snapshot().plan_version(), 2); let statuses = lifecycle.statuses(); assert_eq!(statuses.len(), 2); assert_eq!(statuses[0].phase, PhysicalPlanPhase::Draining); assert_eq!(statuses[1].phase, PhysicalPlanPhase::Active); - lifecycle.retire_drained(7, 1); + lifecycle.mark_drained_plan_retired(7, 1); assert_eq!(lifecycle.statuses()[0].phase, PhysicalPlanPhase::Retired); } #[test] fn failed_activation_preparation_keeps_active_and_staged_generations() { - let active = HotReloadActivePhysicalPlan::new(physical_plan(7, 1, 100, None)); - let held_reader = active.snapshot(); + let active = ActivePhysicalPlanHandle::new(physical_plan(7, 1, 100, None)); + let held_reader = active.active_snapshot(); let lifecycle = PhysicalPlanLifecycle::new(active.clone()); lifecycle .stage(physical_plan(7, 2, 200, None), 150) @@ -859,7 +907,7 @@ mod tests { error, PhysicalPlanLifecycleError::Prepare("catalog rejected".into()) ); - assert_eq!(active.snapshot().plan_version(), 1); + assert_eq!(active.active_snapshot().plan_version(), 1); assert_eq!(held_reader.plan_version(), 1); assert!(lifecycle.statuses().iter().any(|status| { status.plan_version == 2 && status.phase == PhysicalPlanPhase::Staged @@ -868,14 +916,14 @@ mod tests { lifecycle .activate_with_prepare(7, 2, 200, |_| Ok::<(), String>(())) .unwrap(); - assert_eq!(active.snapshot().plan_version(), 2); + assert_eq!(active.active_snapshot().plan_version(), 2); } // Failed publication releases only its staging slot; active readers remain valid. #[test] fn discard_staged_allows_retry_and_never_discards_active() { - let active = HotReloadActivePhysicalPlan::new(physical_plan(7, 1, 100, None)); - let held_reader = active.snapshot(); + let active = ActivePhysicalPlanHandle::new(physical_plan(7, 1, 100, None)); + let held_reader = active.active_snapshot(); let lifecycle = PhysicalPlanLifecycle::new(active.clone()); lifecycle .stage(physical_plan(7, 2, 200, None), 150) @@ -886,13 +934,13 @@ mod tests { .unwrap(); lifecycle.activate(7, 2, 300).unwrap(); assert!(lifecycle.discard_staged(7, 2).is_err()); - assert_eq!(active.snapshot().plan_version(), 2); + assert_eq!(active.active_snapshot().plan_version(), 2); assert_eq!(held_reader.plan_version(), 1); } #[test] fn materialization_readiness_is_generation_scoped_and_monotonic() { - let active = HotReloadActivePhysicalPlan::new(physical_plan(7, 1, 100, None)); + let active = ActivePhysicalPlanHandle::new(physical_plan(7, 1, 100, None)); let fingerprint = asap_types::PolicyFingerprint(41); active.readiness.lock().unwrap().statuses.insert( fingerprint, @@ -923,7 +971,7 @@ mod tests { #[test] fn physical_plan_rejects_stale_and_expired_generations() { - let active = HotReloadActivePhysicalPlan::new(physical_plan(7, 2, 100, None)); + let active = ActivePhysicalPlanHandle::new(physical_plan(7, 2, 100, None)); let lifecycle = PhysicalPlanLifecycle::new(active); assert!(matches!( lifecycle.stage(physical_plan(7, 1, 100, None), 150), @@ -937,7 +985,7 @@ mod tests { #[test] fn activation_rechecks_version_and_cannot_downgrade_across_plan_ids() { - let active = HotReloadActivePhysicalPlan::new(physical_plan(7, 1, 100, None)); + let active = ActivePhysicalPlanHandle::new(physical_plan(7, 1, 100, None)); let lifecycle = PhysicalPlanLifecycle::new(active.clone()); lifecycle .stage(physical_plan(8, 3, 100, None), 100) @@ -955,6 +1003,6 @@ mod tests { .. }) )); - assert_eq!(active.snapshot().plan_version(), 3); + assert_eq!(active.active_snapshot().plan_version(), 3); } } diff --git a/data_plane/src/storage_engines/types/streaming_config.rs b/data_plane/src/storage_engines/types/streaming_config.rs index 74c6b2da9..6db358911 100644 --- a/data_plane/src/storage_engines/types/streaming_config.rs +++ b/data_plane/src/storage_engines/types/streaming_config.rs @@ -26,7 +26,11 @@ use super::storage_backend::StorageBackend; /// from_configs` primitive this method now calls directly. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct StreamingConfig { - pub aggregation_configs: HashMap, + #[serde( + rename = "aggregation_configs", + alias = "materializations_by_policy_fingerprint" + )] + pub materializations_by_policy_fingerprint: HashMap, /// Phase-5 capability-routing axis: which storage tier serves this /// per-metric runtime config. The controller pushes this when planning /// (see `docs/design-gorilla-s3-cold-engine.md` §8); pre-Phase-5 @@ -42,9 +46,9 @@ pub struct StreamingConfig { } impl StreamingConfig { - pub fn new(aggregation_configs: HashMap) -> Self { + pub fn new(materializations_by_policy_fingerprint: HashMap) -> Self { Self { - aggregation_configs, + materializations_by_policy_fingerprint, storage_backend: StorageBackend::default(), monitors: Vec::new(), } @@ -59,11 +63,11 @@ impl StreamingConfig { /// Used by the controller-driven plan-push path; tests typically /// stay on `Self::new(...)` and let the default land. pub fn with_storage_backend( - aggregation_configs: HashMap, + materializations_by_policy_fingerprint: HashMap, storage_backend: StorageBackend, ) -> Self { Self { - aggregation_configs, + materializations_by_policy_fingerprint, storage_backend, monitors: Vec::new(), } @@ -77,21 +81,23 @@ impl StreamingConfig { } pub fn get_aggregation_config(&self, aggregation_id: u64) -> Option<&AggregationConfig> { - self.aggregation_configs.get(&aggregation_id) + self.materializations_by_policy_fingerprint + .get(&aggregation_id) } - pub fn get_all_aggregation_configs(&self) -> &HashMap { - &self.aggregation_configs + pub fn materializations(&self) -> &HashMap { + &self.materializations_by_policy_fingerprint } pub fn contains(&self, aggregation_id: u64) -> bool { - self.aggregation_configs.contains_key(&aggregation_id) + self.materializations_by_policy_fingerprint + .contains_key(&aggregation_id) } /// Derived content-addressed view. Builds a [`PolicyRegistry`] keyed /// on [`asap_types::PolicyFingerprint`] — the merged-sid-identity-chain /// replacement for the `aggregation_id`-keyed lookup. Cheap (O(N) - /// over `aggregation_configs.len()`); call at swap time, not per + /// over `materializations_by_policy_fingerprint.len()`); call at swap time, not per /// query, if it shows up in hot-path profiles. /// /// Dual-keyed transition: this method exists alongside the legacy @@ -99,7 +105,11 @@ impl StreamingConfig { /// one at a time. The two views are derived from the same source — /// they can never disagree. pub fn policy_registry(&self) -> PolicyRegistry { - PolicyRegistry::from_configs(self.aggregation_configs.values().cloned()) + PolicyRegistry::from_configs( + self.materializations_by_policy_fingerprint + .values() + .cloned(), + ) } pub fn from_yaml_file(yaml_file: &str) -> Result { @@ -116,7 +126,8 @@ impl StreamingConfig { /// (operator-authored query→agg_ids YAML feeding a retention_map) /// is gone — the controller drives capability matching dynamically. pub fn from_yaml_data(data: &Value) -> Result { - let mut aggregation_configs: HashMap = HashMap::new(); + let mut materializations_by_policy_fingerprint: HashMap = + HashMap::new(); if let Some(aggregations) = data.get("aggregations").and_then(|v| v.as_sequence()) { for aggregation_data in aggregations { @@ -143,11 +154,11 @@ impl StreamingConfig { // PR 5: the map key IS the policy-fingerprint u64. // `AggregationConfig::policy_fp_u64()` is the canonical // accessor for this value. - aggregation_configs.insert(config.policy_fp_u64(), config); + materializations_by_policy_fingerprint.insert(config.policy_fp_u64(), config); } } - let mut config = Self::new(aggregation_configs); + let mut config = Self::new(materializations_by_policy_fingerprint); // Continuous-monitoring (CDM) specs: a top-level `monitors:` array, each // entry deserializing into a MonitorSpec. Absent → empty (the common // case). The data-plane monitor coordinator reads these. @@ -167,7 +178,7 @@ impl Index for StreamingConfig { type Output = AggregationConfig; fn index(&self, aggregation_id: u64) -> &Self::Output { - &self.aggregation_configs[&aggregation_id] + &self.materializations_by_policy_fingerprint[&aggregation_id] } } @@ -177,6 +188,13 @@ impl Default for StreamingConfig { } } +impl StreamingConfig { + #[deprecated(note = "Use materializations")] + pub fn get_all_aggregation_configs(&self) -> &HashMap { + self.materializations() + } +} + #[cfg(test)] mod tests { use super::*; @@ -247,8 +265,12 @@ aggregations:\n\ - aggregationType: DDSketch\n aggregationSubType: ''\n metric: cpu_seconds\n labels:\n grouping: [host]\n rollup: []\n aggregated: []\n parameters:\n relative_accuracy: 0.01\n windowSize: 30\n windowType: tumbling\n spatialFilter: ''\n"; let data: Value = serde_yaml::from_str(yaml).expect("yaml ok"); let cfg = StreamingConfig::from_yaml_data(&data).expect("decode without id"); - assert_eq!(cfg.aggregation_configs.len(), 1); - let (k, v) = cfg.aggregation_configs.iter().next().unwrap(); + assert_eq!(cfg.materializations_by_policy_fingerprint.len(), 1); + let (k, v) = cfg + .materializations_by_policy_fingerprint + .iter() + .next() + .unwrap(); assert_ne!(*k, 0, "derived id is not the 0 sentinel"); assert_eq!(*k, v.policy_fp_u64(), "map key equals fingerprint u64"); assert_eq!(v.metric, "cpu_seconds"); @@ -269,8 +291,16 @@ aggregations:\n\ let wo: Value = serde_yaml::from_str(without).expect("without yaml ok"); let cw = StreamingConfig::from_yaml_data(&w).expect("with"); let cwo = StreamingConfig::from_yaml_data(&wo).expect("without"); - let (kw, _) = cw.aggregation_configs.iter().next().unwrap(); - let (kwo, _) = cwo.aggregation_configs.iter().next().unwrap(); + let (kw, _) = cw + .materializations_by_policy_fingerprint + .iter() + .next() + .unwrap(); + let (kwo, _) = cwo + .materializations_by_policy_fingerprint + .iter() + .next() + .unwrap(); assert_eq!( kw, kwo, "explicit aggregationId in YAML must not change identity" diff --git a/data_plane/src/tests/capability_miss_http_e2e_tests.rs b/data_plane/src/tests/capability_miss_http_e2e_tests.rs index 2930ea74d..c313e8f7f 100644 --- a/data_plane/src/tests/capability_miss_http_e2e_tests.rs +++ b/data_plane/src/tests/capability_miss_http_e2e_tests.rs @@ -3,7 +3,7 @@ //! //! The in-process version of this loop already lives in //! `simple_engine.rs::e2e_feedback_loop_tests` — it swaps the -//! `HotReloadStreamingConfig` handle directly from a mock +//! `StreamingConfigHandle` handle directly from a mock //! `ControlPlaneClient`. What was missing, and what this file adds, //! is the **real HTTP round-trip**: //! @@ -18,7 +18,7 @@ //! │ 3. receive miss → craft config YAML //! ▼ //! POST backend:/api/v1/streaming-config (real HTTP) -//! │ 4. HotReloadStreamingConfig.swap +//! │ 4. StreamingConfigHandle.swap //! ▼ //! GET backend:/api/v1/streaming-config //! → aggregation_count ≥ 1 (loop closed) @@ -39,7 +39,7 @@ use crate::drivers::query::adapters::AdapterConfig; use crate::drivers::query::servers::http::{HttpServer, HttpServerConfig}; use crate::query_engines::ASAPQueryEngine; #[cfg(test)] -use crate::storage_engines::types::{HotReloadStreamingConfig, StreamingConfig}; +use crate::storage_engines::types::{StreamingConfig, StreamingConfigHandle}; use axum::{extract::State, routing::post, Router}; use reqwest::Client; use serde_json::Value; @@ -99,7 +99,7 @@ fn expected_fp_for(metric: &str) -> u64 { let data: serde_yaml::Value = serde_yaml::from_str(&yaml).expect("yaml parses"); let sc = crate::storage_engines::types::StreamingConfig::from_yaml_data(&data) .expect("yaml decodes"); - *sc.aggregation_configs + *sc.materializations_by_policy_fingerprint .keys() .next() .expect("one agg in the canned plan") @@ -162,7 +162,7 @@ async fn start_mock_control_plane(state: MockControlPlaneState) -> u16 { port } -async fn start_backend(control_plane_url: String, hot_reload: HotReloadStreamingConfig) -> u16 { +async fn start_backend(control_plane_url: String, hot_reload: StreamingConfigHandle) -> u16 { let _streaming_config = hot_reload.snapshot(); let engine = Arc::new( ASAPQueryEngine::new(15_000) @@ -226,7 +226,7 @@ async fn poll_until_plan_active( async fn spin_up_loop( metric: &str, expected_agg_id: u64, -) -> (String, MockControlPlaneState, HotReloadStreamingConfig) { +) -> (String, MockControlPlaneState, StreamingConfigHandle) { let control_plane_state = MockControlPlaneState { received_count: Arc::new(AtomicUsize::new(0)), pushed_plan_ts: Arc::new(Mutex::new(None)), @@ -240,7 +240,7 @@ async fn spin_up_loop( let control_plane_url = format!("http://127.0.0.1:{control_plane_port}/api/v1/plan"); // 2. backend up, with control-plane URL baked in - let hot_reload = HotReloadStreamingConfig::new(StreamingConfig::default()); + let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); let backend_port = start_backend(control_plane_url, hot_reload.clone()).await; let backend_url = format!("http://127.0.0.1:{backend_port}"); diff --git a/data_plane/src/tests/test_utilities/engine_factories.rs b/data_plane/src/tests/test_utilities/engine_factories.rs index 203330293..5158398bd 100644 --- a/data_plane/src/tests/test_utilities/engine_factories.rs +++ b/data_plane/src/tests/test_utilities/engine_factories.rs @@ -21,14 +21,14 @@ use std::collections::HashMap; /// Each factory gets its own resolver instance; tests are isolated so /// the `next_sid = 1, 2, ...` counter doesn't bleed between fixtures. fn ingest_with_fresh_resolver( - sketch_index: &crate::storage_engines::sketch_db::index::SketchStore, + summary_store: &crate::storage_engines::sketch_db::index::SketchStore, resolver: &std::sync::Arc, agg_cfg: &AggregationConfig, output: &PrecomputedOutput, accumulator: &dyn AggregateCore, ) -> Option { let resolver = resolver.clone(); - sketch_index.ingest_precompute_for_agg_config( + summary_store.ingest_precompute_for_agg_config( |m, fp, ak| resolver.resolve(m, fp, ak), agg_cfg, output, @@ -88,7 +88,7 @@ pub fn create_engine_single_pop_with_aggregated( .cloned() .collect(); - let mut aggregation_configs = HashMap::new(); + let mut materializations_by_policy_fingerprint = HashMap::new(); let agg_config = AggregationConfig { population_key_encoding: Default::default(), aggregation_type, @@ -116,15 +116,15 @@ pub fn create_engine_single_pop_with_aggregated( value_source_column: None, }; let agg_id = agg_config.policy_fp_u64(); - aggregation_configs.insert(agg_id, agg_config); + materializations_by_policy_fingerprint.insert(agg_id, agg_config); let streaming_config = Arc::new(StreamingConfig { - aggregation_configs, + materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), }); - let sketch_index = + let summary_store = std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); let resolver = std::sync::Arc::new(SeriesIdResolver::new()); @@ -142,10 +142,10 @@ pub fn create_engine_single_pop_with_aggregated( key, asap_types::PolicyFingerprint(agg_id), ); - ingest_with_fresh_resolver(&sketch_index, &resolver, &agg_cfg, &output, acc.as_ref()); + ingest_with_fresh_resolver(&summary_store, &resolver, &agg_cfg, &output, acc.as_ref()); } - ASAPQueryEngine::new(1).with_sketch_index(sketch_index) + ASAPQueryEngine::new(1).with_sketch_index(summary_store) } /// Build a test engine with separate value and key aggregations. Each input @@ -172,7 +172,7 @@ pub fn create_engine_dual_input( .cloned() .collect(); - let mut aggregation_configs = HashMap::new(); + let mut materializations_by_policy_fingerprint = HashMap::new(); // Value aggregation let value_agg_config = AggregationConfig { @@ -202,7 +202,7 @@ pub fn create_engine_dual_input( value_source_column: None, }; let value_id = value_agg_config.policy_fp_u64(); - aggregation_configs.insert(value_id, value_agg_config); + materializations_by_policy_fingerprint.insert(value_id, value_agg_config); // Keys aggregation let keys_agg_config = AggregationConfig { @@ -232,15 +232,15 @@ pub fn create_engine_dual_input( value_source_column: None, }; let keys_id = keys_agg_config.policy_fp_u64(); - aggregation_configs.insert(keys_id, keys_agg_config); + materializations_by_policy_fingerprint.insert(keys_id, keys_agg_config); let streaming_config = Arc::new(StreamingConfig { - aggregation_configs, + materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), }); - let sketch_index = + let summary_store = std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); let resolver = std::sync::Arc::new(SeriesIdResolver::new()); @@ -261,7 +261,7 @@ pub fn create_engine_dual_input( key, asap_types::PolicyFingerprint(value_id), ); - ingest_with_fresh_resolver(&sketch_index, &resolver, &agg_cfg_1, &output, acc.as_ref()); + ingest_with_fresh_resolver(&summary_store, &resolver, &agg_cfg_1, &output, acc.as_ref()); } for (label_values_opt, acc) in keys_data { let key = label_values_opt.map(|labels| KeyByLabelValues { labels }); @@ -271,10 +271,10 @@ pub fn create_engine_dual_input( key, asap_types::PolicyFingerprint(keys_id), ); - ingest_with_fresh_resolver(&sketch_index, &resolver, &agg_cfg_2, &output, acc.as_ref()); + ingest_with_fresh_resolver(&summary_store, &resolver, &agg_cfg_2, &output, acc.as_ref()); } - ASAPQueryEngine::new(1).with_sketch_index(sketch_index) + ASAPQueryEngine::new(1).with_sketch_index(summary_store) } /// Creates a ASAPQueryEngine with two independent metrics, each with their own @@ -298,7 +298,7 @@ pub fn create_engine_two_metrics( let labels_a: Vec = grouping_labels_a.iter().map(|s| s.to_string()).collect(); let labels_b: Vec = grouping_labels_b.iter().map(|s| s.to_string()).collect(); - let mut aggregation_configs = HashMap::new(); + let mut materializations_by_policy_fingerprint = HashMap::new(); let agg_config_a = AggregationConfig { population_key_encoding: Default::default(), @@ -327,7 +327,7 @@ pub fn create_engine_two_metrics( value_source_column: None, }; let id_a = agg_config_a.policy_fp_u64(); - aggregation_configs.insert(id_a, agg_config_a); + materializations_by_policy_fingerprint.insert(id_a, agg_config_a); let agg_config_b = AggregationConfig { population_key_encoding: Default::default(), @@ -356,15 +356,15 @@ pub fn create_engine_two_metrics( value_source_column: None, }; let id_b = agg_config_b.policy_fp_u64(); - aggregation_configs.insert(id_b, agg_config_b); + materializations_by_policy_fingerprint.insert(id_b, agg_config_b); let streaming_config = Arc::new(StreamingConfig { - aggregation_configs, + materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), }); - let sketch_index = + let summary_store = std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); let resolver = std::sync::Arc::new(SeriesIdResolver::new()); let agg_cfg_1 = streaming_config @@ -384,7 +384,7 @@ pub fn create_engine_two_metrics( key, asap_types::PolicyFingerprint(id_a), ); - ingest_with_fresh_resolver(&sketch_index, &resolver, &agg_cfg_1, &output, acc.as_ref()); + ingest_with_fresh_resolver(&summary_store, &resolver, &agg_cfg_1, &output, acc.as_ref()); } for (label_values_opt, acc) in data_b { let key = label_values_opt.map(|labels| KeyByLabelValues { labels }); @@ -394,10 +394,10 @@ pub fn create_engine_two_metrics( key, asap_types::PolicyFingerprint(id_b), ); - ingest_with_fresh_resolver(&sketch_index, &resolver, &agg_cfg_2, &output, acc.as_ref()); + ingest_with_fresh_resolver(&summary_store, &resolver, &agg_cfg_2, &output, acc.as_ref()); } let _ = (query_a, query_b); - ASAPQueryEngine::new(1).with_sketch_index(sketch_index) + ASAPQueryEngine::new(1).with_sketch_index(summary_store) } /// Creates a ASAPQueryEngine with three independent metrics, each with their own @@ -426,7 +426,7 @@ pub fn create_engine_three_metrics( let labels_b: Vec = grouping_labels_b.iter().map(|s| s.to_string()).collect(); let labels_c: Vec = grouping_labels_c.iter().map(|s| s.to_string()).collect(); - let mut aggregation_configs = HashMap::new(); + let mut materializations_by_policy_fingerprint = HashMap::new(); let mut ids: Vec = Vec::new(); for (agg_type, labels, metric) in [ @@ -462,16 +462,16 @@ pub fn create_engine_three_metrics( }; let id = cfg.policy_fp_u64(); ids.push(id); - aggregation_configs.insert(id, cfg); + materializations_by_policy_fingerprint.insert(id, cfg); } let streaming_config = Arc::new(StreamingConfig { - aggregation_configs, + materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), }); - let sketch_index = + let summary_store = std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); let resolver = std::sync::Arc::new(SeriesIdResolver::new()); let agg_cfgs: Vec<_> = ids @@ -495,12 +495,12 @@ pub fn create_engine_three_metrics( key, asap_types::PolicyFingerprint(agg_id), ); - ingest_with_fresh_resolver(&sketch_index, &resolver, agg_cfg, &output, acc.as_ref()); + ingest_with_fresh_resolver(&summary_store, &resolver, agg_cfg, &output, acc.as_ref()); } } let _ = (labels_a, labels_b, labels_c, query_a, query_b, query_c); - ASAPQueryEngine::new(1).with_sketch_index(sketch_index) + ASAPQueryEngine::new(1).with_sketch_index(summary_store) } /// Creates a single-pop engine with data at multiple timestamps for testing merge. @@ -515,7 +515,7 @@ pub fn create_engine_multi_timestamp( let grouping_label_strings: Vec = grouping_labels.iter().map(|s| s.to_string()).collect(); - let mut aggregation_configs = HashMap::new(); + let mut materializations_by_policy_fingerprint = HashMap::new(); let agg_config = AggregationConfig { population_key_encoding: Default::default(), aggregation_type, @@ -543,15 +543,15 @@ pub fn create_engine_multi_timestamp( value_source_column: None, }; let agg_id = agg_config.policy_fp_u64(); - aggregation_configs.insert(agg_id, agg_config); + materializations_by_policy_fingerprint.insert(agg_id, agg_config); let streaming_config = Arc::new(StreamingConfig { - aggregation_configs, + materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), }); - let sketch_index = + let summary_store = std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); let resolver = std::sync::Arc::new(SeriesIdResolver::new()); let agg_cfg = streaming_config @@ -566,9 +566,9 @@ pub fn create_engine_multi_timestamp( key, asap_types::PolicyFingerprint(agg_id), ); - ingest_with_fresh_resolver(&sketch_index, &resolver, &agg_cfg, &output, acc.as_ref()); + ingest_with_fresh_resolver(&summary_store, &resolver, &agg_cfg, &output, acc.as_ref()); } - ASAPQueryEngine::new(1).with_sketch_index(sketch_index) + ASAPQueryEngine::new(1).with_sketch_index(summary_store) } /// Creates a single-pop engine with data at multiple timestamps and configurable window. @@ -589,7 +589,7 @@ pub fn create_engine_multi_timestamp_with_window( let grouping_label_strings: Vec = grouping_labels.iter().map(|s| s.to_string()).collect(); - let mut aggregation_configs = HashMap::new(); + let mut materializations_by_policy_fingerprint = HashMap::new(); let agg_config = AggregationConfig { population_key_encoding: Default::default(), aggregation_type, @@ -617,15 +617,15 @@ pub fn create_engine_multi_timestamp_with_window( value_source_column: None, }; let agg_id = agg_config.policy_fp_u64(); - aggregation_configs.insert(agg_id, agg_config); + materializations_by_policy_fingerprint.insert(agg_id, agg_config); let streaming_config = Arc::new(StreamingConfig { - aggregation_configs, + materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), }); - let sketch_index = + let summary_store = std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); let resolver = std::sync::Arc::new(SeriesIdResolver::new()); let agg_cfg = streaming_config @@ -640,7 +640,7 @@ pub fn create_engine_multi_timestamp_with_window( key, asap_types::PolicyFingerprint(agg_id), ); - ingest_with_fresh_resolver(&sketch_index, &resolver, &agg_cfg, &output, acc.as_ref()); + ingest_with_fresh_resolver(&summary_store, &resolver, &agg_cfg, &output, acc.as_ref()); } - ASAPQueryEngine::new(1).with_sketch_index(sketch_index) + ASAPQueryEngine::new(1).with_sketch_index(summary_store) } diff --git a/data_plane/src/tests/test_utilities/planning.rs b/data_plane/src/tests/test_utilities/planning.rs index 8e6bb709c..f05dac22e 100644 --- a/data_plane/src/tests/test_utilities/planning.rs +++ b/data_plane/src/tests/test_utilities/planning.rs @@ -1,25 +1,28 @@ //! Synthetic complete quotes for deployment fixtures; never used in production. use control_plane::physical::compiler::{ - BackendLocalPlanningSnapshot, PhysicalCompiler, BACKEND_REVISION, PLANNER_REVISION, + BackendLocalPlanningInput, PhysicalPlanCompiler, BACKEND_REVISION, PLANNER_REVISION, }; pub(crate) fn quoted_snapshot( - mut snapshot: BackendLocalPlanningSnapshot, + mut snapshot: BackendLocalPlanningInput, metricsql: bool, -) -> BackendLocalPlanningSnapshot { +) -> BackendLocalPlanningInput { use control_plane::physical::workload_cost::{ - manifest, with_exact_alternative, WorkloadCostEvidence, WorkloadQuote, + enumerate_exact_and_materialized_candidates, manifest, WorkloadCostEvidence, WorkloadQuote, }; - let (request, environment) = snapshot.clone().planning_request().unwrap(); - let quotes = with_exact_alternative(request) + let (request, environment) = snapshot + .clone() + .into_physical_compilation_request() + .unwrap(); + let quotes = enumerate_exact_and_materialized_candidates(request) .unwrap() .into_iter() .enumerate() .filter_map(|(index, candidate)| { let plan = if metricsql { - PhysicalCompiler.compile_metricsql(candidate.clone(), environment.clone()) + PhysicalPlanCompiler.compile_metricsql(candidate.clone(), environment.clone()) } else { - PhysicalCompiler.compile(candidate.clone(), environment.clone()) + PhysicalPlanCompiler.compile_promql(candidate.clone(), environment.clone()) } .ok()?; let manifest = manifest(&plan, &candidate.queries).unwrap(); diff --git a/data_plane/src/monitor/coordinator.rs b/data_plane/src/update_sampling/coordinator.rs similarity index 100% rename from data_plane/src/monitor/coordinator.rs rename to data_plane/src/update_sampling/coordinator.rs diff --git a/data_plane/src/monitor/epoch.rs b/data_plane/src/update_sampling/epoch.rs similarity index 100% rename from data_plane/src/monitor/epoch.rs rename to data_plane/src/update_sampling/epoch.rs diff --git a/data_plane/src/monitor/mod.rs b/data_plane/src/update_sampling/mod.rs similarity index 100% rename from data_plane/src/monitor/mod.rs rename to data_plane/src/update_sampling/mod.rs diff --git a/data_plane/src/monitor/sampling_alloc.rs b/data_plane/src/update_sampling/sampling_alloc.rs similarity index 100% rename from data_plane/src/monitor/sampling_alloc.rs rename to data_plane/src/update_sampling/sampling_alloc.rs diff --git a/data_plane/src/monitor/server.rs b/data_plane/src/update_sampling/server.rs similarity index 99% rename from data_plane/src/monitor/server.rs rename to data_plane/src/update_sampling/server.rs index b5b8d9fe0..c05025458 100644 --- a/data_plane/src/monitor/server.rs +++ b/data_plane/src/update_sampling/server.rs @@ -338,7 +338,7 @@ impl MonitorService for MonitorServiceImpl { #[cfg(test)] mod reconfigure_tests { use super::MonitorCoordinator; - use crate::monitor::coordinator::MonitorConfig; + use crate::update_sampling::coordinator::MonitorConfig; fn cfg(agg_id: u64, key: &str, tau: f64) -> MonitorConfig { MonitorConfig { diff --git a/data_plane/src/utils/file_io.rs b/data_plane/src/utils/file_io.rs index 7e90183d1..5150ddf0e 100644 --- a/data_plane/src/utils/file_io.rs +++ b/data_plane/src/utils/file_io.rs @@ -48,12 +48,8 @@ aggregations: write!(streaming_temp_file, "{streaming_yaml_content}").unwrap(); let config = read_streaming_config(streaming_temp_file.path().to_str().unwrap()).unwrap(); - assert!(!config.aggregation_configs.is_empty()); - let agg = config - .get_all_aggregation_configs() - .values() - .next() - .expect("one agg"); + assert!(!config.materializations_by_policy_fingerprint.is_empty()); + let agg = config.materializations().values().next().expect("one agg"); assert_eq!(agg.num_aggregates_to_retain, Some(6)); } } diff --git a/data_plane/tests/asapquery_compatibility_process_e2e.rs b/data_plane/tests/asapquery_compatibility_process_e2e.rs index a50a3452c..a942f428f 100644 --- a/data_plane/tests/asapquery_compatibility_process_e2e.rs +++ b/data_plane/tests/asapquery_compatibility_process_e2e.rs @@ -30,29 +30,32 @@ mod immutable_maintenance_process; // Test-only quotes preserve the fixture's local candidate without a production bypass. fn quote_snapshot_for_test( - snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot, -) -> control_plane::physical::compiler::BackendLocalPlanningSnapshot { + snapshot: control_plane::physical::compiler::BackendLocalPlanningInput, +) -> control_plane::physical::compiler::BackendLocalPlanningInput { quote_snapshot_for_frontend_test(snapshot, false) } fn quote_snapshot_for_frontend_test( - mut snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot, + mut snapshot: control_plane::physical::compiler::BackendLocalPlanningInput, metricsql: bool, -) -> control_plane::physical::compiler::BackendLocalPlanningSnapshot { +) -> control_plane::physical::compiler::BackendLocalPlanningInput { use control_plane::physical::{ - compiler::{PhysicalCompiler, BACKEND_REVISION, PLANNER_REVISION}, + compiler::{PhysicalPlanCompiler, BACKEND_REVISION, PLANNER_REVISION}, workload_cost::{self, WorkloadCostEvidence, WorkloadQuote}, }; - let (request, environment) = snapshot.clone().planning_request().unwrap(); + let (request, environment) = snapshot + .clone() + .into_physical_compilation_request() + .unwrap(); let mut preferred = true; - let quotes = workload_cost::with_exact_alternative(request) + let quotes = workload_cost::enumerate_exact_and_materialized_candidates(request) .unwrap() .into_iter() .filter_map(|candidate| { let plan = if metricsql { - PhysicalCompiler.compile_metricsql(candidate.clone(), environment.clone()) + PhysicalPlanCompiler.compile_metricsql(candidate.clone(), environment.clone()) } else { - PhysicalCompiler.compile(candidate.clone(), environment.clone()) + PhysicalPlanCompiler.compile_promql(candidate.clone(), environment.clone()) } .ok()?; let unit_cost = if preferred { 1.0 } else { 1e12 }; @@ -204,7 +207,7 @@ fn is_warm(response: &Value) -> bool { #[tokio::test] #[ignore = "requires ASAPCollector CollectorPlan schema compatibility; run explicitly after Collector is updated"] async fn erp_measured_kll_collector_to_query_oracle() { - use control_plane::physical::compiler::{BackendLocalPlanningSnapshot, PhysicalCompiler}; + use control_plane::physical::compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}; const QUERY: &str = "quantile_over_time(0.9, erp_latency[5s])"; let artifact: Value = serde_json::from_str(include_str!( "../../control_plane/tests/fixtures/erp-kll-measured.json" @@ -226,10 +229,10 @@ async fn erp_measured_kll_collector_to_query_oracle() { "byte_second_weight": 1e-9, "mode": "hybrid", "runtime": {"allowed_algorithms": ["Kll"], "max_memory_bytes": null} }); - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_value(fixture).unwrap(); - let (mut request, mut environment) = snapshot.planning_request().unwrap(); - request.hybrid_execution = false; - request.queries[0].group_by = vec!["service".into()]; + let snapshot: BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); + let (mut request, mut environment) = snapshot.into_physical_compilation_request().unwrap(); + request.allow_mixed_summary_and_exact_execution = false; + request.queries[0].group_by_labels = vec!["service".into()]; let lifecycle_entry = &mut request .query_workload .as_mut() @@ -240,8 +243,10 @@ async fn erp_measured_kll_collector_to_query_oracle() { lifecycle_entry.time_selection.scope = planner_types::workload::QueryTimeScope::Unknown; environment.target = control_plane::physical::compiler::PhysicalDeploymentTarget::DistributedCollectors; - environment.collector_ids = vec!["erp-collector".into()]; - let plan = PhysicalCompiler.compile(request, environment).unwrap(); + environment.target_collector_ids = vec!["erp-collector".into()]; + let plan = PhysicalPlanCompiler + .compile_promql(request, environment) + .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); assert_eq!(plan.precompute_plan.materializations[0].parameters["k"], 32); let collector = serde_json::to_value(&plan.collector_plans[0]).unwrap(); @@ -486,7 +491,7 @@ async fn registered_temporal_topk_count_sketch_heap() { } async fn registered_temporal_topk(algorithm: planner_types::post_asap::SketchAlgorithm) { - use control_plane::physical::compiler::{BackendLocalPlanningSnapshot, PhysicalCompiler}; + use control_plane::physical::compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}; use planner_types::post_asap::{CompositionOperator, SketchQuery, SummaryFamilyType}; const QUERY: &str = "topk(3, count_over_time(top_endpoint_qps[5s]))"; struct Evidence; @@ -525,17 +530,19 @@ async fn registered_temporal_topk(algorithm: planner_types::post_asap::SketchAlg "source": "deterministic-count-ranking-fixture" } }); - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_value(fixture).unwrap(); - let (mut request, environment) = snapshot.planning_request().unwrap(); + let snapshot: BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); + let (mut request, environment) = snapshot.into_physical_compilation_request().unwrap(); let query = &mut request.queries[0]; - let expr = - control_plane::query_parser::parse_query_expr_canonical(QUERY, query.accuracy.clone()) - .unwrap(); + let expr = control_plane::query_parser::parse_query_expr_canonical( + QUERY, + query.accuracy_target.clone(), + ) + .unwrap(); let model = control_plane::physical::post_asap::cost_model::ForcedFamilyCostModel::new( - query.accuracy.clone(), + query.accuracy_target.clone(), algorithm.clone(), ); - query.post_asap = control_plane::planner_selection::select_summary_with_evidence( + query.selected_plan_root = control_plane::planner_selection::select_summary_with_evidence( &expr, &model, &asap_aware_mapping::DefaultAccuracyModel, @@ -543,7 +550,9 @@ async fn registered_temporal_topk(algorithm: planner_types::post_asap::SketchAlg &Evidence, ) .unwrap(); - let plan = PhysicalCompiler.compile(request, environment).unwrap(); + let plan = PhysicalPlanCompiler + .compile_promql(request, environment) + .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); use data_plane::storage_engines::types::AggregationType; let expected_type = match algorithm { @@ -783,7 +792,7 @@ async fn run_shared_dashboard(multi_pane: bool) { }) .collect(), ); - let mut typed: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let mut typed: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_value(snapshot.clone()).unwrap(); if multi_pane { for entry in typed.query_workload.repeating_queries.as_mut().unwrap() { @@ -796,15 +805,18 @@ async fn run_shared_dashboard(multi_pane: bool) { }; } } - let (request, environment) = typed.clone().planning_request().unwrap(); + let (request, environment) = typed.clone().into_physical_compilation_request().unwrap(); let candidates = - control_plane::physical::workload_cost::with_exact_alternative(request).unwrap(); + control_plane::physical::workload_cost::enumerate_exact_and_materialized_candidates( + request, + ) + .unwrap(); let quotes = candidates .into_iter() .enumerate() .map(|(index, candidate)| { - let plan = control_plane::physical::compiler::PhysicalCompiler - .compile(candidate.clone(), environment.clone()) + let plan = control_plane::physical::compiler::PhysicalPlanCompiler + .compile_promql(candidate.clone(), environment.clone()) .unwrap(); let manifest = control_plane::physical::workload_cost::manifest(&plan, &candidate.queries) @@ -821,7 +833,7 @@ async fn run_shared_dashboard(multi_pane: bool) { } }) .collect(); - typed.snapshot_version = 2; + typed.schema_version = 2; typed.workload_cost_evidence = Some( control_plane::physical::workload_cost::WorkloadCostEvidence { backend_revision: control_plane::physical::compiler::BACKEND_REVISION.into(), @@ -834,13 +846,13 @@ async fn run_shared_dashboard(multi_pane: bool) { }, ); snapshot = serde_json::to_value(&typed).unwrap(); - let plan = typed.compile().unwrap(); + let plan = typed.compile_promql().unwrap(); assert!(plan.cost_comparison.is_some()); assert_eq!(plan.precompute_plan.materializations.len(), 1); assert_eq!(plan.query_plan.entries.len(), 3); if multi_pane { assert!(plan.lifecycle_estimates[0] - .window_implementation_id + .window_realization_id .contains("pane")); for entry in plan.query_plan.entries.values() { assert_eq!(entry.instant.lookback_ms, 10_000); @@ -1641,9 +1653,9 @@ async fn collector_free_profile_serves_complete_matrix_and_falls_back_exactly() let materializations = status["materializations"] .as_array() .expect("materialization statuses"); - let planned_snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let planned_snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(&std::fs::read_to_string(&snapshot).unwrap()).unwrap(); - let planned = planned_snapshot.compile().unwrap(); + let planned = planned_snapshot.compile_promql().unwrap(); // Every selected state must be serving; the Planner may share or separate // physical populations, so compare identities rather than a frozen count. let expected = planned @@ -1674,7 +1686,7 @@ async fn collector_free_profile_serves_complete_matrix_and_falls_back_exactly() "collector_plans": invalid.collector_plans, "precompute_plan": invalid.precompute_plan, "transmission_plan": invalid.transmission_plan, "query_plan": invalid.query_plan, "storage_routing": null, "adaptation_evidence": []}); - let built = data_plane::drivers::query::servers::http::build_active_physical_plan( + let built = data_plane::drivers::query::servers::http::validate_and_build_runtime_plan( serde_json::from_value(artifact.clone()).unwrap(), std::sync::Arc::new(data_plane::storage_engines::types::BackendStorageRouting::empty()), ); diff --git a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs index ec107e025..1f156aacc 100644 --- a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs +++ b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs @@ -8,7 +8,7 @@ //! ─PromQL /api/v1/query─► //! query answer //! -//! The control plane drives the streaming-config: a `QueryWorkload` +//! The control plane drives the streaming-config: a `RegisteredWorkload` //! goes through `bind_workload_typed` → `split_typed_three_stage` → //! `emit_backend_streaming_config_json`, the resulting JSON is posted //! to the backend's `/api/v1/streaming-config` endpoint in parser tests. @@ -34,7 +34,6 @@ //! PromQL, asserts the response is well-formed for the planned //! metric. -use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; #[path = "support/physical_fixture.rs"] @@ -69,14 +68,14 @@ async fn post_full_config(client: &reqwest::Client, stack: &FullStack, json: &Js .unwrap(); // The transport payloads below contain one-second states. The legacy // streaming emitter's default window is not their physical layout. - for config in runtime.aggregation_configs.values_mut() { + for config in runtime.materializations_by_policy_fingerprint.values_mut() { config.window_size = 1; config.slide_interval = 1; config.window_layout = asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }; } let mut artifact = physical_fixture::artifact(&runtime); if runtime - .aggregation_configs + .materializations_by_policy_fingerprint .values() .any(|c| c.metric == "http_requests_total_latency_ms") { @@ -134,8 +133,8 @@ async fn post_full_config(client: &reqwest::Client, stack: &FullStack, json: &Js .insert(stack.otlp_http_port, plan); } -use control_plane::types::{AggType, QueryWorkload, WorkloadCharacteristics}; -use data_plane::storage_engines::types::HotReloadStreamingConfig; +use control_plane::types::{AggType, RegisteredWorkload, WorkloadCharacteristics}; +use data_plane::storage_engines::types::StreamingConfigHandle; use serde_json::Value as JsonValue; use asap_otel_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; @@ -156,7 +155,7 @@ use prost::Message; // ── Helpers ───────────────────────────────────────────────────────────────── -/// Build a `QueryWorkload` with the given parameters. Mirrors the +/// Build a `RegisteredWorkload` with the given parameters. Mirrors the /// `WorkloadAnalyzer` output shape but constructed directly for tests. fn build_workload_with_override( metric_name: &str, @@ -166,21 +165,34 @@ fn build_workload_with_override( group_by_labels: Vec, quantiles: Vec, sketch_type_override: Option, -) -> QueryWorkload { - QueryWorkload { - metric_name: metric_name.to_string(), - label_filters: HashMap::new(), - group_by_labels, - aggregations, - time_window, - repeat_every: None, - accuracy: control_plane::types::AccuracyTarget::Epsilon(accuracy_sla), - accuracy_sla, - latency_sla: None, - sketch_type_override, - exact_required: false, - quantiles, - } +) -> RegisteredWorkload { + let query = match aggregations.as_slice() { + [AggType::Quantile] => format!( + "quantile_over_time({}, {metric_name}[{}s])", + quantiles.first().copied().unwrap_or(0.99), + time_window.as_secs() + ), + [AggType::Cardinality] => format!( + "distinct_over_time({metric_name}[{}s])", + time_window.as_secs() + ), + [AggType::Frequency] => { + format!("count_over_time({metric_name}[{}s])", time_window.as_secs()) + } + _ => panic!("fixture requires one canonical aggregation"), + }; + control_plane::pipeline::Analyzer::new() + .analyze( + serde_json::from_value(serde_json::json!({ + "query_string": query, + "group_by_labels": group_by_labels, + "accuracy_sla": 1.0 - accuracy_sla, + "accuracy": {"Epsilon": accuracy_sla}, + "sketch_type": sketch_type_override, + })) + .unwrap(), + ) + .unwrap() } /// Convenience wrapper — no sketch_type_override. @@ -191,7 +203,7 @@ fn build_workload( time_window: Duration, group_by_labels: Vec, quantiles: Vec, -) -> QueryWorkload { +) -> RegisteredWorkload { build_workload_with_override( metric_name, aggregations, @@ -203,7 +215,7 @@ fn build_workload( ) } -/// Run the controller's planning pipeline end-to-end on a `QueryWorkload` +/// Run the controller's planning pipeline end-to-end on a `RegisteredWorkload` /// and return the `BackendStageConfig` the controller would emit from /// for it — the same object both `emit_backend_streaming_config_json` /// (legacy JSON) and the catalog-backed physical-plan compiler @@ -211,11 +223,11 @@ fn build_workload( /// /// Mirrors the `handle_plan` flow's `StageConfig::Backend(mut be)` /// branch — including the post-emit grouping patch (#245) so the config -/// carries `grouping` from `workload.group_by_labels`. +/// carries `grouping` from `workload.group_by_labels()`. fn plan_backend_stage_config( - workload: &QueryWorkload, + workload: &RegisteredWorkload, ) -> control_plane::physical::colored_dag::BackendStageConfig { - let deployment_expr = if workload.metric_name == "top_endpoint_qps" { + let deployment_expr = if workload.metric_name() == "top_endpoint_qps" { let evidence = control_plane::physical::compiler::TopKMembershipEvidence { selected_lower_bound: 101.0, excluded_upper_bound: 100.0, @@ -249,20 +261,20 @@ fn plan_backend_stage_config( // workload directly, the same way #245 patches grouping. for agg in &mut backend_cfg.aggregations { if agg.metric_name.is_empty() { - agg.metric_name = workload.metric_name.clone(); + agg.metric_name = workload.metric_name().clone(); } if agg.window_secs == 0 { - agg.window_secs = workload.time_window.as_secs(); + agg.window_secs = workload.time_window().as_secs(); } - agg.grouping = workload.group_by_labels.clone(); + agg.grouping = workload.group_by_labels().clone(); } backend_cfg } -/// Run the controller's planning pipeline end-to-end on a `QueryWorkload` +/// Run the controller's planning pipeline end-to-end on a `RegisteredWorkload` /// and return the streaming-config JSON document the controller would /// POST to the backend's `/api/v1/streaming-config` endpoint. -fn plan_streaming_config_json(workload: &QueryWorkload) -> JsonValue { +fn plan_streaming_config_json(workload: &RegisteredWorkload) -> JsonValue { let backend_cfg = plan_backend_stage_config(workload); // No continuous-monitoring (CDM) intents in these tests — pass an empty // slice (the `&[MonitorIntent]` arg added when CDM monitor specs landed). @@ -270,21 +282,21 @@ fn plan_streaming_config_json(workload: &QueryWorkload) -> JsonValue { .expect("emit_backend_streaming_config_json must succeed") } -/// Spin up an in-process backend HTTP server with `HotReloadStreamingConfig` +/// Spin up an in-process backend HTTP server with `StreamingConfigHandle` /// wired through both the query engine and the POST `/api/v1/streaming-config` /// handler. Returns `(port, hot_reload_handle)` — the latter so tests can /// also inspect the current config from the controller's side. -async fn start_backend_http_server() -> (u16, HotReloadStreamingConfig) { +async fn start_backend_http_server() -> (u16, StreamingConfigHandle) { use data_plane::drivers::query::adapters::config::AdapterConfig; use data_plane::drivers::query::servers::{HttpServer, HttpServerConfig}; use data_plane::query_engines::asap_query_engine::engine::ASAPQueryEngine; use data_plane::storage_engines::sketch_db::index::SketchStore; use data_plane::storage_engines::types::StreamingConfig; - let hot_reload = HotReloadStreamingConfig::new(StreamingConfig::default()); - let sketch_index = Arc::new(SketchStore::new()); + let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); + let summary_store = Arc::new(SketchStore::new()); let query_engine = - Arc::new(ASAPQueryEngine::new(15_000).with_sketch_index(sketch_index.clone())); + Arc::new(ASAPQueryEngine::new(15_000).with_sketch_index(summary_store.clone())); let adapter_config = AdapterConfig::prometheus_promql( "http://127.0.0.1:9999".to_string(), // unused — no forwarding in this test @@ -296,7 +308,7 @@ async fn start_backend_http_server() -> (u16, HotReloadStreamingConfig) { adapter_config, }; - let server = HttpServer::new(http_config, query_engine, sketch_index) + let server = HttpServer::new(http_config, query_engine, summary_store) .with_hot_reload_config(hot_reload.clone()); let port = server @@ -352,7 +364,7 @@ fn _wc_anchor() -> WorkloadCharacteristics { /// Full test stack: PrecomputeEngine + SketchStoreSink + OtlpReceiver + /// HttpServer, all sharing the same `SketchStore` and -/// `HotReloadStreamingConfig` so a controller-posted streaming-config +/// `StreamingConfigHandle` so a controller-posted streaming-config /// is visible to the engine's accumulator routing, the engine's window /// outputs land in `SketchStore`, and the query engine reads from the /// same store. @@ -380,17 +392,17 @@ async fn start_full_stack(otlp_http_port: u16, otlp_grpc_port: u16) -> FullStack use data_plane::query_engines::asap_query_engine::engine::ASAPQueryEngine; use data_plane::storage_engines::sketch_db::index::SketchStore; - let sketch_index = Arc::new(SketchStore::new()); - let active = data_plane::storage_engines::types::HotReloadActivePhysicalPlan::new( + let summary_store = Arc::new(SketchStore::new()); + let active = data_plane::storage_engines::types::ActivePhysicalPlanHandle::new( physical_fixture::bootstrap(), ); - let hot_reload = HotReloadStreamingConfig::from_active(active.clone()); + let hot_reload = StreamingConfigHandle::from_active_physical_plan(active.clone()); let series_resolver = Arc::new(SeriesIdResolver::new()); // SketchStoreSink writes precompute output back into SketchStore so // the query engine can find it. let sink = Arc::new(SketchStoreSink::new( - sketch_index.clone(), + summary_store.clone(), hot_reload.clone(), series_resolver.clone(), )); @@ -413,7 +425,7 @@ async fn start_full_stack(otlp_http_port: u16, otlp_grpc_port: u16) -> FullStack hot_reload.clone(), sink, series_resolver.clone(), - sketch_index.clone(), + summary_store.clone(), ); let ingest_state = engine.ingest_state(); tokio::spawn(async move { @@ -448,10 +460,10 @@ async fn start_full_stack(otlp_http_port: u16, otlp_grpc_port: u16) -> FullStack // `sketch_index` via OTLP ingest (the engine's // `precompute_engine` shares the Arc), but the query // path can't see them without this binding. - .with_sketch_index(sketch_index.clone()) + .with_sketch_index(summary_store.clone()) .with_active_physical_plan(active.clone()), ); - let server = HttpServer::new(http_config, query_engine, sketch_index) + let server = HttpServer::new(http_config, query_engine, summary_store) .with_hot_reload_config(hot_reload.clone()) .with_active_physical_plan(active); let backend_port = server @@ -953,7 +965,7 @@ async fn controller_plans_with_grouping_and_backend_parses_grouping_labels() { assert_eq!( names, vec!["zone"], - "controller must thread workload.group_by_labels → labels.grouping (#245)\n\ + "controller must thread workload.group_by_labels() → labels.grouping (#245)\n\ {streaming_config_json}" ); @@ -965,7 +977,7 @@ async fn controller_plans_with_grouping_and_backend_parses_grouping_labels() { assert_eq!(active["aggregation_count"], 1); // Walk the streaming_config object to find the registered grouping - // labels. The snapshot path is `streaming_config.aggregation_configs. + // labels. The snapshot path is `streaming_config.materializations_by_policy_fingerprint. // .grouping_labels.`. let cfgs = active["streaming_config"]["aggregation_configs"] .as_object() @@ -1015,7 +1027,7 @@ async fn controller_plans_with_grouping_and_backend_parses_grouping_labels() { // * Modified-OTLP `DdSketchDataPoint` wire encoding + the backend's // OTLP HTTP receiver accept the payload (no 4xx/5xx). // * The full stack (PrecomputeEngine + SketchStoreSink + OtlpReceiver -// + HttpServer all sharing SketchStore + HotReloadStreamingConfig) +// + HttpServer all sharing SketchStore + StreamingConfigHandle) // comes up and stays up under POST + query traffic. // * The OTLP-ingested sketch lands in `SketchStore` keyed by the // right `PolicyFingerprint` (or via the `instances_matching` diff --git a/data_plane/tests/monitor_grpc.rs b/data_plane/tests/monitor_grpc.rs index 31b1d5652..b8901f014 100644 --- a/data_plane/tests/monitor_grpc.rs +++ b/data_plane/tests/monitor_grpc.rs @@ -22,7 +22,7 @@ use asap_otel_proto::monitor::v1::{ edge_to_coord, monitor_service_client::MonitorServiceClient, EdgeToCoord, MonitorRegister, MonitorReport, }; -use data_plane::monitor::{MonitorConfig, MonitorCoordinator, MonitorServiceImpl}; +use data_plane::update_sampling::{MonitorConfig, MonitorCoordinator, MonitorServiceImpl}; async fn start_server(cfgs: Vec) -> String { let coord = MonitorCoordinator::new(cfgs); diff --git a/data_plane/tests/support/distinct_planning_process.rs b/data_plane/tests/support/distinct_planning_process.rs index a67a197ad..058ca9d7b 100644 --- a/data_plane/tests/support/distinct_planning_process.rs +++ b/data_plane/tests/support/distinct_planning_process.rs @@ -1,5 +1,5 @@ use super::*; -use control_plane::physical::compiler::BackendLocalPlanningSnapshot; +use control_plane::physical::compiler::BackendLocalPlanningInput; /// The production compiler, ingest engine and query DAG preserve distinct populations. #[tokio::test] @@ -24,9 +24,9 @@ async fn distinct_range_uses_planner_selected_hll_and_source_labels() { entry["requirements"]["accuracy"] = serde_json::json!({"explicit": {"Epsilon": 0.05}}); fixture["query_workload"]["repeating_queries"] = serde_json::json!([entry]); let plan = quote_snapshot_for_test( - serde_json::from_value::(fixture.clone()).unwrap(), + serde_json::from_value::(fixture.clone()).unwrap(), ) - .compile() + .compile_promql() .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); assert_eq!( @@ -81,7 +81,7 @@ async fn distinct_range_uses_planner_selected_hll_and_source_labels() { wait_until_ready(&client, &format!("{backend}/api/v1/health"), &mut child.0).await; // Source syntax uses the shared parser fork; serving semantics and exact // routing belong to the MetricsQL adapter and its installed query entries. - let snapshot = serde_json::from_value::(fixture).unwrap(); + let snapshot = serde_json::from_value::(fixture).unwrap(); let mut snapshot = snapshot; snapshot.environment.plan_version = 2; let compiled = quote_snapshot_for_frontend_test(snapshot, true) diff --git a/data_plane/tests/support/durable_summary_process.rs b/data_plane/tests/support/durable_summary_process.rs index f8a704906..b79aad913 100644 --- a/data_plane/tests/support/durable_summary_process.rs +++ b/data_plane/tests/support/durable_summary_process.rs @@ -16,9 +16,9 @@ async fn persisted_summary_restarts_without_live_reregistration() { // This restart fixture persists one complete five-second population. fixture["query_workload"]["repeating_queries"][0]["demand"]["fixed_interval_at"]["interval"] = serde_json::json!(5000); - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); - let plan = quote_snapshot_for_test(snapshot).compile().unwrap(); + let plan = quote_snapshot_for_test(snapshot).compile_promql().unwrap(); let install = data_plane::drivers::query::servers::http::PhysicalPlanInstallRequest { summary_catalog: plan.summary_catalog, collector_plans: plan.collector_plans, diff --git a/data_plane/tests/support/erp_planning_process.rs b/data_plane/tests/support/erp_planning_process.rs index 6e88beac7..59b32c89b 100644 --- a/data_plane/tests/support/erp_planning_process.rs +++ b/data_plane/tests/support/erp_planning_process.rs @@ -1,5 +1,5 @@ use super::*; -use control_plane::physical::{compiler::BackendLocalPlanningSnapshot, erp::ErpShapeObserver}; +use control_plane::physical::{compiler::BackendLocalPlanningInput, erp::ErpShapeObserver}; fn measured_profiles(raw: &[f64]) -> Value { let mut records = Vec::new(); @@ -102,9 +102,8 @@ async fn observed_shape_selects_installed_parameters_and_executes_remote_write() ), control_plane::physical::erp::ErpParameterDecision::Empirical { .. } )); - let snapshot: BackendLocalPlanningSnapshot = - serde_json::from_value(fixture.clone()).unwrap(); - let plan = quote_snapshot_for_test(snapshot).compile().unwrap(); + let snapshot: BackendLocalPlanningInput = serde_json::from_value(fixture.clone()).unwrap(); + let plan = quote_snapshot_for_test(snapshot).compile_promql().unwrap(); assert_eq!( plan.precompute_plan.materializations.len(), 1, diff --git a/data_plane/tests/support/immutable_maintenance_process.rs b/data_plane/tests/support/immutable_maintenance_process.rs index f105cdfa6..a1bbf43fa 100644 --- a/data_plane/tests/support/immutable_maintenance_process.rs +++ b/data_plane/tests/support/immutable_maintenance_process.rs @@ -33,9 +33,9 @@ async fn run_maintenance_process(multi_source: bool, distinct_groups: bool) { entry["demand"]["fixed_interval_at"]["evaluation_phase"] = 0.into(); entry["time_selection"]["lookback"] = 60_000.into(); fixture["query_workload"]["repeating_queries"] = serde_json::json!([entry]); - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_value(fixture.clone()).unwrap(); - let plan = quote_snapshot_for_test(snapshot).compile().unwrap(); + let plan = quote_snapshot_for_test(snapshot).compile_promql().unwrap(); assert_eq!( plan.precompute_plan.materializations.len(), if multi_source { 3 } else { 2 } @@ -303,15 +303,14 @@ async fn run_maintenance_process(multi_source: bool, distinct_groups: bool) { // singleton-derived output while new source populations can arrive. let mut next_fixture = fixture.clone(); next_fixture["environment"]["plan_version"] = 2.into(); - let next = - quote_snapshot_for_test( - serde_json::from_value::< - control_plane::physical::compiler::BackendLocalPlanningSnapshot, - >(next_fixture) - .unwrap(), + let next = quote_snapshot_for_test( + serde_json::from_value::( + next_fixture, ) - .compile() - .unwrap(); + .unwrap(), + ) + .compile_promql() + .unwrap(); let next_install = data_plane::drivers::query::servers::http::PhysicalPlanInstallRequest { summary_catalog: next.summary_catalog, collector_plans: next.collector_plans, diff --git a/data_plane/tests/support/physical_fixture.rs b/data_plane/tests/support/physical_fixture.rs index b06e8bd2b..4c799b901 100644 --- a/data_plane/tests/support/physical_fixture.rs +++ b/data_plane/tests/support/physical_fixture.rs @@ -4,7 +4,7 @@ use control_plane::{physical::compiler::*, query_plan::*}; use data_plane::{ drivers::query::servers::http::PhysicalPlanInstallRequest, - storage_engines::types::{ActivePhysicalPlan, BackendStorageRouting, StreamingConfig}, + storage_engines::types::{BackendStorageRouting, RuntimePhysicalPlan, StreamingConfig}, }; use std::{collections::BTreeMap, sync::Arc}; @@ -20,7 +20,7 @@ pub fn artifact(config: &StreamingConfig) -> PhysicalPlanInstallRequest { capability_snapshot_id: "transport-fixture".into(), }; let mut configs = config - .aggregation_configs + .materializations_by_policy_fingerprint .values() .cloned() .collect::>(); @@ -37,7 +37,7 @@ pub fn artifact(config: &StreamingConfig) -> PhysicalPlanInstallRequest { let mut precompute = PrecomputePlan::build(envelope.clone(), configs, &["fixture".into()]).unwrap(); precompute.summary_catalog = Some(catalog.reference().unwrap()); - let mut transmission = control_plane::physical::compiler::compile_transmission_plan( + let mut transmission = control_plane::physical::compiler::build_transmission_plan( envelope, &precompute, &BTreeMap::new(), @@ -158,8 +158,8 @@ pub fn artifact(config: &StreamingConfig) -> PhysicalPlanInstallRequest { } #[allow(dead_code)] -pub fn bootstrap() -> ActivePhysicalPlan { - let mut plan = data_plane::drivers::query::servers::http::build_active_physical_plan( +pub fn bootstrap() -> RuntimePhysicalPlan { + let mut plan = data_plane::drivers::query::servers::http::validate_and_build_runtime_plan( artifact(&StreamingConfig::default()), Arc::new(BackendStorageRouting::empty()), ) diff --git a/data_plane/tests/support/univmon_erp_process.rs b/data_plane/tests/support/univmon_erp_process.rs index 29b5d572e..3bba79280 100644 --- a/data_plane/tests/support/univmon_erp_process.rs +++ b/data_plane/tests/support/univmon_erp_process.rs @@ -1,5 +1,5 @@ use super::*; -use control_plane::physical::{compiler::BackendLocalPlanningSnapshot, erp::ErpShapeObserver}; +use control_plane::physical::{compiler::BackendLocalPlanningInput, erp::ErpShapeObserver}; use data_plane::precompute_engine::operators::univmon_accumulator::UnivMonAccumulator; use data_plane::storage_engines::types::{AggregateCore, SerializableToSink}; @@ -126,8 +126,8 @@ async fn measured_readout_evidence_selects_and_executes_univmon() { "minimum_confidence": 0.7, "minimum_confidence_margin": 0.05}, "runtime": {"allowed_algorithms": ["Hll", "Kll", "UnivMon"], "max_memory_bytes": null} }); - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_value(fixture.clone()).unwrap(); - let plan = quote_snapshot_for_test(snapshot).compile().unwrap(); + let snapshot: BackendLocalPlanningInput = serde_json::from_value(fixture.clone()).unwrap(); + let plan = quote_snapshot_for_test(snapshot).compile_promql().unwrap(); eprintln!( "UNIVMON_PLANNED {}", serde_json::json!({"query_plan": plan.query_plan, "materializations": plan.precompute_plan.materializations, "lifecycle_estimates": plan.lifecycle_estimates, "executable_dags": plan.precompute_plan.executable_dags, "observation": observation}) @@ -151,9 +151,9 @@ async fn measured_readout_evidence_selects_and_executes_univmon() { .remove("max_frequency_entropy_absolute_bits_error"); } let missing = quote_snapshot_for_test( - serde_json::from_value::(missing_entropy).unwrap(), + serde_json::from_value::(missing_entropy).unwrap(), ) - .compile() + .compile_promql() .unwrap(); use control_plane::query_plan::{QueryPlanNode, QueryReadout}; assert!(missing @@ -306,9 +306,9 @@ async fn measured_readout_evidence_selects_and_executes_univmon() { plan.summary_catalog.reference().unwrap() ); if key.sketch == "univmon" { - let mut live_snapshot: BackendLocalPlanningSnapshot = + let mut live_snapshot: BackendLocalPlanningInput = serde_json::from_value(fixture.clone()).unwrap(); - let policy = live_snapshot.implementation.erp.as_mut().unwrap(); + let policy = live_snapshot.physical_inputs.erp.as_mut().unwrap(); policy.observed_shape_source = Some(control_plane::physical::erp::ErpObservedShapeSource { source: key.source.clone(), @@ -334,7 +334,9 @@ async fn measured_readout_evidence_selects_and_executes_univmon() { .unwrap() .invalid_reason .is_none()); - let replanned = quote_snapshot_for_test(live_snapshot).compile().unwrap(); + let replanned = quote_snapshot_for_test(live_snapshot) + .compile_promql() + .unwrap(); assert!( replanned .precompute_plan diff --git a/docs/developer_docs/README.md b/docs/developer_docs/README.md index 183524ca3..cc35304e9 100644 --- a/docs/developer_docs/README.md +++ b/docs/developer_docs/README.md @@ -27,6 +27,8 @@ cross-repository interfaces live in ## Control plane - [Physical compiler](control-plane/physical-compiler.md) +- [Planning terminology and architecture](control-plane/planning-terminology.md) +- [Architecture naming review (中文)](control-plane/architecture-naming-review.zh.md) - [Plan publication](control-plane/plan-publication.md) - [Workload inputs](control-plane/workload-inputs.md) - [Runtime accuracy feedback](control-plane/runtime-accuracy-feedback.md) diff --git a/docs/developer_docs/control-plane/architecture-naming-review.zh.md b/docs/developer_docs/control-plane/architecture-naming-review.zh.md new file mode 100644 index 000000000..ef6ae4c64 --- /dev/null +++ b/docs/developer_docs/control-plane/architecture-naming-review.zh.md @@ -0,0 +1,219 @@ +# ASAPQuery-backend 架构与命名审查 + +审查基线：GitHub `main`，`b1a58ca810d7b7347f2cc5f30924db56532bbb6b`，2026-09-13。关联 Issue：[#709](https://github.com/ProjectASAP/ASAPQuery-backend/issues/709)。面向维护规划、共享契约、安装及执行代码的开发者。 + +结论：#709 应从“术语替换表”改为“组件边界和数据语义对齐”。优先澄清逻辑候选、物理候选、报价对象、部署状态和存储身份，再决定具体命名。不能因为几个对象都包含计划字段，就把它们合成一个类型；也不能因为都叫 `SummaryNode`，就认为它们只表示近似查询。 + +本文件保留实施前的架构审查与决策依据。当前实现及迁移规则见 [Planning terminology and architecture](planning-terminology.md)。其中明确列出此次命名迁移与保留为后续工作的结构调整。 + +**追加实施：删除扁平 workload，统一 canonical 模型** + +[注册模型](../../../control_plane/src/registered_workload.rs)、[输入转换](../../../control_plane/src/pipeline.rs) 和 [注册存储](../../../control_plane/src/store/workload.rs) 已改为直接使用 ASAPPlanner 的 `QueryWorkload` 及内嵌 `DataWorkload`。`LegacyMetricWorkload` 与旧 `types::QueryWorkload` 别名删除。`RegisteredWorkload` 只组合 canonical workload 与部署选项，不保存第二份扁平查询字段或 `ParsedQuery`。 + +| 原字段 / 对象 | canonical 或部署归属 | 重复判断与迁移含义 | +|---|---|---| +| `metric_name`、过滤条件、聚合、quantiles | `QueryWorkload` 中的完整查询表达式 | 旧字段是表达式的重复投影；现在按需解析，不独立存储 | +| `exact_required` | 从 canonical 表达式推导当前 stage 编译能力 | 不再独立存储，也不替代用户的 accuracy requirement | +| `accuracy` / `accuracy_sla` | entry 的 `requirements.accuracy` | 删除可变标量镜像；入口兼容旧 SLA，显式 `EpsilonDelta` 完整保留 | +| `latency_sla` | `requirements.response_latency` | 查询响应要求，不是数据到达频率 | +| `repeat_every` / `shape` | batch entry 或 repeating entry 的 demand | 明确一次执行与固定周期；毫秒转换检查溢出，不推导数据到达模式 | +| `time_window` | 查询 range 与一致的 `time_selection.lookback` | 不把执行周期、历史时间点和 lookback 混成一个窗口 | +| `series_count`、每系列采样速率 | `DataWorkload.input_cardinality`、`ingestion_rate` | 本单 metric 适配器约定 cardinality 为活跃 series；10 series × 5 Hz = 50 samples/s | +| 分布、来源与证据时效 | `DataWorkload.distribution` / `Evidence` | 存储时保留；未知或过期证据不可退回已知默认值 | +| sketch pin、保留标签、wire bytes、memory cap | `DeploymentOptions` | 是部署约束，不属于查询语义或数据到达事实 | +| `distinct_keys_per_window` | 部署侧每次 flush 的 item 基数估计 | 与 series cardinality 不是同一总体，不合并 | +| `WorkloadCharacteristics` | 兼容输入 DTO / 临时成本投影 | 不再与查询模型一起重复存储；重规划从 canonical evidence 获取有效值 | +| `QueryId` / deployment model | 注册元数据 | 保留标识和路由提示，不伪称已经执行额外约束 | + +迁移包含有意的入口行为变化：字段式 quantile 生成显式 0.99 分位查询，frequency 生成时间范围内的 item count；不再给已有查询硬塞 `quantile`。查询与显式 metric/window/aggregation/filter 冲突时返回错误，要求调用者修改查询表达式。旧 `group_by_labels` 中用于 collector 保留的标签放在部署选项，不改写查询 GROUP BY。 + +单 metric 注册接口拒绝不能无损投影的多 selector、非等值过滤、offset/@、子查询、`without`、历史 time selection、未支持的 cadence 以及未实施的 dollars 约束。更完整的查询继续走已有完整编译接口。数据到达模式与查询周期独立；Batch 表示静态数据，持续 ingestion rate 为零。公开 canonical 构造器也验证这些边界，不能绕过入口校验后静默降级。 + +测试覆盖 canonical 查询与精度/周期/数据证据的存储往返、aggregate sample rate、过期/未知证据、cadence 范围检查及不支持的投影。英文流程架构图见 [Planning terminology and architecture](planning-terminology.md)。下文中提出临时命名 `LegacyMetricWorkload` 的内容保留为审查历史，已被本节的实际删除方案取代。 + +**本 PR 的最终落地与迁移方式** + +下表记录最终采用的名称；后文保留审查时的备选与理由。实施基线为 `cb3153a8`（#710），继续复用 Planner 的精度类型；随后合并 `47d4332f`（#711）与 `ca944a6a`（#712、#713、#717），保留主分支的窗口规划、storage routing 和接口清理。 + +| 所属边界 / 源码 | 原名称 → 最终名称 | +|---|---| +| [编译输入与编译器](../../../control_plane/src/physical/compiler.rs) | `BackendLocalPlanningSnapshot` → `BackendLocalPlanningInput`；`BackendLocalImplementation` → `BackendLocalPhysicalInputs`；`PlanningQuery` → `QueryCompilationInput`；`PlanningRequest` → `PhysicalCompilationRequest`；`PhysicalCompiler` → `PhysicalPlanCompiler`；`PhysicalPlan` → `CompiledPhysicalPlan` | +| 同上：查询语义 | `post_asap` → `selected_plan_root`；`source` → `legacy_query_source`；`window_secs` → `query_lookback_seconds`；`group_by` → `group_by_labels`；`accuracy` → `accuracy_target`；`lifecycle` → `summary_lifecycle_inputs`；`runtime_policy` → `materialization_runtime_policy` | +| 同上：候选与证据 | `logical_selection` → `planner_selection_trace`；`materialization_policy` → `enabled_materialization_keys`；`evidence` → `topk_membership_evidence_by_query_id`；`hybrid_execution` → `allow_mixed_summary_and_exact_execution`；`synthesized_window_queries` 删除：编译器来源标记改为每个候选的 `derived` / `cohort_only`，不接受序列化输入 | +| 同上：窗口与成本 | `WindowImplementationCandidate` → `WindowRealizationCandidate`；`ImplementationCostEvidence` → `WindowRealizationCostQuote`；`LifecycleCostEvidence` → `LifecycleUnitCosts`；`LifecyclePlanningInput` → `SummaryLifecyclePlanningInputs`；`window_implementations` → `window_realization_candidates` | +| 同上：外层配置 | `snapshot_version` → `schema_version`；`implementation` → `physical_inputs`；外部窗口候选与默认窗口字段随 #712 删除，改用 `window_cost_model`；编译器在逻辑选择后按 cadence / phase 生成候选；`max_retained_summary_bytes` → `retained_summary_memory_budget_bytes` | +| 同上：部署与保留 | `DeploymentEnvironment` → `PhysicalDeploymentContext`；`collector_ids` → `target_collector_ids`；`query_staleness_margin_ms` → `query_retention_margin_ms` | +| [候选定价与选择](../../../control_plane/src/physical/workload_cost.rs) | `CostDemand` → `CostComponentDemand`；`unit` → `pricing_basis`；`multiplicity` → `occurrences_per_horizon`；`AlternativeCost` → `CandidatePlanEvaluation`；`WorkloadCostComparison` → `CandidatePlanSelectionReport`；`alternative_id` → `candidate_id`；`physical_alternative_id` → `physical_candidate_id`；`alternatives` → `candidate_evaluations` | +| 同上：方法 | `with_exact_alternative` → `enumerate_exact_and_materialized_candidates`；`bind_alternative` → `compile_candidate_for_pricing`；`prepare_manifests` → `compile_candidates_for_pricing`；`select` → `select_lowest_cost_candidate`；`select_metricsql` → `select_lowest_cost_metricsql_candidate` | +| [诊断状态](../../../control_plane/src/physical/workload_cost/status.rs) | `status` 使用 `CandidateEvaluationStatus`；覆盖范围使用 `CandidateSearchScope`；兼容未知字符串与原缺省值。原 `PhysicalQueryFrontend` 移到共享编译入口并命名为 `QueryFrontend`，替代内部布尔参数 | +| [共享运行时算子](../../../crates/asap_types/src/query_plan/residual.rs) | `query_plan::logical::LogicalOperator` → `query_plan::residual::ResidualQueryOperator`；backend 对应模块迁移到 `residual`，保留旧模块导出 | +| [运行时计划与句柄](../../../data_plane/src/storage_engines/types/hot_reload_config.rs) | `ActivePhysicalPlan` → `RuntimePhysicalPlan`；`HotReloadActivePhysicalPlan` → `ActivePhysicalPlanHandle`；`HotReloadStreamingConfig` → `StreamingConfigHandle`；`runtime_config` → `streaming_config`；active handle 的 `snapshot` → `active_snapshot`；`from_active` → `from_active_physical_plan`；`retire_drained` → `mark_drained_plan_retired` | +| [存储元数据](../../../data_plane/src/storage_engines/sketch_db/index/mod.rs) | `SketchInstanceMetadata` → `SummarySeriesMetadata`；`sketch_index` 字段与变量 → `summary_store`；保留 `SketchStore` 类型 | +| 其余调用边界 | `types::QueryWorkload` / `LegacyMetricWorkload` 删除，统一到 Planner `QueryWorkload` + `DataWorkload`；`ClickHouseSqlWorkload.sds` → `summary_catalog`；`aggregation_configs` → `materializations_by_policy_fingerprint`；`aggregation_id_for_key/value` → `key_policy_fingerprint/value_policy_fingerprint`；`data_plane::monitor` → `update_sampling` | + +主分支集成补充：共享 pane 保留 cadence、phase 与 evaluation alignment；发布的 storage routing 从选中物理计划生成；`types_v2` 已删除，定义合入 `types`。随后合入 #715（`e25d53b1`），保留未使用的 auto/Pareto 规划、rollback/diff HTTP 接口及辅助模块删除；canonical 注册与重规划仍走保留入口。这些上游删除的接口不提供旧字段兼容。下文涉及旧窗口模板和 query 级来源集合的建议仅属历史审查。 + +迁移规则：**保留字段的 Rust 名称更新，输出 wire 名称保持原样**；反序列化接受新名称作为 alias。旧公共类型导入及主要入口提供 deprecated 转发，但 Rust struct literal 的旧字段拼写无法通过类型别名兼容，源码消费者需要按表迁移。`None` / 空候选集合、浮点频次、报价身份、候选排序、严格小于的选择规则及计划生命周期均保持原义。 + +`erp` 保留并明确为 Error–Resource Profile；`WorkloadQuote.executable` 保留，因为它不代表安装或部署已经通过验证。双模式 streaming handle 的读写行为只补充说明；拆分写 API、删除 legacy 路径、统一 SQL 定价以及 typed ID / 时间单位迁移留作独立工作。英文流程图与边界说明见 [Planning terminology and architecture](planning-terminology.md)。 + +**一、组件职责与当前真实路径** + +| 组件 | 实际职责 / 输入输出 | 命名判断 | +|---|---|---| +| 外部 ASAPPlanner | 解析与语义 IR，合法 summary/exact 候选，精度推理，逻辑选择；backend 提供具体成本和能力约束 | Planner 的语义选择与 backend 的物理候选比较是不同层次，不是两个重复 planner | +| `control_plane::planner_selection` | 适配 Planner 的选择调用、精度及证据；输出语义 DAG 与诊断 trace | `selection` 必须说明是 logical 还是 physical；trace 不是决定执行行为的配置 | +| `physical::compiler` | 输入规范化、窗口候选校验、调用逻辑选择，以及绑定具体物理实现，生成多个一致的计划投影 | `PhysicalPlanCompiler` 比 `PhysicalCompiler` 清楚；整个模块当前职责仍比单纯 lowering 更宽 | +| `physical::workload_cost` | 枚举工作负载级候选、编译、生成报价清单、核验报价、选择最低成本可行候选 | manifest、quote、evaluation、selection report 不应相互替代 | +| `physical::erp` | Error–Resource Profile 的部署适配、分布匹配、经验参数与资源估计 | `empirical_runtime_profile` 是错误展开；输入还包含策略与观测，不只一个 profile | +| `control_plane::clickhouse` | SQL frontend、逻辑选择、物理绑定；支持已有 catalog 输入与自动生成 materialization 两条路径 | `ClickHouseSqlWorkload.sds` 实际是 `SummaryCatalog`；SQL 当前没有走同一套 `workload_cost::select` 整计划报价流程 | +| `asap_types` | 跨组件共享的 catalog、SDS、生产/传输/预计算/查询/发布契约与验证逻辑 | 是共享契约的实际定义方；`control_plane` 中若仅 re-export，不是重复类型 | +| `physical::publication` + backend client + OpAMP | 从编译结果构造发布制品，安装请求、collector 发布与应用确认、backend 激活 | publication 是制品或发布过程；install request 是命令；不是 active runtime 对象 | +| data-plane HTTP 安装 + `PhysicalPlanLifecycle` | 验证跨计划一致性，stage、activate、drain、retire | `ActivePhysicalPlan` 被用于 staged map，名称把类型和生命周期阶段混在一起 | +| OTLP / Remote Write drivers + `SeriesIdResolver` | 协议接入、生产者/世代/帧校验、分配并解析物理 series 身份 | `collector_id` 是特定生产者身份；不能把所有 producer 无条件改叫 collector | +| `PrecomputeEngine` + workers + maintenance runtime | 按安装的 materialization 更新状态；派生维护消费已完成的源状态 | `PrecomputeEngineConfig` 是 worker/队列等引擎设置，`StreamingConfig` 是 materialization 的运行时视图，不是重复配置 | +| `SketchStore` + backfill / persistence | 保存 sketch 和精确聚合状态；SID 索引、覆盖与完整性、恢复和历史填充 | `SketchStore` 已比名称更广；领域表述宜用 summary store。全仓类型迁移可独立进行 | +| query engines / routing / summary execution | 用同一计划快照执行已安装 DAG、读取指定状态、执行精确子树或回退 | 生产路径不能在读请求里重新选择 materialization；legacy helper 应标明边界 | +| `control_plane::monitor` / `replan` | 旧工作负载路径上的指标抓取、违规/过期触发重规划与发布 | 与运行时 ERP 观测不是同一条自动闭环，不应画成一个通用 feedback planner | +| `data_plane::monitor` | 给边缘生产者协调 update-sampling grant | `monitor` 太宽，与 CP 的指标/SLA monitor 不同；建议模块领域名 `update_sampling` | +| `asap_otel_proto` | OTLP 等生成协议类型 | 保持协议边界；不能把协议 DTO 当成语义 catalog | +| `tools` / `scripts` / demos | 校准、离线证据、回放、评测与操作工具；有 JSON 消费者 | 序列化迁移必须审计这些调用方；它们不是运行时执行组件 | + +英文流程架构图见 [Planning terminology and architecture](planning-terminology.md)。下文表格的“当前名称”指审查基线上的名称，右栏是迁移建议。 + +**二、不要把系统画成一条所有入口都相同的流水线** + +PromQL / MetricsQL 的主要计划路径是： + +`工作负载与证据 → Planner 语义选择 → 工作负载级物理候选 → 编译结果与 manifest → provider quote → 最低成本可行候选 → publication → runtime generation` + +这里有三个不同范围的选择：语义 DAG 的选择、单个窗口物理实现的选择、整个工作负载物理候选的成本比较。它们并非重复实现。`with_exact_alternative` 的有界枚举也不保证找到所有未枚举实现的全局最优值。相同成本时，当前严格 `<` 比较保留先出现的候选。[编译器][C1]、[候选与定价][C2] + +`prepare_manifests()` 编译后丢弃 plan，仅返回 manifests 和诊断行；部署选择阶段重新编译候选，再与证据中的 manifest 做完整相等比较。这是“发现/采集成本”和“使用成本部署”两个阶段，不应因为函数都做 compile 就直接删除其中一个。[C2] + +ClickHouse 的自动路径直接从 SQL 选择与绑定构建 `PhysicalPlanPublication`；已有 catalog 路径接收 `sds + precompute_plan + transmission_plan`。它们与 PromQL 共用安装及运行时契约，但不能画成已经共用相同的整计划报价选择过程。建议两个请求对象分别叫 `ClickHouseCompilationInput`、`ClickHouseBindingInput`，或保留现名并明确 automatic / catalog-bound 区别。[C3] + +旧 flat workload → `CollectionPlan` / stage emission 路径仍有实际调用；`PhysicalPlanner`、`PlanNode`、`PostAsapPlan` 等旧模块也留在公开模块树中。不要据名字推断它们都与当前 `PhysicalPlan` 等价，也不要据旧模块存在就断定它们全部处于主路径。[C4] + +**三、#709 中应该修正或收紧的映射** + +| 当前值 / 提议 | 实际语义 | 建议 | +|---|---|---| +| `erp` → `empirical_runtime_profile` | ERP = **Error–Resource Profile**；`ErpPlanningInput` 还含观测、匹配策略、mode、权重和能力 | 保留 `erp` 并写明定义，或字段 `error_resource_planning_input`；不要使用错误全称 | +| `PlanningQuery` → `SelectedQueryInput` | 对象先构造，再被 `select_workload_roots_with_trace(&mut queries, ...)` 更新；并非整个生命周期都已 selected | 倾向 `QueryCompilationInput`，不用类型名承诺当前代码没有保证的阶段 | +| `post_asap` → `selected_summary_plan_root` | Planner 选择的语义 DAG 根，也可装 `KeepPreAsap` exact 根 | `selected_plan_root` / `selected_logical_root`；不要暗示必定 materialized 或 approximate | +| `materialization_candidate_keys()` → `optional_materialization_ids()` | 编译前枚举用的候选 key，与 catalog 中的 `SummaryDefinitionId` 不是一类身份 | `eligible_materialization_keys()`；保留 key 与已绑定 ID 的区别 | +| `materialization_policy` → `enabled_optional_materializations` | `Option>`：`None` 启用全部 eligible，`Some(empty)` 不启用任何可选项 | `enabled_materialization_keys`，保留并明确三态语义；不要默认成普通空集合 | +| `materialization_leaf_contract` → 候选 keys | 函数实际上返回原始 source 的 metric、可选 window、filter contract | `raw_materialization_input_contract`；真正枚举 keys 的函数另行命名 | +| 所有 `leaf` 改为 materialization | DAG 结构叶节点、物化候选、执行中的 exact subtree 是不同对象 | 按消费点改；真正的图论 leaf 可保留，不能机械替换 Planner 术语 | +| `CostDemand.unit` → `cost_unit` | 值为 `horizon` / `query_evaluation`，表达报价的工作量基准，并非 CPU 秒、字节或货币单位 | `pricing_basis` / `demand_basis`；可定义 `PricingBasis` 枚举 | +| `multiplicity` → `occurrences_per_horizon` | 基准工作在 horizon 内的乘数；允许按频率得到浮点估计 | 支持；不可顺手改为整数。总成本是 quote × occurrences | +| `LifecycleCostEvidence` → `LifecycleCostRates` | 同时包含 build/read/retirement 单次成本和 maintenance/retention rate | `LifecycleCostModel` / `LifecycleUnitCosts`，不应全称 rates | +| `WorkloadQuote.executable` → `deployable` | provider 对匹配 manifest 的执行可行性声明；还需 compiler、证据、安装与发布验证 | 保留 `executable` 或用 `provider_feasible`；不要暗示已通过端到端部署检查 | +| `synthesized_window_queries` → `queries_with_derived_window_candidates` | 编译器生成报价的 query ID 集合，决定能否做共享 pane 重定价 | `compiler_priced_window_query_ids`；derived 已用于“由 summary 派生 summary”，易混淆 | +| `collector_ids` → `eligible_collector_ids` | 分布式模式下，每个 ID 都生成 `CollectorPlan` | `target_collector_ids`；代码没有在这些 eligible collector 中再选子集 | +| `query_staleness_margin_ms` → `max_query_staleness_ms` | 为滞后的查询增加保留状态量；自身不实现请求拒绝规则 | `query_retention_margin_ms`，或保留现名并说明用途；避免暗示已有 admission enforcement | +| `BackendLocalImplementation` → `BackendLocalPlanningInputs` | 外层也是 planning input；内层含成本、窗口候选、证据、ERP 与保留约束 | 外层 `BackendLocalPlanningInput`，内层倾向 `BackendLocalPhysicalInputs`；不为这些分类额外创建一组 wrapper | +| `ActivePhysicalPlan` 保持不动，只改 handle | 同一类型也用于 staged 计划和 draining 的旧快照 | 类型倾向 `RuntimePhysicalPlan`；handle 用 `ActivePhysicalPlanHandle`，phase 由 lifecycle 管理 | +| `build_active_physical_plan` → `validate_and_build_active_physical_plan` | 校验与构建，但不执行 activation | `validate_and_build_runtime_plan`；否则名字仍把构建和激活混淆 | +| 泛化的 `snapshot()` → `active_snapshot()` | active-plan handle 与独立 streaming-config handle 的快照含义不同 | 只在能保证 active 语义的 handle 上使用；不可全仓替换 | +| `runtime_policy` → `materialization_runtime_policy` | `RuntimeRulePolicy` 具体控制 sampling、delta、GOS 与 adaptation | 可接受；跨组件说明是 producer/update/transmission policy，不能与候选选择策略混淆 | + +依据：[编译输入与选择 C1][C1]、[候选与定价 C2][C2]、[ERP C5][C5]、[候选 key C6][C6]、[pane 重定价 C7][C7]、[运行时生命周期 C8][C8]。 + +以下映射方向正确，可以作为同一轮小范围迁移： + +| 当前名称 | 建议名称 / 约束 | +|---|---| +| `PlanningRequest` | `PhysicalCompilationRequest`；包含 workload 上下文，不是单 query | +| `PhysicalCompiler` | `PhysicalPlanCompiler` | +| `PhysicalPlan` | `CompiledPhysicalPlan`；候选和获选结果可继续复用此类型，无须新增 `SelectedPhysicalPlan` wrapper | +| `logical_selection` | `planner_selection_trace`；说明只做诊断 | +| `window_implementations` | `window_realization_candidates`；对象 `WindowImplementationCandidate` 也应相应命名 | +| `ImplementationCostEvidence` | `WindowRealizationCostQuote`；保留 model、时效与 workload scope | +| `window_candidates` | `window_realization_candidates_by_query`；注明 key 为注册查询文本，不是 query ID | +| `window_implementation_id` | 默认模板处用 `default_window_realization_id`，已选估计项用 `window_realization_id`，不能全仓统一加 default | +| `PlanningQuery.window_secs` | `query_lookback_seconds`；不可连带重命名其他类型的所有 window 字段 | +| `PlanningQuery.source` | `legacy_query_source` 可接受，但该值也写入 workload cost manifest，不能借改名删除或改变其报价身份 | +| `group_by` / `accuracy` | 在相应输入上用 `group_by_labels` / `accuracy_target`；输出精度结果不能改叫 target | +| `lifecycle` / `LifecyclePlanningInput` | `summary_lifecycle_inputs` / `SummaryLifecyclePlanningInputs` | +| `PlanningRequest.evidence` | `topk_membership_evidence_by_query_id`；输入 map 的 `topk_evidence` 是按查询文本，不应加 by_query_id | +| `hybrid_execution` | `allow_mixed_summary_and_exact_execution`；是允许编译的模式，不是本次请求必然采用混合执行 | +| `bind_alternative` / `prepare_manifests` | `compile_candidate_for_pricing` / `compile_candidates_for_pricing`；后者返回清单及诊断，不返回编译 plans | +| `with_exact_alternative` | `enumerate_exact_and_materialized_candidates`；保留有界搜索与枚举顺序 | +| `AlternativeCost` | `CandidatePlanEvaluation` 可接受，但必须允许尚未定价或编译失败；更中性的 `CandidatePlanDiagnostic` 也符合现有全阶段用途 | +| `WorkloadCostComparison` | `CandidatePlanSelectionReport` | +| `select` / `select_metricsql` | 可用 `select_lowest_cost_candidate` / `select_lowest_cost_metricsql_candidate`；文档写明 feasible 与枚举范围 | +| `metricsql: bool` | PromQL/MetricQL 路径使用 `QueryFrontend`；先审计已有 `PhysicalQueryFrontend`，不要再建第三份枚举；SQL 并未因此自动适配此入口 | +| `CostDemand` | `CostComponentDemand` | +| `publication()` | `to_publication_artifact()`，或沿用 `publication()`；它构造并校验制品，不执行 HTTP 发布 | +| `compile_transmission_plan` | `build_transmission_plan`；从 precompute 和 runtime policies 构造，不一定要把全部参数编码进长函数名 | +| `DeploymentEnvironment` | `PhysicalDeploymentContext`；target、capability、time、generation 都是上下文 | +| `max_retained_summary_bytes` | `retained_summary_memory_budget_bytes`，保留旧缺省值、别名与具体计量定义 | +| `ActivePhysicalPlan.runtime_config` | `streaming_config`，准确表达持有的配置类型 | +| `aggregation_configs` | `materializations_by_policy_fingerprint`；目前 key 是 `u64` 形式 fingerprint，改名不等于已经升级 typed key | +| `get_all_aggregation_configs()` | `materializations()`；明确当前仍返回 map | +| `from_active()` | `from_active_physical_plan()` | +| `retire_drained()` | `mark_drained_plan_retired()`；它标记 lifecycle status，不执行存储 GC | + +**四、全仓中 #709 漏掉的歧义与重复** + +| 对象 | 判断 | 最小处理 | +|---|---|---| +| backend `types::QueryWorkload` 与 Planner `workload::QueryWorkload` | **同名不同义**：前者是单 metric 的 flat legacy 意图；后者是 canonical 工作负载 | legacy 侧叫 `LegacyMetricWorkload` 或导入时显式 alias | +| `PlanNode` / `CollectionPlan` / `PhysicalPlan` / `QueryPlan` | **不同层级**：旧 annotated stage tree、旧采集方案、完整物理 bundle、工作负载的查询执行条目集合 | 优先给旧类型/模块加 legacy 或 stage 语义；保留新契约的区别 | +| `QueryPlan` 与 `QueryPlanEntry` | 前者实际上是按 canonical identity 查找的一组查询执行计划 | 可将前者描述为 query execution catalog；若做 API 迁移再考虑 `QueryExecutionCatalog` / `QueryExecutionPlan`，#709 不必强行扩展 | +| `query_plan::logical::LogicalOperator` | **阶段误导**：内容是已安装的 residual 运行时算子，包括 exact subquery、scan、aggregate | `ResidualQueryOperator`，模块 `residual`；对应 `prepare_logical` 按实际职责命名 | +| `ClickHouseSqlWorkload.sds` | **层级混淆**：保存整个 `SummaryCatalog`，不是单个 Self-Describing Summary | `summary_catalog` | +| `AggregationConfig` 与 `PrecomputeMaterialization` | **同一个类型的兼容别名**，不是两种配置对象 | 新代码用 `PrecomputeMaterialization`；保留旧 alias 过渡，不复制定义 | +| `AggregationIdInfo.aggregation_id_for_key/value` | **历史身份名**：注释明确已是 policy fingerprint 的 u64 | 使用 `key_policy_fingerprint` / `value_policy_fingerprint`；typed ID 迁移单独评估 | +| `SummaryDefinitionId` 与 `PolicyFingerprint` | **合理的语义 wrapper**：前者是 catalog 中的定义引用，后者是兼容内容 fingerprint | 跨 catalog API 优先前者；不能把它与 SID 或 descriptor ID 合并 | +| `SketchInstanceMetadata` | **instance 的粒度错误**：按 SID 管理长期 series 元数据；真正 SDS instance 另外包含具体时间范围 | `SummarySeriesMetadata`，避免与 `SummaryInstance` 混为一个窗口对象 | +| `SketchStore` / `sketch_index` | **名称比职责窄**：对象还保存 exact accumulator、持久化、生命周期及状态读取 | 文档统一称 summary store；`sketch_index` 变量可先改 `summary_store`，全面类型迁移另行处理 | +| `SummaryCatalog` / `sketch_catalog` / SQL catalog / `PolicyRegistry` | **不是重复 catalog**：已安装定义、算法候选/默认参数、源表 schema、运行时 config 派生 lookup | 分别明确 `summary catalog`、`sketch capabilities`、`source schema catalog`、`materialization lookup` | +| `PrecomputePlan.materializations` 与 `StreamingConfig.aggregation_configs` | **合理投影**：安装验证后构建的运行时 map，不是两个应独立编辑的配置真源 | 文档声明 runtime config 来源；保证同一世代，不因字段重复删除验证 | +| `PhysicalPlan` / Publication / InstallRequest / Runtime plan | **合理边界重复**：编译诊断、共享发布制品、带 routing/evidence 的安装命令、持有 Arc 的运行时对象 | 继续使用边界构造和交叉验证；无需一个承载全部阶段的大对象 | +| `HotReloadStreamingConfig.inner` 与 `.active` | **真实双来源**：active 模式读 active plan，`swap` 仍写 inner，写入不成为 snapshot 的结果 | 先将模式写清；若拆为 legacy 可写句柄与 active 只读视图，必须独立定义 API 行为与测试 | +| `summary_exec` / `summary_executor` | 通用执行接口/调度与具体 store-backed 实现，名称近似但不是可直接去重的函数集合 | 更明确的 module docs；若迁移可用 `summary_execution` / `store_summary_reader`，保持执行语义边界 | +| `control_plane::monitor` 与 `data_plane::monitor` | **同名不同功能**：指标/违规监控 vs 更新采样分配 | 用职责名区分，不将它们都当作 ERP 实时反馈 | +| `summary_catalog` / `sds` / `canonical` 中的 re-export | **兼容入口，不是重复实现** | 建立 canonical import 路径后逐步减少旧入口；不要复制 shared DTO | + +依据：[旧 workload C4][C4]、[共享 materialization C9][C9]、[SDS C10][C10]、[catalog C11][C11]、[runtime config C12][C12]、[存储元数据 C13][C13]、[residual operators C14][C14]、[shared publication C15][C15]。 + +**五、必须保留的身份、时间和生命周期区别** + +`plan_id / plan_version` 标识部署决定与世代；`CatalogGeneration` 额外携带 catalog 的摘要。`SummaryDescriptorId` 表达算法/状态语义，`DataDescriptorId` 表达输入 population；`SummaryDefinitionId` 绑定这些定义与物理布局。SID 表达具体物理 series 生命周期，而 `SummaryInstance` 表达具体时间范围和 group 的状态，并携带物理引用、来源与完整性。不能因为底层有些都用 u64，就全部命名为 `id` 或 `materialization_id`。[SDS][C10]、[Catalog][C11]、[存储元数据][C13] + +`window` 至少有 query lookback、evaluation interval、stored pane duration、slide、origin、retention、producer emission cadence 等含义。`PrecomputeMaterialization::stored_window_ms()` 对 FullWindow 与 pane layout 的处理不同，不能全仓把 `window_size` 替换成 `query_lookback_seconds`。改名应保留各字段当前单位；单位转换属于额外行为变更。[C9] + +生命周期至少有三条轴：计划 `Staged / Active / Draining / Retired`，materialization `Materializing / Ready / Serving`，以及具体实例的 completeness。一个计划已 Active，并不表示其每个查询范围都已完整；读路径仍需校验状态。`retire_drained` 只是状态标记，不表示已经回收所有持久化数据。[计划生命周期][C8]、[SDS][C10] + +`snapshot` 对 immutable catalog 和 `Arc` 读取是合适术语，不应一律删除。对于 `BackendLocalPlanningSnapshot`，代码主要将其消费为输入，改成 input 合理；但要同时决定 `snapshot_version` 的 schema 名称和兼容策略。 + +**六、迁移与验收建议** + +1. 先写下字段语义、所属阶段、key 范围和单位；修正 ERP、成本基准、runtime/staged、target collectors 等会误导实现的名称。把 #709 正文与补充评论合并为一份不冲突的映射。 +2. 做内部函数/局部变量改名，优先保持数据结构和算法原样。不要为每个生命周期步骤引入一个新 wrapper。`physical` 的旧路径隔离、hot-reload handle 拆分和 shared schema 版本升级分别处理。 +3. 公共 Rust 字段改名不能靠 type alias 或 forwarding method 保持兼容。先识别实际外部消费者；crate 根文档也声明部分 public module 仅供 workspace 内使用，因此不需要不加区分地给每个 public symbol 建长期兼容层。 +4. 保留 wire 输出时，用 `serde(rename = "old_name")` 固定旧名称，视需要接受新名 alias。仅 `alias = "old_name"` 无法保护旧消费者读取新输出。还需检查手工 JSON/YAML 生成、Python 校准与回放工具、CLI 输出和已有 fixture。 +5. 名称变化可能影响身份：catalog digest 对序列化内容求 SHA-256，编译计划也存在对序列化 materialization 求 hash 的路径。验收应检查 plan/materialization/catalog identity、component keys、manifest 匹配不变，不只是能够 deserialize。 +6. 枚举 `status` 与 `search_scope` 时保留原 wire 表达，或明确版本迁移。现有 status 含 `bind_failed`、`bound`、`evidence_missing`、`rejected`、`evidence_invalid`、`unselected`、`selected`，且 `status` 缺省为空字符串。typed scope 还应保留“bounded inventory 不承诺未枚举最优”的解释信息。 +7. 运行既有 compiler、cost selection、publication、安装 lifecycle、query execution 与 backend process E2E。针对实际 wire/identity 变动补兼容断言：双向读写、缺省行为、None/empty 集合、同成本选择顺序、staged 不生效、旧读者 drain。纯局部改名无需逐字段补镜像测试。 + +额外维护问题：`docs/developer_docs/data-structure-ownership-audit.md` 仍说多个 wire DTO 在 control_plane、尚待搬迁，但当前已经在 `asap_types`；`PrecomputeEngine` 注释还称 Remote Write 已删除，而 driver 与启动代码中已存在。命名 PR 应同步修正相关陈旧说明，否则新的词汇表仍会被旧架构描述覆盖。 + +[C1]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/control_plane/src/physical/compiler.rs +[C2]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/control_plane/src/physical/workload_cost.rs +[C3]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/control_plane/src/clickhouse.rs +[C4]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/control_plane/src/types.rs#L266 +[C5]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/control_plane/src/physical/erp.rs +[C6]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/control_plane/src/query_plan/logical.rs#L1105 +[C7]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/control_plane/src/physical/pane_reuse.rs +[C8]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/data_plane/src/storage_engines/types/hot_reload_config.rs +[C9]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/crates/asap_types/src/aggregation_config.rs +[C10]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/crates/asap_types/src/sds.rs +[C11]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/crates/asap_types/src/summary_catalog.rs +[C12]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/data_plane/src/storage_engines/types/streaming_config.rs +[C13]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/data_plane/src/storage_engines/sketch_db/index/mod.rs#L209 +[C14]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/crates/asap_types/src/query_plan/logical.rs +[C15]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/crates/asap_types/src/plan_publication.rs diff --git a/docs/developer_docs/control-plane/physical-compiler.md b/docs/developer_docs/control-plane/physical-compiler.md index 7bccd4a4c..44f9e3ae8 100644 --- a/docs/developer_docs/control-plane/physical-compiler.md +++ b/docs/developer_docs/control-plane/physical-compiler.md @@ -8,7 +8,7 @@ The compiler consumes ASAPPlanner types pinned to the revision exposed as `physical::compiler::PLANNER_REVISION`, selects from Planner's legal candidate space with backend-owned cost and evidence -inputs, and emits one `PhysicalPlan`. The plan contains one SummaryCatalog plus +inputs, and emits one `CompiledPhysicalPlan`. The plan contains one SummaryCatalog plus CollectorPlan, PrecomputePlan, TransmissionPlan, and QueryPlan projections compiled from the same decision for every target collector. Legacy @@ -28,14 +28,14 @@ compiler never invents a framework or assigns it an optimistic zero cost. The control plane has three public layers: ```text -PlanningRequest + DataWorkload + concrete implementation evidence +PhysicalCompilationRequest + DataWorkload + concrete implementation evidence | ^ | abstract candidates | complete physical costs v | -ASAPPlanner selection <---------- PhysicalCompiler +ASAPPlanner selection <---------- PhysicalPlanCompiler | v -PhysicalCompiler -------> PhysicalPlan +PhysicalPlanCompiler -------> CompiledPhysicalPlan | | | | v v v v Collector Precompute Backend Query @@ -48,7 +48,7 @@ PhysicalCompiler -------> PhysicalPlan - **Physical compiler** enumerates concrete window/pane/state-layout, placement, transport, and runtime implementations without changing the Planner-owned abstract framework. -- **PhysicalPlan** is the only output passed to publication. Its SummaryCatalog, +- **CompiledPhysicalPlan** is the only output passed to publication. Its SummaryCatalog, CollectorPlan, PrecomputePlan, TransmissionPlan, and QueryPlan are created together and share identities. Logical query parsing, summary alternatives, guarantees, and candidate search @@ -59,13 +59,13 @@ remain public ASAPPlanner interfaces. Runtime publication is documented in ### Planning request -`PlanningRequest` is backend-owned request context around Planner's canonical +`PhysicalCompilationRequest` is backend-owned request context around Planner's canonical per-query IR: ```rust -pub struct PlanningRequest { - pub queries: Vec, - pub evidence: HashMap, +pub struct PhysicalCompilationRequest { + pub queries: Vec, + pub topk_membership_evidence_by_query_id: HashMap, pub planner_revision: String, } ``` @@ -74,13 +74,13 @@ Input definitions: | Field | Definition | | --- | --- | -| `queries` | Canonical `QueryExpr`, source, grouping labels, accuracy, stable query ID, `DataWorkload`, and executor-feasible `window_implementations`. | -| `evidence` | Optional typed TopK membership certificates keyed by query ID. | +| `queries` | Selected Planner DAG roots, query identities, source metadata, grouping labels, accuracy targets, lifecycle inputs, and executor-feasible `window_realization_candidates`. | +| `topk_membership_evidence_by_query_id` | Optional typed TopK membership certificates keyed by query ID. | | `planner_revision` | Immutable Planner build/revision used for reproducibility. | TopK evidence is accepted only when its selected lower bound is strictly above the excluded upper bound, its failure probability is valid, its source is -non-empty, and its observation is fresh under `DeploymentEnvironment`. +non-empty, and its observation is fresh under `PhysicalDeploymentContext`. ```rust pub struct TopKMembershipEvidence { @@ -101,7 +101,7 @@ requirements and deployment target, then derives or matches layout-specific cost See [repeated window planning](../planning/repeated-dashboard-panes.md) for inputs, migration and cadence examples. -Each internally generated `WindowImplementationCandidate` carries a backend-owned implementation +Each internally generated `WindowRealizationCandidate` carries a backend-owned implementation identity, its Planner `SummaryWindowFramework`, concrete window/pane/state layout, and versioned workload-specific CPU, peak-memory, network, storage, scan, and calibrated weighted-cost evidence. The compiler collapses several @@ -112,18 +112,18 @@ identity. Physical identities never enter post-ASAP IR. ### Physical compiler ```rust -impl PhysicalCompiler { - pub fn compile( +impl PhysicalPlanCompiler { + pub fn compile_promql( &self, - request: PlanningRequest, - environment: DeploymentEnvironment, - ) -> Result; + request: PhysicalCompilationRequest, + environment: PhysicalDeploymentContext, + ) -> Result; } ``` ```rust -pub struct DeploymentEnvironment { - pub collector_ids: Vec, +pub struct PhysicalDeploymentContext { + pub target_collector_ids: Vec, pub capability_snapshot_id: String, pub observed_at_unix_ms: u64, pub max_evidence_age_ms: u64, @@ -133,7 +133,7 @@ pub struct DeploymentEnvironment { pub backend_compat: String, } -pub struct PhysicalPlan { +pub struct CompiledPhysicalPlan { pub envelope: PlanEnvelope, pub summary_catalog: SummaryCatalog, pub collector_plans: Vec, // complete per-target projections @@ -147,7 +147,7 @@ Supporting public types: | Type | Definition | | --- | --- | -| `DeploymentEnvironment` | Target collector IDs, capability snapshot identity, planning time, and evidence freshness policy. | +| `PhysicalDeploymentContext` | Target collector IDs, capability snapshot identity, planning time, and evidence freshness policy. | | `PlanEnvelope` | Shared deterministic `plan_id`, generation time, capability snapshot, and Planner revision. | | `CollectorPlan` | Serializable execution projection consumed by ASAPCollector. | | `PrecomputePlan` | Authoritative materialization, ingest, state-schema, and producer contract consumed directly by the backend runtime. | @@ -337,8 +337,8 @@ identity. ### Add a deployment topology -1. Add a public `DeploymentTopology` variant and its required target fields. -2. Teach `PhysicalCompiler::compile` how selected operators can be placed on it. +1. Add a public `PhysicalDeploymentTarget` variant and its required target fields. +2. Teach `PhysicalPlanCompiler::compile_promql` how selected operators can be placed on it. 3. Reject plans requiring an unavailable stage/capability. 4. Verify the output contains one complete CollectorPlan for every producer and one SummaryCatalog and matching QueryPlan referencing all produced materializations. diff --git a/docs/developer_docs/control-plane/planning-terminology.md b/docs/developer_docs/control-plane/planning-terminology.md new file mode 100644 index 000000000..3a4dbcd97 --- /dev/null +++ b/docs/developer_docs/control-plane/planning-terminology.md @@ -0,0 +1,178 @@ +# Planning terminology and architecture + +Audience: developers changing planning inputs, pricing, publication, or runtime +plan consumers. The [Chinese architecture review](architecture-naming-review.zh.md) +records the reasoning and the original-to-proposed naming checklist for #709. + +```mermaid +flowchart TB + subgraph CP[Control plane] + DTO["JSON / YAML compatibility input"] + NORMALIZE["Analyzer
Validate and normalize once"] + WORKLOAD["ASAPPlanner QueryWorkload
Query + requirements + recurrence + time selection"] + DATA["ASAPPlanner DataWorkload
Arrival + rate + cardinality + evidence freshness"] + OPTIONS["DeploymentOptions
Sketch constraint + retained labels + wire size + memory cap"] + REGISTRY["WorkloadStore
Canonical workload + deployment options per metric and role"] + METRIC["Metric deployment planning and replanning
Derive query metadata and fresh cost inputs"] + DTO --> NORMALIZE --> WORKLOAD + DATA --> WORKLOAD + WORKLOAD --> REGISTRY + OPTIONS --> REGISTRY + REGISTRY --> METRIC + INPUT["BackendLocalPlanningInput
Workload demand + physical inputs + deployment context"] + LOGICAL["ASAPPlanner + selection adapter
Legal semantic DAG selection"] + REQUEST["PhysicalCompilationRequest
QueryCompilationInput + enabled materialization keys"] + WINDOWS["Generate window candidates
Cadence + evaluation phase + WindowCostModel"] + COMPILE["PhysicalPlanCompiler
Compile concrete candidate plans"] + MANIFEST["WorkloadCostManifest
Component implementations and pricing basis"] + QUOTE["WorkloadQuote
Provider feasibility and component prices"] + EVALUATE["CandidatePlanEvaluation
Select the lowest-cost feasible enumerated candidate"] + SQL["ClickHouse SQL selection and binding
Separate compilation path"] + PUBLICATION["PhysicalPlanPublication
PhysicalPlanInstallRequest"] + WORKLOAD --> INPUT + INPUT --> LOGICAL --> REQUEST --> WINDOWS --> COMPILE --> MANIFEST + MANIFEST --> QUOTE --> EVALUATE --> PUBLICATION + SQL --> PUBLICATION + end + CONTRACT["Shared asap_types contracts
SummaryCatalog + CollectorPlan + PrecomputePlan
TransmissionPlan + QueryPlan + StorageRouting"] + subgraph DP[Data plane] + STAGE["Validate and stage RuntimePhysicalPlan"] + ACTIVE["Activate via ActivePhysicalPlanHandle
One immutable generation for all readers"] + MAINTAIN["Ingest and precompute
StreamingConfigHandle materialization view"] + QUERY["Execute installed query DAG
Residual operators, bound reads, exact subqueries"] + STORE["Summary storage
SummarySeriesMetadata + physical series state"] + STAGE --> ACTIVE + ACTIVE --> MAINTAIN --> STORE + ACTIVE --> QUERY --> STORE + end + PUBLICATION --> CONTRACT --> STAGE + PUBLICATION -->|OpAMP| COLLECTOR[ASAPCollector] + METRIC -->|Stage configuration via OpAMP| COLLECTOR + COLLECTOR -->|OTLP frames| MAINTAIN + RAW[Raw time-series samples] -->|Remote Write| MAINTAIN + QUERY -->|Exact execution or fallback| EXACT[Prometheus / VictoriaMetrics / ClickHouse] +``` + +SQL shares publication and runtime contracts; it does not currently use the +time-series workload quote-selection path. Metric stage emission remains a +separate deployment path, but its stored semantic input now uses the same Planner +`QueryWorkload` and embedded `DataWorkload` types. `LegacyMetricWorkload` and its +old `types::QueryWorkload` alias are deleted. + +## Domain boundaries + +| Value | Meaning | +|---|---| +| `QueryCompilationInput.selected_plan_root` | Selected semantic DAG root, including exact or mixed execution; not necessarily an approximate materialization | +| `WindowRealizationCandidate` | One concrete window framework/layout choice, not a whole-workload candidate | +| `PhysicalCompilationRequest.enabled_materialization_keys` | Optional candidate-key set: `None` enables all eligible keys; an empty set enables none | +| `CompiledPhysicalPlan` | A complete compiled candidate; selection can attach a `CandidatePlanSelectionReport` to the same type | +| `CostComponentDemand.pricing_basis` | Work priced per horizon or per query evaluation, not a currency/resource unit | +| `occurrences_per_horizon` | Floating-point expected occurrences multiplying a component's unit quote | +| `CandidatePlanEvaluation` | Diagnostic state throughout compilation and pricing, including failures and candidates awaiting quotes | +| `LifecycleUnitCosts` | Both one-time costs and per-update/per-second costs; not exclusively rates | +| `erp` | Error–Resource Profile planning inputs, including distribution evidence and matching policy | +| `PhysicalDeploymentContext.target_collector_ids` | Every targeted collector receives a plan; this is not an eligibility pool | +| `query_retention_margin_ms` | Extra retained history for lagging query evaluation; not a separate query-age rejection check | +| `RuntimePhysicalPlan` | Immutable runtime representation, also used while staged and draining | +| `ActivePhysicalPlanHandle` | Shared pointer to the currently active runtime generation | +| `StreamingConfigHandle` | Legacy owned config or a view projected from the active runtime plan | +| `SummarySeriesMetadata` | Metadata for a physical SID; distinct from a time-scoped SDS `SummaryInstance` | + +`SummaryDefinitionId`, `PolicyFingerprint`, descriptor IDs, physical SIDs, +instance IDs, and catalog generations retain their distinct identities. +Likewise, plan activation, materialization readiness, and instance completeness +remain separate conditions. Retiring a drained plan marks lifecycle state; it +does not perform storage garbage collection. + +## Compatibility transition + +New Rust names retain the previous JSON/YAML field names through explicit serde +renames. New spellings are accepted as aliases where fields are deserialized. +This preserves old consumers, exact manifest comparisons, and serialized +identity inputs. Diagnostic enums retain their old string representations, +including unknown strings and the missing-status default. + +Other old public type imports and selected method/function names remain deprecated +forwarders. The flat workload type is removed without a compatibility alias. This does **not** preserve old Rust struct-literal field names; +workspace callers migrate with the definitions. External Rust source consumers +must update fields using the review's mapping before these compatibility +imports are removed. No removal date is set until consumer migration is known. + +Wire names containing historical terms, such as `enumerated_local_masks`, are +intentionally retained. New Rust set variables use candidate-key terminology. +Candidate ordering, bounded coverage, strict-less-than tie handling, pricing, +publication validation, and activation behavior are unchanged. + +## Scope of this migration + +The implementation clarifies planning/compiler types, candidate pricing and +diagnostics, publication conversion, runtime generations, streaming views, +series metadata, canonical workload registration, residual query operators, and the +data-plane update-sampling module. Existing re-exports remain compatibility +entrypoints rather than duplicate implementations. + +The review also identifies follow-up work that is intentionally separate: + +- Splitting the two `StreamingConfigHandle` modes and changing their write API. + Active-plan snapshots read the runtime plan; `swap` still affects only the + legacy backing slot. Plan activation changes authoritative active config. +- Removing legacy stage planning/emission, or unifying SQL and time-series + candidate pricing. +- Renaming the full `SketchStore` public type or redesigning `QueryPlan` as a + differently shaped execution catalog. +- Changing window units, schema versions, identity encodings, or typed ID + ownership. + +These remain separate from the naming changes and the canonical workload migration. + +## Canonical workload registration + +`RegisteredWorkload` is a backend registration envelope, containing only the +Planner `QueryWorkload` and `DeploymentOptions`. It does not store a parsed query +or a second set of metric, accuracy, window, cadence or data-rate fields. +`ParsedQuery` is an ephemeral adapter view; `WorkloadCharacteristics` is an input +DTO and a transient cost projection. The cached planner obtains that projection +from canonical evidence, and unavailable evidence skips rate-based cost selection. + +For this single-metric adapter, `input_cardinality` denotes active time series and +`ingestion_rate` is the aggregate sample rate: 10 series at 5 Hz produces 50 +samples/s. Distinct item keys per flush remain a separate deployment estimate. +Evidence provenance and freshness survive registry round trips. Unknown or stale +facts do not become declared defaults. Batch data has `AtRest` arrival and no +ongoing ingestion; repetition remains independent of arrival. + +Compatibility field-only inputs generate explicit queries: quantile uses +`quantile_over_time(0.99, ...)`, cardinality uses `distinct_over_time(...)`, and +frequency uses a temporal item count. Filter values are JSON-escaped before query +parsing. Source and label identifiers must be accepted by the PromQL frontend. +The old `group_by_labels` input also supplies collector retention labels; these +are deployment options and do not introduce a query GROUP BY. + +The single-metric registration route accepts one query entry, one selector, +equality filters, one-shot or fixed-interval demand, and supported real-time +selection. It rejects conflicting query/field overrides, multi-selector input, +regex or negative matching, offset/@, subqueries, `without`, unsupported cadence +forms, historical time-selection metadata and unenforced dollar constraints. +Cadences are checked before conversion to Planner milliseconds. The full query +compilation APIs retain their broader query support. + +Both field-only and string requests bind from the registered canonical expression. +Explicit typed accuracy, including epsilon and delta, remains authoritative; +sketch choices remain physical constraints subject to legal binding. IDs and +routing hints are retained as registration metadata. + +## Latest main integration + +Window candidates are generated after semantic DAG selection from cadence, evaluation +phase and `WindowCostModel` quotes. Compiler provenance stays on each candidate +(`derived` / `cohort_only`) and is not accepted from serialized input. The old +external candidate/default-window fields and query-level provenance set are removed +by upstream #712. Shared panes preserve cadence, phase and evaluation alignment. +Publication includes storage routing derived from the selected physical plan (#713). +The former `types_v2` definitions now live in `types` (#717). These upstream API +removals also apply here; wire-name compatibility covers retained naming-only fields. + +Upstream #715 removes unused auto/Pareto planning and rollback/diff HTTP routes +and their auxiliary modules. This integration retains those removals; canonical +workload registration and replanning continue through the retained interfaces. diff --git a/docs/developer_docs/data-structure-ownership-audit.md b/docs/developer_docs/data-structure-ownership-audit.md index 9174b411d..807ca849c 100644 --- a/docs/developer_docs/data-structure-ownership-audit.md +++ b/docs/developer_docs/data-structure-ownership-audit.md @@ -19,15 +19,17 @@ It classifies operators such as `topk`, `rate`, and `quantile`; the control-plan `QueryShape` describes evaluation lifecycle (`one_shot`, `streaming`, or `periodic`). The distinct names prevent accidental cross-layer use. -## Remaining migration - -`QueryPlan`, `PrecomputePlan`, `TransmissionPlan`, and `CollectorPlan` are wire -contracts currently defined in `control_plane`, so the data-plane crate depends -on the whole control-plane crate to deserialize and execute them. A later change -should move only their serde DTOs and validation-independent identifiers into -`asap_types`, leaving selection, compilation, publication validation, HTTP -handlers, and execution in their current owners. That migration should preserve -the JSON schema and use compile-time conversion at the compiler boundary. +## Shared contracts and compatibility imports + +`QueryPlan`, `PrecomputePlan`, `TransmissionPlan`, `CollectorPlan`, +`SummaryCatalog`, and the publication/install envelopes are now defined in +`asap_types`. Control-plane modules retain compatibility re-exports; these are +not independent DTO implementations. Selection and compilation stay in the +control plane, while installation, ingestion, and query execution stay in the +data plane. Shared contracts retain their validation methods. + +See [planning terminology](control-plane/planning-terminology.md) for the +compiled-plan/runtime-plan distinction and the wire-preserving naming migration. Planner's `planner_types::workload::QueryLanguage` and backend `asap_types::QueryLanguage` also have different scopes. They should remain