From 4ee65f68501e8bfec56eae9a9aa65f0fcf586259 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 13 Sep 2026 11:13:56 -0600 Subject: [PATCH 1/3] refactor: align planning and runtime names with domain semantics --- .../examples/audit_clickhouse_corpus.rs | 4 +- .../examples/calibration_candidates.rs | 24 +- .../examples/compile_workload_artifact.rs | 8 +- .../examples/workload_cost_manifest.rs | 12 +- control_plane/src/backend_client.rs | 22 +- control_plane/src/clickhouse.rs | 32 +- control_plane/src/emit/backend_push.rs | 2 +- control_plane/src/emit/mod.rs | 20 +- control_plane/src/lib.rs | 13 +- control_plane/src/main.rs | 201 +- control_plane/src/opamp/mod.rs | 2 +- control_plane/src/physical/colored_dag/mod.rs | 6 +- control_plane/src/physical/compiler.rs | 1647 ++++++++++------- .../src/physical/deployment_cost/delta.rs | 10 +- .../src/physical/deployment_cost/mod.rs | 28 +- .../src/physical/deployment_cost/pareto.rs | 10 +- control_plane/src/physical/erp.rs | 11 +- control_plane/src/physical/pane_reuse.rs | 31 +- control_plane/src/physical/plan_cache.rs | 8 +- control_plane/src/physical/post_asap/mod.rs | 6 +- control_plane/src/physical/post_asap/tests.rs | 2 +- control_plane/src/physical/publication.rs | 13 +- control_plane/src/physical/realization.rs | 38 +- control_plane/src/physical/stage_split.rs | 6 +- control_plane/src/physical/workload_cost.rs | 701 ++++--- .../materialization_candidates.rs | 81 +- .../src/physical/workload_cost/status.rs | 124 ++ .../src/physical/workload_planner.rs | 28 +- control_plane/src/pipeline.rs | 10 +- control_plane/src/query_parser/mod.rs | 138 +- control_plane/src/query_plan.rs | 395 ++-- .../src/query_plan/clickhouse_exact.rs | 186 +- .../query_plan/{logical.rs => residual.rs} | 435 ++--- control_plane/src/replan.rs | 14 +- control_plane/src/store/workload.rs | 16 +- control_plane/src/types.rs | 9 +- control_plane/src/workload.rs | 12 +- crates/asap_types/src/aggregation_config.rs | 10 +- crates/asap_types/src/executable_plan.rs | 2 +- crates/asap_types/src/grouping_projection.rs | 102 +- crates/asap_types/src/policy_registry.rs | 2 +- crates/asap_types/src/producer_plan.rs | 5 +- crates/asap_types/src/query_plan.rs | 9 +- .../query_plan/{logical.rs => residual.rs} | 8 +- crates/asap_types/src/routing_index.rs | 2 +- crates/asap_types/src/sds.rs | 52 +- data_plane/benches/sketch_db.rs | 14 +- .../examples/audit_clickhouse_fallback.rs | 10 +- data_plane/examples/sketch_db_diag.rs | 8 +- .../src/bin/monitor_coordinator_harness.rs | 2 +- data_plane/src/drivers/ingest/otel.rs | 110 +- .../drivers/ingest/prometheus_remote_write.rs | 76 +- .../src/drivers/ingest/series_resolver.rs | 196 +- .../drivers/query/adapters/prometheus_http.rs | 4 +- .../src/drivers/query/adapters/traits.rs | 6 +- data_plane/src/drivers/query/servers/http.rs | 263 +-- data_plane/src/lib.rs | 10 +- data_plane/src/main.rs | 80 +- data_plane/src/precompute_engine/engine.rs | 21 +- .../src/precompute_engine/ingest_handler.rs | 21 +- .../precompute_engine/maintenance_runtime.rs | 114 +- .../multisource_coordinator.rs | 4 +- .../src/precompute_engine/output_sink.rs | 67 +- data_plane/src/precompute_engine/worker.rs | 18 +- .../accelerator.rs | 36 +- .../clickhouse_result_adapter.rs | 214 +-- .../relational_adapter.rs | 354 ++-- .../asap_clickhouse_query_engine/server.rs | 202 +- .../asap_query_engine/catalog_resolver.rs | 8 +- .../query_engines/asap_query_engine/engine.rs | 132 +- .../asap_query_engine/exact_subqueries.rs | 41 +- .../asap_query_engine/live_serve.rs | 8 +- .../asap_query_engine/logical_dag.rs | 48 +- .../query_engines/asap_query_engine/mod.rs | 4 +- .../asap_query_engine/post_asap_readout.rs | 22 +- .../asap_query_engine/summary_executor.rs | 22 +- .../routing/backend_storage_routing.rs | 10 +- data_plane/src/storage_engines/mod.rs | 2 +- .../storage_engines/sketch_db/backfill/mod.rs | 46 +- .../sketch_db/backfill/processor.rs | 44 +- .../sketch_db/backfill/prometheus_reader.rs | 146 +- .../sketch_db/backfill/service.rs | 26 +- .../src/storage_engines/sketch_db/data/mod.rs | 2 +- .../sketch_db/index/epoch_columnar.rs | 2 +- .../sketch_db/index/maintenance.rs | 16 +- .../storage_engines/sketch_db/index/mod.rs | 112 +- .../sketch_db/lifecycle/eviction.rs | 58 +- .../sketch_db/lifecycle/mod.rs | 2 +- .../sketch_db/lifecycle/reconcile.rs | 16 +- .../sketch_db/persistence/metadata.rs | 2 +- .../sketch_db/persistence/source.rs | 2 +- .../sketch_db/query/timeline.rs | 12 +- .../src/storage_engines/sketch_db/sds.rs | 12 +- .../types/hot_reload_config.rs | 196 +- .../storage_engines/types/streaming_config.rs | 68 +- .../tests/capability_miss_http_e2e_tests.rs | 14 +- .../tests/test_utilities/engine_factories.rs | 84 +- .../src/tests/test_utilities/planning.rs | 19 +- .../coordinator.rs | 0 .../src/{monitor => update_sampling}/epoch.rs | 0 .../src/{monitor => update_sampling}/mod.rs | 0 .../sampling_alloc.rs | 0 .../{monitor => update_sampling}/server.rs | 2 +- data_plane/src/utils/file_io.rs | 8 +- .../asapquery_compatibility_process_e2e.rs | 94 +- ...e2e_controller_plans_and_backend_serves.rs | 36 +- data_plane/tests/monitor_grpc.rs | 2 +- .../support/distinct_planning_process.rs | 8 +- .../tests/support/durable_summary_process.rs | 4 +- .../tests/support/erp_planning_process.rs | 7 +- .../support/immutable_maintenance_process.rs | 19 +- data_plane/tests/support/physical_fixture.rs | 10 +- .../tests/support/univmon_erp_process.rs | 18 +- 113 files changed, 4251 insertions(+), 3460 deletions(-) create mode 100644 control_plane/src/physical/workload_cost/status.rs rename control_plane/src/query_plan/{logical.rs => residual.rs} (91%) rename crates/asap_types/src/query_plan/{logical.rs => residual.rs} (94%) rename data_plane/src/{monitor => update_sampling}/coordinator.rs (100%) rename data_plane/src/{monitor => update_sampling}/epoch.rs (100%) rename data_plane/src/{monitor => update_sampling}/mod.rs (100%) rename data_plane/src/{monitor => update_sampling}/sampling_alloc.rs (100%) rename data_plane/src/{monitor => update_sampling}/server.rs (99%) diff --git a/control_plane/examples/audit_clickhouse_corpus.rs b/control_plane/examples/audit_clickhouse_corpus.rs index e7497b8d1..dc34920b7 100644 --- a/control_plane/examples/audit_clickhouse_corpus.rs +++ b/control_plane/examples/audit_clickhouse_corpus.rs @@ -56,7 +56,7 @@ fn publication_inputs(schema: &Schema, sql: String) -> ClickHouseSqlWorkload { let mut precompute_plan = PrecomputePlan::build_backend_local(envelope.clone(), vec![materialization.clone()]) .unwrap(); - let mut transmission_plan = control_plane::physical::compiler::compile_transmission_plan( + let mut transmission_plan = control_plane::physical::compiler::build_transmission_plan( envelope, &precompute_plan, &Default::default(), @@ -72,7 +72,7 @@ fn publication_inputs(schema: &Schema, sql: String) -> ClickHouseSqlWorkload { precompute_plan.summary_catalog = Some(reference.clone()); transmission_plan.summary_catalog = Some(reference); ClickHouseSqlWorkload { - sds, + summary_catalog: sds, precompute_plan, transmission_plan, tables: std::collections::HashMap::from([("raw_samples".into(), schema.clone())]), diff --git a/control_plane/examples/calibration_candidates.rs b/control_plane/examples/calibration_candidates.rs index 65fcb1369..f2894c4fb 100644 --- a/control_plane/examples/calibration_candidates.rs +++ b/control_plane/examples/calibration_candidates.rs @@ -1,6 +1,6 @@ //! Export every bindable candidate for isolated measurement, without selecting a winner. use control_plane::physical::{ - compiler::{BackendLocalPlanningSnapshot, PhysicalCompiler}, + compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}, workload_cost, }; use planner_types::post_asap::{SummaryExpr, SummaryNode}; @@ -9,7 +9,7 @@ use std::{collections::BTreeMap, rc::Rc}; // Planner IR does not implement Serialize. Preserve actual DAG identity and // typed variant/edges; leaf metadata uses explicitly labelled Debug encoding. -fn planner_forest(queries: &[control_plane::physical::compiler::PlanningQuery]) -> Value { +fn planner_forest(queries: &[control_plane::physical::compiler::QueryCompilationInput]) -> Value { fn visit( node: &Rc, seen: &mut BTreeMap, @@ -118,7 +118,7 @@ fn planner_forest(queries: &[control_plane::physical::compiler::PlanningQuery]) } let mut seen = BTreeMap::new(); let mut nodes = BTreeMap::new(); - let roots:Vec<_>=queries.iter().map(|q|json!({"query_id":q.query_id,"original_promql":q.query_string,"root":visit(&q.post_asap,&mut seen,&mut nodes)})).collect(); + let roots:Vec<_>=queries.iter().map(|q|json!({"query_id":q.query_id,"original_promql":q.query_string,"root":visit(&q.selected_plan_root,&mut seen,&mut nodes)})).collect(); json!({"encoding":"structured_graph_with_debug_metadata_v1","scope":"actual candidate Planner post-ASAP input before physical binding; not reconstructed from installed nodes","roots":roots,"nodes":nodes}) } @@ -127,26 +127,26 @@ fn main() -> Result<(), Box> { .nth(1) .ok_or("usage: calibration_candidates SNAPSHOT.json [--metricsql]")?; let metricsql = std::env::args().skip(2).any(|arg| arg == "--metricsql"); - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_slice(&std::fs::read(path)?)?; - let (request, environment) = snapshot.planning_request()?; + let snapshot: BackendLocalPlanningInput = serde_json::from_slice(&std::fs::read(path)?)?; + let (request, environment) = snapshot.into_physical_compilation_request()?; let mut results = Vec::new(); - for (index, candidate) in workload_cost::with_exact_alternative(request)? + for (index, candidate) in workload_cost::enumerate_exact_and_materialized_candidates(request)? .into_iter() .enumerate() { let queries = candidate.queries.clone(); - let materialization_policy = candidate.materialization_policy.clone(); + let enabled_materialization_keys = candidate.enabled_materialization_keys.clone(); let planner_selected_queries = planner_forest(&queries); let compiled = if metricsql { - PhysicalCompiler.compile_metricsql(candidate, environment.clone()) + PhysicalPlanCompiler.compile_metricsql(candidate, environment.clone()) } else { - PhysicalCompiler.compile(candidate, environment.clone()) + PhysicalPlanCompiler.compile_promql(candidate, environment.clone()) }; let plan = match compiled { Ok(plan) => plan, Err(error) => { results.push( - json!({"candidate_index": index, "materialization_policy": materialization_policy, "planner_selected_queries": planner_selected_queries, "unavailable_reason": error.to_string()}), + json!({"candidate_index": index, "materialization_policy": enabled_materialization_keys, "planner_selected_queries": planner_selected_queries, "unavailable_reason": error.to_string()}), ); continue; } @@ -155,14 +155,14 @@ fn main() -> Result<(), Box> { Ok(manifest) => manifest, Err(error) => { results.push( - json!({"candidate_index": index, "materialization_policy": materialization_policy, "planner_selected_queries": planner_selected_queries, "unavailable_reason": error.to_string()}), + json!({"candidate_index": index, "materialization_policy": enabled_materialization_keys, "planner_selected_queries": planner_selected_queries, "unavailable_reason": error.to_string()}), ); continue; } }; results.push(json!({ "candidate_index": index, - "materialization_policy": materialization_policy, + "materialization_policy": enabled_materialization_keys, "planner_selected_queries": planner_selected_queries, "manifest": manifest, "lifecycle_estimates": plan.lifecycle_estimates, diff --git a/control_plane/examples/compile_workload_artifact.rs b/control_plane/examples/compile_workload_artifact.rs index fa6ceed58..392ba0292 100644 --- a/control_plane/examples/compile_workload_artifact.rs +++ b/control_plane/examples/compile_workload_artifact.rs @@ -1,5 +1,5 @@ //! Control-plane entry point: cost-select a workload and emit its atomic install request. -use control_plane::physical::compiler::BackendLocalPlanningSnapshot; +use control_plane::physical::compiler::BackendLocalPlanningInput; use serde_json::json; fn main() -> Result<(), Box> { @@ -15,12 +15,12 @@ fn main() -> Result<(), Box> { if args.next().is_some() { return Err("unexpected arguments".into()); } - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_slice(&std::fs::read(path)?)?; + let snapshot: BackendLocalPlanningInput = serde_json::from_slice(&std::fs::read(path)?)?; let start = std::time::Instant::now(); let plan = if metricsql { snapshot.compile_metricsql()? } else { - snapshot.compile()? + snapshot.compile_promql()? }; let elapsed = start.elapsed().as_nanos(); let comparison = plan @@ -33,7 +33,7 @@ fn main() -> Result<(), Box> { "planning_elapsed_ns": elapsed, "envelope": plan.envelope, "cost_comparison": comparison, - "logical_selection": plan.logical_selection, + "logical_selection": plan.planner_selection_trace, "backend_revision": control_plane::physical::compiler::BACKEND_REVISION, "planner_revision": control_plane::physical::compiler::PLANNER_REVISION, "lifecycle_estimates": plan.lifecycle_estimates, diff --git a/control_plane/examples/workload_cost_manifest.rs b/control_plane/examples/workload_cost_manifest.rs index 723fe8ed2..142f4a07c 100644 --- a/control_plane/examples/workload_cost_manifest.rs +++ b/control_plane/examples/workload_cost_manifest.rs @@ -1,21 +1,21 @@ //! Emit pricing requirements; never fabricate quotes or publish a plan. use control_plane::physical::{ - compiler::BackendLocalPlanningSnapshot, compiler::PhysicalCompiler, workload_cost, + compiler::BackendLocalPlanningInput, compiler::PhysicalPlanCompiler, workload_cost, }; fn main() -> Result<(), Box> { let path = std::env::args() .nth(1) .ok_or("usage: workload_cost_manifest SNAPSHOT.json")?; - let snapshot: BackendLocalPlanningSnapshot = + let snapshot: BackendLocalPlanningInput = serde_json::from_str(&std::fs::read_to_string(path)?)?; - let (request, environment) = snapshot.planning_request()?; - let manifests = workload_cost::with_exact_alternative(request)? + let (request, environment) = snapshot.into_physical_compilation_request()?; + let manifests = workload_cost::enumerate_exact_and_materialized_candidates(request)? .into_iter() .filter_map(|candidate| { let queries = candidate.queries.clone(); - PhysicalCompiler - .compile(candidate, environment.clone()) + PhysicalPlanCompiler + .compile_promql(candidate, environment.clone()) .and_then(|plan| workload_cost::manifest(&plan, &queries)) .ok() }) diff --git a/control_plane/src/backend_client.rs b/control_plane/src/backend_client.rs index 75841913c..fcce53262 100644 --- a/control_plane/src/backend_client.rs +++ b/control_plane/src/backend_client.rs @@ -596,16 +596,18 @@ mod tests { #[tokio::test] async fn catalog_publication_posts_canonical_document_without_legacy_bytes() { - let snapshot: crate::physical::compiler::BackendLocalPlanningSnapshot = - serde_json::from_str(include_str!( - "../../docs/examples/asapquery-planning-snapshot.json" - )) - .unwrap(); - let publication = crate::physical::compiler::tests::quoted_snapshot(snapshot, false) - .compile() - .unwrap() - .publication() - .unwrap(); + let snapshot: crate::physical::compiler::BackendLocalPlanningInput = serde_json::from_str( + include_str!("../../docs/examples/asapquery-planning-snapshot.json"), + ) + .unwrap(); + let publication = crate::physical::compiler::tests::quoted_snapshot( + snapshot, + crate::physical::compiler::QueryFrontend::PromQl, + ) + .compile_promql() + .unwrap() + .to_publication_artifact() + .unwrap(); let hits: StdArc>> = StdArc::new(Mutex::new(Vec::new())); let route_hits = hits.clone(); let app = Router::new().route( diff --git a/control_plane/src/clickhouse.rs b/control_plane/src/clickhouse.rs index 4abd29e6c..6c0181b98 100644 --- a/control_plane/src/clickhouse.rs +++ b/control_plane/src/clickhouse.rs @@ -204,7 +204,8 @@ pub use asap_frontend_sql::SqlCatalog as ClickHouseSqlCatalog; #[derive(Debug, Deserialize)] pub struct ClickHouseSqlWorkload { - pub sds: SummaryCatalog, + #[serde(rename = "sds", alias = "summary_catalog")] + pub summary_catalog: SummaryCatalog, pub precompute_plan: PrecomputePlan, pub transmission_plan: TransmissionPlan, pub tables: HashMap, @@ -295,7 +296,7 @@ pub async fn compile_automatic_clickhouse_workload( .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?, ); precompute.executable_dags = installed_dags; - let mut transmission = crate::physical::compiler::compile_transmission_plan( + let mut transmission = crate::physical::compiler::build_transmission_plan( request.envelope.clone(), &precompute, &std::collections::BTreeMap::new(), @@ -404,7 +405,7 @@ pub async fn compile_clickhouse_workload( ) -> Result { request .precompute_plan - .validate_against_catalog(&request.sds) + .validate_against_catalog(&request.summary_catalog) .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; request .transmission_plan @@ -435,13 +436,13 @@ pub async fn compile_clickhouse_workload( let mut precompute_plan = request.precompute_plan.clone(); precompute_plan.executable_dags = installed_dags; let publication = crate::physical::publication::PhysicalPlanPublication { - summary_catalog: request.sds.clone(), + summary_catalog: request.summary_catalog.clone(), precompute_plan, collector_plans: Vec::new(), transmission_plan: request.transmission_plan.clone(), query_plan: QueryPlan { - plan_id: request.sds.plan_id, - plan_version: request.sds.plan_version, + plan_id: request.summary_catalog.plan_id, + plan_version: request.summary_catalog.plan_version, clickhouse_context: Some(ClickHousePlanningContext { window_templates, tables: request.tables.clone(), @@ -1394,7 +1395,7 @@ mod tests { let mut precompute = PrecomputePlan::build_backend_local(envelope.clone(), vec![config]).unwrap(); precompute.summary_catalog = Some(sds.reference().unwrap()); - let mut transmission = crate::physical::compiler::compile_transmission_plan( + let mut transmission = crate::physical::compiler::build_transmission_plan( envelope, &precompute, &std::collections::BTreeMap::new(), @@ -1412,7 +1413,7 @@ mod tests { ) }; let mut request = ClickHouseSqlWorkload { - sds, + summary_catalog: sds, precompute_plan: precompute, transmission_plan: transmission, tables: HashMap::from([ @@ -1442,7 +1443,7 @@ mod tests { let simple_sql = "SELECT sum(value) FROM telemetry WHERE timestamp_ms >= 0 AND timestamp_ms < 2000"; let simple = compile_clickhouse_workload(&ClickHouseSqlWorkload { - sds: request.sds.clone(), + summary_catalog: request.summary_catalog.clone(), precompute_plan: request.precompute_plan.clone(), transmission_plan: request.transmission_plan.clone(), tables: request.tables.clone(), @@ -1487,7 +1488,7 @@ mod tests { ] }; let multiple = compile_clickhouse_workload(&ClickHouseSqlWorkload { - sds: request.sds.clone(), + summary_catalog: request.summary_catalog.clone(), precompute_plan: request.precompute_plan.clone(), transmission_plan: request.transmission_plan.clone(), tables: request.tables.clone(), @@ -1700,18 +1701,21 @@ mod tests { value: planner_types::pre_asap::ScalarValue::Utf8("requests".into()), }], }); - request.sds = SummaryCatalog::from_materializations(71, 1, &[config.clone()]).unwrap(); + request.summary_catalog = + SummaryCatalog::from_materializations(71, 1, &[config.clone()]).unwrap(); let envelope = request.precompute_plan.envelope.clone(); request.precompute_plan = PrecomputePlan::build_backend_local(envelope.clone(), vec![config]).unwrap(); - request.precompute_plan.summary_catalog = Some(request.sds.reference().unwrap()); - request.transmission_plan = crate::physical::compiler::compile_transmission_plan( + request.precompute_plan.summary_catalog = + Some(request.summary_catalog.reference().unwrap()); + request.transmission_plan = crate::physical::compiler::build_transmission_plan( envelope, &request.precompute_plan, &std::collections::BTreeMap::new(), ) .unwrap(); - request.transmission_plan.summary_catalog = Some(request.sds.reference().unwrap()); + request.transmission_plan.summary_catalog = + Some(request.summary_catalog.reference().unwrap()); assert!(compile_clickhouse_workload(&request).await.is_ok()); } } diff --git a/control_plane/src/emit/backend_push.rs b/control_plane/src/emit/backend_push.rs index 9be93c761..03113fadc 100644 --- a/control_plane/src/emit/backend_push.rs +++ b/control_plane/src/emit/backend_push.rs @@ -260,7 +260,7 @@ async fn push_documents_coupled( warn!(%error, "failed to bind compatibility PrecomputePlan to SummaryCatalog"); return (false, false, 0); } - let transmission_plan = match crate::physical::compiler::compile_transmission_plan( + let transmission_plan = match crate::physical::compiler::build_transmission_plan( precompute_plan.envelope.clone(), &precompute_plan, &Default::default(), diff --git a/control_plane/src/emit/mod.rs b/control_plane/src/emit/mod.rs index 8a7e1db49..67dfe23ce 100644 --- a/control_plane/src/emit/mod.rs +++ b/control_plane/src/emit/mod.rs @@ -637,7 +637,7 @@ pub fn collect_cumulative_counter_metrics( let mut seen: std::collections::BTreeSet = std::collections::BTreeSet::new(); for entry in registry.entries() { // `derive_agg_role` reads the WorkloadEntry directly (query - // string + family override), not the lowered QueryWorkload, so + // string + family override), not the lowered LegacyMetricWorkload, so // we classify the registry entry. We still consult the // workload_store to confirm the metric was successfully // pre-populated (matching the contract of the sibling @@ -900,10 +900,10 @@ mod runtime_tests { // Reproduces the live demo gap: 3 of 6 (HLL, CountSketch, CMS) silently // drop because the analyzer pre-population path doesn't propagate // `sketch_family_override` from the workload YAML into - // `QueryWorkload::sketch_type_override`. + // `LegacyMetricWorkload::sketch_type_override`. /// Mimics the pre-population loop in `main()` — turns each - /// `WorkloadEntry` into a `QueryWorkload` via the shared `Analyzer`. + /// `WorkloadEntry` into a `LegacyMetricWorkload` via the shared `Analyzer`. fn populate_store_from_registry(registry: &WorkloadRegistry, store: &WorkloadStore) { use crate::pipeline::Analyzer; use crate::types; @@ -1124,12 +1124,12 @@ mod runtime_tests { /// fan into three pipelines. /// /// We populate the store directly with three `(metric, role)` - /// `QueryWorkload`s — one per capability — so the test pins + /// `LegacyMetricWorkload`s — one per capability — so the test pins /// `collect_metric_to_family`'s union semantics independently of the /// analyzer's query-string → AggType parsing. #[test] fn collect_metric_to_family_unions_multiple_capabilities_per_metric() { - use crate::types::{AggType, QueryWorkload, SketchType, WorkloadCharacteristics}; + use crate::types::{AggType, LegacyMetricWorkload, SketchType, WorkloadCharacteristics}; use crate::workload::AggRole; use planner_types::post_asap::SketchAlgorithm; use std::collections::BTreeSet; @@ -1156,8 +1156,8 @@ mod runtime_tests { let mk = |agg: AggType, override_family: Option, quantiles: Vec| - -> QueryWorkload { - QueryWorkload { + -> LegacyMetricWorkload { + LegacyMetricWorkload { metric_name: METRIC.to_string(), label_filters: Default::default(), group_by_labels: Vec::new(), @@ -1221,7 +1221,7 @@ mod runtime_tests { // Pre-B3 the WorkloadEntry YAML had no way to declare grouping // labels — the analyzer pulled them only from PromQL `by (...)` // clauses. Bare `quantile_over_time(0.99, metric[30s])` carries no - // `by`, so `QueryWorkload.group_by_labels` ended up empty, so + // `by`, so `LegacyMetricWorkload.group_by_labels` ended up empty, so // `collect_metric_to_grouping_labels` returned `{metric: vec![]}`, // so the 5-sketch routing emitter wrote // `keep_keys(datapoint.attributes, [])` — stripping ALL attrs @@ -1230,7 +1230,7 @@ mod runtime_tests { // // Post-B3 a declarative `grouping_labels: [zone]` on WorkloadEntry // is threaded through the pre-pop QuerySpec → analyzer → - // QueryWorkload.group_by_labels → collect_metric_to_grouping_labels + // LegacyMetricWorkload.group_by_labels → collect_metric_to_grouping_labels // → the emitter's keep_keys list. Without this round-trip the // end-to-end test's sid catalog stays empty-per-zone. #[test] @@ -1256,7 +1256,7 @@ mod runtime_tests { populate_store_from_registry(®istry, &store); // The analyzer must have threaded grouping_labels into - // QueryWorkload.group_by_labels. + // LegacyMetricWorkload.group_by_labels. let map = collect_metric_to_grouping_labels(®istry, &store); assert_eq!( map.get("http_requests_total_latency_ms"), diff --git a/control_plane/src/lib.rs b/control_plane/src/lib.rs index c61639e5c..868e2b556 100644 --- a/control_plane/src/lib.rs +++ b/control_plane/src/lib.rs @@ -1,7 +1,14 @@ -//! Control-plane planning, configuration emission, and deployment services. +//! Backend control-plane library and standalone service. //! -//! These modules support in-process integration and the standalone control-plane -//! binary. Internal module paths are not a wire-protocol compatibility contract. +//! ASAPPlanner owns canonical semantic IR and legal logical selection. This +//! crate adapts workload/evidence inputs, compiles physical candidates, checks +//! provider quotes, and publishes one consistent catalog-backed generation. +//! Shared execution and installation contracts live in `asap_types`. +//! +//! `physical::compiler`, `physical::workload_cost`, and `clickhouse` are the +//! current compilation paths. Flat metric workloads and stage emission remain +//! compatibility paths. Their public modules are workspace integration APIs, +//! not an independent wire schema or a second semantic planner. #![allow( clippy::collapsible_match, diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index 0ad1a2f8a..c9d4cbd80 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -301,7 +301,7 @@ async fn main() { // // Critical: thread `sketch_family_override` from each registry entry // into the QuerySpec's `sketch_type` field — that's what populates - // `QueryWorkload::sketch_type_override`, which the typed planner + // `LegacyMetricWorkload::sketch_type_override`, which the typed planner // (`bind_workload_typed`) reads to honour MVP §46 entries 5–8 (HLL / // CountSketch / CountMinSketch). Without this stitch the workloads // round-trip through the analyzer with a None override and the @@ -572,8 +572,12 @@ struct PhysicalPlanQueryRequest { #[serde(default)] group_by: Vec, accuracy: types_v2::AccuracyTarget, - lifecycle: physical::compiler::LifecyclePlanningInput, - window_implementations: Vec, + lifecycle: physical::compiler::SummaryLifecyclePlanningInputs, + #[serde( + rename = "window_implementations", + alias = "window_realization_candidates" + )] + window_realization_candidates: Vec, #[serde(default)] runtime_policy: physical::compiler::RuntimeRulePolicy, } @@ -589,7 +593,8 @@ struct CompileAndPublishPhysicalPlanRequest { #[serde(default)] workload_cost_evidence: Option, queries: Vec, - collector_ids: Vec, + #[serde(rename = "collector_ids", alias = "target_collector_ids")] + target_collector_ids: Vec, capability_snapshot_id: String, #[serde(default)] evidence: HashMap, @@ -618,48 +623,19 @@ fn default_physical_plan_timeout_ms() -> u64 { 10_000 } -#[derive(Clone, Copy)] -enum PhysicalQueryFrontend { - PromQl, - MetricsQl, -} - -impl PhysicalQueryFrontend { - fn parse( - self, - query: &str, - accuracy: types_v2::AccuracyTarget, - ) -> Result { - match self { - Self::PromQl => parse_query_expr_canonical(query, accuracy) - .map_err(|e| format!("frontend.promql: {e}")), - Self::MetricsQl => parse_query_expr_canonical(query, accuracy) - .map_err(|e| format!("victoriametrics.promql_subset: {e}")), - } - } - fn compile( - self, - request: physical::compiler::PlanningRequest, - environment: physical::compiler::DeploymentEnvironment, - ) -> Result { - match self { - Self::PromQl => physical::compiler::PhysicalCompiler.compile(request, environment), - Self::MetricsQl => { - physical::compiler::PhysicalCompiler.compile_metricsql(request, environment) - } - } - } -} +use physical::compiler::QueryFrontend; #[derive(Debug, Serialize)] struct CompileAndPublishPhysicalPlanResponse { - cost_comparison: Option, - logical_selection: Vec, + cost_comparison: Option, + #[serde(rename = "logical_selection", alias = "planner_selection_trace")] + planner_selection_trace: Vec, plan_id: u64, plan_version: u64, status: &'static str, generated_at_unix_ms: u64, - collector_ids: Vec, + #[serde(rename = "collector_ids", alias = "target_collector_ids")] + target_collector_ids: Vec, lifecycle_estimates: Vec, } @@ -669,20 +645,20 @@ async fn handle_compile_and_publish_physical_plan( State(st): State, Json(request): Json, ) -> Response { - compile_and_publish_physical_plan(st, request, PhysicalQueryFrontend::PromQl).await + compile_and_publish_physical_plan(st, request, QueryFrontend::PromQl).await } async fn handle_compile_and_publish_metricsql_physical_plan( State(st): State, Json(request): Json, ) -> Response { - compile_and_publish_physical_plan(st, request, PhysicalQueryFrontend::MetricsQl).await + compile_and_publish_physical_plan(st, request, QueryFrontend::MetricsQl).await } async fn compile_and_publish_physical_plan( st: AppState, mut request: CompileAndPublishPhysicalPlanRequest, - frontend: PhysicalQueryFrontend, + frontend: QueryFrontend, ) -> Response { // Serialize typed activations so an older response cannot overwrite the // catalog recorded after a newer backend activation. @@ -694,7 +670,7 @@ async fn compile_and_publish_physical_plan( let catalog = active_catalog.clone(); erp.resolve_population_data_descriptor(catalog.as_deref()); } - let (bundle, collector_ids, apply_timeout, adaptation_evidence, _) = + let (bundle, target_collector_ids, apply_timeout, adaptation_evidence, _) = match compile_physical_plan_request(request, false, frontend) { Ok((Some(bundle), ids, timeout, adaptation, manifests)) => { (bundle, ids, timeout, adaptation, manifests) @@ -727,7 +703,7 @@ async fn compile_and_publish_physical_plan( ) .into_response(); } - let publication = match bundle.publication() { + let publication = match bundle.to_publication_artifact() { Ok(publication) => publication, Err(error) => { return ( @@ -791,12 +767,12 @@ async fn compile_and_publish_physical_plan( Json(CompileAndPublishPhysicalPlanResponse { cost_comparison: bundle.cost_comparison, - logical_selection: bundle.logical_selection, + planner_selection_trace: bundle.planner_selection_trace, plan_id: bundle.envelope.plan_id, plan_version: bundle.envelope.plan_version, status: "active", generated_at_unix_ms: bundle.envelope.generated_at_unix_ms, - collector_ids, + target_collector_ids, lifecycle_estimates: bundle.lifecycle_estimates, }) .into_response() @@ -867,16 +843,16 @@ async fn publish_clickhouse_plan( fn compile_physical_plan_request( request: CompileAndPublishPhysicalPlanRequest, manifests_only: bool, - frontend: PhysicalQueryFrontend, + frontend: QueryFrontend, ) -> Result< ( - Option, + Option, Vec, Duration, Vec, ( Vec, - Vec, + Vec, Vec, ), ), @@ -884,9 +860,9 @@ fn compile_physical_plan_request( > { if request.queries.is_empty() || (request.target == physical::compiler::PhysicalDeploymentTarget::DistributedCollectors - && request.collector_ids.is_empty()) + && request.target_collector_ids.is_empty()) || (request.target == physical::compiler::PhysicalDeploymentTarget::BackendLocalRemoteWrite - && !request.collector_ids.is_empty()) + && !request.target_collector_ids.is_empty()) { return Err(( StatusCode::UNPROCESSABLE_ENTITY, @@ -943,23 +919,23 @@ fn compile_physical_plan_request( Err(error) => return Err((StatusCode::UNPROCESSABLE_ENTITY, error.to_string().into())), }; canonical_roots.push(std::rc::Rc::new(expr)); - queries.push(physical::compiler::PlanningQuery { + queries.push(physical::compiler::QueryCompilationInput { query_id: query.query_id, query_string: query.query_string, - post_asap, - source: planner_types::pre_asap::Source::TimeSeries { + selected_plan_root: post_asap, + legacy_query_source: planner_types::pre_asap::Source::TimeSeries { metric: query.metric, }, - window_secs: query.window_secs, - group_by: query.group_by, - accuracy: query.accuracy, - lifecycle: query.lifecycle, - window_implementations: query.window_implementations, - runtime_policy: query.runtime_policy, + query_lookback_seconds: query.window_secs, + group_by_labels: query.group_by, + accuracy_target: query.accuracy, + summary_lifecycle_inputs: query.lifecycle, + window_realization_candidates: query.window_realization_candidates, + materialization_runtime_policy: query.runtime_policy, }); } - let logical_selection = match physical::compiler::select_workload_roots_with_trace( + let planner_selection_trace = match physical::compiler::select_logical_roots_with_trace( &mut queries, canonical_roots, &request.evidence, @@ -970,25 +946,25 @@ fn compile_physical_plan_request( Err(error) => return Err((StatusCode::UNPROCESSABLE_ENTITY, error.to_string().into())), }; - let planning_request = physical::compiler::PlanningRequest { - synthesized_window_queries: Default::default(), - logical_selection, + let compilation_request = physical::compiler::PhysicalCompilationRequest { + compiler_priced_window_query_ids: Default::default(), + planner_selection_trace, query_workload: None, queries, - hybrid_execution: request.target + allow_mixed_summary_and_exact_execution: request.target == physical::compiler::PhysicalDeploymentTarget::BackendLocalRemoteWrite, - materialization_policy: None, - evidence: request.evidence, + enabled_materialization_keys: None, + topk_membership_evidence_by_query_id: request.evidence, exact_composition_costs: request.exact_composition_costs, erp: request.erp, planner_revision: request.planner_revision, source_sample_interval_ms: None, - query_staleness_margin_ms: 0, + query_retention_margin_ms: 0, retained_summary_memory_budget_bytes: None, }; - let environment = physical::compiler::DeploymentEnvironment { + let environment = physical::compiler::PhysicalDeploymentContext { target: request.target, - collector_ids: request.collector_ids.clone(), + target_collector_ids: request.target_collector_ids.clone(), capability_snapshot_id: request.capability_snapshot_id, observed_at_unix_ms: now, max_evidence_age_ms: request.max_evidence_age_ms, @@ -997,13 +973,15 @@ fn compile_physical_plan_request( expiry_unix_ms: request.expiry_unix_ms, backend_compat: request.backend_compat, }; - let candidates = physical::workload_cost::with_exact_alternative(planning_request.clone()) - .map_err(|error| (StatusCode::UNPROCESSABLE_ENTITY, error.to_string().into()))?; - let logical_selection = planning_request.logical_selection.clone(); - let (manifests, alternatives) = physical::workload_cost::prepare_manifests( + let candidates = physical::workload_cost::enumerate_exact_and_materialized_candidates( + compilation_request.clone(), + ) + .map_err(|error| (StatusCode::UNPROCESSABLE_ENTITY, error.to_string().into()))?; + let planner_selection_trace = compilation_request.planner_selection_trace.clone(); + let (manifests, alternatives) = physical::workload_cost::compile_candidates_for_pricing( candidates.clone(), environment.clone(), - matches!(frontend, PhysicalQueryFrontend::MetricsQl), + frontend, ); let apply_timeout = Duration::from_millis(request.apply_timeout_ms); // Quote preparation enumerates feasible bindings; it does not select the @@ -1013,27 +991,33 @@ fn compile_physical_plan_request( return Err(( StatusCode::UNPROCESSABLE_ENTITY, serde_json::json!({"status": "all_infeasible", "alternatives": alternatives, - "logical_selection": planning_request.logical_selection}), + "logical_selection": compilation_request.planner_selection_trace}), )); } return Ok(( None, - request.collector_ids, + request.target_collector_ids, apply_timeout, request.runtime_adaptation_evidence, - (manifests, alternatives, logical_selection), + (manifests, alternatives, planner_selection_trace), )); } let compiled = match request.workload_cost_evidence { Some(evidence) => match frontend { - PhysicalQueryFrontend::PromQl => { - physical::workload_cost::select(candidates, environment, &evidence) - } - PhysicalQueryFrontend::MetricsQl => { - physical::workload_cost::select_metricsql(candidates, environment, &evidence) + QueryFrontend::PromQl => physical::workload_cost::select_lowest_cost_candidate( + candidates, + environment, + &evidence, + ), + QueryFrontend::MetricsQl => { + physical::workload_cost::select_lowest_cost_metricsql_candidate( + candidates, + environment, + &evidence, + ) } }, - None => frontend.compile(planning_request, environment), + None => frontend.compile(compilation_request, environment), }; let bundle = match compiled { Ok(bundle) => bundle, @@ -1044,10 +1028,10 @@ fn compile_physical_plan_request( }; Ok(( Some(bundle), - request.collector_ids, + request.target_collector_ids, apply_timeout, request.runtime_adaptation_evidence, - (manifests, alternatives, logical_selection), + (manifests, alternatives, planner_selection_trace), )) } @@ -1055,13 +1039,13 @@ fn compile_physical_plan_request( async fn handle_workload_cost_manifests( Json(request): Json, ) -> impl IntoResponse { - workload_cost_manifests(request, PhysicalQueryFrontend::PromQl) + workload_cost_manifests(request, QueryFrontend::PromQl) } async fn handle_metricsql_workload_cost_manifests( Json(request): Json, ) -> impl IntoResponse { - workload_cost_manifests(request, PhysicalQueryFrontend::MetricsQl) + workload_cost_manifests(request, QueryFrontend::MetricsQl) } fn physical_compile_failure((status, report): (StatusCode, serde_json::Value)) -> Response { @@ -1073,7 +1057,7 @@ fn physical_compile_failure((status, report): (StatusCode, serde_json::Value)) - fn workload_cost_manifests( request: CompileAndPublishPhysicalPlanRequest, - frontend: PhysicalQueryFrontend, + frontend: QueryFrontend, ) -> Response { if request.workload_cost_evidence.is_some() { return ( @@ -1084,9 +1068,9 @@ fn workload_cost_manifests( } let explain = request.explain; match compile_physical_plan_request(request, true, frontend) { - Ok((_, _, _, _, (manifests, alternatives, logical_selection))) => { + Ok((_, _, _, _, (manifests, alternatives, planner_selection_trace))) => { if explain { - Json(serde_json::json!({"manifests": manifests, "alternatives": alternatives, "logical_selection": logical_selection})) + Json(serde_json::json!({"manifests": manifests, "alternatives": alternatives, "logical_selection": planner_selection_trace})) .into_response() } else { Json(manifests).into_response() @@ -1200,7 +1184,7 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> // optimised L3 tree run through `physical::post_asap::bind_query_expr`. // For specs that supply only explicit fields (no `query_string` to // parse), there is no L3 tree to bind, so we fall back to - // `bind_workload_typed`, which lowers the flat `QueryWorkload` + // `bind_workload_typed`, which lowers the flat `LegacyMetricWorkload` // summary to a `PhysicalExpr` directly. if let Some(configs) = stage_configs { for (stage_id, stage_cfg) in configs { @@ -1292,7 +1276,7 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> // columns (open-set label naming is // a Step γ TODO in // `intent_algebra::column_resolution`). - // `QueryWorkload` carries both unambiguously, + // `LegacyMetricWorkload` carries both unambiguously, // and every aggregation under one workload // shares them — so the patch is uniform. let item_labels = emit::collect_metric_to_item_label( @@ -1512,7 +1496,7 @@ async fn handle_plan_auto( // the data-plane coordinator derives the live ε-floor p. st.workload_registry.insert_runtime(entry); // (2) Best-effort SKETCH registration: analyze a query for this metric - // and register the QueryWorkload (with the chosen sketch) so replan + // and register the LegacyMetricWorkload (with the chosen sketch) so replan // emits a fresh sketch plan. The pipeline analyzer accepts a narrower // grammar than the planner's, so this is non-fatal on rejection. for q in &req.queries { @@ -2272,11 +2256,11 @@ mod api_tests { // A missing warm implementation must not hide the executable exact quote. #[tokio::test] async fn cost_manifests_survive_unavailable_warm_candidate() { - let snapshot: physical::compiler::BackendLocalPlanningSnapshot = serde_json::from_str( + let snapshot: physical::compiler::BackendLocalPlanningInput = serde_json::from_str( include_str!("../../docs/examples/asapquery-planning-snapshot.json"), ) .unwrap(); - let (planning, _) = snapshot.planning_request().unwrap(); + let (planning, _) = snapshot.into_physical_compilation_request().unwrap(); let query = &planning.queries[0]; let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -2285,8 +2269,8 @@ mod api_tests { let mut request_body = serde_json::json!({ "queries": [{ "query_id": query.query_id, "query_string": query.query_string, - "metric": "m", "window_secs": 60, "accuracy": query.accuracy, - "lifecycle": query.lifecycle, "window_implementations": [] + "metric": "m", "window_secs": 60, "accuracy": query.accuracy_target, + "lifecycle": query.summary_lifecycle_inputs, "window_implementations": [] }], "collector_ids": ["test"], "capability_snapshot_id": "test", "planner_revision": physical::compiler::PLANNER_REVISION, @@ -2336,29 +2320,30 @@ mod api_tests { #[test] fn backend_local_typed_request_compiles_without_collectors() { - let snapshot: physical::compiler::BackendLocalPlanningSnapshot = serde_json::from_str( + let snapshot: physical::compiler::BackendLocalPlanningInput = serde_json::from_str( include_str!("../../docs/examples/asapquery-compatibility-demo-snapshot.json"), ) .unwrap(); - let (planning, _) = snapshot.planning_request().unwrap(); + let (planning, _) = snapshot.into_physical_compilation_request().unwrap(); let mut query = planning.queries[0].clone(); let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_millis() as u64; - query.lifecycle.evidence_observed_at_unix_ms = now; - for implementation in &mut query.window_implementations { + query.summary_lifecycle_inputs.evidence_observed_at_unix_ms = now; + for implementation in &mut query.window_realization_candidates { implementation.cost.observed_at_unix_ms = now; } - let planner_types::pre_asap::Source::TimeSeries { metric } = &query.source else { + let planner_types::pre_asap::Source::TimeSeries { metric } = &query.legacy_query_source + else { panic!("expected time series fixture"); }; let value = serde_json::json!({ "target": "backend_local_remote_write", "queries": [{ "query_id": query.query_id, "query_string": query.query_string, - "metric": metric, "window_secs": query.window_secs, "accuracy": query.accuracy, - "lifecycle": query.lifecycle, "window_implementations": query.window_implementations + "metric": metric, "window_secs": query.query_lookback_seconds, "accuracy": query.accuracy_target, + "lifecycle": query.summary_lifecycle_inputs, "window_implementations": query.window_realization_candidates }], "collector_ids": [], "capability_snapshot_id": "test", "planner_revision": physical::compiler::PLANNER_REVISION, @@ -2367,7 +2352,7 @@ mod api_tests { }); let request = serde_json::from_value(value.clone()).unwrap(); let (plan, collectors, _, _, _) = - compile_physical_plan_request(request, false, PhysicalQueryFrontend::PromQl).unwrap(); + compile_physical_plan_request(request, false, QueryFrontend::PromQl).unwrap(); let plan = plan.unwrap(); assert!(collectors.is_empty()); assert!(plan.collector_plans.is_empty()); @@ -2381,7 +2366,7 @@ mod api_tests { assert!(compile_physical_plan_request( serde_json::from_value(distributed).unwrap(), false, - PhysicalQueryFrontend::PromQl + QueryFrontend::PromQl ) .is_err()); } @@ -2572,7 +2557,7 @@ mod api_tests { let (st, app) = test_app(); // Seed one plan directly. use crate::physical::workload_planner::DeploymentPlanCompiler; - let wl = crate::types::QueryWorkload { + let wl = crate::types::LegacyMetricWorkload { metric_name: "m".into(), label_filters: std::collections::HashMap::new(), group_by_labels: vec![], diff --git a/control_plane/src/opamp/mod.rs b/control_plane/src/opamp/mod.rs index 1e2db151a..5cb98e5ad 100644 --- a/control_plane/src/opamp/mod.rs +++ b/control_plane/src/opamp/mod.rs @@ -1026,7 +1026,7 @@ mod tests { window_secs: 60, abstract_window_framework: planner_types::post_asap::SummaryWindowFramework::Tumbling, - window_implementation_id: "collector-tumbling-v1".into(), + window_realization_id: "collector-tumbling-v1".into(), slide_secs: 60, pane_origin_ms: Some(0), window_layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 60 }, diff --git a/control_plane/src/physical/colored_dag/mod.rs b/control_plane/src/physical/colored_dag/mod.rs index ed51d564b..844c3186c 100644 --- a/control_plane/src/physical/colored_dag/mod.rs +++ b/control_plane/src/physical/colored_dag/mod.rs @@ -40,9 +40,6 @@ pub mod dag; pub mod emitter; pub mod stage_id; -#[cfg(test)] -mod tests; - // Re-exports — `crate::physical::colored_dag::*` for downstream callers. // The historical `crate::physical::colored_dag::*` path is preserved via a // re-export in `lib.rs` so existing callers see no source-level break. @@ -54,3 +51,6 @@ pub use emitter::{ PrometheusArchiveMetric, StageConfig, ThreeStageEmitter, }; pub use stage_id::{StageId, Topology}; + +#[cfg(test)] +mod tests; diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 2ea40ab23..6b608e4d3 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -53,38 +53,38 @@ fn is_default_retained_summary_memory_budget_bytes(value: &u64) -> bool { } #[derive(Debug, Clone)] -pub struct PlanningQuery { +pub struct QueryCompilationInput { pub query_id: String, /// Catalog expression used only to build the stable QueryPlan identity. - /// The selected implementation comes from `post_asap`, never this text. + /// The selected implementation comes from `selected_plan_root`, never this text. pub query_string: String, /// Planner-selected post-ASAP DAG. The physical compiler must not /// re-select a summary family from pre-ASAP input. - pub post_asap: Rc, + pub selected_plan_root: Rc, /// Legacy catalog source retained for request compatibility. Physical /// materialization sources are derived from each post-ASAP SummaryAgg; - /// this field is only used to reject table execution in the MVP. - pub source: Source, - pub window_secs: u64, + /// it also remains part of the cost manifest workload identity. + pub legacy_query_source: Source, + pub query_lookback_seconds: u64, /// Label names are deployment metadata because Planner's canonical IR /// currently carries positional column IDs at this boundary. - pub group_by: Vec, - pub accuracy: AccuracyTarget, - pub lifecycle: LifecyclePlanningInput, + pub group_by_labels: Vec, + pub accuracy_target: AccuracyTarget, + pub summary_lifecycle_inputs: SummaryLifecyclePlanningInputs, /// Executor-feasible concrete realizations offered to Planner for its /// abstract window-framework decision. The compiler retains physical /// identities and exposes only framework + complete weighted cost to /// Planner. An empty or stale set fails closed. - pub window_implementations: Vec, + pub window_realization_candidates: Vec, /// Physical runtime policy selected for this Planner materialization. /// It is validated against the selected summary family during physical /// compilation and becomes part of the immutable plan generation. - pub runtime_policy: RuntimeRulePolicy, + pub materialization_runtime_policy: RuntimeRulePolicy, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] -pub struct ImplementationCostEvidence { +pub struct WindowRealizationCostQuote { pub model_version: String, pub workload_fingerprint: String, pub observed_at_unix_ms: u64, @@ -106,19 +106,20 @@ pub struct ImplementationCostEvidence { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] -pub struct WindowImplementationCandidate { +pub struct WindowRealizationCandidate { /// Backend-owned identity; never copied into Planner IR. - pub implementation_id: String, + #[serde(rename = "implementation_id", alias = "realization_id")] + pub realization_id: String, pub framework: SummaryWindowFramework, pub window_secs: u64, pub slide_secs: u64, pub layout: asap_types::WindowMaterializationLayout, - pub cost: ImplementationCostEvidence, + pub cost: WindowRealizationCostQuote, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] -pub struct LifecycleCostEvidence { +pub struct LifecycleUnitCosts { pub build: f64, pub maintenance_per_update: f64, pub read: f64, @@ -128,33 +129,34 @@ pub struct LifecycleCostEvidence { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] -pub struct LifecyclePlanningInput { +pub struct SummaryLifecyclePlanningInputs { pub evaluation_interval_ms: u32, pub ingestion_rate_per_second: f64, pub evidence_observed_at_unix_ms: u64, pub evidence_valid_for_ms: u64, pub horizon_seconds: f64, - pub costs: LifecycleCostEvidence, + pub costs: LifecycleUnitCosts, } #[derive(Debug, Clone, Default)] -pub struct PlanningRequest { +pub struct PhysicalCompilationRequest { /// Diagnostic projections of the original Planner search; never consumed by selection. - pub logical_selection: Vec, + pub planner_selection_trace: Vec, /// Enable a composable DAG with SummaryStore materializations and Prometheus exact subtrees. - pub hybrid_execution: bool, - /// Allowed materialization leaf contracts; None enables every eligible leaf. - pub materialization_policy: Option>, + pub allow_mixed_summary_and_exact_execution: bool, + /// Enabled optional candidate keys: None enables all eligible keys; an + /// explicitly empty set enables none. These are not catalog definition IDs. + pub enabled_materialization_keys: Option>, /// Original dashboard demand, in the same order as queries. None is legacy input. pub query_workload: Option, - pub queries: Vec, + pub queries: Vec, /// Compiler-owned quotes eligible for joint pane repricing; never inferred from model labels. - pub synthesized_window_queries: BTreeSet, - pub evidence: HashMap, + pub compiler_priced_window_query_ids: BTreeSet, + pub topk_membership_evidence_by_query_id: HashMap, /// Fresh measured costs for Planner exact/summary composition sites, /// scoped to query IDs just like accuracy evidence. pub exact_composition_costs: HashMap>, - /// Optional distribution-conditioned empirical sizing policy. Hybrid + /// Optional Error–Resource Profile inputs and distribution-conditioned sizing. Hybrid /// mode falls back to theoretical sizing and then exact execution. pub erp: Option, pub planner_revision: String, @@ -164,7 +166,7 @@ pub struct PlanningRequest { /// How far behind the newest ingested sample an admitted query may be /// evaluated. This extends physical retention only; it never changes the /// PromQL range selector used for readout. - pub query_staleness_margin_ms: u64, + pub query_retention_margin_ms: u64, /// Maximum aggregate encoded footprint of every retained summary pane. /// `None` uses the backend's default persistence-memory limit. pub retained_summary_memory_budget_bytes: Option, @@ -181,9 +183,10 @@ pub struct TopKMembershipEvidence { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] -pub struct DeploymentEnvironment { +pub struct PhysicalDeploymentContext { pub target: PhysicalDeploymentTarget, - pub collector_ids: Vec, + #[serde(rename = "collector_ids", alias = "target_collector_ids")] + pub target_collector_ids: Vec, pub capability_snapshot_id: String, pub observed_at_unix_ms: u64, pub max_evidence_age_ms: u64, @@ -207,33 +210,48 @@ pub enum PhysicalDeploymentTarget { /// identity required to choose a concrete physical realization. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] -pub struct BackendLocalPlanningSnapshot { - pub snapshot_version: u32, +pub struct BackendLocalPlanningInput { + #[serde(rename = "snapshot_version", alias = "schema_version")] + pub schema_version: u32, /// May be absent during candidate discovery, never during deployment. #[serde(default, skip_serializing_if = "Option::is_none")] pub workload_cost_evidence: Option, pub query_workload: QueryWorkload, pub data_workload: DataWorkload, - pub implementation: BackendLocalImplementation, - pub environment: DeploymentEnvironment, + #[serde(rename = "implementation", alias = "physical_inputs")] + pub physical_inputs: BackendLocalPhysicalInputs, + pub environment: PhysicalDeploymentContext, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] -pub struct BackendLocalImplementation { +pub struct BackendLocalPhysicalInputs { /// Provider-priced concrete pane choices keyed by the registered PromQL text. #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub window_candidates: HashMap>, - pub lifecycle_costs: LifecycleCostEvidence, + #[serde( + rename = "window_candidates", + alias = "window_realization_candidates_by_query" + )] + pub window_realization_candidates_by_query: HashMap>, + pub lifecycle_costs: LifecycleUnitCosts, pub evidence_observed_at_unix_ms: u64, pub evidence_valid_for_ms: u64, pub horizon_seconds: f64, - pub window_implementation_id: String, - pub implementation_cost: ImplementationCostEvidence, + #[serde( + rename = "window_implementation_id", + alias = "default_window_realization_id" + )] + pub default_window_realization_id: String, + #[serde(rename = "implementation_cost", alias = "default_window_cost_quote")] + pub default_window_cost_quote: WindowRealizationCostQuote, #[serde(default, skip_serializing_if = "Option::is_none")] pub source_sample_interval_ms: Option, #[serde(default, skip_serializing_if = "u64_is_zero")] - pub query_staleness_margin_ms: u64, + #[serde( + rename = "query_staleness_margin_ms", + alias = "query_retention_margin_ms" + )] + pub query_retention_margin_ms: u64, /// Admission budget for all retained panes and estimated partitions. /// Missing legacy snapshots inherit the backend default. #[serde( @@ -241,7 +259,11 @@ pub struct BackendLocalImplementation { alias = "maxRetainedSummaryBytes", skip_serializing_if = "is_default_retained_summary_memory_budget_bytes" )] - pub max_retained_summary_bytes: u64, + #[serde( + rename = "max_retained_summary_bytes", + alias = "retained_summary_memory_budget_bytes" + )] + pub retained_summary_memory_budget_bytes: u64, /// Certificates keyed by exact registered PromQL; converted to root IDs /// before workload selection so one query cannot borrow another's evidence. #[serde(default, skip_serializing_if = "HashMap::is_empty")] @@ -332,7 +354,7 @@ pub fn gos_policy_from_accuracy_budget( }) } -pub fn compile_transmission_plan( +pub fn build_transmission_plan( envelope: PlanEnvelope, precompute: &PrecomputePlan, runtime_policies: &BTreeMap, @@ -399,7 +421,7 @@ pub fn compile_transmission_plan( /// Complete physical projection of one post-ASAP planning decision. /// All three child plans share the same envelope and are compiled together. #[derive(Debug, Clone)] -pub struct PhysicalPlan { +pub struct CompiledPhysicalPlan { pub envelope: PlanEnvelope, pub summary_catalog: super::summary_catalog::SummaryCatalog, pub collector_plans: Vec, @@ -408,15 +430,16 @@ pub struct PhysicalPlan { pub query_plan: QueryPlan, /// Lifecycle component only, not a complete physical-plan comparison. pub lifecycle_estimates: Vec, - pub cost_comparison: Option, - pub logical_selection: Vec, + pub cost_comparison: Option, + pub planner_selection_trace: Vec, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct MaterializationLifecycleEstimate { pub materialization: asap_types::sds::SummaryDefinitionId, pub consumer_query_ids: Vec, - pub window_implementation_id: String, + #[serde(rename = "window_implementation_id", alias = "window_realization_id")] + pub window_realization_id: String, pub horizon_seconds: f64, pub expected_reads: f64, pub expected_updates: f64, @@ -468,44 +491,52 @@ impl AccuracyEvidenceProvider for QueryEvidence<'_> { } #[derive(Debug, Default)] -pub struct PhysicalCompiler; +pub struct PhysicalPlanCompiler; -impl BackendLocalPlanningSnapshot { +impl BackendLocalPlanningInput { /// Invoke the pinned Planner from canonical startup workloads and compile - /// one backend-local PhysicalPlan. No CollectorPlan is produced and no + /// one backend-local CompiledPhysicalPlan. No CollectorPlan is produced and no /// precompiled serving artifact is accepted at this boundary. - pub fn compile(self) -> Result { - self.compile_frontend(false) + pub fn compile_promql(self) -> Result { + self.compile_frontend(QueryFrontend::PromQl) } /// Use the shared parser subset with MetricsQL serving and exact routing. - pub fn compile_metricsql(self) -> Result { - self.compile_frontend(true) + pub fn compile_metricsql(self) -> Result { + self.compile_frontend(QueryFrontend::MetricsQl) } - fn compile_frontend(self, metricsql: bool) -> Result { + fn compile_frontend( + self, + frontend: QueryFrontend, + ) -> Result { let evidence = self.workload_cost_evidence.clone().ok_or_else(|| { CompileError::Snapshot( "deployment requires complete workload cost evidence; export candidates and price them before compiling".into(), ) })?; - let (request, environment) = self.planning_request()?; - let candidates = super::workload_cost::with_exact_alternative(request)?; - if metricsql { - super::workload_cost::select_metricsql(candidates, environment, &evidence) + let (request, environment) = self.into_physical_compilation_request()?; + let candidates = + super::workload_cost::enumerate_exact_and_materialized_candidates(request)?; + if frontend == QueryFrontend::MetricsQl { + super::workload_cost::select_lowest_cost_metricsql_candidate( + candidates, + environment, + &evidence, + ) } else { - super::workload_cost::select(candidates, environment, &evidence) + super::workload_cost::select_lowest_cost_candidate(candidates, environment, &evidence) } } /// Build Planner-authorized candidates for evidence collection without publishing. - pub fn planning_request( + pub fn into_physical_compilation_request( self, - ) -> Result<(PlanningRequest, DeploymentEnvironment), CompileError> { - if self.snapshot_version != 2 { + ) -> Result<(PhysicalCompilationRequest, PhysicalDeploymentContext), CompileError> { + if self.schema_version != 2 { return Err(CompileError::Snapshot(format!( "unsupported workload snapshot version {}; only version 2 is supported", - self.snapshot_version + self.schema_version ))); } if self.environment.target != PhysicalDeploymentTarget::BackendLocalRemoteWrite { @@ -544,7 +575,11 @@ impl BackendLocalPlanningSnapshot { "QueryWorkload must contain at least one query".into(), )); } - for query in self.implementation.window_candidates.keys() { + for query in self + .physical_inputs + .window_realization_candidates_by_query + .keys() + { if !entries.iter().any(|entry| &entry.query.0 == query) { return Err(CompileError::Snapshot(format!( "window candidates reference unregistered query `{query}`" @@ -587,60 +622,60 @@ impl BackendLocalPlanningSnapshot { let source_hint = source_metrics.iter().next().cloned().ok_or_else(|| { CompileError::Snapshot(format!("query {index} has no named time-series source")) })?; - let lifecycle = LifecyclePlanningInput { + let lifecycle = SummaryLifecyclePlanningInputs { evaluation_interval_ms, ingestion_rate_per_second: ingestion_rate.0, - evidence_observed_at_unix_ms: self.implementation.evidence_observed_at_unix_ms, - evidence_valid_for_ms: self.implementation.evidence_valid_for_ms, - horizon_seconds: self.implementation.horizon_seconds, - costs: self.implementation.lifecycle_costs.clone(), + evidence_observed_at_unix_ms: self.physical_inputs.evidence_observed_at_unix_ms, + evidence_valid_for_ms: self.physical_inputs.evidence_valid_for_ms, + horizon_seconds: self.physical_inputs.horizon_seconds, + costs: self.physical_inputs.lifecycle_costs.clone(), }; let derived_lifecycle = lifecycle.clone(); let post_asap = crate::planner_selection::keep_pre_asap(&parsed) .map_err(|error| CompileError::Snapshot(format!("query {index}: {error}")))?; canonical_roots.push(Rc::new(parsed)); - let mut cost = self.implementation.implementation_cost.clone(); + let mut cost = self.physical_inputs.default_window_cost_quote.clone(); cost.workload_fingerprint = canonical_promql(&query_string).map_err(CompileError::QueryPlan)?; - cost.horizon_seconds = self.implementation.horizon_seconds; + cost.horizon_seconds = self.physical_inputs.horizon_seconds; let query_id = format!("compat-query-{index}"); - if let Some(evidence) = self.implementation.topk_evidence.get(&query_string) { + if let Some(evidence) = self.physical_inputs.topk_evidence.get(&query_string) { topk_evidence_by_id.insert(query_id.clone(), evidence.clone()); } - queries.push(PlanningQuery { + queries.push(QueryCompilationInput { query_id, query_string: query_string.clone(), - post_asap, - source: Source::TimeSeries { + selected_plan_root: post_asap, + legacy_query_source: Source::TimeSeries { metric: source_hint, }, - window_secs: lookback_ms / 1_000, - group_by: metadata.group_by_labels, - accuracy, - lifecycle, - window_implementations: self - .implementation - .window_candidates + query_lookback_seconds: lookback_ms / 1_000, + group_by_labels: metadata.group_by_labels, + accuracy_target: accuracy, + summary_lifecycle_inputs: lifecycle, + window_realization_candidates: self + .physical_inputs + .window_realization_candidates_by_query .get(&query_string) .cloned() .unwrap_or_else(|| { derived_window_candidates( - &self.implementation.window_implementation_id, + &self.physical_inputs.default_window_realization_id, canonical_roots.last().expect("root pushed above"), lookback_ms, evaluation_interval_ms, cost, &derived_lifecycle, - self.implementation.query_staleness_margin_ms, + self.physical_inputs.query_retention_margin_ms, ) }), - runtime_policy: RuntimeRulePolicy::default(), + materialization_runtime_policy: RuntimeRulePolicy::default(), }); } let mut exact_costs_by_id = HashMap::new(); for (index, entry) in workload.entries().enumerate() { if let Some(rows) = self - .implementation + .physical_inputs .exact_composition_costs .get(&entry.query.0) { @@ -658,34 +693,32 @@ impl BackendLocalPlanningSnapshot { exact_costs_by_id.insert(format!("compat-query-{index}"), rows.clone()); } } - let logical_selection = select_workload_roots_with_trace( + let planner_selection_trace = select_logical_roots_with_trace( &mut queries, canonical_roots, &topk_evidence_by_id, &exact_costs_by_id, - self.implementation.erp.as_ref(), + self.physical_inputs.erp.as_ref(), )?; // Derived maintenance currently consumes full, non-overlapping source cohorts. // Restrict only synthesized candidates; deployment-supplied evidence is authoritative. for query in &mut queries { if self - .implementation - .window_candidates + .physical_inputs + .window_realization_candidates_by_query .contains_key(&query.query_string) { continue; } - let states = - collect_selected_materializations(&query.post_asap, true).map_err(|reason| { - CompileError::Query { - query_id: query.query_id.clone(), - reason, - } + let states = collect_selected_materializations(&query.selected_plan_root, true) + .map_err(|reason| CompileError::Query { + query_id: query.query_id.clone(), + reason, })?; let mut full_windows = BTreeSet::new(); for state in &states { if let Some(sources) = immutable_materialization_sources(&state.node) { - full_windows.insert(state.window_secs.unwrap_or(query.window_secs)); + full_windows.insert(state.window_secs.unwrap_or(query.query_lookback_seconds)); for source in sources { let (_, window, _) = selected_input_contract(&source).map_err(|reason| { @@ -694,12 +727,12 @@ impl BackendLocalPlanningSnapshot { reason, } })?; - full_windows.insert(window.unwrap_or(query.window_secs)); + full_windows.insert(window.unwrap_or(query.query_lookback_seconds)); } } } let mut seen = BTreeSet::new(); - query.window_implementations.retain_mut(|candidate| { + query.window_realization_candidates.retain_mut(|candidate| { if !full_windows.contains(&candidate.window_secs) { return true; } @@ -711,47 +744,47 @@ impl BackendLocalPlanningSnapshot { candidate.layout = asap_types::WindowMaterializationLayout::Pane { pane_secs: candidate.window_secs, }; - candidate.implementation_id = format!( + candidate.realization_id = format!( "{}-{}s-derived-cohort", - self.implementation.window_implementation_id, candidate.window_secs + self.physical_inputs.default_window_realization_id, candidate.window_secs ); candidate.cost = derived_window_cost( &candidate.cost, - &query.lifecycle, + &query.summary_lifecycle_inputs, candidate.window_secs, candidate.slide_secs, &candidate.layout, - self.implementation.query_staleness_margin_ms, + self.physical_inputs.query_retention_margin_ms, ); true }); } // Composable lowering residualizes unsafe leaves individually; retain Planner siblings. Ok(( - PlanningRequest { - logical_selection, - hybrid_execution: true, - materialization_policy: None, + PhysicalCompilationRequest { + planner_selection_trace, + allow_mixed_summary_and_exact_execution: true, + enabled_materialization_keys: None, query_workload: Some(workload), - synthesized_window_queries: queries + compiler_priced_window_query_ids: queries .iter() .filter(|q| { !self - .implementation - .window_candidates + .physical_inputs + .window_realization_candidates_by_query .contains_key(&q.query_string) }) .map(|q| q.query_id.clone()) .collect(), queries, - evidence: topk_evidence_by_id, + topk_membership_evidence_by_query_id: topk_evidence_by_id, exact_composition_costs: exact_costs_by_id, - erp: self.implementation.erp, + erp: self.physical_inputs.erp, planner_revision: PLANNER_REVISION.into(), - source_sample_interval_ms: self.implementation.source_sample_interval_ms, - query_staleness_margin_ms: self.implementation.query_staleness_margin_ms, + source_sample_interval_ms: self.physical_inputs.source_sample_interval_ms, + query_retention_margin_ms: self.physical_inputs.query_retention_margin_ms, retained_summary_memory_budget_bytes: Some( - self.implementation.max_retained_summary_bytes, + self.physical_inputs.retained_summary_memory_budget_bytes, ), }, self.environment, @@ -825,14 +858,14 @@ fn has_unsafe_raw_entity_leaf( } /// Preserve native execution for raw states that cannot preserve source semantics. -fn preserve_native_unsafe_raw_roots(queries: &mut [PlanningQuery]) -> Result<(), CompileError> { +fn preserve_native_unsafe_raw_roots( + queries: &mut [QueryCompilationInput], +) -> Result<(), CompileError> { for query in queries { - let selected = - collect_selected_materializations(&query.post_asap, false).map_err(|reason| { - CompileError::Query { - query_id: query.query_id.clone(), - reason, - } + let selected = collect_selected_materializations(&query.selected_plan_root, false) + .map_err(|reason| CompileError::Query { + query_id: query.query_id.clone(), + reason, })?; if selected.is_empty() { continue; @@ -841,17 +874,18 @@ fn preserve_native_unsafe_raw_roots(queries: &mut [PlanningQuery]) -> Result<(), .iter() .map(|state| Rc::clone(&state.node)) .collect::>(); - let unsafe_entities = has_unsafe_raw_entity_leaf(&query.post_asap, &selected_nodes, false); + let unsafe_entities = + has_unsafe_raw_entity_leaf(&query.selected_plan_root, &selected_nodes, false); if unsafe_entities { let parsed = crate::query_parser::parse_query_expr_canonical( &query.query_string, - query.accuracy.clone(), + query.accuracy_target.clone(), ) .map_err(|error| CompileError::Query { query_id: query.query_id.clone(), reason: error.to_string(), })?; - query.post_asap = + query.selected_plan_root = crate::planner_selection::keep_pre_asap(&parsed).map_err(|error| { CompileError::Query { query_id: query.query_id.clone(), @@ -868,29 +902,29 @@ fn preserve_native_unsafe_raw_roots(queries: &mut [PlanningQuery]) -> Result<(), /// intentionally unsupported (and therefore invalid as an executable /// maintenance DAG), but the original query remains a valid exact plan. fn preserve_invalid_exact_fallback_roots( - queries: &mut [PlanningQuery], + queries: &mut [QueryCompilationInput], composable: bool, ) -> Result<(), CompileError> { for query in queries { - let selected = - collect_selected_materializations(&query.post_asap, composable).map_err(|reason| { - CompileError::Query { - query_id: query.query_id.clone(), - reason, - } + let selected = collect_selected_materializations(&query.selected_plan_root, composable) + .map_err(|reason| CompileError::Query { + query_id: query.query_id.clone(), + reason, })?; let invalid_executable = - selected.is_empty() && validate_executable_subdag(&query.post_asap).is_err(); - if invalid_executable && !matches!(query.post_asap.expr, SummaryExpr::KeepPreAsap(_)) { + selected.is_empty() && validate_executable_subdag(&query.selected_plan_root).is_err(); + if invalid_executable + && !matches!(query.selected_plan_root.expr, SummaryExpr::KeepPreAsap(_)) + { let parsed = crate::query_parser::parse_query_expr_canonical( &query.query_string, - query.accuracy.clone(), + query.accuracy_target.clone(), ) .map_err(|error| CompileError::Query { query_id: query.query_id.clone(), reason: error.to_string(), })?; - query.post_asap = + query.selected_plan_root = crate::planner_selection::keep_pre_asap(&parsed).map_err(|error| { CompileError::Query { query_id: query.query_id.clone(), @@ -907,16 +941,14 @@ fn preserve_invalid_exact_fallback_roots( /// query as one native exact root. Mixed queries retain their other selected /// summaries and let residual lowering cut only the counter branches. fn preserve_metricsql_counter_only_roots( - queries: &mut [PlanningQuery], + queries: &mut [QueryCompilationInput], composable: bool, ) -> Result<(), CompileError> { for query in queries { - let selected = - collect_selected_materializations(&query.post_asap, composable).map_err(|reason| { - CompileError::Query { - query_id: query.query_id.clone(), - reason, - } + let selected = collect_selected_materializations(&query.selected_plan_root, composable) + .map_err(|reason| CompileError::Query { + query_id: query.query_id.clone(), + reason, })?; if selected.is_empty() || !selected.iter().all(|state| { @@ -934,46 +966,47 @@ fn preserve_metricsql_counter_only_roots( } let parsed = crate::query_parser::parse_query_expr_canonical( &query.query_string, - query.accuracy.clone(), + query.accuracy_target.clone(), ) .map_err(|error| CompileError::Query { query_id: query.query_id.clone(), reason: error.to_string(), })?; - query.post_asap = crate::planner_selection::keep_pre_asap(&parsed).map_err(|error| { - CompileError::Query { - query_id: query.query_id.clone(), - reason: error.to_string(), - } - })?; + query.selected_plan_root = + crate::planner_selection::keep_pre_asap(&parsed).map_err(|error| { + CompileError::Query { + query_id: query.query_id.clone(), + reason: error.to_string(), + } + })?; } Ok(()) } -impl PhysicalCompiler { - pub fn compile( +impl PhysicalPlanCompiler { + pub fn compile_promql( &self, - request: PlanningRequest, - environment: DeploymentEnvironment, - ) -> Result { - self.compile_language(request, environment, false) + request: PhysicalCompilationRequest, + environment: PhysicalDeploymentContext, + ) -> Result { + self.compile_for_frontend(request, environment, QueryFrontend::PromQl) } pub fn compile_metricsql( &self, - request: PlanningRequest, - environment: DeploymentEnvironment, - ) -> Result { - self.compile_language(request, environment, true) + request: PhysicalCompilationRequest, + environment: PhysicalDeploymentContext, + ) -> Result { + self.compile_for_frontend(request, environment, QueryFrontend::MetricsQl) } - fn compile_language( + pub fn compile_for_frontend( &self, - mut request: PlanningRequest, - environment: DeploymentEnvironment, - metricsql: bool, - ) -> Result { - if request.hybrid_execution + mut request: PhysicalCompilationRequest, + environment: PhysicalDeploymentContext, + frontend: QueryFrontend, + ) -> Result { + if request.allow_mixed_summary_and_exact_execution && environment.target != PhysicalDeploymentTarget::BackendLocalRemoteWrite { return Err(CompileError::Snapshot( @@ -987,7 +1020,7 @@ impl PhysicalCompiler { }); } if environment.target == PhysicalDeploymentTarget::BackendLocalRemoteWrite - && !environment.collector_ids.is_empty() + && !environment.target_collector_ids.is_empty() { return Err(CompileError::Query { query_id: "deployment-target".into(), @@ -1017,13 +1050,19 @@ impl PhysicalCompiler { } } - if metricsql { - preserve_metricsql_counter_only_roots(&mut request.queries, request.hybrid_execution)?; + if frontend == QueryFrontend::MetricsQl { + preserve_metricsql_counter_only_roots( + &mut request.queries, + request.allow_mixed_summary_and_exact_execution, + )?; } - preserve_invalid_exact_fallback_roots(&mut request.queries, request.hybrid_execution)?; + preserve_invalid_exact_fallback_roots( + &mut request.queries, + request.allow_mixed_summary_and_exact_execution, + )?; if environment.target == PhysicalDeploymentTarget::BackendLocalRemoteWrite - && !request.hybrid_execution + && !request.allow_mixed_summary_and_exact_execution { preserve_native_unsafe_raw_roots(&mut request.queries)?; } @@ -1032,10 +1071,10 @@ impl PhysicalCompiler { .queries .iter() .enumerate() - .map(|(id, query)| (id, Rc::clone(&query.post_asap))) + .map(|(id, query)| (id, Rc::clone(&query.selected_plan_root))) .collect(); for (id, root) in planner_types::post_asap::share_common_summary_subtrees(roots) { - request.queries[id].post_asap = root; + request.queries[id].selected_plan_root = root; } let mut compiled_materializations = Vec::with_capacity(request.queries.len()); let mut collector_materializations = Vec::with_capacity(request.queries.len()); @@ -1054,22 +1093,27 @@ impl PhysicalCompiler { let consumers = materialization_consumers( &request.queries, environment.target, - request.hybrid_execution, + request.allow_mixed_summary_and_exact_execution, )?; let mut lifecycle_estimates = BTreeMap::::new(); for (query_index, query) in request.queries.iter().enumerate() { - let evidence = request.evidence.get(&query.query_id); + let evidence = request + .topk_membership_evidence_by_query_id + .get(&query.query_id); if let Some(e) = evidence { validate_evidence(&query.query_id, e, &environment)?; } - let node = query.post_asap.clone(); - let selected = collect_selected_materializations(&node, request.hybrid_execution) - .map_err(|reason| CompileError::Query { - query_id: query.query_id.clone(), - reason, - })?; + let node = query.selected_plan_root.clone(); + let selected = collect_selected_materializations( + &node, + request.allow_mixed_summary_and_exact_execution, + ) + .map_err(|reason| CompileError::Query { + query_id: query.query_id.clone(), + reason, + })?; let selected = selected .into_iter() .filter(|state| { @@ -1077,14 +1121,14 @@ impl PhysicalCompiler { // MetricsQL includes boundary samples differently, so retain // these leaves as external exact dependencies until its // counter semantics have a dedicated implementation. - !(metricsql && matches!(state.family, + !(frontend == QueryFrontend::MetricsQl && matches!(state.family, SummaryFamilyType::ExactAggregate( planner_types::post_asap::ExactKind::Rate | planner_types::post_asap::ExactKind::Increase, _))) - && (!request.hybrid_execution + && (!request.allow_mixed_summary_and_exact_execution || state.window_secs.is_none_or(|window| { query - .window_implementations + .window_realization_candidates .iter() .any(|candidate| candidate.window_secs == window) })) @@ -1094,22 +1138,22 @@ impl PhysicalCompiler { planner_types::post_asap::ExactKind::MinMax, _ ) - ) || crate::query_plan::logical::selected_range_max_materialization( + ) || crate::query_plan::residual::selected_range_max_materialization( &query.query_string, &state.node, ) .ok() .flatten() .is_some()) - && request.materialization_policy.as_ref().is_none_or(|policy| { - let key = crate::query_plan::logical::selected_counter_materialization( + && request.enabled_materialization_keys.as_ref().is_none_or(|policy| { + let key = crate::query_plan::residual::selected_counter_materialization( &query.query_string, &state.node, ) .ok() .flatten() .or_else(|| { - crate::query_plan::logical::selected_range_max_materialization( + crate::query_plan::residual::selected_range_max_materialization( &query.query_string, &state.node, ) @@ -1127,14 +1171,15 @@ impl PhysicalCompiler { if selected.is_empty() { continue; } - let executable = - planner_types::post_asap::compile_executable_dag_with_node_ids(&query.post_asap) - .map_err(|error| CompileError::Query { - query_id: query.query_id.clone(), - reason: format!("invalid executable subDAG: {error}"), - })?; + let executable = planner_types::post_asap::compile_executable_dag_with_node_ids( + &query.selected_plan_root, + ) + .map_err(|error| CompileError::Query { + query_id: query.query_id.clone(), + reason: format!("invalid executable subDAG: {error}"), + })?; executable_dags[query_index] = Some(executable); - validate_lifecycle_input(&query.query_id, &query.lifecycle)?; + validate_lifecycle_input(&query.query_id, &query.summary_lifecycle_inputs)?; if environment.target == PhysicalDeploymentTarget::DistributedCollectors && selected.iter().any(|state| { matches!( @@ -1152,7 +1197,7 @@ impl PhysicalCompiler { .into(), }); } - match &query.source { + match &query.legacy_query_source { Source::TimeSeries { .. } => {} Source::Table { .. } => { return Err(CompileError::Query { @@ -1181,30 +1226,35 @@ impl PhysicalCompiler { } for (ordinal, selected) in selected.into_iter().enumerate() { let mut branch_query = query.clone(); - branch_query.window_secs = selected.window_secs.unwrap_or(query.window_secs); - branch_query.group_by = selected + branch_query.query_lookback_seconds = + selected.window_secs.unwrap_or(query.query_lookback_seconds); + branch_query.group_by_labels = selected .group_by .clone() - .unwrap_or_else(|| query.group_by.clone()); + .unwrap_or_else(|| query.group_by_labels.clone()); branch_query - .window_implementations - .retain(|candidate| candidate.window_secs == branch_query.window_secs); + .window_realization_candidates + .retain(|candidate| { + candidate.window_secs == branch_query.query_lookback_seconds + }); let query = &branch_query; let lifecycle_costs = SummaryMaintenanceLifecycleCostInputs { - build_cost: Some(Cost(query.lifecycle.costs.build)), + build_cost: Some(Cost(query.summary_lifecycle_inputs.costs.build)), maintenance_cost_per_update: Some(Cost( - query.lifecycle.costs.maintenance_per_update, + query.summary_lifecycle_inputs.costs.maintenance_per_update, + )), + summary_read_cost: Some(Cost(query.summary_lifecycle_inputs.costs.read)), + retention_cost_rate: Some(CostRate( + query.summary_lifecycle_inputs.costs.retention_per_second, )), - summary_read_cost: Some(Cost(query.lifecycle.costs.read)), - retention_cost_rate: Some(CostRate(query.lifecycle.costs.retention_per_second)), - retirement_cost: Some(Cost(query.lifecycle.costs.retirement)), + retirement_cost: Some(Cost(query.summary_lifecycle_inputs.costs.retirement)), }; let window_costs = super::realization::RealizationProvider::windows( &super::realization::ExistingRealizations, query, &environment, )?; - let model = ControlPlaneCostModel::new(query.accuracy.clone()) + let model = ControlPlaneCostModel::new(query.accuracy_target.clone()) .with_summary_maintenance( lifecycle_costs, SummaryMaintenanceCapabilities { @@ -1253,8 +1303,8 @@ impl PhysicalCompiler { ) }), )?; - let window_implementation = query.window_implementations.iter() - .find(|candidate| candidate.implementation_id == planner_selection.window_implementation_id + let window_implementation = query.window_realization_candidates.iter() + .find(|candidate| candidate.realization_id == planner_selection.window_realization_id && candidate.framework == planner_selection.window_framework) .ok_or_else(|| CompileError::Lifecycle { query_id: query.query_id.clone(), @@ -1399,8 +1449,8 @@ impl PhysicalCompiler { .or_insert_with(|| MaterializationLifecycleEstimate { materialization: materialization.into(), consumer_query_ids, - window_implementation_id: window_implementation.implementation_id.clone(), - horizon_seconds: query.lifecycle.horizon_seconds, + window_realization_id: window_implementation.realization_id.clone(), + horizon_seconds: query.summary_lifecycle_inputs.horizon_seconds, expected_reads: planner_selection.expected_reads, expected_updates: planner_selection.expected_updates, lifecycle_cost: planner_selection.lifecycle_cost, @@ -1425,10 +1475,11 @@ impl PhysicalCompiler { }); } } - if let Some(existing) = - runtime_policies.insert(materialization, query.runtime_policy.clone()) - { - if existing != query.runtime_policy { + if let Some(existing) = runtime_policies.insert( + materialization, + query.materialization_runtime_policy.clone(), + ) { + if existing != query.materialization_runtime_policy { return Err(CompileError::Query { query_id: query.query_id.clone(), reason: "queries sharing one materialization specify different runtime policies" @@ -1447,7 +1498,7 @@ impl PhysicalCompiler { group_by: physical_group_by, window_secs: runtime_materialization.window_size, abstract_window_framework: planner_selection.window_framework.clone(), - window_implementation_id: window_implementation.implementation_id.clone(), + window_realization_id: window_implementation.realization_id.clone(), slide_secs: runtime_materialization.slide_interval, pane_origin_ms: runtime_materialization.pane_origin_ms, window_layout: window_implementation.layout.clone(), @@ -1493,14 +1544,14 @@ impl PhysicalCompiler { ); } - let plan_id = if request.hybrid_execution { + let plan_id = if request.allow_mixed_summary_and_exact_execution { use std::hash::{Hash, Hasher}; let mut hash = std::collections::hash_map::DefaultHasher::new(); stable_workload_plan_id(&plan_materializations, &request.queries).hash(&mut hash); "typed-local-residual-v3-counter-index".hash(&mut hash); - request.materialization_policy.hash(&mut hash); + request.enabled_materialization_keys.hash(&mut hash); for query in &request.queries { - format!("{:?}", query.post_asap).hash(&mut hash); + format!("{:?}", query.selected_plan_root).hash(&mut hash); } hash.finish() } else { @@ -1517,7 +1568,9 @@ impl PhysicalCompiler { capability_snapshot_id: environment.capability_snapshot_id, }; let producer_ids = match environment.target { - PhysicalDeploymentTarget::DistributedCollectors => environment.collector_ids.clone(), + PhysicalDeploymentTarget::DistributedCollectors => { + environment.target_collector_ids.clone() + } PhysicalDeploymentTarget::BackendLocalRemoteWrite => Vec::new(), }; // Several queries/readouts may intentionally share one maintained @@ -1601,7 +1654,7 @@ impl PhysicalCompiler { let window_ms = materialization.window_size.saturating_mul(1_000); if materialization_family != physical_materialization_family(node_family) || window_ms == 0 - || source_window.unwrap_or(query.window_secs).saturating_mul(1_000) + || source_window.unwrap_or(query.query_lookback_seconds).saturating_mul(1_000) % window_ms != 0 { return Err(crate::query_plan::QueryPlanError::Invalid(format!( @@ -1621,25 +1674,25 @@ impl PhysicalCompiler { }) }; let instant = InstantExecution { - lookback_ms: query.window_secs.saturating_mul(1_000), + lookback_ms: query.query_lookback_seconds.saturating_mul(1_000), full_history: false, cumulative_readout: true, }; // A whole-query native fallback need not be expressible in the local // residual algebra (for example an ERP-rejected entropy readout). // Retain its native boundary without discarding other workload roots. - let native_root = request.hybrid_execution - && if let SummaryExpr::KeepPreAsap(expr) = &query.post_asap.expr { + let native_root = request.allow_mixed_summary_and_exact_execution + && if let SummaryExpr::KeepPreAsap(expr) = &query.selected_plan_root.expr { let original = crate::query_parser::parse_query_expr_canonical( &query.query_string, - query.accuracy.clone(), + query.accuracy_target.clone(), ) .map_err(|error| CompileError::Query { query_id: query.query_id.clone(), reason: error.to_string(), })?; expr.as_ref() == &original - && crate::query_plan::logical::compile_logical( + && crate::query_plan::residual::compile_logical( query.query_id.clone(), canonical.clone(), instant, @@ -1649,11 +1702,11 @@ impl PhysicalCompiler { } else { false }; - let mut entry = if request.hybrid_execution && !native_root { + let mut entry = if request.allow_mixed_summary_and_exact_execution && !native_root { crate::query_plan::compile_bound_composable_mapped( query.query_id.clone(), canonical.clone(), - &query.post_asap, + &query.selected_plan_root, instant, FallbackPolicy::ExactBackend, binding, @@ -1670,7 +1723,7 @@ impl PhysicalCompiler { crate::query_plan::compile_bound_mapped( query.query_id.clone(), canonical.clone(), - &query.post_asap, + &query.selected_plan_root, instant, FallbackPolicy::ExactBackend, binding, @@ -1684,13 +1737,13 @@ impl PhysicalCompiler { }, ) }?; - if request.hybrid_execution { + if request.allow_mixed_summary_and_exact_execution { // Any Planner-selected leaf without a physical summary binding // is an exact subtree boundary. Deployed plans never retain a // backend-local range index leaf. - crate::query_plan::logical::finalize_residuals(&mut entry)?; + crate::query_plan::residual::finalize_residuals(&mut entry)?; } - if metricsql { + if frontend == QueryFrontend::MetricsQl { entry.language = crate::query_plan::QueryLanguage::MetricsQl; } let catalog_key = QueryPlan::catalog_key(entry.language, &canonical); @@ -1747,7 +1800,7 @@ impl PhysicalCompiler { if let Some(lookback_ms) = max_lookback_ms { materialization.num_aggregates_to_retain = Some(retained_state_count( lookback_ms, - request.query_staleness_margin_ms, + request.query_retention_margin_ms, materialization.slide_interval.saturating_mul(1_000), &materialization.window_layout, )); @@ -1807,7 +1860,7 @@ impl PhysicalCompiler { query_id: "precompute-plan".into(), reason: error.to_string(), })?; - let mut transmission_plan = crate::physical::compiler::compile_transmission_plan( + let mut transmission_plan = crate::physical::compiler::build_transmission_plan( envelope.clone(), &precompute_plan, &runtime_policies, @@ -1865,7 +1918,7 @@ impl PhysicalCompiler { })?; } query_plan.validate_against_catalog(&summary_catalog)?; - Ok(PhysicalPlan { + Ok(CompiledPhysicalPlan { envelope, summary_catalog, collector_plans, @@ -1874,7 +1927,7 @@ impl PhysicalCompiler { query_plan, lifecycle_estimates: lifecycle_estimates.into_values().collect(), cost_comparison: None, - logical_selection: request.logical_selection, + planner_selection_trace: request.planner_selection_trace, }) } } @@ -1930,13 +1983,13 @@ fn summary_agg_metric(node: &SummaryNode) -> Option { /// Shared selection boundary for canonical startup and compile-and-publish. /// Certificate-bearing roots stay isolated: equal certificate values do not /// establish that the certificate's source scope covers another query. -pub fn select_workload_roots( - queries: &mut [PlanningQuery], +pub fn select_logical_roots_for_queries( + queries: &mut [QueryCompilationInput], roots: Vec>, evidence: &HashMap, exact_costs: &HashMap>, ) -> Result<(), CompileError> { - select_workload_roots_with_erp(queries, roots, evidence, exact_costs, None) + select_logical_roots_with_error_resource_profiles(queries, roots, evidence, exact_costs, None) } fn observed_population_matches_root( @@ -1986,18 +2039,18 @@ fn observed_population_matches_root( == i64::try_from(window.saturating_mul(1000)).ok() } -pub fn select_workload_roots_with_erp( - queries: &mut [PlanningQuery], +pub fn select_logical_roots_with_error_resource_profiles( + queries: &mut [QueryCompilationInput], roots: Vec>, evidence: &HashMap, exact_costs: &HashMap>, erp: Option<&super::erp::ErpPlanningInput>, ) -> Result<(), CompileError> { - select_workload_roots_with_trace(queries, roots, evidence, exact_costs, erp).map(|_| ()) + select_logical_roots_with_trace(queries, roots, evidence, exact_costs, erp).map(|_| ()) } -pub fn select_workload_roots_with_trace( - queries: &mut [PlanningQuery], +pub fn select_logical_roots_with_trace( + queries: &mut [QueryCompilationInput], roots: Vec>, evidence: &HashMap, exact_costs: &HashMap>, @@ -2013,7 +2066,7 @@ pub fn select_workload_roots_with_trace( Vec::new(); let original_roots = roots.clone(); for (index, root) in roots.into_iter().enumerate() { - let accuracy = &queries[index].accuracy; + let accuracy = &queries[index].accuracy_target; let certificate_scope = (evidence.contains_key(&queries[index].query_id) || exact_costs.contains_key(&queries[index].query_id)) .then(|| queries[index].query_id.clone()); @@ -2088,7 +2141,7 @@ pub fn select_workload_roots_with_trace( let selected_indices = selected.iter().map(|(index, _)| *index).collect::>(); for (index, node) in selected { if erp.is_some_and(|policy| { - requires_exact_erp_fallback(&node, &queries[index].accuracy, policy) + requires_exact_erp_fallback(&node, &queries[index].accuracy_target, policy) }) { if let Some(values) = trace["deployment_overrides"].as_array_mut() { values.push( @@ -2096,16 +2149,16 @@ pub fn select_workload_roots_with_trace( "reason": "ERP requires exact fallback"}), ); } - queries[index].post_asap = + queries[index].selected_plan_root = crate::planner_selection::keep_pre_asap(&original_roots[index]) .map_err(|error| CompileError::Snapshot(error.to_string()))?; } else { - queries[index].post_asap = node; + queries[index].selected_plan_root = node; } } trace["committed_roots"] = serde_json::json!(selected_indices.into_iter().map(|index| serde_json::json!({"query_index": index, - "logical_root_id": crate::planner_selection::explained_root_id(&queries[index].post_asap, &queries[index].accuracy) + "logical_root_id": crate::planner_selection::explained_root_id(&queries[index].selected_plan_root, &queries[index].accuracy_target) })).collect::>()); traces.push(trace); } @@ -2202,11 +2255,11 @@ fn requires_exact_erp_fallback( /// Planner-adapter selection step used before physical compilation. Keeping /// this separate makes the ownership boundary explicit: callers supply the -/// selected post-ASAP DAG to [`PhysicalCompiler::compile`]. +/// selected post-ASAP DAG to [`PhysicalPlanCompiler::compile`]. pub fn select_post_asap( expr: &QueryExpr, accuracy: AccuracyTarget, - lifecycle: &LifecyclePlanningInput, + lifecycle: &SummaryLifecyclePlanningInputs, evidence: Option<&TopKMembershipEvidence>, ) -> Result, crate::planner_selection::SelectionError> { let model = ControlPlaneCostModel::new(accuracy).with_summary_maintenance( @@ -2235,7 +2288,7 @@ pub fn select_post_asap( fn validate_evidence( query_id: &str, evidence: &TopKMembershipEvidence, - env: &DeploymentEnvironment, + env: &PhysicalDeploymentContext, ) -> Result<(), CompileError> { let age = env .observed_at_unix_ms @@ -2259,7 +2312,7 @@ fn validate_evidence( fn validate_lifecycle_input( query_id: &str, - input: &LifecyclePlanningInput, + input: &SummaryLifecyclePlanningInputs, ) -> Result<(), CompileError> { let costs = [ input.costs.build, @@ -2286,12 +2339,12 @@ fn validate_lifecycle_input( /// Price one derived layout from the snapshot's own lifecycle unit costs. /// -/// `ImplementationCostEvidence` is normally measured evidence, and its +/// `WindowRealizationCostQuote` is normally measured evidence, and its /// `weighted_cost` doc puts pricing update CPU, query-time merges, retained /// memory, storage, scans and network on the evidence producer. When a /// snapshot prices no candidate, the control plane becomes that producer for /// the derived shapes — and it does so without inventing a single magnitude. -/// Every unit cost below is supplied evidence (`LifecycleCostEvidence`, from +/// Every unit cost below is supplied evidence (`LifecycleUnitCosts`, from /// `implementation.lifecycle_costs`); every multiplier is a structural count /// that follows from the layout's definition. Nothing here is a measurement. /// @@ -2320,13 +2373,13 @@ fn validate_lifecycle_input( /// by `validate_window_implementations` and by the `min_by` that ranks /// candidates. `model_version` records that the quote is derived. pub(super) fn derived_window_cost( - template: &ImplementationCostEvidence, - lifecycle: &LifecyclePlanningInput, + template: &WindowRealizationCostQuote, + lifecycle: &SummaryLifecyclePlanningInputs, window_secs: u64, slide_secs: u64, layout: &asap_types::WindowMaterializationLayout, staleness_margin_ms: u64, -) -> ImplementationCostEvidence { +) -> WindowRealizationCostQuote { let costs = &lifecycle.costs; let horizon = lifecycle.horizon_seconds.max(0.0); let window = window_secs.max(1) as f64; @@ -2368,7 +2421,7 @@ pub(super) fn derived_window_cost( let cpu_cost = build + maintenance; let weighted_cost = cpu_cost + read + retention + retirement; - ImplementationCostEvidence { + WindowRealizationCostQuote { model_version: format!( "{}+derived-window-layout-v1", template @@ -2437,15 +2490,15 @@ fn derived_window_candidates( expr: &QueryExpr, lookback_ms: u64, evaluation_interval_ms: u32, - cost: ImplementationCostEvidence, - lifecycle: &LifecyclePlanningInput, + cost: WindowRealizationCostQuote, + lifecycle: &SummaryLifecyclePlanningInputs, staleness_margin_ms: u64, -) -> Vec { +) -> Vec { let mut windows = range_selector_windows_secs(expr); if windows.is_empty() { windows.insert(lookback_ms / 1_000); } - // `window_implementation_id` reaches lifecycle estimates and cost + // `window_realization_id` reaches lifecycle estimates and cost // manifests, so one label must not describe several shapes. A query with a // single window keeps the snapshot's identity untouched. let distinct = windows.len() > 1; @@ -2493,8 +2546,8 @@ fn derived_window_candidates( }; layouts .into_iter() - .map(|(id, layout)| WindowImplementationCandidate { - implementation_id: id, + .map(|(id, layout)| WindowRealizationCandidate { + realization_id: id, framework: if advances_within_window { SummaryWindowFramework::Sliding } else { @@ -2518,30 +2571,31 @@ fn derived_window_candidates( } pub(super) fn validate_window_implementations( - query: &PlanningQuery, - environment: &DeploymentEnvironment, + query: &QueryCompilationInput, + environment: &PhysicalDeploymentContext, ) -> Result, CompileError> { let mut ids = BTreeSet::new(); let mut candidates = Vec::new(); - for candidate in &query.window_implementations { + for candidate in &query.window_realization_candidates { let evidence = &candidate.cost; let age = environment .observed_at_unix_ms .saturating_sub(evidence.observed_at_unix_ms); let valid = evidence.observed_at_unix_ms <= environment.observed_at_unix_ms - && !candidate.implementation_id.trim().is_empty() - && ids.insert(candidate.implementation_id.clone()) + && !candidate.realization_id.trim().is_empty() + && ids.insert(candidate.realization_id.clone()) && !evidence.model_version.trim().is_empty() && !evidence.workload_fingerprint.trim().is_empty() && evidence.valid_for_ms != 0 && age <= environment.max_evidence_age_ms.min(evidence.valid_for_ms) && evidence.horizon_seconds.is_finite() - && (evidence.horizon_seconds - query.lifecycle.horizon_seconds).abs() <= f64::EPSILON + && (evidence.horizon_seconds - query.summary_lifecycle_inputs.horizon_seconds).abs() + <= f64::EPSILON && evidence.cpu_cost.is_finite() && evidence.cpu_cost >= 0.0 && evidence.weighted_cost.is_finite() && evidence.weighted_cost >= 0.0 - && candidate.window_secs == query.window_secs + && candidate.window_secs == query.query_lookback_seconds && candidate .layout .validate(candidate.window_secs, candidate.slide_secs) @@ -2575,12 +2629,12 @@ pub(super) fn validate_window_implementations( query_id: query.query_id.clone(), reason: format!( "window implementation `{}` has incomplete, stale, incompatible, or duplicate physical evidence", - candidate.implementation_id + candidate.realization_id ), }); } candidates.push(( - candidate.implementation_id.clone(), + candidate.realization_id.clone(), candidate.framework.clone(), Cost(evidence.weighted_cost), )); @@ -2716,7 +2770,7 @@ fn validate_retained_summary_footprint( } struct PlannerPhysicalSelection { - window_implementation_id: String, + window_realization_id: String, lifecycle: CollectorLifecycle, window_framework: SummaryWindowFramework, expected_reads: f64, @@ -2725,20 +2779,20 @@ struct PlannerPhysicalSelection { } fn select_lifecycle( - query: &PlanningQuery, + query: &QueryCompilationInput, node: &SummaryNode, model: &ControlPlaneCostModel, - environment: &DeploymentEnvironment, - consumers: &[&PlanningQuery], + environment: &PhysicalDeploymentContext, + consumers: &[&QueryCompilationInput], original_workload: Option<(&QueryWorkload, Vec)>, ) -> Result { // Current lifecycle evidence is per producer with one unit read cost. // Conflicting source/rate/horizon/cost snapshots cannot be averaged into // invented evidence. Only recurrence may differ between consumers. - let mut common = query.lifecycle.clone(); + let mut common = query.summary_lifecycle_inputs.clone(); common.evaluation_interval_ms = 0; for consumer in consumers { - let mut input = consumer.lifecycle.clone(); + let mut input = consumer.summary_lifecycle_inputs.clone(); input.evaluation_interval_ms = 0; if input != common { return Err(CompileError::Lifecycle { @@ -2756,10 +2810,10 @@ fn select_lifecycle( .map(|query| RepeatingEntry { query: Query(query.query_id.clone()), demand: RepeatedDemand::FixedInterval(RepetitionInterval( - query.lifecycle.evaluation_interval_ms, + query.summary_lifecycle_inputs.evaluation_interval_ms, )), requirements: QueryRequirements { - accuracy: AccuracyRequirement::Explicit(query.accuracy.clone()), + accuracy: AccuracyRequirement::Explicit(query.accuracy_target.clone()), ..QueryRequirements::default() }, predictability: Predictability::Predictable { known_at: None }, @@ -2768,10 +2822,10 @@ fn select_lifecycle( // claim deletion support for moving-window retractions. scope: QueryTimeScope::Unknown, lookback: Some(DurationMs( - materialization_leaf_contract(node) + raw_materialization_input_contract(node) .ok() .and_then(|(_, window, _)| window) - .unwrap_or(query.window_secs) + .unwrap_or(query.query_lookback_seconds) .saturating_mul(1_000), )), as_of: None, @@ -2782,10 +2836,12 @@ fn select_lifecycle( data_workload: Some(DataWorkload { arrival: DataArrival::ContinuouslyIngesting, ingestion_rate: Evidence { - value: Some(Rate(query.lifecycle.ingestion_rate_per_second)), + value: Some(Rate( + query.summary_lifecycle_inputs.ingestion_rate_per_second, + )), source: EvidenceSource::Observed, - observed_at_ms: Some(query.lifecycle.evidence_observed_at_unix_ms), - valid_for_ms: Some(query.lifecycle.evidence_valid_for_ms), + observed_at_ms: Some(query.summary_lifecycle_inputs.evidence_observed_at_unix_ms), + valid_for_ms: Some(query.summary_lifecycle_inputs.evidence_valid_for_ms), }, ..DataWorkload::default() }), @@ -2796,7 +2852,7 @@ fn select_lifecycle( Rc::new(node.clone()), WorkloadDemand::new(workload, &indices), environment.observed_at_unix_ms, - Some(Horizon(query.lifecycle.horizon_seconds)), + Some(Horizon(query.summary_lifecycle_inputs.horizon_seconds)), SummaryMaintenanceLifecycleCapabilities { supports_ephemeral: false, supports_prepared: false, @@ -2826,12 +2882,13 @@ fn select_lifecycle( reason: "latest ASAPPlanner selected no window framework from the supplied physical evidence".into(), })?; Ok(PlannerPhysicalSelection { - window_implementation_id: plan.selected_window_implementation_id.clone().ok_or_else( - || CompileError::Lifecycle { + window_realization_id: plan + .selected_window_implementation_id + .clone() + .ok_or_else(|| CompileError::Lifecycle { query_id: query.query_id.clone(), reason: "Planner returned no concrete window implementation identity".into(), - }, - )?, + })?, expected_reads: plan.expected_reads.ok_or_else(|| CompileError::Lifecycle { query_id: query.query_id.clone(), reason: "missing joint read demand".into(), @@ -2843,7 +2900,7 @@ fn select_lifecycle( reason: "missing source update demand".into(), })? .0 - * query.lifecycle.horizon_seconds, + * query.summary_lifecycle_inputs.horizon_seconds, lifecycle_cost: plan.deployments[0] .alternatives .iter() @@ -2890,7 +2947,7 @@ fn select_lifecycle( /// A warm producer may consume only a source whose semantics its precompute accumulator /// implements. Predicates and shifted ranges remain executable residual nodes. -pub(crate) fn materialization_leaf_contract( +pub(crate) fn raw_materialization_input_contract( node: &SummaryNode, ) -> Result<(String, Option, String), String> { let SummaryExpr::SummaryAgg { child, .. } = &node.expr else { @@ -3041,9 +3098,9 @@ fn immutable_materialization_sources(node: &SummaryNode) -> Option Result<(String, Option, String), String> { if let Some(source) = immutable_materialization_sources(node) { - materialization_leaf_contract(&source[0]) + raw_materialization_input_contract(&source[0]) } else { - materialization_leaf_contract(node) + raw_materialization_input_contract(node) } } @@ -3074,7 +3131,7 @@ fn validate_executable_subdag(node: &Rc) -> Result<(), String> { } fn physical_aggregation( - query: &PlanningQuery, + query: &QueryCompilationInput, selected: &SelectedMaterialization, aggregation_id: String, target: PhysicalDeploymentTarget, @@ -3083,12 +3140,12 @@ fn physical_aggregation( aggregation_id, metric_name: selected.metric.clone(), family: physical_materialization_family(&selected.family), - window_secs: selected.window_secs.unwrap_or(query.window_secs), + window_secs: selected.window_secs.unwrap_or(query.query_lookback_seconds), spatial_filter: selected.spatial_filter.clone(), grouping: selected .group_by .clone() - .unwrap_or_else(|| query.group_by.clone()), + .unwrap_or_else(|| query.group_by_labels.clone()), item_label: selected.item_label.clone(), heap_update_mode: selected.parameters.get("weight_mode").and_then(|mode| { match mode.as_str() { @@ -3159,7 +3216,7 @@ pub(crate) fn aggregation_config_for_materialization( } fn materialization_consumers( - queries: &[PlanningQuery], + queries: &[QueryCompilationInput], target: PhysicalDeploymentTarget, composable: bool, ) -> Result>, CompileError> { @@ -3169,18 +3226,16 @@ fn materialization_consumers( let mut cohort_programs = BTreeMap::>::new(); for (index, query) in queries.iter().enumerate() { - let states = - collect_selected_materializations(&query.post_asap, composable).map_err(|reason| { - CompileError::Query { - query_id: query.query_id.clone(), - reason, - } + let states = collect_selected_materializations(&query.selected_plan_root, composable) + .map_err(|reason| CompileError::Query { + query_id: query.query_id.clone(), + reason, })?; for state in states { if composable && state.window_secs.is_some_and(|window| { !query - .window_implementations + .window_realization_candidates .iter() .any(|candidate| candidate.window_secs == window) }) @@ -3223,7 +3278,7 @@ fn shared_pane_origin_ms( let pane_width = i64::try_from(pane_width_ms) .map_err(|_| "materialized pane width exceeds runtime timestamp range".to_string())?; let Some(workload) = workload else { - // Compatibility-only PlanningRequest callers do not claim a certified + // Compatibility-only PhysicalCompilationRequest callers do not claim a certified // phase. The read path rejects this binding and uses exact fallback. return Ok(None); }; @@ -3553,7 +3608,7 @@ fn stable_plan_id(materializations: &[CollectorMaterialization]) -> u64 { fn stable_workload_plan_id( materializations: &[CollectorMaterialization], - queries: &[PlanningQuery], + queries: &[QueryCompilationInput], ) -> u64 { use std::hash::{Hash, Hasher}; let mut hasher = std::collections::hash_map::DefaultHasher::new(); @@ -3565,6 +3620,89 @@ fn stable_workload_plan_id( hasher.finish() } +// Compatibility imports; new callers use the domain names above. +#[deprecated(note = "Use BackendLocalPhysicalInputs")] +pub use BackendLocalPhysicalInputs as BackendLocalImplementation; +#[deprecated(note = "Use BackendLocalPlanningInput")] +pub use BackendLocalPlanningInput as BackendLocalPlanningSnapshot; +#[deprecated(note = "Use CompiledPhysicalPlan")] +pub use CompiledPhysicalPlan as PhysicalPlan; +#[deprecated(note = "Use LifecycleUnitCosts")] +pub use LifecycleUnitCosts as LifecycleCostEvidence; +#[deprecated(note = "Use PhysicalCompilationRequest")] +pub use PhysicalCompilationRequest as PlanningRequest; +#[deprecated(note = "Use PhysicalDeploymentContext")] +pub use PhysicalDeploymentContext as DeploymentEnvironment; +#[deprecated(note = "Use PhysicalPlanCompiler")] +pub use PhysicalPlanCompiler as PhysicalCompiler; +#[deprecated(note = "Use QueryCompilationInput")] +pub use QueryCompilationInput as PlanningQuery; +#[deprecated(note = "Use SummaryLifecyclePlanningInputs")] +pub use SummaryLifecyclePlanningInputs as LifecyclePlanningInput; +#[deprecated(note = "Use WindowRealizationCandidate")] +pub use WindowRealizationCandidate as WindowImplementationCandidate; +#[deprecated(note = "Use WindowRealizationCostQuote")] +pub use WindowRealizationCostQuote as ImplementationCostEvidence; + +/// Parser/executor frontend for the time-series physical compiler. SQL has its +/// own compilation input and must not silently enter this path. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum QueryFrontend { + PromQl, + MetricsQl, +} +impl QueryFrontend { + pub fn parse( + self, + query: &str, + accuracy: crate::types_v2::AccuracyTarget, + ) -> Result { + crate::query_parser::parse_query_expr_canonical(query, accuracy).map_err(|error| match self + { + Self::PromQl => format!("frontend.promql: {error}"), + Self::MetricsQl => format!("victoriametrics.promql_subset: {error}"), + }) + } + pub fn compile( + self, + request: PhysicalCompilationRequest, + environment: PhysicalDeploymentContext, + ) -> Result { + PhysicalPlanCompiler.compile_for_frontend(request, environment, self) + } +} +impl BackendLocalPlanningInput { + #[deprecated(note = "Use compile_promql")] + pub fn compile(self) -> Result { + self.compile_promql() + } + #[deprecated(note = "Use into_physical_compilation_request")] + pub fn planning_request( + self, + ) -> Result<(PhysicalCompilationRequest, PhysicalDeploymentContext), CompileError> { + self.into_physical_compilation_request() + } +} +impl PhysicalPlanCompiler { + #[deprecated(note = "Use compile_promql")] + pub fn compile( + &self, + request: PhysicalCompilationRequest, + environment: PhysicalDeploymentContext, + ) -> Result { + self.compile_promql(request, environment) + } +} + +#[deprecated(note = "Use build_transmission_plan")] +pub use build_transmission_plan as compile_transmission_plan; +#[deprecated(note = "Use select_logical_roots_for_queries")] +pub use select_logical_roots_for_queries as select_workload_roots; +#[deprecated(note = "Use select_logical_roots_with_error_resource_profiles")] +pub use select_logical_roots_with_error_resource_profiles as select_workload_roots_with_erp; +#[deprecated(note = "Use select_logical_roots_with_trace")] +pub use select_logical_roots_with_trace as select_workload_roots_with_trace; + #[cfg(test)] pub(crate) mod tests { use super::*; @@ -3576,7 +3714,9 @@ pub(crate) mod tests { let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; entry.query = Query("sum_over_time(a[1m]) / sum_over_time(a[10m])".into()); entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let plan = quoted_snapshot(snapshot, false).compile().unwrap(); + let plan = quoted_snapshot(snapshot, crate::physical::compiler::QueryFrontend::PromQl) + .compile_promql() + .unwrap(); assert!(plan.cost_comparison.is_some()); assert_eq!(plan.precompute_plan.materializations.len(), 1); let bindings = plan @@ -3598,22 +3738,26 @@ pub(crate) mod tests { // Synthetic quotes exercise deployment selection in tests, never production defaults. pub(crate) fn quoted_snapshot( - mut snapshot: BackendLocalPlanningSnapshot, - metricsql: bool, - ) -> BackendLocalPlanningSnapshot { + mut snapshot: BackendLocalPlanningInput, + frontend: QueryFrontend, + ) -> BackendLocalPlanningInput { use super::super::workload_cost::{ - manifest, with_exact_alternative, WorkloadCostEvidence, WorkloadQuote, + enumerate_exact_and_materialized_candidates, manifest, WorkloadCostEvidence, + WorkloadQuote, }; - let (request, environment) = snapshot.clone().planning_request().unwrap(); - let quotes = with_exact_alternative(request) + let (request, environment) = snapshot + .clone() + .into_physical_compilation_request() + .unwrap(); + let quotes = enumerate_exact_and_materialized_candidates(request) .unwrap() .into_iter() .enumerate() .filter_map(|(index, candidate)| { - let plan = if metricsql { - PhysicalCompiler.compile_metricsql(candidate.clone(), environment.clone()) + let plan = if frontend == QueryFrontend::MetricsQl { + PhysicalPlanCompiler.compile_metricsql(candidate.clone(), environment.clone()) } else { - PhysicalCompiler.compile(candidate.clone(), environment.clone()) + PhysicalPlanCompiler.compile_promql(candidate.clone(), environment.clone()) } .ok()?; let manifest = manifest(&plan, &candidate.queries).unwrap(); @@ -3643,11 +3787,13 @@ pub(crate) mod tests { /// Optional counter masks must retain the workload's mandatory sketch bindings. #[test] fn costed_mixed_workload_retains_sketches_and_counter_readouts() { - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + let snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-compatibility-demo-snapshot.json" )) .unwrap(); - let plan = quoted_snapshot(snapshot, false).compile().unwrap(); + let plan = quoted_snapshot(snapshot, crate::physical::compiler::QueryFrontend::PromQl) + .compile_promql() + .unwrap(); assert!(!plan.precompute_plan.materializations.is_empty()); for entry in plan.query_plan.entries.values() { assert!(!entry.materialization_bindings().is_empty(), "{entry:#?}"); @@ -3657,31 +3803,31 @@ pub(crate) mod tests { /// A schema marker cannot opt into a legacy deployment policy. #[test] fn only_current_snapshot_schema_is_accepted() { - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + let snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); for version in [0, 1, 3] { let mut old = snapshot.clone(); - old.snapshot_version = version; + old.schema_version = version; assert!(old .clone() - .planning_request() + .into_physical_compilation_request() .unwrap_err() .to_string() .contains("only version 2")); - assert!(old.compile().is_err()); + assert!(old.compile_promql().is_err()); } - assert!(snapshot.planning_request().is_ok()); + assert!(snapshot.into_physical_compilation_request().is_ok()); } #[test] fn installed_partition_must_match_the_bound_dag_reduction() { let mut env = environment(10_000); env.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - env.collector_ids.clear(); - let mut plan = PhysicalCompiler - .compile(request("scope", "sum_over_time(m[1m])"), env) + env.target_collector_ids.clear(); + let mut plan = PhysicalPlanCompiler + .compile_promql(request("scope", "sum_over_time(m[1m])"), env) .unwrap(); let installed = plan .precompute_plan @@ -3728,9 +3874,9 @@ pub(crate) mod tests { ] { let mut environment = environment(10_000); environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - environment.collector_ids.clear(); - let plan = PhysicalCompiler - .compile(request("per-entity", query), environment) + environment.target_collector_ids.clear(); + let plan = PhysicalPlanCompiler + .compile_promql(request("per-entity", query), environment) .unwrap(); assert!( plan.precompute_plan @@ -3747,9 +3893,9 @@ pub(crate) mod tests { ] { let mut environment = environment(10_000); environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - environment.collector_ids.clear(); - let plan = PhysicalCompiler - .compile(request("reduced", query), environment) + environment.target_collector_ids.clear(); + let plan = PhysicalPlanCompiler + .compile_promql(request("reduced", query), environment) .unwrap(); assert!( !plan.precompute_plan.materializations.is_empty(), @@ -3763,8 +3909,10 @@ pub(crate) mod tests { let request = request("counter", "sum(rate(m[1m]))"); let mut environment = environment(10_000); environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - environment.collector_ids.clear(); - let plan = PhysicalCompiler.compile(request, environment).unwrap(); + environment.target_collector_ids.clear(); + let plan = PhysicalPlanCompiler + .compile_promql(request, environment) + .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); assert_eq!( plan.precompute_plan.materializations[0].num_aggregates_to_retain, @@ -3786,8 +3934,8 @@ pub(crate) mod tests { #[test] fn raw_counter_artifact_is_valid_for_backend_precompute() { - let plan = PhysicalCompiler - .compile(request("counter", "rate(m[1m])"), environment(10_000)) + let plan = PhysicalPlanCompiler + .compile_promql(request("counter", "rate(m[1m])"), environment(10_000)) .unwrap(); plan.precompute_plan.validate().unwrap(); let catalog = plan.summary_catalog.clone(); @@ -3812,8 +3960,10 @@ pub(crate) mod tests { .extend(request("gauge", "sum(sum_over_time(g[1m]))").queries); let mut environment = environment(10_000); environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - environment.collector_ids.clear(); - let plan = PhysicalCompiler.compile(workload, environment).unwrap(); + environment.target_collector_ids.clear(); + let plan = PhysicalPlanCompiler + .compile_promql(workload, environment) + .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 2); assert_eq!( plan.query_plan @@ -3836,16 +3986,16 @@ pub(crate) mod tests { // Capability normalization precedes candidate enumeration, avoiding duplicate exact quotes. #[test] fn counter_only_snapshot_has_distinct_local_and_native_cost_alternatives() { - let mut snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; entry.query = Query("rate(m[1m])".into()); entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let (request, _) = snapshot.planning_request().unwrap(); + let (request, _) = snapshot.into_physical_compilation_request().unwrap(); assert_eq!( - super::super::workload_cost::with_exact_alternative(request) + super::super::workload_cost::enumerate_exact_and_materialized_candidates(request) .unwrap() .len(), 3 @@ -3867,8 +4017,8 @@ pub(crate) mod tests { source: "unit-fixture".into(), }; let request = request_with_evidence("topk", query, Some(evidence)).unwrap(); - let plan = PhysicalCompiler - .compile(request, environment(10000)) + let plan = PhysicalPlanCompiler + .compile_promql(request, environment(10000)) .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1, "{query}"); assert_eq!( @@ -3889,8 +4039,8 @@ pub(crate) mod tests { source: "unit-fixture".into(), }; let request = request_with_evidence("topk-rate", query, Some(evidence)).unwrap(); - let plan = PhysicalCompiler - .compile(request, environment(10_000)) + let plan = PhysicalPlanCompiler + .compile_promql(request, environment(10_000)) .unwrap(); let entry = plan.query_plan.entries.values().next().unwrap(); let crate::query_plan::QueryPlanNode::CandidateTopK { inputs, .. } = @@ -3967,7 +4117,7 @@ pub(crate) mod tests { #[test] fn hybrid_weighted_topk_installs_only_candidates_and_delegates_filtered_exact_values() { use crate::query_plan::{ - logical::LogicalOperator, ExternalExactInput, ExternalExactOutput, QueryPlanNode, + logical::ResidualQueryOperator, ExternalExactInput, ExternalExactOutput, QueryPlanNode, }; let query = "topk(2, sum by (job) (rate(m[1m])))"; let evidence = TopKMembershipEvidence { @@ -3978,12 +4128,14 @@ pub(crate) mod tests { source: "unit-fixture".into(), }; let mut request = request_with_evidence("topk-rate", query, Some(evidence)).unwrap(); - request.hybrid_execution = true; + request.allow_mixed_summary_and_exact_execution = true; let mut environment = environment(10_000); environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - environment.collector_ids.clear(); + environment.target_collector_ids.clear(); - let plan = PhysicalCompiler.compile(request, environment).unwrap(); + let plan = PhysicalPlanCompiler + .compile_promql(request, environment) + .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); assert_eq!( @@ -4011,7 +4163,7 @@ pub(crate) mod tests { node, QueryPlanNode::ExactReadout { .. } | QueryPlanNode::Logical { - operator: LogicalOperator::Scan { .. }, + operator: ResidualQueryOperator::Scan { .. }, .. } ))); @@ -4046,8 +4198,8 @@ pub(crate) mod tests { observed_at_unix_ms: 9_500, source: "unit-fixture".into(), }; - let plan = PhysicalCompiler - .compile( + let plan = PhysicalPlanCompiler + .compile_promql( request_with_evidence( "topk-rate", "topk(2, sum by (job) (rate(m[1m])))", @@ -4096,8 +4248,8 @@ pub(crate) mod tests { let mut request = request("bounded", "sum(sum_over_time(m[1m]))"); request.retained_summary_memory_budget_bytes = Some(1); - let error = PhysicalCompiler - .compile(request, environment(10_000)) + let error = PhysicalPlanCompiler + .compile_promql(request, environment(10_000)) .unwrap_err(); assert!(error.to_string().contains("retained summary footprint")); } @@ -4105,22 +4257,29 @@ pub(crate) mod tests { #[test] fn legacy_backend_snapshot_gets_explicit_retained_memory_default_and_alias() { let source = include_str!("../../../docs/examples/asapquery-planning-snapshot.json"); - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(source).unwrap(); + let snapshot: BackendLocalPlanningInput = serde_json::from_str(source).unwrap(); assert_eq!( - snapshot.implementation.max_retained_summary_bytes, + snapshot + .physical_inputs + .retained_summary_memory_budget_bytes, DEFAULT_RETAINED_SUMMARY_MEMORY_BUDGET_BYTES ); let mut value: Value = serde_json::from_str(source).unwrap(); value["implementation"]["maxRetainedSummaryBytes"] = json!(123_456); - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_value(value).unwrap(); - assert_eq!(snapshot.implementation.max_retained_summary_bytes, 123_456); + let snapshot: BackendLocalPlanningInput = serde_json::from_value(value).unwrap(); + assert_eq!( + snapshot + .physical_inputs + .retained_summary_memory_budget_bytes, + 123_456 + ); } - fn environment(now: u64) -> DeploymentEnvironment { - DeploymentEnvironment { + fn environment(now: u64) -> PhysicalDeploymentContext { + PhysicalDeploymentContext { target: PhysicalDeploymentTarget::DistributedCollectors, - collector_ids: vec!["edge-a".into(), "edge-b".into()], + target_collector_ids: vec!["edge-a".into(), "edge-b".into()], capability_snapshot_id: "caps-7".into(), observed_at_unix_ms: now, max_evidence_age_ms: 60_000, @@ -4135,20 +4294,20 @@ pub(crate) mod tests { query_id: &str, promql: &str, evidence: Option, - ) -> Result { + ) -> Result { let accuracy = AccuracyTarget::EpsilonDelta { epsilon: 0.01, delta: 0.01, }; let parsed = crate::query_parser::parse_query_expr_canonical(promql, accuracy.clone()) .expect("canonical query"); - let lifecycle = LifecyclePlanningInput { + let lifecycle = SummaryLifecyclePlanningInputs { evaluation_interval_ms: 10_000, ingestion_rate_per_second: 100.0, evidence_observed_at_unix_ms: 9_500, evidence_valid_for_ms: 60_000, horizon_seconds: 300.0, - costs: LifecycleCostEvidence { + costs: LifecycleUnitCosts { build: 10.0, maintenance_per_update: 0.001, read: 0.1, @@ -4161,28 +4320,28 @@ pub(crate) mod tests { if let Some(evidence) = evidence { evidence_by_query.insert(query_id.to_string(), evidence); } - Ok(PlanningRequest { - logical_selection: Vec::new(), - synthesized_window_queries: BTreeSet::new(), - hybrid_execution: false, - materialization_policy: None, + Ok(PhysicalCompilationRequest { + planner_selection_trace: Vec::new(), + compiler_priced_window_query_ids: BTreeSet::new(), + allow_mixed_summary_and_exact_execution: false, + enabled_materialization_keys: None, query_workload: None, - queries: vec![PlanningQuery { + queries: vec![QueryCompilationInput { query_id: query_id.into(), query_string: promql.into(), - post_asap, - source: Source::TimeSeries { metric: "m".into() }, - window_secs: 60, - group_by: vec![], - accuracy, - lifecycle, - window_implementations: vec![WindowImplementationCandidate { - implementation_id: "collector-tumbling-v1".into(), + selected_plan_root: post_asap, + legacy_query_source: Source::TimeSeries { metric: "m".into() }, + query_lookback_seconds: 60, + group_by_labels: vec![], + accuracy_target: accuracy, + summary_lifecycle_inputs: lifecycle, + window_realization_candidates: vec![WindowRealizationCandidate { + realization_id: "collector-tumbling-v1".into(), framework: SummaryWindowFramework::Tumbling, window_secs: 60, slide_secs: 60, layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 60 }, - cost: ImplementationCostEvidence { + cost: WindowRealizationCostQuote { model_version: "test-cost-v1".into(), workload_fingerprint: "test-workload".into(), observed_at_unix_ms: 9_500, @@ -4196,19 +4355,19 @@ pub(crate) mod tests { weighted_cost: 1.0, }, }], - runtime_policy: RuntimeRulePolicy::default(), + materialization_runtime_policy: RuntimeRulePolicy::default(), }], - evidence: evidence_by_query, + topk_membership_evidence_by_query_id: evidence_by_query, erp: None, exact_composition_costs: HashMap::new(), planner_revision: PLANNER_REVISION.into(), source_sample_interval_ms: None, - query_staleness_margin_ms: 0, + query_retention_margin_ms: 0, retained_summary_memory_budget_bytes: None, }) } - fn request(query_id: &str, promql: &str) -> PlanningRequest { + fn request(query_id: &str, promql: &str) -> PhysicalCompilationRequest { request_with_evidence(query_id, promql, None).expect("post-ASAP selection") } @@ -4234,16 +4393,19 @@ pub(crate) mod tests { #[test] fn immutable_nested_summary_keeps_actual_source_and_derived_bindings() { let mut workload = request("nested", "quantile(0.9, sum_over_time(m[1m]))"); - workload.hybrid_execution = true; + workload.allow_mixed_summary_and_exact_execution = true; let states = - collect_selected_materializations(&workload.queries[0].post_asap, true).unwrap(); + collect_selected_materializations(&workload.queries[0].selected_plan_root, true) + .unwrap(); assert_eq!(states.len(), 2, "source and consumer must both be selected"); assert!(immutable_materialization_sources(&states[0].node).is_none()); assert!(immutable_materialization_sources(&states[1].node).is_some()); let mut deployment = environment(10_000); deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - deployment.collector_ids.clear(); - let plan = PhysicalCompiler.compile(workload, deployment).unwrap(); + deployment.target_collector_ids.clear(); + let plan = PhysicalPlanCompiler + .compile_promql(workload, deployment) + .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 2); let derived = plan .precompute_plan @@ -4277,16 +4439,19 @@ pub(crate) mod tests { "nested", "quantile(0.9, sum_over_time(m[1m]) + sum_over_time(n[1m]))", ); - workload.hybrid_execution = true; + workload.allow_mixed_summary_and_exact_execution = true; let states = - collect_selected_materializations(&workload.queries[0].post_asap, true).unwrap(); + collect_selected_materializations(&workload.queries[0].selected_plan_root, true) + .unwrap(); assert_eq!(states.len(), 3, "source and consumer must both be selected"); assert!(immutable_materialization_sources(&states[0].node).is_none()); assert!(immutable_materialization_sources(&states[2].node).is_some()); let mut deployment = environment(10_000); deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - deployment.collector_ids.clear(); - let plan = PhysicalCompiler.compile(workload, deployment).unwrap(); + deployment.target_collector_ids.clear(); + let plan = PhysicalPlanCompiler + .compile_promql(workload, deployment) + .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 3); assert!(plan .precompute_plan @@ -4336,19 +4501,24 @@ pub(crate) mod tests { fn distinct_range_compiles_to_partitioned_hll() { let mut deployment = environment(10_000); deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - deployment.collector_ids.clear(); + deployment.target_collector_ids.clear(); let mut workload = request("distinct", "distinct_over_time(m{job=\"api\"}[1m])"); let query = &mut workload.queries[0]; // HLL's modeled RSE does not certify a failure probability. - query.accuracy = AccuracyTarget::Epsilon(0.05); + query.accuracy_target = AccuracyTarget::Epsilon(0.05); let parsed = crate::query_parser::parse_query_expr_canonical( &query.query_string, - query.accuracy.clone(), + query.accuracy_target.clone(), ) .unwrap(); - query.post_asap = - select_post_asap(&parsed, query.accuracy.clone(), &query.lifecycle, None).unwrap(); - let plan = PhysicalCompiler + query.selected_plan_root = select_post_asap( + &parsed, + query.accuracy_target.clone(), + &query.summary_lifecycle_inputs, + None, + ) + .unwrap(); + let plan = PhysicalPlanCompiler .compile_metricsql(workload, deployment) .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); @@ -4380,20 +4550,25 @@ pub(crate) mod tests { fn unsupported_cardinality_family_fails_admission() { let mut deployment = environment(10_000); deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - deployment.collector_ids.clear(); + deployment.target_collector_ids.clear(); let mut workload = request("confidence", "distinct_over_time(m[1m])"); - workload.hybrid_execution = true; - let result = PhysicalCompiler.compile_metricsql(workload, deployment); + workload.allow_mixed_summary_and_exact_execution = true; + let result = PhysicalPlanCompiler.compile_metricsql(workload, deployment); assert!(matches!(result, Err(CompileError::QueryPlan(_)))); } #[test] fn snapshot_metricsql_entry_uses_the_shared_serving_language_contract() { - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + let snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-compatibility-demo-snapshot.json" )) .unwrap(); - let plan = quoted_snapshot(snapshot, true).compile_metricsql().unwrap(); + let plan = quoted_snapshot( + snapshot, + crate::physical::compiler::QueryFrontend::MetricsQl, + ) + .compile_metricsql() + .unwrap(); assert!(!plan.query_plan.entries.is_empty()); assert!(plan .query_plan @@ -4409,8 +4584,8 @@ pub(crate) mod tests { second.query_string = "max_over_time(b[1m])".into(); workload.queries.push(second); - let error = PhysicalCompiler - .compile(workload, environment(10_000)) + let error = PhysicalPlanCompiler + .compile_promql(workload, environment(10_000)) .unwrap_err(); assert!(matches!( error, @@ -4426,11 +4601,11 @@ pub(crate) mod tests { "increase(counter_probe{case=\"reset\"}[5s])", ] { let mut workload = request("counter", text); - workload.hybrid_execution = true; + workload.allow_mixed_summary_and_exact_execution = true; let mut deployment = environment(10_000); deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - deployment.collector_ids.clear(); - let plan = PhysicalCompiler + deployment.target_collector_ids.clear(); + let plan = PhysicalPlanCompiler .compile_metricsql(workload, deployment) .unwrap(); assert!(plan.precompute_plan.materializations.is_empty()); @@ -4443,11 +4618,11 @@ pub(crate) mod tests { #[test] fn metricsql_counter_gate_preserves_an_independent_summary_sibling() { let mut workload = request("mixed", "max_over_time(m[1m]) + rate(m[1m])"); - workload.hybrid_execution = true; + workload.allow_mixed_summary_and_exact_execution = true; let mut deployment = environment(10_000); deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - deployment.collector_ids.clear(); - let plan = PhysicalCompiler + deployment.target_collector_ids.clear(); + let plan = PhysicalPlanCompiler .compile_metricsql(workload, deployment) .unwrap(); assert!(!plan.precompute_plan.materializations.is_empty()); @@ -4462,26 +4637,29 @@ pub(crate) mod tests { ))); let entry = plan.query_plan.entries.values().next().unwrap(); assert!(!entry.materialization_bindings().is_empty()); - assert!(entry.nodes.values().any(|node| matches!( - node, - crate::query_plan::QueryPlanNode::ExternalExact { .. } - | crate::query_plan::QueryPlanNode::Logical { - operator: crate::query_plan::logical::LogicalOperator::ExactSubquery { .. }, - .. - } - ))); + assert!( + entry.nodes.values().any(|node| matches!( + node, + crate::query_plan::QueryPlanNode::ExternalExact { .. } + | crate::query_plan::QueryPlanNode::Logical { + operator: + crate::query_plan::residual::ResidualQueryOperator::ExactSubquery { .. }, + .. + } + )) + ); } #[test] fn metricsql_compilation_publishes_a_language_tagged_query_entry() { let query = "mad_over_time(m[1m])"; let mut workload = request("vm-q", "last_over_time(m[1m])"); - let accuracy = workload.queries[0].accuracy.clone(); + let accuracy = workload.queries[0].accuracy_target.clone(); let canonical = asap_frontend_promql::lower_promql(query, accuracy.clone()).unwrap(); workload.queries[0].query_string = query.into(); - workload.queries[0].post_asap = + workload.queries[0].selected_plan_root = crate::planner_selection::keep_pre_asap(&canonical).unwrap(); - let plan = PhysicalCompiler + let plan = PhysicalPlanCompiler .compile_metricsql(workload, environment(10_000)) .unwrap(); let identity = canonical_promql(query).unwrap(); @@ -4499,7 +4677,7 @@ pub(crate) mod tests { let roots = vec![Rc::new( crate::query_parser::parse_query_expr_canonical( &workload.queries[0].query_string, - workload.queries[0].accuracy.clone(), + workload.queries[0].accuracy_target.clone(), ) .unwrap(), )]; @@ -4530,16 +4708,16 @@ pub(crate) mod tests { max_memory_bytes: None, }, }; - select_workload_roots_with_erp( + select_logical_roots_with_error_resource_profiles( &mut workload.queries, roots, - &workload.evidence, + &workload.topk_membership_evidence_by_query_id, &workload.exact_composition_costs, Some(&erp), ) .unwrap(); assert!(matches!( - workload.queries[0].post_asap.expr, + workload.queries[0].selected_plan_root.expr, SummaryExpr::KeepPreAsap(_) )); } @@ -4593,18 +4771,18 @@ pub(crate) mod tests { ); let mut workload = request("q", "quantile_over_time(0.9, m[1m])"); - workload.queries[0].accuracy = AccuracyTarget::Epsilon(0.06); + workload.queries[0].accuracy_target = AccuracyTarget::Epsilon(0.06); let root = Rc::new( crate::query_parser::parse_query_expr_canonical( &workload.queries[0].query_string, - workload.queries[0].accuracy.clone(), + workload.queries[0].accuracy_target.clone(), ) .unwrap(), ); - select_workload_roots_with_erp( + select_logical_roots_with_error_resource_profiles( &mut workload.queries, vec![root], - &workload.evidence, + &workload.topk_membership_evidence_by_query_id, &workload.exact_composition_costs, Some(&erp), ) @@ -4626,15 +4804,19 @@ pub(crate) mod tests { } } assert!( - contains_measured_kll(&workload.queries[0].post_asap), + contains_measured_kll(&workload.queries[0].selected_plan_root), "ERP hit was lost before physical compilation: {:#?}", - workload.queries[0].post_asap + workload.queries[0].selected_plan_root ); - let guarantee = workload.queries[0].post_asap.guarantee.as_ref().unwrap(); + let guarantee = workload.queries[0] + .selected_plan_root + .guarantee + .as_ref() + .unwrap(); assert_eq!(guarantee.failure_probability.evaluate(), None); assert!(!guarantee.is_exact()); - let plan = PhysicalCompiler - .compile(workload, environment(10000)) + let plan = PhysicalPlanCompiler + .compile_promql(workload, environment(10000)) .unwrap(); assert_eq!(plan.precompute_plan.materializations[0].parameters["k"], 32); let empirical_identity = plan.precompute_plan.materializations[0].policy_fingerprint(); @@ -4659,7 +4841,7 @@ pub(crate) mod tests { (unsupported, AccuracyTarget::Epsilon(0.06), true), ] { let mut workload = request("q", "quantile_over_time(0.9, m[1m])"); - workload.queries[0].accuracy = accuracy.clone(); + workload.queries[0].accuracy_target = accuracy.clone(); let root = Rc::new( crate::query_parser::parse_query_expr_canonical( &workload.queries[0].query_string, @@ -4667,23 +4849,25 @@ pub(crate) mod tests { ) .unwrap(), ); - select_workload_roots_with_erp( + select_logical_roots_with_error_resource_profiles( &mut workload.queries, vec![root], - &workload.evidence, + &workload.topk_membership_evidence_by_query_id, &workload.exact_composition_costs, Some(&policy), ) .unwrap(); if exact { assert!(matches!( - workload.queries[0].post_asap.expr, + workload.queries[0].selected_plan_root.expr, SummaryExpr::KeepPreAsap(_) )); } else { - assert!(!contains_measured_kll(&workload.queries[0].post_asap)); - let plan = PhysicalCompiler - .compile(workload, environment(10000)) + assert!(!contains_measured_kll( + &workload.queries[0].selected_plan_root + )); + let plan = PhysicalPlanCompiler + .compile_promql(workload, environment(10000)) .unwrap(); let state = &plan.precompute_plan.materializations[0]; assert!(state.parameters["k"].as_u64().unwrap() > 32); @@ -4762,11 +4946,11 @@ pub(crate) mod tests { fn sum_rate_uses_summary_child_only_with_measured_composition_costs() { let promql = "sum by (job) (rate(m[1m]))"; let mut with_evidence = request("topk-rate", promql); - with_evidence.hybrid_execution = true; + with_evidence.allow_mixed_summary_and_exact_execution = true; let root = Rc::new( crate::query_parser::parse_query_expr_canonical( promql, - with_evidence.queries[0].accuracy.clone(), + with_evidence.queries[0].accuracy_target.clone(), ) .unwrap(), ); @@ -4774,17 +4958,19 @@ pub(crate) mod tests { "topk-rate".into(), measured_exact_composition_rows(&root, 9_500), ); - select_workload_roots( + select_logical_roots_for_queries( &mut with_evidence.queries, vec![root], - &with_evidence.evidence, + &with_evidence.topk_membership_evidence_by_query_id, &with_evidence.exact_composition_costs, ) .unwrap(); let mut backend = environment(10_000); backend.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - backend.collector_ids.clear(); - let plan = PhysicalCompiler.compile(with_evidence, backend).unwrap(); + backend.target_collector_ids.clear(); + let plan = PhysicalPlanCompiler + .compile_promql(with_evidence, backend) + .unwrap(); assert!( !plan.summary_catalog.materializations.is_empty(), "measured exact-composition evidence must expose the rate child as a SummaryStore binding" @@ -4795,25 +4981,27 @@ pub(crate) mod tests { fn sum_rate_without_composition_costs_does_not_invent_a_composition_cost() { let promql = "sum by (job) (rate(m[1m]))"; let mut unavailable = request("topk-rate", promql); - unavailable.hybrid_execution = true; + unavailable.allow_mixed_summary_and_exact_execution = true; let root = Rc::new( crate::query_parser::parse_query_expr_canonical( promql, - unavailable.queries[0].accuracy.clone(), + unavailable.queries[0].accuracy_target.clone(), ) .unwrap(), ); - select_workload_roots( + select_logical_roots_for_queries( &mut unavailable.queries, vec![root], - &unavailable.evidence, + &unavailable.topk_membership_evidence_by_query_id, &unavailable.exact_composition_costs, ) .unwrap(); let mut backend = environment(10_000); backend.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - backend.collector_ids.clear(); - let plan = PhysicalCompiler.compile(unavailable, backend).unwrap(); + backend.target_collector_ids.clear(); + let plan = PhysicalPlanCompiler + .compile_promql(unavailable, backend) + .unwrap(); // Planner 54f can realize this particular shape directly as an exact // counter readout plus a query-time reduce; it does not require an // ExactComposition candidate. The absence of evidence must therefore @@ -4914,21 +5102,21 @@ pub(crate) mod tests { Rc::new( crate::query_parser::parse_query_expr_canonical( &query.query_string, - query.accuracy.clone(), + query.accuracy_target.clone(), ) .unwrap(), ) }) .collect(); - select_workload_roots( + select_logical_roots_for_queries( &mut workload.queries, roots, - &workload.evidence, + &workload.topk_membership_evidence_by_query_id, &workload.exact_composition_costs, ) .unwrap(); - let bundle = PhysicalCompiler - .compile(workload, environment(10000)) + let bundle = PhysicalPlanCompiler + .compile_promql(workload, environment(10000)) .unwrap(); assert_eq!(bundle.query_plan.entries.len(), 2); assert_eq!(bundle.collector_plans[0].materializations.len(), 1); @@ -4947,10 +5135,10 @@ pub(crate) mod tests { #[test] fn shared_selection_rejects_incomplete_root_mapping() { let mut workload = request("q", "quantile_over_time(0.9, m[1m])"); - assert!(select_workload_roots( + assert!(select_logical_roots_for_queries( &mut workload.queries, vec![], - &workload.evidence, + &workload.topk_membership_evidence_by_query_id, &workload.exact_composition_costs, ) .is_err()); @@ -4959,8 +5147,8 @@ pub(crate) mod tests { // Adding another readout adds recurring reads, not another update stream. #[test] fn joint_lifecycle_charges_shared_updates_once() { - let baseline = PhysicalCompiler - .compile( + let baseline = PhysicalPlanCompiler + .compile_promql( request("q90", "quantile_over_time(0.9, m[1m])"), environment(10000), ) @@ -4969,10 +5157,10 @@ pub(crate) mod tests { let mut second = request("q99", "quantile_over_time(0.99, m[1m])") .queries .remove(0); - second.lifecycle.evaluation_interval_ms = 20000; + second.summary_lifecycle_inputs.evaluation_interval_ms = 20000; workload.queries.push(second); - let shared = PhysicalCompiler - .compile(workload, environment(10000)) + let shared = PhysicalPlanCompiler + .compile_promql(workload, environment(10000)) .unwrap(); assert_eq!(shared.lifecycle_estimates.len(), 1); let estimate = &shared.lifecycle_estimates[0]; @@ -4997,10 +5185,10 @@ pub(crate) mod tests { let mut second = request("q99", "quantile_over_time(0.99, m[1m])") .queries .remove(0); - second.lifecycle.ingestion_rate_per_second = 200.0; + second.summary_lifecycle_inputs.ingestion_rate_per_second = 200.0; workload.queries.push(second); assert!(matches!( - PhysicalCompiler.compile(workload, environment(10000)), + PhysicalPlanCompiler.compile_promql(workload, environment(10000)), Err(CompileError::Lifecycle { .. }) )); } @@ -5024,10 +5212,10 @@ pub(crate) mod tests { let mut env = environment(10_000); env.target = target; if target == PhysicalDeploymentTarget::BackendLocalRemoteWrite { - env.collector_ids.clear(); + env.target_collector_ids.clear(); } - let bundle = PhysicalCompiler - .compile(workload, env) + let bundle = PhysicalPlanCompiler + .compile_promql(workload, env) .expect("shared compile"); assert_eq!(bundle.query_plan.entries.len(), 2); assert_eq!(bundle.summary_catalog.materializations.len(), 1); @@ -5050,15 +5238,15 @@ pub(crate) mod tests { #[test] fn adding_shared_consumer_changes_plan_identity() { let workload = request("q90", "quantile_over_time(0.90, m[1m])"); - let one = PhysicalCompiler - .compile(workload, environment(10_000)) + let one = PhysicalPlanCompiler + .compile_promql(workload, environment(10_000)) .unwrap(); let mut workload = request("q90", "quantile_over_time(0.90, m[1m])"); workload .queries .extend(request("q99", "quantile_over_time(0.99, m[1m])").queries); - let two = PhysicalCompiler - .compile(workload, environment(10_000)) + let two = PhysicalPlanCompiler + .compile_promql(workload, environment(10_000)) .unwrap(); assert_ne!(one.envelope.plan_id, two.envelope.plan_id); assert_eq!(two.collector_plans[0].materializations.len(), 1); @@ -5068,10 +5256,10 @@ pub(crate) mod tests { fn different_sources_do_not_share_materializations() { let mut workload = request("qm", "quantile_over_time(0.90, m[1m])"); let mut other = request("qn", "quantile_over_time(0.90, n[1m])"); - other.queries[0].source = Source::TimeSeries { metric: "n".into() }; + other.queries[0].legacy_query_source = Source::TimeSeries { metric: "n".into() }; workload.queries.extend(other.queries); - let bundle = PhysicalCompiler - .compile(workload, environment(10_000)) + let bundle = PhysicalPlanCompiler + .compile_promql(workload, environment(10_000)) .unwrap(); assert_eq!(bundle.summary_catalog.materializations.len(), 2); assert_eq!(bundle.precompute_plan.materializations.len(), 2); @@ -5086,8 +5274,8 @@ pub(crate) mod tests { workload .queries .extend(request("increase", "increase(m[1m])").queries); - let bundle = PhysicalCompiler - .compile(workload, environment(10_000)) + let bundle = PhysicalPlanCompiler + .compile_promql(workload, environment(10_000)) .unwrap(); assert_eq!(bundle.query_plan.entries.len(), 2); assert_eq!(bundle.precompute_plan.materializations.len(), 1); @@ -5101,11 +5289,11 @@ pub(crate) mod tests { fn shared_materialization_rejects_conflicting_deployment_contracts() { let mut workload = request("q90", "quantile_over_time(0.90, m[1m])"); let mut other = request("q99", "quantile_over_time(0.99, m[1m])"); - other.queries[0].window_implementations[0].implementation_id = + other.queries[0].window_realization_candidates[0].realization_id = "another-implementation".into(); workload.queries.extend(other.queries); - let error = PhysicalCompiler - .compile(workload, environment(10_000)) + let error = PhysicalPlanCompiler + .compile_promql(workload, environment(10_000)) .expect_err("conflicting shared state must fail before publication"); assert!(error .to_string() @@ -5115,7 +5303,7 @@ pub(crate) mod tests { #[test] fn exact_dashboard_binds_sum_and_count_to_one_local_producer() { // Both dashboard roots use one packed raw accumulator, with explicit readouts. - let mut snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); @@ -5128,8 +5316,8 @@ pub(crate) mod tests { .into(), ); entries.push(mean); - let (request, env) = snapshot.planning_request().unwrap(); - let bundle = PhysicalCompiler.compile(request, env).unwrap(); + let (request, env) = snapshot.into_physical_compilation_request().unwrap(); + let bundle = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!(bundle.precompute_plan.materializations.len(), 1); assert_eq!(bundle.query_plan.entries.len(), 2); for entry in bundle.query_plan.entries.values() { @@ -5149,7 +5337,7 @@ pub(crate) mod tests { .any(|entry| entry.nodes.values().any(|node| matches!( node, crate::query_plan::QueryPlanNode::Logical { - operator: crate::query_plan::logical::LogicalOperator::Binary { .. }, + operator: crate::query_plan::residual::ResidualQueryOperator::Binary { .. }, .. } )))); @@ -5157,7 +5345,7 @@ pub(crate) mod tests { #[test] fn invalid_unselected_candidate_is_canonicalized_to_exact_fallback() { - let mut snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); @@ -5165,16 +5353,16 @@ pub(crate) mod tests { entry.query = Query("sum by (service) (sum_over_time(m[1m]) / count_over_time(m[1m]))".into()); entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let (mut request, _) = snapshot.planning_request().unwrap(); + let (mut request, _) = snapshot.into_physical_compilation_request().unwrap(); assert!(!matches!( - request.queries[0].post_asap.expr, + request.queries[0].selected_plan_root.expr, SummaryExpr::KeepPreAsap(_) )); preserve_invalid_exact_fallback_roots(&mut request.queries, true).unwrap(); assert!(matches!( - request.queries[0].post_asap.expr, + request.queries[0].selected_plan_root.expr, SummaryExpr::KeepPreAsap(_) )); } @@ -5182,8 +5370,8 @@ pub(crate) mod tests { #[test] fn selected_maintenance_dependency_still_requires_a_valid_executable_dag() { let mut request = request("invalid-dependency", "sum(sum_over_time(m[1m]))"); - let selected = request.queries[0].post_asap.clone(); - request.queries[0].post_asap = Rc::new(SummaryNode { + let selected = request.queries[0].selected_plan_root.clone(); + request.queries[0].selected_plan_root = Rc::new(SummaryNode { expr: SummaryExpr::BinaryOp { timing: planner_types::post_asap::ExecutionTiming::ReadTime, lhs: selected.clone(), @@ -5198,12 +5386,14 @@ pub(crate) mod tests { schema: selected.schema.clone(), guarantee: None, }); - request.hybrid_execution = true; + request.allow_mixed_summary_and_exact_execution = true; let mut environment = environment(10_000); environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - environment.collector_ids.clear(); + environment.target_collector_ids.clear(); - let error = PhysicalCompiler.compile(request, environment).unwrap_err(); + let error = PhysicalPlanCompiler + .compile_promql(request, environment) + .unwrap_err(); assert!(error.to_string().contains("invalid executable subDAG")); } @@ -5219,16 +5409,18 @@ pub(crate) mod tests { "sum_over_time(m[1m]) / count_over_time(m[5m])", "sum_over_time(m[1m] offset 1h) / count_over_time(m[1m] offset 1h)", ] { - let mut snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); let entries = snapshot.query_workload.repeating_queries.as_mut().unwrap(); entries[0].query = Query(query.into()); entries[0].requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let (mut request, environment) = snapshot.planning_request().unwrap(); - request.hybrid_execution = false; - let bundle = PhysicalCompiler.compile(request, environment).unwrap(); + let (mut request, environment) = snapshot.into_physical_compilation_request().unwrap(); + request.allow_mixed_summary_and_exact_execution = false; + let bundle = PhysicalPlanCompiler + .compile_promql(request, environment) + .unwrap(); assert!(bundle.precompute_plan.materializations.is_empty()); assert!(bundle.query_plan.entries.values().all(|entry| matches!( entry.nodes[&entry.root], @@ -5240,7 +5432,7 @@ pub(crate) mod tests { // Local composition must evaluate per-series division before the outer sum. #[test] fn composable_non_additive_rollup_does_not_pool_raw_producers() { - let mut snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); @@ -5248,16 +5440,18 @@ pub(crate) mod tests { entry.query = Query("sum by (service) (sum_over_time(m[1m]) / count_over_time(m[1m]))".into()); entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let (request, environment) = snapshot.planning_request().unwrap(); - let candidates = super::super::workload_cost::with_exact_alternative(request).unwrap(); - match PhysicalCompiler.compile(candidates[0].clone(), environment.clone()) { + let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); + let candidates = + super::super::workload_cost::enumerate_exact_and_materialized_candidates(request) + .unwrap(); + match PhysicalPlanCompiler.compile_promql(candidates[0].clone(), environment.clone()) { Ok(plan) => assert!(plan.precompute_plan.materializations.is_empty()), Err(error) => assert!(error .to_string() .contains("semantically identical original subtree witness")), } - let native = PhysicalCompiler - .compile(candidates.last().unwrap().clone(), environment) + let native = PhysicalPlanCompiler + .compile_promql(candidates.last().unwrap().clone(), environment) .unwrap(); assert!(native.precompute_plan.materializations.is_empty()); } @@ -5265,15 +5459,15 @@ pub(crate) mod tests { #[test] fn composable_per_entity_window_installs_isolated_state() { use crate::query_plan::QueryPlanNode; - let mut snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; entry.query = Query("sum_over_time(m[1m])".into()); entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let (request, env) = snapshot.planning_request().unwrap(); - let plan = PhysicalCompiler.compile(request, env).unwrap(); + let (request, env) = snapshot.into_physical_compilation_request().unwrap(); + let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); assert_eq!( plan.precompute_plan.materializations[0].partitioning, @@ -5290,7 +5484,7 @@ pub(crate) mod tests { // remains valid for both readouts. #[test] fn composable_binary_binds_independent_source_windows() { - let mut snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); @@ -5299,8 +5493,8 @@ pub(crate) mod tests { entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); // The derivation now covers both range selectors, so this no longer // needs a hand-supplied 5m candidate to keep `b` from falling back. - let (request, env) = snapshot.planning_request().unwrap(); - let plan = PhysicalCompiler.compile(request, env).unwrap(); + let (request, env) = snapshot.into_physical_compilation_request().unwrap(); + let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); let bindings = plan .query_plan .entries @@ -5343,9 +5537,9 @@ pub(crate) mod tests { json!("quantile(0.9, sum_over_time(m[1m]))"); value["data_workload"]["ingestion_rate"]["value"] = json!(rate); value["query_workload"]["data_workload"]["ingestion_rate"]["value"] = json!(rate); - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_value(value).unwrap(); - let (request, env) = snapshot.planning_request().unwrap(); - let plan = PhysicalCompiler.compile(request, env).unwrap(); + let snapshot: BackendLocalPlanningInput = serde_json::from_value(value).unwrap(); + let (request, env) = snapshot.into_physical_compilation_request().unwrap(); + let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); assert!(plan .precompute_plan .materializations @@ -5362,9 +5556,11 @@ pub(crate) mod tests { // A full-window producer keeps overlapping accumulators alive even before publication. #[test] fn derived_window_regression_resident_cost() { - let template = planning_snapshot().implementation.implementation_cost; + let template = planning_snapshot() + .physical_inputs + .default_window_cost_quote; let mut lifecycle = planning_lifecycle(); - lifecycle.costs = LifecycleCostEvidence { + lifecycle.costs = LifecycleUnitCosts { build: 0.0, maintenance_per_update: 0.0, read: 0.0, @@ -5396,8 +5592,8 @@ pub(crate) mod tests { interval: RepetitionInterval(interval), evaluation_phase: planner_types::workload::TimestampMs(0), }; - let (request, env) = snapshot.planning_request().unwrap(); - let plan = PhysicalCompiler.compile(request, env).unwrap(); + let (request, env) = snapshot.into_physical_compilation_request().unwrap(); + let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), expected_states); let entry = plan.query_plan.entries.values().next().unwrap(); let bindings = entry.materialization_bindings(); @@ -5434,8 +5630,8 @@ pub(crate) mod tests { evaluation_phase: planner_types::workload::TimestampMs(phase), }; entries.push(second); - let (request, env) = snapshot.planning_request().unwrap(); - let plan = PhysicalCompiler.compile(request, env).unwrap(); + let (request, env) = snapshot.into_physical_compilation_request().unwrap(); + let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!( plan.precompute_plan.materializations.len(), if phase == 0 { 1 } else { 2 } @@ -5454,11 +5650,11 @@ pub(crate) mod tests { #[test] fn derived_cost_overflow_is_rejected() { let snapshot = planning_snapshot(); - let (mut request, env) = snapshot.planning_request().unwrap(); + let (mut request, env) = snapshot.into_physical_compilation_request().unwrap(); let query = &mut request.queries[0]; - let mut lifecycle = query.lifecycle.clone(); + let mut lifecycle = query.summary_lifecycle_inputs.clone(); lifecycle.costs.build = f64::MAX; - let candidate = &mut query.window_implementations[0]; + let candidate = &mut query.window_realization_candidates[0]; candidate.cost = derived_window_cost( &candidate.cost, &lifecycle, @@ -5478,24 +5674,34 @@ pub(crate) mod tests { let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; entry.query = Query(query.into()); entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let (derived, _) = snapshot.clone().planning_request().unwrap(); - snapshot.implementation.window_candidates.insert( - query.into(), - derived.queries[0].window_implementations.clone(), - ); - let (request, env) = snapshot.planning_request().unwrap(); - assert!(request.synthesized_window_queries.is_empty()); - let plan = PhysicalCompiler.compile(request, env).unwrap(); + let (derived, _) = snapshot + .clone() + .into_physical_compilation_request() + .unwrap(); + snapshot + .physical_inputs + .window_realization_candidates_by_query + .insert( + query.into(), + derived.queries[0].window_realization_candidates.clone(), + ); + let (request, env) = snapshot.into_physical_compilation_request().unwrap(); + assert!(request.compiler_priced_window_query_ids.is_empty()); + let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 2); } - fn planning_lifecycle() -> LifecyclePlanningInput { - planning_snapshot().planning_request().unwrap().0.queries[0] - .lifecycle + fn planning_lifecycle() -> SummaryLifecyclePlanningInputs { + planning_snapshot() + .into_physical_compilation_request() + .unwrap() + .0 + .queries[0] + .summary_lifecycle_inputs .clone() } - fn planning_snapshot() -> BackendLocalPlanningSnapshot { + fn planning_snapshot() -> BackendLocalPlanningInput { serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) @@ -5507,7 +5713,9 @@ pub(crate) mod tests { // answers with results that only change once per window. #[test] fn derived_window_candidate_follows_the_evaluation_cadence() { - let cost = planning_snapshot().implementation.implementation_cost; + let cost = planning_snapshot() + .physical_inputs + .default_window_cost_quote; let expr = crate::query_parser::parse_query_expr_canonical( "quantile_over_time(0.5, data[5m])", AccuracyTarget::Exact, @@ -5520,7 +5728,7 @@ pub(crate) mod tests { assert_eq!( derived .iter() - .map(|c| (c.implementation_id.as_str(), c.layout.clone())) + .map(|c| (c.realization_id.as_str(), c.layout.clone())) .collect::>(), vec![ ( @@ -5550,7 +5758,9 @@ pub(crate) mod tests { // not the magnitudes. #[test] fn derived_window_layout_prices_write_against_read_amplification() { - let cost = planning_snapshot().implementation.implementation_cost; + let cost = planning_snapshot() + .physical_inputs + .default_window_cost_quote; let expr = crate::query_parser::parse_query_expr_canonical( "quantile_over_time(0.5, data[5m])", AccuracyTarget::Exact, @@ -5598,7 +5808,9 @@ pub(crate) mod tests { // framework/layout table, so there is no alternative to price against it. #[test] fn tumbling_shapes_have_no_layout_alternative_to_rank() { - let cost = planning_snapshot().implementation.implementation_cost; + let cost = planning_snapshot() + .physical_inputs + .default_window_cost_quote; let expr = crate::query_parser::parse_query_expr_canonical( "quantile_over_time(0.5, data[5m])", AccuracyTarget::Exact, @@ -5614,7 +5826,7 @@ pub(crate) mod tests { 0, ); assert_eq!(derived.len(), 1); - assert_eq!(derived[0].implementation_id, "id"); + assert_eq!(derived[0].realization_id, "id"); assert_eq!(derived[0].framework, SummaryWindowFramework::Tumbling); } @@ -5623,8 +5835,10 @@ pub(crate) mod tests { #[test] fn derived_window_candidate_shapes_are_accepted_by_validation() { let snapshot = planning_snapshot(); - let (request, environment) = snapshot.planning_request().unwrap(); - let cost = planning_snapshot().implementation.implementation_cost; + let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); + let cost = planning_snapshot() + .physical_inputs + .default_window_cost_quote; for (lookback_ms, evaluation_ms) in [ (300_000, 30_000), (300_000, 300_000), @@ -5632,13 +5846,13 @@ pub(crate) mod tests { (60_000, 90_000), ] { let mut query = request.queries[0].clone(); - query.window_secs = lookback_ms / 1_000; + query.query_lookback_seconds = lookback_ms / 1_000; let expr = crate::query_parser::parse_query_expr_canonical( &format!("quantile_over_time(0.5, data[{}s])", lookback_ms / 1_000), AccuracyTarget::Exact, ) .unwrap(); - query.window_implementations = derived_window_candidates( + query.window_realization_candidates = derived_window_candidates( "derived", &expr, lookback_ms, @@ -5658,7 +5872,9 @@ pub(crate) mod tests { // previous tumbling shape rather than emitting something unschedulable. #[test] fn derived_window_candidate_stays_tumbling_without_a_dividing_cadence() { - let cost = planning_snapshot().implementation.implementation_cost; + let cost = planning_snapshot() + .physical_inputs + .default_window_cost_quote; let expr = crate::query_parser::parse_query_expr_canonical( "quantile_over_time(0.5, data[5m])", AccuracyTarget::Exact, @@ -5710,7 +5926,7 @@ pub(crate) mod tests { &expr, 60_000, 60_000, - snapshot.implementation.implementation_cost.clone(), + snapshot.physical_inputs.default_window_cost_quote.clone(), &planning_lifecycle(), 0, ) @@ -5719,11 +5935,14 @@ pub(crate) mod tests { supplied.slide_secs = 20; supplied.layout = asap_types::WindowMaterializationLayout::Pane { pane_secs: 20 }; snapshot - .implementation - .window_candidates + .physical_inputs + .window_realization_candidates_by_query .insert(query_string, vec![supplied.clone()]); - let (request, _) = snapshot.planning_request().unwrap(); - assert_eq!(request.queries[0].window_implementations, vec![supplied]); + let (request, _) = snapshot.into_physical_compilation_request().unwrap(); + assert_eq!( + request.queries[0].window_realization_candidates, + vec![supplied] + ); } // End to end: the retained-state count is derived from the pane width, so @@ -5732,8 +5951,10 @@ pub(crate) mod tests { #[test] fn retained_state_count_follows_the_derived_pane_width() { let snapshot = planning_snapshot(); - let (request, environment) = snapshot.planning_request().unwrap(); - let plan = PhysicalCompiler.compile(request, environment).unwrap(); + let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); + let plan = PhysicalPlanCompiler + .compile_promql(request, environment) + .unwrap(); assert_eq!( plan.precompute_plan.materializations[0].num_aggregates_to_retain, Some(7) @@ -5756,8 +5977,10 @@ pub(crate) mod tests { evaluation_phase: planner_types::workload::TimestampMs(0), }; } - let (request, environment) = snapshot.planning_request().unwrap(); - let plan = PhysicalCompiler.compile(request, environment).unwrap(); + let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); + let plan = PhysicalPlanCompiler + .compile_promql(request, environment) + .unwrap(); let materialization = &plan.precompute_plan.materializations[0]; assert_eq!( ( @@ -5782,8 +6005,8 @@ pub(crate) mod tests { // with each operand keeping its own range. #[test] fn composable_binary_summarizes_each_prometheus_filtered_operand() { - use crate::query_plan::{logical::LogicalOperator, QueryPlanNode}; - let mut snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + use crate::query_plan::{logical::ResidualQueryOperator, QueryPlanNode}; + let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); @@ -5791,8 +6014,8 @@ pub(crate) mod tests { entry.query = Query("sum(sum_over_time(a[1m])) / sum(sum_over_time(b{job!=\"x\"}[5m]))".into()); entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let (request, env) = snapshot.planning_request().unwrap(); - let plan = PhysicalCompiler.compile(request, env).unwrap(); + let (request, env) = snapshot.into_physical_compilation_request().unwrap(); + let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); let query = plan.query_plan.entries.values().next().unwrap(); let bindings = query.materialization_bindings(); // Both operands now hold a summary. The filtered denominator is no @@ -5827,7 +6050,7 @@ pub(crate) mod tests { assert!(!query.nodes.values().any(|node| matches!( node, QueryPlanNode::Logical { - operator: LogicalOperator::Scan { .. }, + operator: ResidualQueryOperator::Scan { .. }, .. } ))); @@ -5843,13 +6066,15 @@ pub(crate) mod tests { "sum_over_time(m{job!~\"a.*\"}[1m])", ] { let request = request("scope", query); - let selected = collect_selected_materializations(&request.queries[0].post_asap, false); + let selected = + collect_selected_materializations(&request.queries[0].selected_plan_root, false); let selected = selected.unwrap(); assert_eq!(selected.len(), 1, "{query}"); assert!(!selected[0].spatial_filter.is_empty(), "{query}"); } let request = request("scope", "sum_over_time(m[1m] offset 1h)"); - let selected = collect_selected_materializations(&request.queries[0].post_asap, false); + let selected = + collect_selected_materializations(&request.queries[0].selected_plan_root, false); assert!(selected.is_err() || selected.unwrap().is_empty()); } @@ -5862,19 +6087,22 @@ pub(crate) mod tests { "sum(avg_over_time(m{job=~\".+\"}[6h]))", "sum(rate(m[5m] offset 1h))", ] { - let mut snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; entry.query = Query(query.into()); entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let (mut request, environment) = snapshot.planning_request().unwrap(); - request = super::super::workload_cost::with_exact_alternative(request) - .unwrap() - .pop() + let (mut request, environment) = snapshot.into_physical_compilation_request().unwrap(); + request = + super::super::workload_cost::enumerate_exact_and_materialized_candidates(request) + .unwrap() + .pop() + .unwrap(); + let bundle = PhysicalPlanCompiler + .compile_promql(request, environment) .unwrap(); - let bundle = PhysicalCompiler.compile(request, environment).unwrap(); assert!(bundle.precompute_plan.materializations.is_empty()); assert!(bundle.query_plan.entries.values().all(|entry| matches!( entry.nodes[&entry.root], @@ -5886,11 +6114,13 @@ pub(crate) mod tests { // Projections reference one immutable snapshot and reject drift or foreign state. #[test] fn catalog_projection_rejects_missing_stale_and_foreign_references() { - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + let snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); - let bundle = quoted_snapshot(snapshot, false).compile().unwrap(); + let bundle = quoted_snapshot(snapshot, crate::physical::compiler::QueryFrontend::PromQl) + .compile_promql() + .unwrap(); let catalog = &bundle.summary_catalog; let mut transmission = bundle.transmission_plan.clone(); transmission.validate_against_catalog(catalog).unwrap(); @@ -5949,7 +6179,7 @@ pub(crate) mod tests { #[test] fn canonical_snapshot_preserves_shared_bindings_after_serialization() { // Two different registered readouts survive publication with one state. - let mut snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); @@ -5958,7 +6188,9 @@ pub(crate) mod tests { let mut second = entries[0].clone(); second.query = Query("sum(sum_over_time(m[1m])) * 2".into()); entries.push(second); - let bundle = quoted_snapshot(snapshot, false).compile().unwrap(); + let bundle = quoted_snapshot(snapshot, crate::physical::compiler::QueryFrontend::PromQl) + .compile_promql() + .unwrap(); assert_eq!(bundle.query_plan.entries.len(), 2); assert_eq!(bundle.precompute_plan.materializations.len(), 1); let query_plan: QueryPlan = @@ -5975,8 +6207,8 @@ pub(crate) mod tests { #[test] fn precompute_catalog_validates_without_backend_projection() { - let bundle = PhysicalCompiler - .compile( + let bundle = PhysicalPlanCompiler + .compile_promql( request("catalog", "quantile_over_time(0.99, m[1m])"), environment(10_000), ) @@ -6032,13 +6264,13 @@ pub(crate) mod tests { #[test] fn publication_is_catalog_authoritative_and_round_trips() { - let bundle = PhysicalCompiler - .compile( + let bundle = PhysicalPlanCompiler + .compile_promql( request("publication", "quantile_over_time(0.99, m[1m])"), environment(10_000), ) .unwrap(); - let publication = bundle.publication().unwrap(); + let publication = bundle.to_publication_artifact().unwrap(); let json = serde_json::to_value(&publication).unwrap(); let mut decoded: super::super::publication::PhysicalPlanPublication = serde_json::from_value(json).unwrap(); @@ -6057,8 +6289,8 @@ pub(crate) mod tests { #[test] fn compiles_one_decision_into_matching_collector_and_backend_views() { - let bundle = PhysicalCompiler - .compile( + let bundle = PhysicalPlanCompiler + .compile_promql( request("q-quantile", "quantile_over_time(0.99, m[1m])"), environment(10_000), ) @@ -6174,7 +6406,7 @@ pub(crate) mod tests { SummaryWindowFramework::Tumbling ); assert_eq!( - plan.materializations[0].window_implementation_id, + plan.materializations[0].window_realization_id, "collector-tumbling-v1" ); assert_eq!(plan.materializations[0].slide_secs, 60); @@ -6196,8 +6428,8 @@ pub(crate) mod tests { #[test] fn backend_local_hll_and_envelope_ingest_are_supported() { - let bundle = PhysicalCompiler - .compile( + let bundle = PhysicalPlanCompiler + .compile_promql( request("q", "quantile_over_time(0.99, m[1m])"), environment(10_000), ) @@ -6226,8 +6458,8 @@ pub(crate) mod tests { #[test] fn backend_local_precompute_contract_has_no_collector_producers() { - let bundle = PhysicalCompiler - .compile( + let bundle = PhysicalPlanCompiler + .compile_promql( request("q-quantile", "quantile_over_time(0.99, m[1m])"), environment(10_000), ) @@ -6246,7 +6478,7 @@ pub(crate) mod tests { assert_eq!(plan.ingest.timestamp_unit, TimestampUnit::UnixMilliseconds); assert!(plan.producers.is_empty()); plan.validate().expect("valid backend-local projection"); - crate::physical::compiler::compile_transmission_plan( + crate::physical::compiler::build_transmission_plan( bundle.envelope, &plan, &BTreeMap::new(), @@ -6295,26 +6527,26 @@ pub(crate) mod tests { }; let mut environment = environment(10_000); environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - environment.collector_ids.clear(); + environment.target_collector_ids.clear(); let template = request("template", "sum(sum_over_time(m[1m]))") .queries .remove(0); - let snapshot = BackendLocalPlanningSnapshot { - snapshot_version: 2, + let snapshot = BackendLocalPlanningInput { + schema_version: 2, workload_cost_evidence: None, query_workload, data_workload, - implementation: BackendLocalImplementation { - window_candidates: HashMap::new(), - lifecycle_costs: template.lifecycle.costs, + physical_inputs: BackendLocalPhysicalInputs { + window_realization_candidates_by_query: HashMap::new(), + lifecycle_costs: template.summary_lifecycle_inputs.costs, evidence_observed_at_unix_ms: 9_500, evidence_valid_for_ms: 60_000, horizon_seconds: 300.0, - window_implementation_id: "backend-tumbling-v1".into(), - implementation_cost: template.window_implementations[0].cost.clone(), + default_window_realization_id: "backend-tumbling-v1".into(), + default_window_cost_quote: template.window_realization_candidates[0].cost.clone(), source_sample_interval_ms: None, - query_staleness_margin_ms: 0, - max_retained_summary_bytes: DEFAULT_RETAINED_SUMMARY_MEMORY_BUDGET_BYTES, + query_retention_margin_ms: 0, + retained_summary_memory_budget_bytes: DEFAULT_RETAINED_SUMMARY_MEMORY_BUDGET_BYTES, topk_evidence: HashMap::new(), exact_composition_costs: HashMap::new(), erp: None, @@ -6324,19 +6556,25 @@ pub(crate) mod tests { assert_eq!( snapshot .clone() - .planning_request() + .into_physical_compilation_request() .unwrap() .0 .query_workload .as_ref(), Some(&snapshot.query_workload) ); - let first = quoted_snapshot(snapshot.clone(), false) - .compile() - .expect("first deterministic plan"); - let second = quoted_snapshot(snapshot.clone(), false) - .compile() - .expect("second deterministic plan"); + let first = quoted_snapshot( + snapshot.clone(), + crate::physical::compiler::QueryFrontend::PromQl, + ) + .compile_promql() + .expect("first deterministic plan"); + let second = quoted_snapshot( + snapshot.clone(), + crate::physical::compiler::QueryFrontend::PromQl, + ) + .compile_promql() + .expect("second deterministic plan"); assert_eq!(first.envelope, second.envelope); assert_eq!(first.summary_catalog, second.summary_catalog); assert_eq!(first.query_plan, second.query_plan); @@ -6347,10 +6585,10 @@ pub(crate) mod tests { ); let encoded = serde_json::to_vec(&snapshot).expect("serialize startup snapshot"); - let decoded: BackendLocalPlanningSnapshot = + let decoded: BackendLocalPlanningInput = serde_json::from_slice(&encoded).expect("deserialize startup snapshot"); - let bundle = quoted_snapshot(decoded, false) - .compile() + let bundle = quoted_snapshot(decoded, crate::physical::compiler::QueryFrontend::PromQl) + .compile_promql() .expect("canonical startup planning"); assert!(bundle.collector_plans.is_empty()); @@ -6405,8 +6643,9 @@ pub(crate) mod tests { ] { let mut deployment = environment(10_000); deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - deployment.collector_ids.clear(); - let compiled = PhysicalCompiler.compile(request(query_id, promql), deployment); + deployment.target_collector_ids.clear(); + let compiled = + PhysicalPlanCompiler.compile_promql(request(query_id, promql), deployment); let plan = compiled.unwrap_or_else(|error| panic!("{promql} must compile: {error}")); assert_eq!(plan.summary_catalog.materializations.len(), 1, "{promql}"); assert_eq!(plan.query_plan.entries.len(), 1, "{promql}"); @@ -6433,31 +6672,35 @@ pub(crate) mod tests { #[test] fn checked_in_per_entity_snapshot_preserves_native_alternative() { let source = include_str!("../../../docs/examples/asapquery-planning-snapshot.json"); - let snapshot: BackendLocalPlanningSnapshot = + let snapshot: BackendLocalPlanningInput = serde_json::from_str(source).expect("strict canonical workload fixture"); let encoded = serde_json::to_value(&snapshot).expect("canonical snapshot value"); let fixture: serde_json::Value = serde_json::from_str(source).expect("fixture JSON"); assert_eq!(encoded, fixture); assert!( - snapshot.clone().compile().is_err(), + snapshot.clone().compile_promql().is_err(), "discovery fixtures must be priced before deployment" ); - let (local, env) = snapshot.clone().planning_request().unwrap(); - let isolated = PhysicalCompiler.compile(local, env).unwrap(); + let (local, env) = snapshot + .clone() + .into_physical_compilation_request() + .unwrap(); + let isolated = PhysicalPlanCompiler.compile_promql(local, env).unwrap(); assert!(!isolated.precompute_plan.materializations.is_empty()); assert!(isolated .precompute_plan .materializations .iter() .all(|state| state.partitioning.is_some())); - let (request, environment) = snapshot.planning_request().unwrap(); - let native = crate::physical::workload_cost::with_exact_alternative(request) - .unwrap() - .pop() - .unwrap(); - let plan = PhysicalCompiler - .compile(native, environment) + let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); + let native = + crate::physical::workload_cost::enumerate_exact_and_materialized_candidates(request) + .unwrap() + .pop() + .unwrap(); + let plan = PhysicalPlanCompiler + .compile_promql(native, environment) .expect("native fixture compiles"); assert!(plan.precompute_plan.materializations.is_empty()); assert!(plan.collector_plans.is_empty()); @@ -6473,27 +6716,31 @@ pub(crate) mod tests { fn compatibility_demo_preserves_complete_native_query_matrix() { let source = include_str!("../../../docs/examples/asapquery-compatibility-demo-snapshot.json"); - let snapshot: BackendLocalPlanningSnapshot = + let snapshot: BackendLocalPlanningInput = serde_json::from_str(source).expect("strict compatibility demo fixture"); assert!( - snapshot.clone().compile().is_err(), + snapshot.clone().compile_promql().is_err(), "discovery fixtures must be priced before deployment" ); - let (local, env) = snapshot.clone().planning_request().unwrap(); - let isolated = PhysicalCompiler.compile(local, env).unwrap(); + let (local, env) = snapshot + .clone() + .into_physical_compilation_request() + .unwrap(); + let isolated = PhysicalPlanCompiler.compile_promql(local, env).unwrap(); assert!(!isolated.precompute_plan.materializations.is_empty()); assert!(isolated .precompute_plan .materializations .iter() .all(|state| state.partitioning.is_some())); - let (request, environment) = snapshot.planning_request().unwrap(); - let native = crate::physical::workload_cost::with_exact_alternative(request) - .unwrap() - .pop() - .unwrap(); - let plan = PhysicalCompiler - .compile(native, environment) + let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); + let native = + crate::physical::workload_cost::enumerate_exact_and_materialized_candidates(request) + .unwrap() + .pop() + .unwrap(); + let plan = PhysicalPlanCompiler + .compile_promql(native, environment) .expect("native demo compiles"); assert!(plan.collector_plans.is_empty()); @@ -6514,15 +6761,15 @@ pub(crate) mod tests { #[test] fn multiple_readouts_share_one_precompute_materialization() { - let mut planning_request = request("q-p90", "quantile_over_time(0.90, m[1m])"); + let mut compilation_request = request("q-p90", "quantile_over_time(0.90, m[1m])"); let second = request("q-p99", "quantile_over_time(0.99, m[1m])") .queries .into_iter() .next() .unwrap(); - planning_request.queries.push(second); - let bundle = PhysicalCompiler - .compile(planning_request, environment(10_000)) + compilation_request.queries.push(second); + let bundle = PhysicalPlanCompiler + .compile_promql(compilation_request, environment(10_000)) .unwrap(); assert_eq!(bundle.query_plan.entries.len(), 2); @@ -6534,10 +6781,10 @@ pub(crate) mod tests { #[test] fn compiles_every_materialization_leaf_in_a_merge_dag() { - let mut planning_request = request("q-merge", "quantile_over_time(0.90, m[1m])"); + let mut compilation_request = request("q-merge", "quantile_over_time(0.90, m[1m])"); let right_request = request("q-right", "quantile_over_time(0.90, n[1m])"); - let left_root = planning_request.queries[0].post_asap.clone(); - let right_root = right_request.queries[0].post_asap.clone(); + let left_root = compilation_request.queries[0].selected_plan_root.clone(); + let right_root = right_request.queries[0].selected_plan_root.clone(); let (left, query) = match &left_root.expr { SummaryExpr::SummaryEstimate { summary_input, @@ -6556,7 +6803,7 @@ pub(crate) mod tests { schema: left.schema.clone(), guarantee: None, }); - planning_request.queries[0].post_asap = Rc::new(SummaryNode { + compilation_request.queries[0].selected_plan_root = Rc::new(SummaryNode { expr: SummaryExpr::SummaryEstimate { summary_input: merge, query, @@ -6565,8 +6812,8 @@ pub(crate) mod tests { guarantee: left_root.guarantee.clone(), }); - let bundle = PhysicalCompiler - .compile(planning_request, environment(10_000)) + let bundle = PhysicalPlanCompiler + .compile_promql(compilation_request, environment(10_000)) .expect("compile merged post-ASAP DAG"); assert_eq!(bundle.summary_catalog.materializations.len(), 2); assert_eq!(bundle.precompute_plan.materializations.len(), 2); @@ -6615,8 +6862,8 @@ pub(crate) mod tests { #[test] fn precompute_schema_must_match_materialization_semantics() { - let bundle = PhysicalCompiler - .compile( + let bundle = PhysicalPlanCompiler + .compile_promql( request("q-quantile", "quantile_over_time(0.99, m[1m])"), environment(10_000), ) @@ -6638,16 +6885,16 @@ pub(crate) mod tests { { let mut request = request("q", "sum(sum_over_time(m[1m]))"); let query = &mut request.queries[0]; - let mut small = query.window_implementations[0].clone(); - small.implementation_id = "small".into(); + let mut small = query.window_realization_candidates[0].clone(); + small.realization_id = "small".into(); small.layout = asap_types::WindowMaterializationLayout::Pane { pane_secs: 10 }; small.cost.weighted_cost = small_cost; - query.window_implementations[0].implementation_id = "large".into(); - query.window_implementations.push(small); + query.window_realization_candidates[0].realization_id = "large".into(); + query.window_realization_candidates.push(small); let mut env = environment(10_000); env.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - env.collector_ids.clear(); - let bundle = PhysicalCompiler.compile(request, env).unwrap(); + env.target_collector_ids.clear(); + let bundle = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); let materialization = bundle.precompute_plan.materializations.first().unwrap(); assert_eq!(materialization.window_size, 60); assert_eq!( @@ -6664,7 +6911,7 @@ pub(crate) mod tests { expected_pane_secs * 1000 ); assert_eq!( - bundle.lifecycle_estimates[0].window_implementation_id, + bundle.lifecycle_estimates[0].window_realization_id, expected_id ); } @@ -6678,8 +6925,8 @@ pub(crate) mod tests { ] { let mut request = request("q", "sum(sum_over_time(m[1m]))"); let query = &mut request.queries[0]; - let mut panes = query.window_implementations[0].clone(); - panes.implementation_id = "mergeable-panes".into(); + let mut panes = query.window_realization_candidates[0].clone(); + panes.realization_id = "mergeable-panes".into(); panes.framework = SummaryWindowFramework::Sliding; panes.slide_secs = 10; panes.layout = asap_types::WindowMaterializationLayout::Pane { pane_secs: 10 }; @@ -6688,7 +6935,7 @@ pub(crate) mod tests { panes.cost.storage_bytes = 1_024; let mut full = panes.clone(); - full.implementation_id = "full-window".into(); + full.realization_id = "full-window".into(); full.layout = asap_types::WindowMaterializationLayout::FullWindow; full.cost.weighted_cost = full_cost; // Full windows spend more update CPU and retained bytes, while @@ -6696,14 +6943,14 @@ pub(crate) mod tests { // includes the workload's measured read frequency and cardinality. full.cost.cpu_cost = 20.0; full.cost.storage_bytes = 64 * 1_024; - query.window_implementations = vec![panes, full]; + query.window_realization_candidates = vec![panes, full]; let mut env = environment(10_000); env.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - env.collector_ids.clear(); - let bundle = PhysicalCompiler.compile(request, env).unwrap(); + env.target_collector_ids.clear(); + let bundle = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!( - bundle.lifecycle_estimates[0].window_implementation_id, + bundle.lifecycle_estimates[0].window_realization_id, expected_id ); assert_eq!( @@ -6715,7 +6962,11 @@ pub(crate) mod tests { ); // Publication validates stored extent independently of slide cadence. // In particular the full-window candidate stores 60s at a 10s slide. - bundle.publication().unwrap().validate().unwrap(); + bundle + .to_publication_artifact() + .unwrap() + .validate() + .unwrap(); assert_eq!(bundle.precompute_plan.materializations[0].window_size, 60); assert_eq!( bundle.precompute_plan.materializations[0].slide_interval, @@ -6732,23 +6983,23 @@ pub(crate) mod tests { let mut second = request("q40", "sum(sum_over_time(m[40s]))") .queries .remove(0); - workload.queries[0].window_secs = 20; - workload.queries[0].window_implementations[0].window_secs = 20; - second.window_secs = 40; - second.lifecycle.evaluation_interval_ms = 20_000; - second.window_implementations[0].window_secs = 40; + workload.queries[0].query_lookback_seconds = 20; + workload.queries[0].window_realization_candidates[0].window_secs = 20; + second.query_lookback_seconds = 40; + second.summary_lifecycle_inputs.evaluation_interval_ms = 20_000; + second.window_realization_candidates[0].window_secs = 40; workload.queries.push(second); for query in &mut workload.queries { - query.window_implementations[0].framework = SummaryWindowFramework::Sliding; - query.window_implementations[0].slide_secs = 10; - query.window_implementations[0].layout = + query.window_realization_candidates[0].framework = SummaryWindowFramework::Sliding; + query.window_realization_candidates[0].slide_secs = 10; + query.window_realization_candidates[0].layout = asap_types::WindowMaterializationLayout::Pane { pane_secs: 10 }; - query.window_implementations[0].implementation_id = "shared-ten-second-pane".into(); + query.window_realization_candidates[0].realization_id = "shared-ten-second-pane".into(); } let mut env = environment(10_000); env.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - env.collector_ids.clear(); - let bundle = PhysicalCompiler.compile(workload, env).unwrap(); + env.target_collector_ids.clear(); + let bundle = PhysicalPlanCompiler.compile_promql(workload, env).unwrap(); assert_eq!(bundle.precompute_plan.materializations.len(), 2); } @@ -6756,27 +7007,27 @@ pub(crate) mod tests { #[test] fn tumbling_sizes_reject_non_divisors() { let mut request = request("q", "sum(sum_over_time(m[1m]))"); - request.queries[0].window_implementations[0].layout = + request.queries[0].window_realization_candidates[0].layout = asap_types::WindowMaterializationLayout::Pane { pane_secs: 7 }; let mut env = environment(10_000); env.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - assert!(PhysicalCompiler.compile(request, env).is_err()); + assert!(PhysicalPlanCompiler.compile_promql(request, env).is_err()); } #[test] fn missing_window_implementation_evidence_fails_closed() { let mut request = request("q-window", "quantile_over_time(0.99, m[1m])"); - request.queries[0].window_implementations.clear(); - let error = PhysicalCompiler - .compile(request, environment(10_000)) + request.queries[0].window_realization_candidates.clear(); + let error = PhysicalPlanCompiler + .compile_promql(request, environment(10_000)) .expect_err("Planner must not receive a zero-cost invented window"); assert!(matches!(error, CompileError::Lifecycle { .. })); } #[test] fn precompute_plan_rejects_schema_or_producer_drift() { - let bundle = PhysicalCompiler - .compile( + let bundle = PhysicalPlanCompiler + .compile_promql( request("q-quantile", "quantile_over_time(0.99, m[1m])"), environment(10_000), ) @@ -6799,8 +7050,8 @@ pub(crate) mod tests { #[test] fn precompute_plan_rejects_empty_or_duplicate_schema_ids() { - let bundle = PhysicalCompiler - .compile( + let bundle = PhysicalPlanCompiler + .compile_promql( request("q-quantile", "quantile_over_time(0.99, m[1m])"), environment(10_000), ) @@ -6840,8 +7091,8 @@ pub(crate) mod tests { }), ) .expect("selection accepts evidence before freshness validation"); - let error = PhysicalCompiler - .compile(request, environment(100_000)) + let error = PhysicalPlanCompiler + .compile_promql(request, environment(100_000)) .expect_err("stale certificate must fail"); assert!(matches!(error, CompileError::InvalidEvidence { .. })); } @@ -6861,16 +7112,16 @@ pub(crate) mod tests { ) .expect("selection occurs before deployment-time freshness validation"); assert!(matches!( - PhysicalCompiler.compile(topk, environment(10_000)), + PhysicalPlanCompiler.compile_promql(topk, environment(10_000)), Err(CompileError::InvalidEvidence { .. }) )); let mut window = request("q-window", "quantile_over_time(0.99, m[1m])"); - window.queries[0].window_implementations[0] + window.queries[0].window_realization_candidates[0] .cost .observed_at_unix_ms = 10_001; assert!(matches!( - PhysicalCompiler.compile(window, environment(10_000)), + PhysicalPlanCompiler.compile_promql(window, environment(10_000)), Err(CompileError::Lifecycle { .. }) )); } @@ -6889,8 +7140,8 @@ pub(crate) mod tests { }), ) .expect("selection accepts valid evidence"); - let bundle = PhysicalCompiler - .compile(request, environment(10_000)) + let bundle = PhysicalPlanCompiler + .compile_promql(request, environment(10_000)) .expect("certified TopK compiles"); assert_eq!(bundle.summary_catalog.materializations.len(), 1); assert_eq!( @@ -6906,7 +7157,7 @@ pub(crate) mod tests { let mut request = request("q", "quantile_over_time(0.9, m[1m])"); request.planner_revision = "different".into(); assert!(matches!( - PhysicalCompiler.compile(request, environment(10_000)), + PhysicalPlanCompiler.compile_promql(request, environment(10_000)), Err(CompileError::PlannerRevision { .. }) )); } @@ -6914,10 +7165,14 @@ pub(crate) mod tests { #[test] fn stale_lifecycle_evidence_fails_closed() { let mut request = request("q", "quantile_over_time(0.9, m[1m])"); - request.queries[0].lifecycle.evidence_observed_at_unix_ms = 1; - request.queries[0].lifecycle.evidence_valid_for_ms = 10; + request.queries[0] + .summary_lifecycle_inputs + .evidence_observed_at_unix_ms = 1; + request.queries[0] + .summary_lifecycle_inputs + .evidence_valid_for_ms = 10; assert!(matches!( - PhysicalCompiler.compile(request, environment(10_000)), + PhysicalPlanCompiler.compile_promql(request, environment(10_000)), Err(CompileError::Lifecycle { .. }) )); } @@ -6943,8 +7198,8 @@ pub(crate) mod tests { #[test] fn runtime_policy_encoding_is_checked() { - let bundle = PhysicalCompiler - .compile( + let bundle = PhysicalPlanCompiler + .compile_promql( request("q", "quantile_over_time(0.99, m[1m])"), environment(10_000), ) @@ -6971,12 +7226,12 @@ pub(crate) mod tests { }), ) .expect("frequency selection"); - request.queries[0].runtime_policy.delta = Some(DeltaPolicy { + request.queries[0].materialization_runtime_policy.delta = Some(DeltaPolicy { absolute_threshold: 0.0, gos: None, }); - let bundle = PhysicalCompiler - .compile(request, environment(10_000)) + let bundle = PhysicalPlanCompiler + .compile_promql(request, environment(10_000)) .expect("delta-capable physical plan"); let rule = &bundle.transmission_plan.rules[0]; assert_eq!(rule.mode, TransmissionMode::Delta); @@ -7010,8 +7265,8 @@ pub(crate) mod tests { #[test] fn runtime_adaptation_requires_fresh_exact_evidence_and_successor_version() { - let bundle = PhysicalCompiler - .compile( + let bundle = PhysicalPlanCompiler + .compile_promql( request("q", "quantile_over_time(0.99, m[1m])"), environment(10_000), ) diff --git a/control_plane/src/physical/deployment_cost/delta.rs b/control_plane/src/physical/deployment_cost/delta.rs index 12f19f0fd..2f3f70366 100644 --- a/control_plane/src/physical/deployment_cost/delta.rs +++ b/control_plane/src/physical/deployment_cost/delta.rs @@ -129,7 +129,7 @@ pub fn delta_benchmark_table() -> HashMap { /// A shorter flush period means: /// • fewer inserts accumulate per period → lower fill rate → better delta /// • more flushes per second → higher total CPU overhead -pub fn flush_period_secs(plan: &CollectionPlan, w: &QueryWorkload) -> f64 { +pub fn flush_period_secs(plan: &CollectionPlan, w: &LegacyMetricWorkload) -> f64 { if let Some(wd) = plan.agent_config.window_duration { return wd.as_secs_f64(); } @@ -196,7 +196,7 @@ fn estimate_distinct_keys(inserts_per_flush: f64, wc: &WorkloadCharacteristics) pub fn estimate_fill_rate( wc: &WorkloadCharacteristics, plan: &CollectionPlan, - w: &QueryWorkload, + w: &LegacyMetricWorkload, ) -> f64 { let flush_secs = flush_period_secs(plan, w); let inserts_per_flush = wc.samples_per_sec_per_series * wc.series_count as f64 * flush_secs; @@ -313,7 +313,7 @@ pub const RAW_PASSTHROUGH_SAMPLE_RATE_THRESHOLD: f64 = 10.0; /// 7. Otherwise → UseDelta. pub fn decide_delta( plan: &CollectionPlan, - w: &QueryWorkload, + w: &LegacyMetricWorkload, wc: &WorkloadCharacteristics, bytes_per_series_per_sec: f64, ) -> (DeltaDecision, TransmissionCostSummary) { @@ -474,8 +474,8 @@ mod tests { use chrono::Utc; use std::collections::HashMap; - fn workload_for(agg: AggType) -> QueryWorkload { - QueryWorkload { + fn workload_for(agg: AggType) -> LegacyMetricWorkload { + LegacyMetricWorkload { metric_name: "m".into(), label_filters: HashMap::new(), group_by_labels: vec![], diff --git a/control_plane/src/physical/deployment_cost/mod.rs b/control_plane/src/physical/deployment_cost/mod.rs index 85b0e5cf0..d55c3f708 100644 --- a/control_plane/src/physical/deployment_cost/mod.rs +++ b/control_plane/src/physical/deployment_cost/mod.rs @@ -101,7 +101,7 @@ pub struct PlanScore { /// Estimates resource costs for a given plan + workload using the provided cost table. pub fn score_with( plan: &CollectionPlan, - w: &QueryWorkload, + w: &LegacyMetricWorkload, table: &HashMap, ) -> PlanScore { let st = &plan.agent_config.sketch_type; @@ -136,7 +136,7 @@ pub fn score_with( } /// Estimates resource costs for a given plan + workload. -pub fn score(plan: &CollectionPlan, w: &QueryWorkload) -> PlanScore { +pub fn score(plan: &CollectionPlan, w: &LegacyMetricWorkload) -> PlanScore { let table = benchmark_table(); let st = &plan.agent_config.sketch_type; @@ -239,7 +239,11 @@ impl DeploymentCostPlanner { /// CPU / memory overhead, and raw vs. sketch bandwidth comparison. /// Pass `None` to use conservative defaults (1 000 series, 100 Hz, /// 100 B/sample, Zipf distribution, no memory budget). - pub fn plan(&self, w: &QueryWorkload, wc: Option<&WorkloadCharacteristics>) -> CollectionPlan { + pub fn plan( + &self, + w: &LegacyMetricWorkload, + wc: Option<&WorkloadCharacteristics>, + ) -> CollectionPlan { if !matches!(w.accuracy, crate::types_v2::AccuracyTarget::Epsilon(epsilon) if epsilon > 0.0) { return self.inner.plan(w); @@ -306,7 +310,7 @@ impl DeploymentCostPlanner { /// Runs the delta cost model and writes the decision into the plan using a provided cost table. fn apply_delta_decision_with( plan: &mut CollectionPlan, - w: &QueryWorkload, + w: &LegacyMetricWorkload, wc: &WorkloadCharacteristics, table: &HashMap, ) { @@ -350,8 +354,8 @@ mod tests { use std::collections::HashMap; use std::time::Duration; - fn workload(aggs: Vec) -> QueryWorkload { - QueryWorkload { + fn workload(aggs: Vec) -> LegacyMetricWorkload { + LegacyMetricWorkload { metric_name: "test".into(), label_filters: HashMap::new(), group_by_labels: vec![], @@ -408,7 +412,7 @@ mod tests { #[test] fn ddsketch_fails_tight_sla() { - let w = QueryWorkload { + let w = LegacyMetricWorkload { accuracy_sla: 0.001, accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.001), ..workload(vec![AggType::Quantile]) @@ -433,11 +437,11 @@ mod tests { #[test] fn dim_multiplier_increases_bandwidth() { - let w_few = QueryWorkload { + let w_few = LegacyMetricWorkload { group_by_labels: vec!["host".into()], ..workload(vec![AggType::Quantile]) }; - let w_many = QueryWorkload { + let w_many = LegacyMetricWorkload { group_by_labels: vec![ "host".into(), "service".into(), @@ -454,7 +458,7 @@ mod tests { #[test] fn kll_error_formula() { - let w = QueryWorkload { + let w = LegacyMetricWorkload { accuracy_sla: 0.02, accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.02), ..workload(vec![AggType::Quantile]) @@ -476,7 +480,7 @@ mod tests { (AggType::Cardinality, 0.01), (AggType::Frequency, 0.02), ] { - let w = QueryWorkload { + let w = LegacyMetricWorkload { accuracy_sla: sla, accuracy: crate::types_v2::AccuracyTarget::Epsilon(sla), ..workload(vec![agg]) @@ -493,7 +497,7 @@ mod tests { #[test] fn cost_model_prefers_lower_bandwidth_for_cardinality() { - let w = QueryWorkload { + let w = LegacyMetricWorkload { accuracy_sla: 0.02, accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.02), ..workload(vec![AggType::Cardinality]) diff --git a/control_plane/src/physical/deployment_cost/pareto.rs b/control_plane/src/physical/deployment_cost/pareto.rs index 11dcf819d..379e7debf 100644 --- a/control_plane/src/physical/deployment_cost/pareto.rs +++ b/control_plane/src/physical/deployment_cost/pareto.rs @@ -95,7 +95,7 @@ pub struct ParetoPoint { /// If `online_store` is `Some` the scoring uses EMA-blended costs; otherwise /// it falls back to the static benchmark table. pub fn pareto_frontier( - workload: &QueryWorkload, + workload: &LegacyMetricWorkload, wc: &WorkloadCharacteristics, weights: ObjectiveWeights, online_store: Option<&online::OnlineMetricsStore>, @@ -182,7 +182,7 @@ fn all_sketch_types() -> Vec { fn apply_delta( st: SketchType, plan: &mut CollectionPlan, - w: &QueryWorkload, + w: &LegacyMetricWorkload, wc: &WorkloadCharacteristics, table: &HashMap, ) { @@ -241,8 +241,8 @@ mod tests { use std::collections::HashMap; use std::time::Duration; - fn quantile_workload() -> QueryWorkload { - QueryWorkload { + fn quantile_workload() -> LegacyMetricWorkload { + LegacyMetricWorkload { metric_name: "latency".into(), label_filters: HashMap::new(), group_by_labels: vec![], @@ -369,7 +369,7 @@ mod tests { #[test] fn tight_sla_excludes_inaccurate_sketches() { - let w = QueryWorkload { + let w = LegacyMetricWorkload { accuracy_sla: 0.001, // very tight — only DDSketch at 0.1% accuracy can meet this accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.001), ..quantile_workload() diff --git a/control_plane/src/physical/erp.rs b/control_plane/src/physical/erp.rs index 9db3de486..2edc111a7 100644 --- a/control_plane/src/physical/erp.rs +++ b/control_plane/src/physical/erp.rs @@ -1457,11 +1457,14 @@ mod tests { query["query"] = "distinct_over_time(asap_demo_latency_ms[5s])".into(); query["requirements"]["accuracy"] = serde_json::json!({"explicit":{"Epsilon":0.05}}); fixture["query_workload"]["repeating_queries"] = serde_json::json!([query]); - let snapshot: crate::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: crate::physical::compiler::BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); - let plan = crate::physical::compiler::tests::quoted_snapshot(snapshot, false) - .compile() - .unwrap(); + let plan = crate::physical::compiler::tests::quoted_snapshot( + snapshot, + crate::physical::compiler::QueryFrontend::PromQl, + ) + .compile_promql() + .unwrap(); let (mut policy, mut observed) = online_population_fixture(); observed.catalog_generation = plan.summary_catalog.reference().unwrap(); observed.summary_definition_id = diff --git a/control_plane/src/physical/pane_reuse.rs b/control_plane/src/physical/pane_reuse.rs index ae34cb15f..f19a31951 100644 --- a/control_plane/src/physical/pane_reuse.rs +++ b/control_plane/src/physical/pane_reuse.rs @@ -1,6 +1,6 @@ use super::compiler::{ derived_window_cost, retained_state_count, CollectorMaterialization, - MaterializationLifecycleEstimate, PlanningRequest, RuntimeRulePolicy, + MaterializationLifecycleEstimate, PhysicalCompilationRequest, RuntimeRulePolicy, }; use planner_types::post_asap::{PostAsapNodeId, SummaryWindowFramework}; use std::collections::{BTreeMap, BTreeSet}; @@ -9,7 +9,7 @@ use std::collections::{BTreeMap, BTreeSet}; /// Only raw additive states are eligible; derived cohorts retain their full-window identity. #[allow(clippy::too_many_arguments)] pub(super) fn share_additive_panes( - request: &PlanningRequest, + request: &PhysicalCompilationRequest, materializations: &mut [asap_types::PrecomputeMaterialization], producers: &mut Vec, plan_producers: &mut [CollectorMaterialization], @@ -57,9 +57,11 @@ pub(super) fn share_additive_panes( // Explicitly priced implementations are not repriced or replaced. if consumers.iter().any(|index| { let q = &request.queries[*index]; - q.lifecycle != query.lifecycle - || q.accuracy != query.accuracy - || !request.synthesized_window_queries.contains(&q.query_id) + q.summary_lifecycle_inputs != query.summary_lifecycle_inputs + || q.accuracy_target != query.accuracy_target + || !request + .compiler_priced_window_query_ids + .contains(&q.query_id) }) { continue; } @@ -72,12 +74,12 @@ pub(super) fn share_additive_panes( }; let key = ( canonical.policy_fingerprint(), - serde_json::to_string(&query.lifecycle).unwrap(), + serde_json::to_string(&query.summary_lifecycle_inputs).unwrap(), serde_json::to_string(policy).unwrap(), ); - let mut maintenance = query.lifecycle.clone(); + let mut maintenance = query.summary_lifecycle_inputs.clone(); maintenance.costs.read = 0.0; - let Some(template) = query.window_implementations.first() else { + let Some(template) = query.window_realization_candidates.first() else { continue; }; let producer_cost = derived_window_cost( @@ -86,11 +88,12 @@ pub(super) fn share_additive_panes( m.window_size, m.slide_interval, &m.window_layout, - request.query_staleness_margin_ms, + request.query_retention_margin_ms, ) .weighted_cost; - let read_cost = query.lifecycle.costs.read * query.lifecycle.horizon_seconds - / (f64::from(query.lifecycle.evaluation_interval_ms) / 1000.0) + let read_cost = query.summary_lifecycle_inputs.costs.read + * query.summary_lifecycle_inputs.horizon_seconds + / (f64::from(query.summary_lifecycle_inputs.evaluation_interval_ms) / 1000.0) * (m.window_size / pane_secs) as f64 * consumers.len() as f64; offers.push(PaneReuseCandidate { @@ -113,7 +116,7 @@ pub(super) fn share_additive_panes( let mut canonical = physical[group.members[0]].1.clone(); canonical.num_aggregates_to_retain = Some(retained_state_count( group.lookback_ms, - request.query_staleness_margin_ms, + request.query_retention_margin_ms, canonical.slide_interval * 1000, &canonical.window_layout, )); @@ -139,7 +142,7 @@ pub(super) fn share_additive_panes( combined.expected_reads = 0.0; combined.expected_updates = 0.0; combined.lifecycle_cost = group.cost; - combined.window_implementation_id = format!("shared-pane-{}", new.0); + combined.window_realization_id = format!("shared-pane-{}", new.0); for index in group.members { let old = physical[index].0; if let Some(estimate) = estimates.remove(&old) { @@ -179,7 +182,7 @@ pub(super) fn share_additive_panes( producer.window_secs = canonical.window_size; producer.slide_secs = canonical.slide_interval; producer.abstract_window_framework = SummaryWindowFramework::Tumbling; - producer.window_implementation_id = estimates[&new].window_implementation_id.clone(); + producer.window_realization_id = estimates[&new].window_realization_id.clone(); } } let mut seen = BTreeSet::new(); diff --git a/control_plane/src/physical/plan_cache.rs b/control_plane/src/physical/plan_cache.rs index 58bbc3451..c8a01c10f 100644 --- a/control_plane/src/physical/plan_cache.rs +++ b/control_plane/src/physical/plan_cache.rs @@ -17,7 +17,7 @@ use std::collections::HashMap; use std::sync::{Arc, RwLock}; use crate::physical::deployment_cost::DeploymentCostPlanner; -use crate::types::{CollectionPlan, QueryWorkload, WorkloadCharacteristics}; +use crate::types::{CollectionPlan, LegacyMetricWorkload, WorkloadCharacteristics}; pub struct CachedDeploymentPlanner { inner: DeploymentCostPlanner, @@ -36,7 +36,7 @@ impl CachedDeploymentPlanner { /// establish a new baseline if this is the first request for the metric. pub fn plan( &self, - workload: &QueryWorkload, + workload: &LegacyMetricWorkload, wc: Option<&WorkloadCharacteristics>, ) -> CollectionPlan { let key = &workload.metric_name; @@ -80,8 +80,8 @@ mod tests { use std::collections::HashMap; use std::time::Duration; - fn workload(metric: &str) -> QueryWorkload { - QueryWorkload { + fn workload(metric: &str) -> LegacyMetricWorkload { + LegacyMetricWorkload { metric_name: metric.into(), label_filters: HashMap::new(), group_by_labels: vec![], diff --git a/control_plane/src/physical/post_asap/mod.rs b/control_plane/src/physical/post_asap/mod.rs index d446bc264..469db23e4 100644 --- a/control_plane/src/physical/post_asap/mod.rs +++ b/control_plane/src/physical/post_asap/mod.rs @@ -13,10 +13,10 @@ pub mod deployment_expr; pub mod lower; pub mod matcher; -#[cfg(test)] -mod tests; - // Re-exports — `crate::physical::post_asap::*` for downstream callers. pub use deployment_expr::{PhysicalExpr, PostAsapPlan}; pub use lower::{bind_query_expr, bind_query_expr_with_cost_model, BindingError}; pub use matcher::SummaryFamilyMatcher; + +#[cfg(test)] +mod tests; diff --git a/control_plane/src/physical/post_asap/tests.rs b/control_plane/src/physical/post_asap/tests.rs index 3735861c5..7f1d00444 100644 --- a/control_plane/src/physical/post_asap/tests.rs +++ b/control_plane/src/physical/post_asap/tests.rs @@ -770,7 +770,7 @@ fn phase_b_e2e_rate_falls_through_to_logical() { /// The legacy single-expression binder cannot choose a frequency sketch for /// value-ranked `topk(sum(rate(...)))` without membership evidence. The /// workload planner handles this query as query-time Sort+Limit over its -/// recursively planned child; its coverage lives in `query_plan::logical`. +/// recursively planned child; its coverage lives in `query_plan::residual`. #[test] fn phase_b_legacy_topk_requires_membership_evidence() { let query = "topk(10, sum by (instance) (rate(http_requests_total[5m])))"; diff --git a/control_plane/src/physical/publication.rs b/control_plane/src/physical/publication.rs index 63081fa5b..b12ac10ce 100644 --- a/control_plane/src/physical/publication.rs +++ b/control_plane/src/physical/publication.rs @@ -1,9 +1,9 @@ //! Control-plane construction of the shared catalog publication contract. -use super::compiler::PhysicalPlan; +use super::compiler::CompiledPhysicalPlan; pub use asap_types::plan_publication::{PhysicalPlanInstallRequest, PhysicalPlanPublication}; -impl PhysicalPlan { - pub fn publication(&self) -> Result { +impl CompiledPhysicalPlan { + pub fn to_publication_artifact(&self) -> Result { let artifact = PhysicalPlanPublication { summary_catalog: self.summary_catalog.clone(), precompute_plan: self.precompute_plan.clone(), @@ -15,3 +15,10 @@ impl PhysicalPlan { Ok(artifact) } } + +impl CompiledPhysicalPlan { + #[deprecated(note = "Use to_publication_artifact")] + pub fn publication(&self) -> Result { + self.to_publication_artifact() + } +} diff --git a/control_plane/src/physical/realization.rs b/control_plane/src/physical/realization.rs index 74d3a8fa6..61672dfa7 100644 --- a/control_plane/src/physical/realization.rs +++ b/control_plane/src/physical/realization.rs @@ -3,8 +3,8 @@ //! Providers may validate and price physical implementations, never rewrite //! selected logical roots or infer a pane width from a query's slide. use super::compiler::{ - CompileError, DeploymentEnvironment, PhysicalCompiler, PhysicalPlan, PlanningQuery, - PlanningRequest, + CompileError, CompiledPhysicalPlan, PhysicalCompilationRequest, PhysicalDeploymentContext, + PhysicalPlanCompiler, QueryCompilationInput, }; use super::workload_cost::{PricedComponents, WorkloadCostEvidence, WorkloadCostManifest}; use asap_aware_mapping::cost_model::Cost; @@ -21,22 +21,22 @@ pub(crate) trait RealizationProvider { fn windows( &self, - query: &PlanningQuery, - environment: &DeploymentEnvironment, + query: &QueryCompilationInput, + environment: &PhysicalDeploymentContext, ) -> Result, CompileError>; fn compile( &self, - request: PlanningRequest, - environment: DeploymentEnvironment, - metricsql: bool, - ) -> Result; + request: PhysicalCompilationRequest, + environment: PhysicalDeploymentContext, + frontend: super::compiler::QueryFrontend, + ) -> Result; fn price( &self, evidence: &WorkloadCostEvidence, manifest: &WorkloadCostManifest, - ) -> Result; + ) -> Result; } /// Only the currently implemented deployment paths. Capability validation @@ -58,22 +58,22 @@ impl RealizationProvider for ExistingRealizations { fn windows( &self, - query: &PlanningQuery, - environment: &DeploymentEnvironment, + query: &QueryCompilationInput, + environment: &PhysicalDeploymentContext, ) -> Result, CompileError> { super::compiler::validate_window_implementations(query, environment) } fn compile( &self, - request: PlanningRequest, - environment: DeploymentEnvironment, - metricsql: bool, - ) -> Result { - if metricsql { - PhysicalCompiler.compile_metricsql(request, environment) + request: PhysicalCompilationRequest, + environment: PhysicalDeploymentContext, + frontend: super::compiler::QueryFrontend, + ) -> Result { + if frontend == super::compiler::QueryFrontend::MetricsQl { + PhysicalPlanCompiler.compile_metricsql(request, environment) } else { - PhysicalCompiler.compile(request, environment) + PhysicalPlanCompiler.compile_promql(request, environment) } } @@ -81,7 +81,7 @@ impl RealizationProvider for ExistingRealizations { &self, evidence: &WorkloadCostEvidence, manifest: &WorkloadCostManifest, - ) -> Result { + ) -> Result { evidence.price(manifest) } } diff --git a/control_plane/src/physical/stage_split.rs b/control_plane/src/physical/stage_split.rs index 34660c3b3..009ad2cdd 100644 --- a/control_plane/src/physical/stage_split.rs +++ b/control_plane/src/physical/stage_split.rs @@ -98,12 +98,12 @@ mod l5_walk_propagation_tests { //! TODO in `intent_algebra::column_resolution`). use crate::physical::colored_dag::StageConfig; - use crate::types::{AggType, QueryWorkload}; + use crate::types::{AggType, LegacyMetricWorkload}; use std::collections::HashMap; use std::time::Duration; - fn workload(metric: &str, group_by: Vec, window: Duration) -> QueryWorkload { - QueryWorkload { + fn workload(metric: &str, group_by: Vec, window: Duration) -> LegacyMetricWorkload { + LegacyMetricWorkload { metric_name: metric.to_string(), label_filters: HashMap::new(), group_by_labels: group_by, diff --git a/control_plane/src/physical/workload_cost.rs b/control_plane/src/physical/workload_cost.rs index d7a8ea3b1..f36c0e0e3 100644 --- a/control_plane/src/physical/workload_cost.rs +++ b/control_plane/src/physical/workload_cost.rs @@ -1,13 +1,15 @@ -//! Complete, provider-priced comparisons of already-bound workload alternatives. +//! Compile physical candidates, export quoteable manifests, and select the +//! lowest-cost feasible candidate from the supplied bounded inventory. //! -//! This is an evidence manifest over the existing physical projection, not a -//! second semantic DAG. Planner supplies legal alternatives; deployment quotes -//! price every reachable operation, and the backend commits one complete plan. +//! Planner owns semantic legality. A manifest describes the exact physical +//! demand to price; provider quotes and candidate evaluations are separate. mod materialization_candidates; +mod status; +pub use status::{CandidateEvaluationStatus, CandidateSearchScope}; #[cfg(test)] -use super::compiler::PhysicalCompiler; +use super::compiler::PhysicalPlanCompiler; use std::collections::{BTreeMap, BTreeSet}; @@ -16,18 +18,21 @@ use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use super::compiler::{ - CompileError, DeploymentEnvironment, PhysicalPlan, PlanningQuery, PlanningRequest, + CompileError, CompiledPhysicalPlan, PhysicalCompilationRequest, PhysicalDeploymentContext, + QueryCompilationInput, }; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] -pub struct CostDemand { +pub struct CostComponentDemand { /// Exact implementation/configuration being priced, not merely a family. pub implementation: Value, /// `horizon` includes all work in the manifest's source/time scope; /// `query_evaluation` is one execution of this bound query operator. - pub unit: String, - pub multiplicity: f64, + #[serde(rename = "unit", alias = "pricing_basis")] + pub pricing_basis: String, + #[serde(rename = "multiplicity", alias = "occurrences_per_horizon")] + pub occurrences_per_horizon: f64, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] @@ -41,7 +46,7 @@ pub struct WorkloadCostManifest { pub horizon_seconds: f64, /// Canonical roots, requirements and demand must match across alternatives. pub workload: BTreeMap, - pub components: BTreeMap, + pub components: BTreeMap, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] @@ -72,14 +77,17 @@ pub struct WorkloadCostEvidence { } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct AlternativeCost { - pub alternative_id: Option, +/// A candidate diagnostic can precede pricing or record compilation failure. +pub struct CandidatePlanEvaluation { + #[serde(rename = "alternative_id", alias = "candidate_id")] + pub candidate_id: Option, #[serde(default)] pub logical_root_ids: Vec, - pub physical_alternative_id: Option, + #[serde(rename = "physical_alternative_id", alias = "physical_candidate_id")] + pub physical_candidate_id: Option, pub identity_unavailable_reason: Option, #[serde(default)] - pub status: String, + pub status: CandidateEvaluationStatus, pub plan_id: Option, pub total_cost: Option, pub unavailable_reason: Option, @@ -87,16 +95,23 @@ pub struct AlternativeCost { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct MaterializationSearchCoverage { - pub eligible_leaves: usize, - pub enumerated_local_masks: usize, + #[serde(rename = "eligible_leaves", alias = "eligible_materialization_count")] + pub eligible_materialization_count: usize, + #[serde( + rename = "enumerated_local_masks", + alias = "enumerated_candidate_key_sets" + )] + pub enumerated_candidate_key_sets: usize, pub exhaustive: bool, - pub scope: String, + #[serde(rename = "scope", alias = "search_scope")] + pub search_scope: CandidateSearchScope, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct WorkloadCostComparison { +pub struct CandidatePlanSelectionReport { #[serde(default)] - pub logical_selection: Vec, + #[serde(rename = "logical_selection", alias = "planner_selection_trace")] + pub planner_selection_trace: Vec, #[serde(default, alias = "index_search_coverage")] pub materialization_search_coverage: Option, pub data_snapshot_id: String, @@ -104,7 +119,8 @@ pub struct WorkloadCostComparison { pub selected_plan_id: u64, pub selected_manifest: WorkloadCostManifest, pub component_costs: BTreeMap, - pub alternatives: Vec, + #[serde(rename = "alternatives", alias = "candidate_evaluations")] + pub candidate_evaluations: Vec, } fn invalid(reason: impl Into) -> CompileError { @@ -112,13 +128,13 @@ fn invalid(reason: impl Into) -> CompileError { } pub fn manifest( - plan: &PhysicalPlan, - queries: &[PlanningQuery], + plan: &CompiledPhysicalPlan, + queries: &[QueryCompilationInput], ) -> Result { let horizon = queries .first() .ok_or_else(|| invalid("empty workload"))? - .lifecycle + .summary_lifecycle_inputs .horizon_seconds; if !horizon.is_finite() || horizon <= 0.0 { return Err(invalid("invalid horizon")); @@ -126,7 +142,8 @@ pub fn manifest( let mut workload = BTreeMap::new(); let mut reads = BTreeMap::new(); for query in queries { - if query.lifecycle.horizon_seconds != horizon || query.lifecycle.evaluation_interval_ms == 0 + if query.summary_lifecycle_inputs.horizon_seconds != horizon + || query.summary_lifecycle_inputs.evaluation_interval_ms == 0 { return Err(invalid("mixed horizons or unknown recurrence")); } @@ -135,9 +152,9 @@ pub fn manifest( .insert( query.query_id.clone(), json!({ - "query": canonical, "accuracy": query.accuracy, - "evaluation_interval_ms": query.lifecycle.evaluation_interval_ms, - "source": query.source, + "query": canonical, "accuracy": query.accuracy_target, + "evaluation_interval_ms": query.summary_lifecycle_inputs.evaluation_interval_ms, + "source": query.legacy_query_source, }), ) .is_some() @@ -146,20 +163,21 @@ pub fn manifest( } reads.insert( query.query_id.clone(), - horizon * 1000.0 / f64::from(query.lifecycle.evaluation_interval_ms), + horizon * 1000.0 / f64::from(query.summary_lifecycle_inputs.evaluation_interval_ms), ); } let mut components = BTreeMap::new(); - let mut add = |id: String, implementation: Value, unit: &str, multiplicity: f64| { - components.insert( - id, - CostDemand { - implementation, - unit: unit.into(), - multiplicity, - }, - ); - }; + let mut add = + |id: String, implementation: Value, pricing_basis: &str, occurrences_per_horizon: f64| { + components.insert( + id, + CostComponentDemand { + implementation, + pricing_basis: pricing_basis.into(), + occurrences_per_horizon, + }, + ); + }; // Backend merge/update, storage, and edge maintenance are separate work. // The raw input is read once per source partition, not once per consumer. for schema in &plan.precompute_plan.schemas { @@ -185,7 +203,7 @@ pub fn manifest( .find(|m| m.policy_fingerprint() == schema.materialization.fingerprint()) .ok_or_else(|| invalid("state has no physical implementation"))?; let identity = json!({"schema": schema, "location": location, "physical": physical, - "window_implementation": plan.lifecycle_estimates.iter().find(|e| e.materialization == schema.materialization).map(|e| &e.window_implementation_id)}); + "window_implementation": plan.lifecycle_estimates.iter().find(|e| e.materialization == schema.materialization).map(|e| &e.window_realization_id)}); for operation in ["build", "update", "residency", "retire"] { add( format!("state:{location}:{}:{operation}", schema.materialization.0), @@ -222,7 +240,7 @@ pub fn manifest( // quoted as already-provisioned/non-incremental for this decision. let parsed = crate::query_parser::parse_query_expr_canonical( &query.query_string, - query.accuracy.clone(), + query.accuracy_target.clone(), ) .map_err(|error| invalid(error.to_string()))?; for metric in exact_source_metrics(&parsed)? { @@ -236,7 +254,7 @@ pub fn manifest( if matches!( node, crate::query_plan::QueryPlanNode::Logical { - operator: crate::query_plan::logical::LogicalOperator::Scan { .. }, + operator: crate::query_plan::residual::ResidualQueryOperator::Scan { .. }, .. } ) { @@ -246,9 +264,10 @@ pub fn manifest( } if let crate::query_plan::QueryPlanNode::Logical { operator: - crate::query_plan::logical::LogicalOperator::ExactSubquery { query } - | crate::query_plan::logical::LogicalOperator::CandidateExactSubquery { - query, .. + crate::query_plan::residual::ResidualQueryOperator::ExactSubquery { query } + | crate::query_plan::residual::ResidualQueryOperator::CandidateExactSubquery { + query, + .. }, .. } = node @@ -355,7 +374,7 @@ pub(crate) fn exact_source_metrics( pub(super) type PricedComponents = (Cost, BTreeMap); impl WorkloadCostEvidence { - fn validate(&self, env: &DeploymentEnvironment) -> Result<(), CompileError> { + fn validate(&self, env: &PhysicalDeploymentContext) -> Result<(), CompileError> { if self.backend_revision != super::compiler::BACKEND_REVISION || self.planner_revision != super::compiler::PLANNER_REVISION { @@ -382,7 +401,7 @@ impl WorkloadCostEvidence { pub(super) fn price( &self, manifest: &WorkloadCostManifest, - ) -> Result { + ) -> Result { let quotes = self .quotes .iter() @@ -390,20 +409,20 @@ impl WorkloadCostEvidence { .collect::>(); if quotes.len() != 1 { return Err(( - "evidence_missing", + CandidateEvaluationStatus::EvidenceMissing, "missing or ambiguous quote for exact manifest".into(), )); } let quote = quotes[0]; if !quote.executable { return Err(( - "rejected", + CandidateEvaluationStatus::ProviderRejected, "provider reports unavailable implementation".into(), )); } if !quote.unit_costs.keys().eq(manifest.components.keys()) { return Err(( - "evidence_invalid", + CandidateEvaluationStatus::EvidenceInvalid, "incomplete or extraneous component evidence".into(), )); } @@ -411,64 +430,82 @@ impl WorkloadCostEvidence { let mut components = BTreeMap::new(); for (id, demand) in &manifest.components { let unit = quote.unit_costs[id]; - let cost = unit * demand.multiplicity; + let cost = unit * demand.occurrences_per_horizon; if !unit.is_finite() || unit < 0.0 || !cost.is_finite() || cost < 0.0 { - return Err(("evidence_invalid", format!("invalid cost for {id}"))); + return Err(( + CandidateEvaluationStatus::EvidenceInvalid, + format!("invalid cost for {id}"), + )); } total += cost; components.insert(id.clone(), cost); } if !total.is_finite() { - return Err(("evidence_invalid", "cost overflow".into())); + return Err(( + CandidateEvaluationStatus::EvidenceInvalid, + "cost overflow".into(), + )); } Ok((Cost(total), components)) } } -fn alternative_description(candidate: &PlanningRequest) -> AlternativeCost { +fn alternative_description(candidate: &PhysicalCompilationRequest) -> CandidatePlanEvaluation { let root_ids = candidate .queries .iter() - .map(|query| crate::planner_selection::explained_root_id(&query.post_asap, &query.accuracy)) + .map(|query| { + crate::planner_selection::explained_root_id( + &query.selected_plan_root, + &query.accuracy_target, + ) + }) .collect::>(); let complete = root_ids.iter().all(Option::is_some); let mut logical_root_ids = root_ids.into_iter().flatten().collect::>(); logical_root_ids.sort(); logical_root_ids.dedup(); - AlternativeCost { - alternative_id: complete.then(|| { + CandidatePlanEvaluation { + candidate_id: complete.then(|| { crate::planner_selection::explain_identity( "alternative", &( &logical_root_ids, - candidate.hybrid_execution, - &candidate.materialization_policy, + candidate.allow_mixed_summary_and_exact_execution, + &candidate.enabled_materialization_keys, ), ) }), logical_root_ids, identity_unavailable_reason: (!complete) .then(|| "lossless canonical executable export unavailable for a logical root".into()), - physical_alternative_id: None, - status: "bind_failed".into(), + physical_candidate_id: None, + status: CandidateEvaluationStatus::CompilationFailed, plan_id: None, total_cost: None, unavailable_reason: None, } } -fn bind_alternative( - candidate: PlanningRequest, - env: DeploymentEnvironment, - metricsql: bool, -) -> Result<(PhysicalPlan, WorkloadCostManifest, AlternativeCost), Box> { +fn compile_candidate_for_pricing( + candidate: PhysicalCompilationRequest, + env: PhysicalDeploymentContext, + frontend: super::compiler::QueryFrontend, +) -> Result< + ( + CompiledPhysicalPlan, + WorkloadCostManifest, + CandidatePlanEvaluation, + ), + Box, +> { let mut description = alternative_description(&candidate); let queries = candidate.queries.clone(); let compiled = super::realization::RealizationProvider::compile( &super::realization::ExistingRealizations, candidate, env, - metricsql, + frontend, ); let plan = match compiled { Ok(plan) => plan, @@ -483,7 +520,7 @@ fn bind_alternative( let mut bindings = plan .lifecycle_estimates .iter() - .map(|item| (item.materialization, item.window_implementation_id.clone())) + .map(|item| (item.materialization, item.window_realization_id.clone())) .collect::>(); bindings.sort(); let mut placement = plan @@ -500,21 +537,25 @@ fn bind_alternative( }) .collect::>(); placement.sort(); - description.physical_alternative_id = description.alternative_id.as_ref().map(|alternative| { - crate::planner_selection::explain_identity( - "physical", - &( - alternative, - bindings, - placement, - &plan.precompute_plan.ingest, - ), - ) - }); + description.physical_candidate_id = + description + .candidate_id + .as_ref() + .map(|logical_candidate_id| { + crate::planner_selection::explain_identity( + "physical", + &( + logical_candidate_id, + bindings, + placement, + &plan.precompute_plan.ingest, + ), + ) + }); match manifest(&plan, &queries) { Ok(manifest) => { - description.status = "bound".into(); + description.status = CandidateEvaluationStatus::AwaitingQuote; Ok((plan, manifest, description)) } Err(error) => { @@ -524,84 +565,99 @@ fn bind_alternative( } } -/// Preserve failed bindings alongside quoteable manifests. This does not select or publish. -pub fn prepare_manifests( - candidates: Vec, - env: DeploymentEnvironment, - metricsql: bool, -) -> (Vec, Vec) { +/// Preserve failed bindings alongside quoteable manifests. This does not select_lowest_cost_candidate or publish. +pub fn compile_candidates_for_pricing( + candidates: Vec, + env: PhysicalDeploymentContext, + frontend: super::compiler::QueryFrontend, +) -> (Vec, Vec) { let mut manifests = Vec::new(); - let mut alternatives = Vec::new(); + let mut candidate_evaluations = Vec::new(); for candidate in candidates { - match bind_alternative(candidate, env.clone(), metricsql) { + match compile_candidate_for_pricing(candidate, env.clone(), frontend) { Ok((_, manifest, description)) => { manifests.push(manifest); - alternatives.push(description); + candidate_evaluations.push(description); } - Err(description) => alternatives.push(*description), + Err(description) => candidate_evaluations.push(*description), } } - (manifests, alternatives) + (manifests, candidate_evaluations) } /// Compare complete Planner-authorized forests after binding. Infeasible or /// uncosted alternatives are retained as unavailable, never assigned zero. -pub fn select( - candidates: Vec, - env: DeploymentEnvironment, +pub fn select_lowest_cost_candidate( + candidates: Vec, + env: PhysicalDeploymentContext, evidence: &WorkloadCostEvidence, -) -> Result { - select_with_frontend(candidates, env, evidence, false) +) -> Result { + select_candidates( + candidates, + env, + evidence, + super::compiler::QueryFrontend::PromQl, + ) } -pub fn select_metricsql( - candidates: Vec, - env: DeploymentEnvironment, +pub fn select_lowest_cost_metricsql_candidate( + candidates: Vec, + env: PhysicalDeploymentContext, evidence: &WorkloadCostEvidence, -) -> Result { - select_with_frontend(candidates, env, evidence, true) +) -> Result { + select_candidates( + candidates, + env, + evidence, + super::compiler::QueryFrontend::MetricsQl, + ) } -fn select_with_frontend( - candidates: Vec, - env: DeploymentEnvironment, +fn select_candidates( + candidates: Vec, + env: PhysicalDeploymentContext, evidence: &WorkloadCostEvidence, - metricsql: bool, -) -> Result { + frontend: super::compiler::QueryFrontend, +) -> Result { evidence.validate(&env)?; if candidates.is_empty() || candidates.len() > 64 { return Err(invalid( "candidate inventory must contain 1..=64 alternatives", )); } - let policies: BTreeSet<_> = candidates + let candidate_key_sets: BTreeSet<_> = candidates .iter() - .filter(|c| c.hybrid_execution) - .filter_map(|c| c.materialization_policy.clone()) + .filter(|c| c.allow_mixed_summary_and_exact_execution) + .filter_map(|c| c.enabled_materialization_keys.clone()) .collect(); - let leaves: BTreeSet<_> = policies.iter().flat_map(|p| p.iter().cloned()).collect(); - let materialization_search_coverage = (!policies.is_empty()).then(|| MaterializationSearchCoverage { - eligible_leaves: leaves.len(), - enumerated_local_masks: policies.len(), - exhaustive: leaves.len() < usize::BITS as usize && policies.len() == (1usize << leaves.len()), - scope: "Backend materialization versus Prometheus exact-subquery masks over Planner-authorized leaves; native alternative separate; bounded inventory does not claim an unenumerated optimum".into(), - }); - let logical_selection = candidates[0].logical_selection.clone(); + let eligible_keys: BTreeSet<_> = candidate_key_sets + .iter() + .flat_map(|p| p.iter().cloned()) + .collect(); + let materialization_search_coverage = + (!candidate_key_sets.is_empty()).then(|| MaterializationSearchCoverage { + eligible_materialization_count: eligible_keys.len(), + enumerated_candidate_key_sets: candidate_key_sets.len(), + exhaustive: eligible_keys.len() < usize::BITS as usize + && candidate_key_sets.len() == (1usize << eligible_keys.len()), + search_scope: CandidateSearchScope::PlannerAuthorizedMaterializations, + }); + let planner_selection_trace = candidates[0].planner_selection_trace.clone(); let mut comparison_workload = None; - let mut alternatives = Vec::new(); + let mut candidate_evaluations = Vec::new(); let mut best_index = 0; let mut best: Option<( Cost, - PhysicalPlan, + CompiledPhysicalPlan, WorkloadCostManifest, BTreeMap, )> = None; for candidate in candidates { let (plan, manifest, mut description) = - match bind_alternative(candidate, env.clone(), metricsql) { + match compile_candidate_for_pricing(candidate, env.clone(), frontend) { Ok(bound) => bound, Err(description) => { - alternatives.push(*description); + candidate_evaluations.push(*description); continue; } }; @@ -621,80 +677,82 @@ fn select_with_frontend( &manifest, ) { Ok((cost, components)) => { - description.status = "unselected".into(); + description.status = CandidateEvaluationStatus::Unselected; description.total_cost = Some(cost.0); - alternatives.push(description); + candidate_evaluations.push(description); if best.as_ref().is_none_or(|(previous, ..)| cost < *previous) { - best_index = alternatives.len() - 1; + best_index = candidate_evaluations.len() - 1; best = Some((cost, plan, manifest, components)); } } Err((status, reason)) => { - description.status = status.into(); + description.status = status; description.unavailable_reason = Some(reason); - alternatives.push(description); + candidate_evaluations.push(description); } } } let (_, mut plan, selected_manifest, component_costs) = best.ok_or_else(|| { CompileError::Alternatives( - json!({"status": "all_infeasible", "logical_selection": logical_selection, - "alternatives": alternatives}), + json!({"status": "all_infeasible", "logical_selection": planner_selection_trace, + "alternatives": candidate_evaluations}), ) })?; // Exactly the winner retained by the existing strict-less-than selector. - alternatives[best_index].status = "selected".into(); - plan.cost_comparison = Some(WorkloadCostComparison { - logical_selection, + candidate_evaluations[best_index].status = CandidateEvaluationStatus::Selected; + plan.cost_comparison = Some(CandidatePlanSelectionReport { + planner_selection_trace, materialization_search_coverage, data_snapshot_id: evidence.data_snapshot_id.clone(), model_version: evidence.model_version.clone(), selected_plan_id: plan.envelope.plan_id, selected_manifest, component_costs, - alternatives, + candidate_evaluations, }); Ok(plan) } /// The current executor exposes continuously maintained state and the native -/// exact backend. Additional Planner-produced forests can use `select` directly. -pub fn with_exact_alternative( - request: PlanningRequest, -) -> Result, CompileError> { +/// exact backend. Additional Planner-produced forests can use `select_lowest_cost_candidate` directly. +pub fn enumerate_exact_and_materialized_candidates( + request: PhysicalCompilationRequest, +) -> Result, CompileError> { let mut exact = request.clone(); - exact.hybrid_execution = false; - exact.materialization_policy = None; + exact.allow_mixed_summary_and_exact_execution = false; + exact.enabled_materialization_keys = None; for query in &mut exact.queries { let parsed = crate::query_parser::parse_query_expr_canonical( &query.query_string, - query.accuracy.clone(), + query.accuracy_target.clone(), ) .map_err(|error| invalid(error.to_string()))?; - query.post_asap = crate::planner_selection::keep_pre_asap(&parsed) + query.selected_plan_root = crate::planner_selection::keep_pre_asap(&parsed) .map_err(|error| invalid(error.to_string()))?; } - if !request.hybrid_execution + if !request.allow_mixed_summary_and_exact_execution && request .queries .iter() .zip(&exact.queries) - .all(|(a, b)| a.post_asap == b.post_asap) + .all(|(a, b)| a.selected_plan_root == b.selected_plan_root) { Ok(vec![request]) } else { - if !request.hybrid_execution || request.materialization_policy.is_some() { + if !request.allow_mixed_summary_and_exact_execution + || request.enabled_materialization_keys.is_some() + { return Ok(vec![request, exact]); } let mut keys = BTreeSet::new(); for query in &request.queries { - match crate::query_plan::logical::materialization_candidate_keys( + match crate::query_plan::residual::eligible_materialization_keys( &query.query_string, - &query.post_asap, + &query.selected_plan_root, ) { Ok(found) => keys.extend(found), // A failed local projection must not make the native alternative - // disappear. Compile/select retains its concrete unavailability. + // disappear. Compile/select_lowest_cost_candidate retains its concrete unavailability. Err(_) => return Ok(vec![request, exact]), } } @@ -702,28 +760,62 @@ pub fn with_exact_alternative( return Ok(vec![request, exact]); } let inventory = materialization_candidates::enumerate(keys); - debug_assert_eq!(inventory.exhaustive, inventory.eligible_leaves <= 4); - let mut alternatives: Vec<_> = inventory - .masks + debug_assert_eq!( + inventory.exhaustive, + inventory.eligible_materialization_count <= 4 + ); + let mut candidate_requests: Vec<_> = inventory + .candidate_key_sets .into_iter() - .map(|mask| { + .map(|enabled_keys| { let mut candidate = request.clone(); - candidate.materialization_policy = Some(mask); + candidate.enabled_materialization_keys = Some(enabled_keys); candidate }) .collect(); - alternatives.push(exact); - Ok(alternatives) + candidate_requests.push(exact); + Ok(candidate_requests) } } +// Compatibility imports; new callers use the domain names above. +#[deprecated(note = "Use CandidatePlanEvaluation")] +pub use CandidatePlanEvaluation as AlternativeCost; +#[deprecated(note = "Use CandidatePlanSelectionReport")] +pub use CandidatePlanSelectionReport as WorkloadCostComparison; +#[deprecated(note = "Use CostComponentDemand")] +pub use CostComponentDemand as CostDemand; + +#[deprecated(note = "Use enumerate_exact_and_materialized_candidates")] +pub use enumerate_exact_and_materialized_candidates as with_exact_alternative; +#[deprecated(note = "Use select_lowest_cost_candidate")] +pub use select_lowest_cost_candidate as select; +#[deprecated(note = "Use select_lowest_cost_metricsql_candidate")] +pub use select_lowest_cost_metricsql_candidate as select_metricsql; +#[deprecated(note = "Use compile_candidates_for_pricing with QueryFrontend")] +pub fn prepare_manifests( + candidates: Vec, + env: PhysicalDeploymentContext, + metricsql: bool, +) -> (Vec, Vec) { + compile_candidates_for_pricing( + candidates, + env, + if metricsql { + super::compiler::QueryFrontend::MetricsQl + } else { + super::compiler::QueryFrontend::PromQl + }, + ) +} + #[cfg(test)] mod tests { - use super::super::compiler::BackendLocalPlanningSnapshot; + use super::super::compiler::BackendLocalPlanningInput; use super::*; - fn fixture() -> BackendLocalPlanningSnapshot { - let mut snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + fn fixture() -> BackendLocalPlanningInput { + let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); @@ -732,46 +824,144 @@ mod tests { snapshot } + // New input aliases must produce the same candidate identities and manifests + // while serialization continues to serve existing evidence producers. + #[test] + fn renamed_inputs_preserve_candidate_manifests_and_wire_names() { + let legacy = serde_json::to_value(fixture()).unwrap(); + assert!(legacy.get("snapshot_version").is_some()); + assert!(legacy.get("physical_inputs").is_none()); + let mut renamed = legacy.clone(); + let root = renamed.as_object_mut().unwrap(); + let version = root.remove("snapshot_version").unwrap(); + root.insert("schema_version".into(), version); + let mut inputs = root.remove("implementation").unwrap(); + let physical = inputs.as_object_mut().unwrap(); + let id = physical.remove("window_implementation_id").unwrap(); + physical.insert("default_window_realization_id".into(), id); + let quote = physical.remove("implementation_cost").unwrap(); + physical.insert("default_window_cost_quote".into(), quote); + root.insert("physical_inputs".into(), inputs); + let environment = root + .get_mut("environment") + .unwrap() + .as_object_mut() + .unwrap(); + let collectors = environment.remove("collector_ids").unwrap(); + environment.insert("target_collector_ids".into(), collectors); + + let old: BackendLocalPlanningInput = serde_json::from_value(legacy.clone()).unwrap(); + let new: BackendLocalPlanningInput = serde_json::from_value(renamed).unwrap(); + assert_eq!(old, new); + assert_eq!(serde_json::to_value(&new).unwrap(), legacy); + // Shared publication fields already had domain names: renaming the + // streaming accessor must not change their wire keys or catalog hash. + let (request, environment) = new.clone().into_physical_compilation_request().unwrap(); + let plan = PhysicalPlanCompiler + .compile_promql(request, environment) + .unwrap(); + for document in [ + serde_json::to_value(&plan.summary_catalog).unwrap(), + serde_json::to_value(&plan.precompute_plan).unwrap(), + ] { + assert!(document.get("materializations").is_some()); + assert!(document.get("get_all_aggregation_configs").is_none()); + } + let compile = |input: BackendLocalPlanningInput| { + let (request, environment) = input.into_physical_compilation_request().unwrap(); + compile_candidates_for_pricing( + enumerate_exact_and_materialized_candidates(request).unwrap(), + environment, + super::super::compiler::QueryFrontend::PromQl, + ) + }; + let old_candidates = compile(old); + let new_candidates = compile(new); + assert!(!old_candidates.0.is_empty()); + assert_eq!(old_candidates, new_candidates); + } + + // A renamed demand remains readable by old quote providers, including + // fractional recurrence; missing and future statuses keep round-tripping. + #[test] + fn demand_and_evaluation_keep_legacy_wire_contracts() { + let old = json!({"implementation": {"op": "read"}, "unit": "query_evaluation", "multiplicity": 2.5}); + let demand: CostComponentDemand = serde_json::from_value(old.clone()).unwrap(); + assert_eq!(demand.pricing_basis, "query_evaluation"); + assert_eq!(demand.occurrences_per_horizon, 2.5); + assert_eq!(serde_json::to_value(demand).unwrap(), old); + let row = json!({ + "alternative_id": null, "physical_alternative_id": null, + "identity_unavailable_reason": null, "plan_id": null, + "total_cost": null, "unavailable_reason": null + }); + let evaluation: CandidatePlanEvaluation = serde_json::from_value(row).unwrap(); + assert_eq!(evaluation.status, CandidateEvaluationStatus::Unspecified); + let encoded = serde_json::to_value(evaluation).unwrap(); + assert_eq!(encoded["status"], ""); + assert!(encoded.get("alternative_id").is_some()); + assert!(encoded.get("candidate_id").is_none()); + } + // IDs describe semantics; activation/version changes do not create new alternatives. #[test] fn explain_identity_is_stable_across_activations_and_distinguishes_native() { - let (request, mut env) = fixture().planning_request().unwrap(); - let candidates = with_exact_alternative(request).unwrap(); - let (_, first) = prepare_manifests(candidates.clone(), env.clone(), false); + let (request, mut env) = fixture().into_physical_compilation_request().unwrap(); + let candidates = enumerate_exact_and_materialized_candidates(request).unwrap(); + let (_, first) = compile_candidates_for_pricing( + candidates.clone(), + env.clone(), + crate::physical::compiler::QueryFrontend::PromQl, + ); env.plan_version += 1; env.activation_unix_ms += 1; - let (_, second) = prepare_manifests(candidates, env, false); + let (_, second) = compile_candidates_for_pricing( + candidates, + env, + crate::physical::compiler::QueryFrontend::PromQl, + ); assert_eq!(first.len(), second.len()); for (a, b) in first.iter().zip(&second) { - assert!(a.alternative_id.is_some()); - assert!(a.physical_alternative_id.is_some()); - assert_eq!(a.alternative_id, b.alternative_id); - assert_eq!(a.physical_alternative_id, b.physical_alternative_id); + assert!(a.candidate_id.is_some()); + assert!(a.physical_candidate_id.is_some()); + assert_eq!(a.candidate_id, b.candidate_id); + assert_eq!(a.physical_candidate_id, b.physical_candidate_id); } assert_ne!( - first.first().unwrap().alternative_id, - first.last().unwrap().alternative_id + first.first().unwrap().candidate_id, + first.last().unwrap().candidate_id ); } // Bind failures remain visible even when the native manifest is usable. #[test] fn explain_retains_failed_bindings_and_all_missing_quotes() { - let (mut request, env) = fixture().planning_request().unwrap(); - request.hybrid_execution = false; - request.queries[0].window_implementations.clear(); - let candidates = with_exact_alternative(request).unwrap(); - let (manifests, explanations) = prepare_manifests(candidates, env, false); + let (mut request, env) = fixture().into_physical_compilation_request().unwrap(); + request.allow_mixed_summary_and_exact_execution = false; + request.queries[0].window_realization_candidates.clear(); + let candidates = enumerate_exact_and_materialized_candidates(request).unwrap(); + let (manifests, explanations) = compile_candidates_for_pricing( + candidates, + env, + crate::physical::compiler::QueryFrontend::PromQl, + ); assert_eq!(manifests.len(), 1); assert_eq!(explanations.len(), 2); - assert_eq!(explanations[0].status, "bind_failed"); + assert_eq!( + explanations[0].status, + CandidateEvaluationStatus::CompilationFailed + ); assert!(explanations[0].unavailable_reason.is_some()); - assert_eq!(explanations[1].status, "bound"); + assert_eq!( + explanations[1].status, + CandidateEvaluationStatus::AwaitingQuote + ); let (candidates, env, mut evidence) = quoted(); let count = candidates.len(); evidence.quotes.clear(); - let CompileError::Alternatives(report) = select(candidates, env, &evidence).unwrap_err() + let CompileError::Alternatives(report) = + select_lowest_cost_candidate(candidates, env, &evidence).unwrap_err() else { panic!("expected structured all-infeasible report") }; @@ -797,19 +987,22 @@ mod tests { }) .min_by(|a, b| a.0.total_cmp(&b.0)) .unwrap(); - let plan = select(candidates, env, &evidence).unwrap(); + let plan = select_lowest_cost_candidate(candidates, env, &evidence).unwrap(); assert_eq!(plan.envelope.plan_id, expected.1); let comparison = plan.cost_comparison.unwrap(); let selected = comparison - .alternatives + .candidate_evaluations .iter() - .filter(|item| item.status == "selected") + .filter(|item| item.status == CandidateEvaluationStatus::Selected) .collect::>(); assert_eq!(selected.len(), 1); assert_eq!(selected[0].total_cost, Some(expected.0)); - assert!(selected[0].alternative_id.is_some()); - assert!(selected[0].physical_alternative_id.is_some()); - assert_eq!(comparison.logical_selection, plan.logical_selection); + assert!(selected[0].candidate_id.is_some()); + assert!(selected[0].physical_candidate_id.is_some()); + assert_eq!( + comparison.planner_selection_trace, + plan.planner_selection_trace + ); } #[test] @@ -828,8 +1021,10 @@ mod tests { "max_over_time(service_retry_queue_depth{job=\"order-service\"}[6h])".into(), ); entries.push(second); - let (request, env) = snapshot.planning_request().unwrap(); - let plan = PhysicalCompiler.compile(request.clone(), env).unwrap(); + let (request, env) = snapshot.into_physical_compilation_request().unwrap(); + let plan = PhysicalPlanCompiler + .compile_promql(request.clone(), env) + .unwrap(); let costs = manifest(&plan, &request.queries).unwrap(); assert_eq!( costs @@ -870,14 +1065,22 @@ mod tests { q.requirements.accuracy = planner_types::workload::AccuracyRequirement::Explicit( crate::types_v2::AccuracyTarget::Exact, ); - let (request, environment) = snapshot.planning_request().unwrap(); - let candidates = with_exact_alternative(request).unwrap(); - assert_eq!(candidates.len(), 5, "four legal masks plus native"); + let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); + let candidates = enumerate_exact_and_materialized_candidates(request).unwrap(); + assert_eq!( + candidates.len(), + 5, + "four legal candidate key sets plus native" + ); let mut identities = BTreeSet::new(); for candidate in &candidates[..4] { - let enabled = candidate.materialization_policy.as_ref().unwrap().len(); - let plan = PhysicalCompiler - .compile(candidate.clone(), environment.clone()) + let enabled = candidate + .enabled_materialization_keys + .as_ref() + .unwrap() + .len(); + let plan = PhysicalPlanCompiler + .compile_promql(candidate.clone(), environment.clone()) .unwrap(); assert!(identities.insert(plan.envelope.plan_id)); let cost = manifest(&plan, &candidate.queries).unwrap(); @@ -898,7 +1101,7 @@ mod tests { assert_eq!( cost.components .values() - .filter(|v| v.unit == "horizon" + .filter(|v| v.pricing_basis == "horizon" && v.implementation.get("location").and_then(Value::as_str) == Some("exact_backend")) .count(), @@ -913,21 +1116,26 @@ mod tests { crate::query_plan::QueryPlanNode::ExactFallback { .. } )))); } - assert!(!candidates.last().unwrap().hybrid_execution); + assert!( + !candidates + .last() + .unwrap() + .allow_mixed_summary_and_exact_execution + ); } fn quoted() -> ( - Vec, - DeploymentEnvironment, + Vec, + PhysicalDeploymentContext, WorkloadCostEvidence, ) { - let (request, env) = fixture().planning_request().unwrap(); - let candidates = with_exact_alternative(request).unwrap(); + let (request, env) = fixture().into_physical_compilation_request().unwrap(); + let candidates = enumerate_exact_and_materialized_candidates(request).unwrap(); let quotes = candidates .iter() .map(|candidate| { - let plan = PhysicalCompiler - .compile(candidate.clone(), env.clone()) + let plan = PhysicalPlanCompiler + .compile_promql(candidate.clone(), env.clone()) .unwrap(); let manifest = manifest(&plan, &candidate.queries).unwrap(); let unit_costs = manifest @@ -963,14 +1171,14 @@ mod tests { entry.query = Query("sum(rate(a{job=\"x\"}[1m])) / sum(rate(a{job!=\"x\"}[5m]))".into()); entry.requirements.accuracy = AccuracyRequirement::Explicit(crate::types_v2::AccuracyTarget::Exact); - let (request, environment) = snapshot.planning_request().unwrap(); - let candidates = with_exact_alternative(request).unwrap(); + let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); + let candidates = enumerate_exact_and_materialized_candidates(request).unwrap(); assert!(candidates.len() >= 2); - let local = PhysicalCompiler - .compile(candidates[0].clone(), environment.clone()) + let local = PhysicalPlanCompiler + .compile_promql(candidates[0].clone(), environment.clone()) .unwrap(); - let native = PhysicalCompiler - .compile(candidates.last().unwrap().clone(), environment) + let native = PhysicalPlanCompiler + .compile_promql(candidates.last().unwrap().clone(), environment) .unwrap(); assert_ne!(local.envelope.plan_id, native.envelope.plan_id); let manifest = manifest(&local, &candidates[0].queries).unwrap(); @@ -1011,7 +1219,7 @@ mod tests { ), ("sum_over_time(m[1m]) + count_over_time(m[1m])", vec!["m"]), ] { - let (mut request, env) = fixture().planning_request().unwrap(); + let (mut request, env) = fixture().into_physical_compilation_request().unwrap(); request.queries[0].query_string = query.into(); request .query_workload @@ -1021,9 +1229,12 @@ mod tests { .as_mut() .unwrap()[0] .query = planner_types::workload::Query(query.into()); - let exact = with_exact_alternative(request).unwrap().pop().unwrap(); - let plan = PhysicalCompiler - .compile(exact.clone(), env.clone()) + let exact = enumerate_exact_and_materialized_candidates(request) + .unwrap() + .pop() + .unwrap(); + let plan = PhysicalPlanCompiler + .compile_promql(exact.clone(), env.clone()) .unwrap(); let manifest = manifest(&plan, &exact.queries).unwrap(); let sources: Vec<_> = manifest @@ -1054,7 +1265,9 @@ mod tests { executable: true, }], }; - assert!(select(vec![exact.clone()], env.clone(), &evidence).is_ok()); + assert!( + select_lowest_cost_candidate(vec![exact.clone()], env.clone(), &evidence).is_ok() + ); let source_id = evidence.quotes[0] .unit_costs .keys() @@ -1063,7 +1276,7 @@ mod tests { .clone(); evidence.quotes[0].unit_costs.remove(&source_id); assert!( - select(vec![exact], env, &evidence).is_err(), + select_lowest_cost_candidate(vec![exact], env, &evidence).is_err(), "missing input upkeep must fail closed" ); } @@ -1072,8 +1285,12 @@ mod tests { // Hidden or unresolved sources must not yield a partially priced manifest. #[test] fn exact_source_discovery_rejects_unresolved_inputs() { - let accuracy = fixture().planning_request().unwrap().0.queries[0] - .accuracy + let accuracy = fixture() + .into_physical_compilation_request() + .unwrap() + .0 + .queries[0] + .accuracy_target .clone(); for query in ["info(m)", "{job=\"api\"}"] { let parsed = @@ -1099,21 +1316,25 @@ mod tests { for cost in evidence.quotes[1].unit_costs.values_mut() { *cost = 1000.0; } - let warm = select(candidates.clone(), env.clone(), &evidence).unwrap(); + let warm = + select_lowest_cost_candidate(candidates.clone(), env.clone(), &evidence).unwrap(); assert_eq!(warm.envelope.plan_id, evidence.quotes[0].manifest.plan_id); let report = warm.cost_comparison.unwrap(); assert_eq!( report.component_costs.len(), report.selected_manifest.components.len() ); - assert_eq!(report.alternatives.len(), 2); - assert!(report.alternatives.iter().all(|a| a.total_cost.is_some())); + assert_eq!(report.candidate_evaluations.len(), 2); + assert!(report + .candidate_evaluations + .iter() + .all(|a| a.total_cost.is_some())); for (id, cost) in &mut evidence.quotes[0].unit_costs { if id.ends_with(":residency") { *cost = 1e9; } } - let raw = select(candidates, env, &evidence).unwrap(); + let raw = select_lowest_cost_candidate(candidates, env, &evidence).unwrap(); assert_eq!(raw.envelope.plan_id, evidence.quotes[1].manifest.plan_id); } @@ -1121,48 +1342,51 @@ mod tests { fn incomplete_unavailable_and_wrong_generation_quotes_are_not_free() { let (candidates, env, mut evidence) = quoted(); evidence.quotes[0].unit_costs.pop_first(); - let plan = select(candidates.clone(), env.clone(), &evidence).unwrap(); - assert!(plan.cost_comparison.unwrap().alternatives[0] + let plan = + select_lowest_cost_candidate(candidates.clone(), env.clone(), &evidence).unwrap(); + assert!(plan.cost_comparison.unwrap().candidate_evaluations[0] .unavailable_reason .is_some()); evidence.quotes[1].executable = false; - assert!(select(candidates.clone(), env.clone(), &evidence).is_err()); + assert!(select_lowest_cost_candidate(candidates.clone(), env.clone(), &evidence).is_err()); let (_, _, mut evidence) = quoted(); evidence .quotes .iter_mut() .for_each(|quote| quote.manifest.capability_snapshot_id.push_str("-wrong")); - assert!(select(candidates.clone(), env.clone(), &evidence).is_err()); + assert!(select_lowest_cost_candidate(candidates.clone(), env.clone(), &evidence).is_err()); let (_, _, mut evidence) = quoted(); evidence.observed_at_unix_ms = env.observed_at_unix_ms + 1; - assert!(select(candidates, env, &evidence).is_err()); + assert!(select_lowest_cost_candidate(candidates, env, &evidence).is_err()); } #[test] fn evidence_from_a_different_compiler_build_is_rejected_before_matching_quotes() { let (candidates, env, mut evidence) = quoted(); evidence.backend_revision = "stale-backend-build".into(); - let error = select(candidates, env, &evidence).unwrap_err().to_string(); + let error = select_lowest_cost_candidate(candidates, env, &evidence) + .unwrap_err() + .to_string(); assert!(error.contains("cost evidence compiler mismatch"), "{error}"); } #[test] fn snapshot_requires_quotes_and_roundtrips_selection() { let mut snapshot = fixture(); - assert!(snapshot.clone().compile().is_err()); + assert!(snapshot.clone().compile_promql().is_err()); let (_, _, evidence) = quoted(); snapshot.workload_cost_evidence = Some(evidence); - let snapshot: BackendLocalPlanningSnapshot = + let snapshot: BackendLocalPlanningInput = serde_json::from_str(&serde_json::to_string(&snapshot).unwrap()).unwrap(); - assert!(snapshot.compile().unwrap().cost_comparison.is_some()); + assert!(snapshot.compile_promql().unwrap().cost_comparison.is_some()); } #[test] fn second_consumer_adds_reads_not_another_shared_state() { - let (request, env) = fixture().planning_request().unwrap(); + let (request, env) = fixture().into_physical_compilation_request().unwrap(); let first = manifest( - &PhysicalCompiler - .compile(request.clone(), env.clone()) + &PhysicalPlanCompiler + .compile_promql(request.clone(), env.clone()) .unwrap(), &request.queries, ) @@ -1189,20 +1413,22 @@ mod tests { std::rc::Rc::new( crate::query_parser::parse_query_expr_canonical( &query.query_string, - query.accuracy.clone(), + query.accuracy_target.clone(), ) .unwrap(), ) }) .collect(); - super::super::compiler::select_workload_roots( + super::super::compiler::select_logical_roots_for_queries( &mut shared.queries, roots, - &shared.evidence, + &shared.topk_membership_evidence_by_query_id, &shared.exact_composition_costs, ) .unwrap(); - let plan = PhysicalCompiler.compile(shared.clone(), env).unwrap(); + let plan = PhysicalPlanCompiler + .compile_promql(shared.clone(), env) + .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); let second = manifest(&plan, &shared.queries).unwrap(); let states = |m: &WorkloadCostManifest| { @@ -1220,9 +1446,14 @@ mod tests { fn exact_alternative_does_not_require_unused_state_implementation_evidence() { let (candidates, env, evidence) = quoted(); let mut exact = candidates[1].clone(); - assert_eq!(with_exact_alternative(exact.clone()).unwrap().len(), 1); - exact.queries[0].window_implementations.clear(); - assert!(select(vec![exact], env, &evidence).is_ok()); + assert_eq!( + enumerate_exact_and_materialized_candidates(exact.clone()) + .unwrap() + .len(), + 1 + ); + exact.queries[0].window_realization_candidates.clear(); + assert!(select_lowest_cost_candidate(vec![exact], env, &evidence).is_ok()); } #[test] @@ -1232,14 +1463,14 @@ mod tests { .quotes .iter_mut() .for_each(|quote| quote.manifest.horizon_seconds += 1.0); - assert!(select(candidates.clone(), env.clone(), &evidence).is_err()); + assert!(select_lowest_cost_candidate(candidates.clone(), env.clone(), &evidence).is_err()); let (_, _, mut evidence) = quoted(); evidence.quotes.extend(evidence.quotes.clone()); - assert!(select(candidates.clone(), env.clone(), &evidence).is_err()); + assert!(select_lowest_cost_candidate(candidates.clone(), env.clone(), &evidence).is_err()); let (_, _, mut evidence) = quoted(); for quote in &mut evidence.quotes { *quote.unit_costs.values_mut().next().unwrap() = -1.0; } - assert!(select(candidates, env, &evidence).is_err()); + assert!(select_lowest_cost_candidate(candidates, env, &evidence).is_err()); } } diff --git a/control_plane/src/physical/workload_cost/materialization_candidates.rs b/control_plane/src/physical/workload_cost/materialization_candidates.rs index 48c9efa71..2344192f2 100644 --- a/control_plane/src/physical/workload_cost/materialization_candidates.rs +++ b/control_plane/src/physical/workload_cost/materialization_candidates.rs @@ -3,53 +3,53 @@ use std::collections::BTreeSet; #[derive(Debug)] -pub(super) struct MaterializationMasks { - pub masks: Vec>, +pub(super) struct MaterializationCandidateSets { + pub candidate_key_sets: Vec>, pub exhaustive: bool, - pub eligible_leaves: usize, + pub eligible_materialization_count: usize, } -/// Enumerate every mask for up to four leaves. Larger forests retain all-materialized, +/// Enumerate every enabled_keys for up to four leaves. Larger forests retain all-materialized, /// all-exact, then singleton/complement pairs in stable key order. The caller must /// disclose bounded coverage; no unenumerated optimum is claimed. Reserve one /// of the selector's 64 candidate slots for native execution. -pub(super) fn enumerate(keys: BTreeSet) -> MaterializationMasks { - let eligible_leaves = keys.len(); +pub(super) fn enumerate(keys: BTreeSet) -> MaterializationCandidateSets { + let eligible_materialization_count = keys.len(); let ordered: Vec<_> = keys.iter().cloned().collect(); - let exhaustive = eligible_leaves <= 4; - let mut masks = vec![keys.clone()]; + let exhaustive = eligible_materialization_count <= 4; + let mut candidate_key_sets = vec![keys.clone()]; if exhaustive { - for bits in 0..(1usize << eligible_leaves) { - let mask = ordered + for bits in 0..(1usize << eligible_materialization_count) { + let enabled_keys = ordered .iter() .enumerate() .filter(|(i, _)| bits & (1 << i) != 0) .map(|(_, key)| key.clone()) .collect(); - if !masks.contains(&mask) { - masks.push(mask); + if !candidate_key_sets.contains(&enabled_keys) { + candidate_key_sets.push(enabled_keys); } } } else { - masks.push(BTreeSet::new()); + candidate_key_sets.push(BTreeSet::new()); for key in ordered { - for mask in [ + for enabled_keys in [ BTreeSet::from([key.clone()]), keys.difference(&BTreeSet::from([key])).cloned().collect(), ] { - if masks.len() >= 63 { + if candidate_key_sets.len() >= 63 { break; } - if !masks.contains(&mask) { - masks.push(mask); + if !candidate_key_sets.contains(&enabled_keys) { + candidate_key_sets.push(enabled_keys); } } } } - MaterializationMasks { - masks, + MaterializationCandidateSets { + candidate_key_sets, exhaustive, - eligible_leaves, + eligible_materialization_count, } } @@ -60,24 +60,45 @@ mod tests { fn small_forests_cover_every_mixed_path_once() { let result = enumerate(BTreeSet::from(["a".into(), "b".into()])); assert!(result.exhaustive); - assert_eq!(result.eligible_leaves, 2); - assert_eq!(result.masks.len(), 4); - assert_eq!(result.masks.iter().collect::>().len(), 4); - assert!(result.masks.contains(&BTreeSet::from(["a".into()]))); - assert!(result.masks.contains(&BTreeSet::from(["b".into()]))); + assert_eq!(result.eligible_materialization_count, 2); + assert_eq!(result.candidate_key_sets.len(), 4); + assert_eq!( + result + .candidate_key_sets + .iter() + .collect::>() + .len(), + 4 + ); + assert!(result + .candidate_key_sets + .contains(&BTreeSet::from(["a".into()]))); + assert!(result + .candidate_key_sets + .contains(&BTreeSet::from(["b".into()]))); } #[test] fn large_inventory_reserves_native_slot_and_discloses_truncation() { let keys = (0..100).map(|i| format!("{i:03}")).collect(); let result = enumerate(keys); assert!(!result.exhaustive); - assert_eq!(result.masks.len(), 63); - assert_eq!(result.masks[0].len(), 100); - assert!(result.masks[1].is_empty()); - assert_eq!(result.masks.iter().collect::>().len(), 63); + assert_eq!(result.candidate_key_sets.len(), 63); + assert_eq!(result.candidate_key_sets[0].len(), 100); + assert!(result.candidate_key_sets[1].is_empty()); + assert_eq!( + result + .candidate_key_sets + .iter() + .collect::>() + .len(), + 63 + ); } #[test] fn no_materialization_has_one_exact_implementation() { - assert_eq!(enumerate(BTreeSet::new()).masks, vec![BTreeSet::new()]); + assert_eq!( + enumerate(BTreeSet::new()).candidate_key_sets, + vec![BTreeSet::new()] + ); } } diff --git a/control_plane/src/physical/workload_cost/status.rs b/control_plane/src/physical/workload_cost/status.rs new file mode 100644 index 000000000..e73051e8a --- /dev/null +++ b/control_plane/src/physical/workload_cost/status.rs @@ -0,0 +1,124 @@ +//! Typed candidate diagnostics with the existing string wire representation. +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(from = "String", into = "String")] +pub enum CandidateEvaluationStatus { + #[default] + Unspecified, + CompilationFailed, + AwaitingQuote, + EvidenceMissing, + ProviderRejected, + EvidenceInvalid, + Unselected, + Selected, + /// Preserve diagnostics from other producer versions during migration. + Other(String), +} + +impl CandidateEvaluationStatus { + pub fn as_str(&self) -> &str { + match self { + Self::Unspecified => "", + Self::CompilationFailed => "bind_failed", + Self::AwaitingQuote => "bound", + Self::EvidenceMissing => "evidence_missing", + Self::ProviderRejected => "rejected", + Self::EvidenceInvalid => "evidence_invalid", + Self::Unselected => "unselected", + Self::Selected => "selected", + Self::Other(value) => value, + } + } +} + +impl From for CandidateEvaluationStatus { + fn from(value: String) -> Self { + match value.as_str() { + "" => Self::Unspecified, + "bind_failed" => Self::CompilationFailed, + "bound" => Self::AwaitingQuote, + "evidence_missing" => Self::EvidenceMissing, + "rejected" => Self::ProviderRejected, + "evidence_invalid" => Self::EvidenceInvalid, + "unselected" => Self::Unselected, + "selected" => Self::Selected, + _ => Self::Other(value), + } + } +} + +impl From for String { + fn from(value: CandidateEvaluationStatus) -> Self { + value.as_str().to_owned() + } +} + +// Preserve the existing report text, including historical terminology, on the +// wire. The typed variant describes the actual scope for new Rust consumers. +const MATERIALIZATION_SEARCH_SCOPE: &str = "Backend materialization versus Prometheus exact-subquery masks over Planner-authorized leaves; native alternative separate; bounded inventory does not claim an unenumerated optimum"; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(from = "String", into = "String")] +pub enum CandidateSearchScope { + PlannerAuthorizedMaterializations, + Other(String), +} + +impl From for CandidateSearchScope { + fn from(value: String) -> Self { + if value == MATERIALIZATION_SEARCH_SCOPE { + Self::PlannerAuthorizedMaterializations + } else { + Self::Other(value) + } + } +} + +impl From for String { + fn from(value: CandidateSearchScope) -> Self { + match value { + CandidateSearchScope::PlannerAuthorizedMaterializations => { + MATERIALIZATION_SEARCH_SCOPE.to_owned() + } + CandidateSearchScope::Other(value) => value, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Renaming diagnostic variants does not change known or future wire values. + #[test] + fn status_wire_values_round_trip() { + for value in [ + "", + "bind_failed", + "bound", + "evidence_missing", + "rejected", + "evidence_invalid", + "unselected", + "selected", + "future_status", + ] { + let status: CandidateEvaluationStatus = + serde_json::from_value(serde_json::json!(value)).unwrap(); + assert_eq!(status.as_str(), value); + assert_eq!(serde_json::to_value(status).unwrap(), value); + } + } + + /// Old search descriptions retain their exact serialized representation. + #[test] + fn scope_wire_values_round_trip() { + for value in [MATERIALIZATION_SEARCH_SCOPE, "future_scope"] { + let scope: CandidateSearchScope = + serde_json::from_value(serde_json::json!(value)).unwrap(); + assert_eq!(serde_json::to_value(scope).unwrap(), value); + } + } +} diff --git a/control_plane/src/physical/workload_planner.rs b/control_plane/src/physical/workload_planner.rs index 1675b8d3a..93fef91bc 100644 --- a/control_plane/src/physical/workload_planner.rs +++ b/control_plane/src/physical/workload_planner.rs @@ -36,7 +36,7 @@ fn mvp_deployment_policy( }) } -/// Bind a `QueryWorkload` into the typed L4 [`crate::physical::post_asap::PhysicalExpr`] +/// Bind a `LegacyMetricWorkload` into the typed L4 [`crate::physical::post_asap::PhysicalExpr`] /// IR, when callers want to inspect the typed binding alongside the /// legacy `CollectionPlan` output. /// @@ -69,7 +69,9 @@ fn mvp_deployment_policy( /// the parallel `USE_TYPED_STAGE_SPLIT` gate is enabled — the bound /// `PhysicalExpr` is then fed into `planner::stage_split::split_typed_three_stage` /// + the per-stage emitters in `config::stage_config`. -pub fn bind_workload_typed(w: &QueryWorkload) -> Option { +pub fn bind_workload_typed( + w: &LegacyMetricWorkload, +) -> Option { bind_workload_typed_with_evidence(w, None, None) } @@ -78,7 +80,7 @@ pub fn bind_workload_typed(w: &QueryWorkload) -> Option Option { bind_workload_typed_with_evidence(w, None, Some(evidence)) @@ -89,21 +91,21 @@ pub fn bind_workload_typed_with_topk_evidence( /// query's actual per-item filter value through to the bound /// `SketchQuery::PointCount` -- `None` (what `bind_workload_typed` itself /// passes) gives the bare bucket total, same as before this parameter -/// existed. `QueryWorkload` itself carries no `item_label` field (adding +/// existed. `LegacyMetricWorkload` itself carries no `item_label` field (adding /// one would break its 30+ struct-literal construction sites across the /// crate), so callers that know a metric's item_label -- e.g. /// `emit::collect_metric_to_family`'s loop, which already has `entry: /// &WorkloadEntry` and `workload.label_filters` in scope -- pass it in /// directly instead. pub fn bind_workload_typed_with_item_filter( - w: &QueryWorkload, + w: &LegacyMetricWorkload, item_filter: Option<(&str, &str)>, ) -> Option { bind_workload_typed_with_evidence(w, item_filter, None) } fn bind_workload_typed_with_evidence( - w: &QueryWorkload, + w: &LegacyMetricWorkload, item_filter: Option<(&str, &str)>, topk_evidence: Option<&crate::physical::compiler::TopKMembershipEvidence>, ) -> Option { @@ -382,7 +384,7 @@ impl DeploymentPlanCompiler { } } - pub fn plan(&self, w: &QueryWorkload) -> CollectionPlan { + pub fn plan(&self, w: &LegacyMetricWorkload) -> CollectionPlan { // This legacy scalar cost path cannot certify a failure probability. // Exact/zero-error and EpsilonDelta use raw; the typed binder independently // checks the full requirement against Planner's family guarantees. @@ -444,7 +446,7 @@ impl DeploymentPlanCompiler { /// Returns a raw-passthrough plan for queries that require exact per-sample /// computation (RSI, MACD, stochastic oscillator, etc.). - fn raw_passthrough_plan(&self, w: &QueryWorkload) -> CollectionPlan { + fn raw_passthrough_plan(&self, w: &LegacyMetricWorkload) -> CollectionPlan { let valid_until = Utc::now() + chrono::Duration::seconds(self.valid_for.as_secs() as i64); let mut label_matchers: Vec = w @@ -492,7 +494,7 @@ pub use crate::physical::sketch_catalog::{build_sketch_params, default_sketch_pa /// /// Rule: if `latency_sla >= time_window` (or unset) → window mode. /// otherwise → batch mode (gateway/backend merges on query). -pub fn select_window_strategy(w: &QueryWorkload) -> (ProcessorMode, Option) { +pub fn select_window_strategy(w: &LegacyMetricWorkload) -> (ProcessorMode, Option) { match w.latency_sla { None => (ProcessorMode::Window, Some(w.time_window)), Some(ls) if ls >= w.time_window => (ProcessorMode::Window, Some(w.time_window)), @@ -507,8 +509,8 @@ mod tests { use super::*; use std::collections::HashMap; - fn workload(aggs: Vec) -> QueryWorkload { - QueryWorkload { + fn workload(aggs: Vec) -> LegacyMetricWorkload { + LegacyMetricWorkload { metric_name: "test".into(), label_filters: HashMap::new(), group_by_labels: vec![], @@ -723,8 +725,8 @@ mod tests { /// contract rows; the AggType still has to be a valid one (the enum /// has no `TopK` variant, so for `top_endpoint_qps` we pass /// `Frequency` and rely on the metric-name reclassification). - fn workload_for(metric: &str, agg: AggType) -> QueryWorkload { - QueryWorkload { + fn workload_for(metric: &str, agg: AggType) -> LegacyMetricWorkload { + LegacyMetricWorkload { metric_name: metric.into(), label_filters: HashMap::new(), group_by_labels: vec![], diff --git a/control_plane/src/pipeline.rs b/control_plane/src/pipeline.rs index 7b34b5ce6..d926a8678 100644 --- a/control_plane/src/pipeline.rs +++ b/control_plane/src/pipeline.rs @@ -4,7 +4,7 @@ use std::collections::{HashMap, HashSet}; use std::time::Duration; use crate::query_parser; -use crate::types::{AggType, QueryWorkload, SketchType, WorkloadCharacteristics}; +use crate::types::{AggType, LegacyMetricWorkload, SketchType, WorkloadCharacteristics}; use crate::types_v2::{AccuracyTarget, DataShape, QueryId, QueryLanguage, QueryShape}; // ── Public API ──────────────────────────────────────────────────────────────── @@ -122,7 +122,7 @@ impl Analyzer { Self } - pub fn analyze(&self, spec: QuerySpec) -> anyhow::Result { + pub fn analyze(&self, spec: QuerySpec) -> anyhow::Result { let accuracy = crate::types_v2::resolve_accuracy_target(spec.accuracy.as_ref(), spec.accuracy_sla) .map_err(|error| anyhow!(error))?; @@ -266,7 +266,7 @@ impl Analyzer { &spec.deployment_model, ); - Ok(QueryWorkload { + Ok(LegacyMetricWorkload { metric_name, label_filters: merged_filters, group_by_labels: all_group_by, @@ -628,14 +628,14 @@ mod tests { /// Typed `accuracy: Some(Epsilon(0.05))` overrides the legacy /// `accuracy_sla: 0.99` (which would translate to `Epsilon(0.01)`), - /// and the resolved value flows through to `QueryWorkload.accuracy_sla`. + /// and the resolved value flows through to `LegacyMetricWorkload.accuracy_sla`. #[test] fn typed_accuracy_overrides_legacy_accuracy_sla() { let mut spec = basic_spec(); spec.accuracy_sla = 0.99; // legacy: ε = 0.01 spec.accuracy = Some(AccuracyTarget::Epsilon(0.05)); let w = Analyzer::new().analyze(spec).unwrap(); - // The resolved 1.0 - 0.05 = 0.95 must reach the QueryWorkload, not + // The resolved 1.0 - 0.05 = 0.95 must reach the LegacyMetricWorkload, not // the legacy 0.99. assert!( (w.accuracy_sla - 0.95).abs() < 1e-9, diff --git a/control_plane/src/query_parser/mod.rs b/control_plane/src/query_parser/mod.rs index 96ec586ee..8f33bfa9e 100644 --- a/control_plane/src/query_parser/mod.rs +++ b/control_plane/src/query_parser/mod.rs @@ -445,75 +445,6 @@ pub(super) fn debs_hint( // ── Tests ───────────────────────────────────────────────────────────────────── -#[cfg(test)] -mod tests { - use super::*; - - const ACC: AccuracyTarget = AccuracyTarget::Epsilon(0.01); - - // Focused unit contracts for the PromQL parse entry point. - - #[test] - fn promql_dispatched_correctly() { - // `by` belongs to the aggregate operator, not the function call. - let pq = parse_query("sum by (host) (quantile_over_time(0.99, latency[5m]))", ACC).unwrap(); - assert!(pq.aggregations.contains(&AggType::Quantile)); - assert_eq!(pq.quantiles, vec![0.99]); - } - - #[test] - fn parse_query_expr_returns_expr() { - let pq = parse_query( - "topk by (symbol) (10, count_over_time(financial_last_trade_price[5m]))", - ACC, - ) - .unwrap(); - // Should parse without error and extract the metric name. - assert_eq!(pq.metric_name, "financial_last_trade_price"); - } - - // ── canonical-IR entry point ───────────────────────────────────────────── - - #[test] - fn canonical_query_quantile_yields_aggregate_over_time_range() { - use planner_types::pre_asap::QueryExpr as CQueryExpr; - // `asap_l2::lower` models a range-vector selector (`m[5m]`) as - // `TimeRange`, not `Window` -- `Window` is reserved for real - // streaming/tumbling windows. `Aggregate` sits directly on top, - // no `Window` wrapper (see this module's own doc for why this - // differs from the retired local parser's shape). - let expr = parse_query_expr_canonical( - "quantile_over_time(0.99, http_request_duration{env=\"prod\"}[5m])", - ACC, - ) - .unwrap(); - match expr { - CQueryExpr::Aggregate { child, .. } => { - assert!(matches!(*child, CQueryExpr::TimeRange { .. })); - } - other => panic!("expected canonical Aggregate, got {other:?}"), - } - } - - #[test] - fn canonical_query_avg_over_time_yields_aggregate_over_time_range() { - use planner_types::pre_asap::QueryExpr as CQueryExpr; - // A bare `avg_over_time(m[w])` (no `by`) lowers to canonical - // `Aggregate { child: TimeRange { child: Scan } }`. - let expr = - parse_query_expr_canonical("avg_over_time(cpu_seconds_total[10m])", ACC).unwrap(); - match expr { - CQueryExpr::Aggregate { child, .. } => match (*child).clone() { - CQueryExpr::TimeRange { child, .. } => { - assert!(matches!(*child, CQueryExpr::Scan { .. })); - } - other => panic!("expected canonical TimeRange, got {other:?}"), - }, - other => panic!("expected canonical Aggregate, got {other:?}"), - } - } -} - #[cfg(test)] mod doc_verify_all { // Pins the design.md §6 worked examples against the canonical IR @@ -596,3 +527,72 @@ mod doc_verify_all { } } } + +#[cfg(test)] +mod tests { + use super::*; + + const ACC: AccuracyTarget = AccuracyTarget::Epsilon(0.01); + + // Focused unit contracts for the PromQL parse entry point. + + #[test] + fn promql_dispatched_correctly() { + // `by` belongs to the aggregate operator, not the function call. + let pq = parse_query("sum by (host) (quantile_over_time(0.99, latency[5m]))", ACC).unwrap(); + assert!(pq.aggregations.contains(&AggType::Quantile)); + assert_eq!(pq.quantiles, vec![0.99]); + } + + #[test] + fn parse_query_expr_returns_expr() { + let pq = parse_query( + "topk by (symbol) (10, count_over_time(financial_last_trade_price[5m]))", + ACC, + ) + .unwrap(); + // Should parse without error and extract the metric name. + assert_eq!(pq.metric_name, "financial_last_trade_price"); + } + + // ── canonical-IR entry point ───────────────────────────────────────────── + + #[test] + fn canonical_query_quantile_yields_aggregate_over_time_range() { + use planner_types::pre_asap::QueryExpr as CQueryExpr; + // `asap_l2::lower` models a range-vector selector (`m[5m]`) as + // `TimeRange`, not `Window` -- `Window` is reserved for real + // streaming/tumbling windows. `Aggregate` sits directly on top, + // no `Window` wrapper (see this module's own doc for why this + // differs from the retired local parser's shape). + let expr = parse_query_expr_canonical( + "quantile_over_time(0.99, http_request_duration{env=\"prod\"}[5m])", + ACC, + ) + .unwrap(); + match expr { + CQueryExpr::Aggregate { child, .. } => { + assert!(matches!(*child, CQueryExpr::TimeRange { .. })); + } + other => panic!("expected canonical Aggregate, got {other:?}"), + } + } + + #[test] + fn canonical_query_avg_over_time_yields_aggregate_over_time_range() { + use planner_types::pre_asap::QueryExpr as CQueryExpr; + // A bare `avg_over_time(m[w])` (no `by`) lowers to canonical + // `Aggregate { child: TimeRange { child: Scan } }`. + let expr = + parse_query_expr_canonical("avg_over_time(cpu_seconds_total[10m])", ACC).unwrap(); + match expr { + CQueryExpr::Aggregate { child, .. } => match (*child).clone() { + CQueryExpr::TimeRange { child, .. } => { + assert!(matches!(*child, CQueryExpr::Scan { .. })); + } + other => panic!("expected canonical TimeRange, got {other:?}"), + }, + other => panic!("expected canonical Aggregate, got {other:?}"), + } + } +} diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs index bcce290d2..400c1cfbb 100644 --- a/control_plane/src/query_plan.rs +++ b/control_plane/src/query_plan.rs @@ -2,12 +2,15 @@ //! Serving consumes asap_types::query_plan; compilation stays in this component. mod clickhouse_exact; -pub mod logical; +pub mod residual; + pub use asap_types::query_plan::*; #[cfg(test)] use asap_types::PolicyFingerprint; use planner_types::post_asap::{SummaryExpr, SummaryFamilyType, SummaryNode}; use planner_types::pre_asap::Reduction; +#[deprecated(note = "Use query_plan::residual")] +pub use residual as logical; use std::collections::BTreeMap; #[cfg(test)] use std::collections::BTreeSet; @@ -210,7 +213,7 @@ where instant, fallback, }; - logical::finalize_residuals(&mut entry)?; + residual::finalize_residuals(&mut entry)?; Ok(entry) } @@ -308,16 +311,16 @@ where self.seen.insert(identity, id); let residual = match (&self.logical_source, &node.expr) { (Some(original), SummaryExpr::KeepPreAsap(expr)) => { - Some(logical::residual_nodes(original, expr)?) + Some(residual::residual_nodes(original, expr)?) } (Some(original), SummaryExpr::SummaryAgg { child, .. }) if matches!(child.expr, SummaryExpr::KeepPreAsap(_)) && !matches!( - crate::physical::compiler::materialization_leaf_contract(node), + crate::physical::compiler::raw_materialization_input_contract(node), Ok((_, Some(_), _)) ) => { - Some(logical::selected_residual_nodes(original, node)?) + Some(residual::selected_residual_nodes(original, node)?) } _ => None, }; @@ -390,11 +393,11 @@ where } if measures.len() == 1 => { use planner_types::pre_asap::AggIntent; let operation = match &measures[0] { - AggIntent::Sum { .. } => logical::Aggregation::Sum, - AggIntent::Count { .. } => logical::Aggregation::Count, - AggIntent::Min { .. } => logical::Aggregation::Min, - AggIntent::Max { .. } => logical::Aggregation::Max, - AggIntent::Avg { .. } => logical::Aggregation::Avg, + AggIntent::Sum { .. } => residual::Aggregation::Sum, + AggIntent::Count { .. } => residual::Aggregation::Count, + AggIntent::Min { .. } => residual::Aggregation::Min, + AggIntent::Max { .. } => residual::Aggregation::Max, + AggIntent::Avg { .. } => residual::Aggregation::Avg, _ => { return Err(QueryPlanError::Invalid( "unsupported exact value aggregation".into(), @@ -423,9 +426,9 @@ where }) .collect::, _>>()?; QueryPlanNode::Logical { - operator: logical::LogicalOperator::Aggregate { + operator: residual::ResidualQueryOperator::Aggregate { operation, - grouping: logical::Grouping { + grouping: residual::Grouping { labels, without: keys.is_without(), }, @@ -487,11 +490,11 @@ where }) .collect::, _>>()?; QueryPlanNode::Logical { - operator: logical::LogicalOperator::TopKSelection { + operator: residual::ResidualQueryOperator::TopKSelection { k: u64::try_from(*n).map_err(|_| { QueryPlanError::Invalid("TopK limit exceeds u64".into()) })?, - grouping: logical::Grouping { + grouping: residual::Grouping { labels, without: partition_by.is_without(), }, @@ -504,7 +507,7 @@ where operation: planner_types::post_asap::ValueOperation::Sort { keys, .. }, timing: planner_types::post_asap::ExecutionTiming::ReadTime, } if keys.len() == 1 => QueryPlanNode::Logical { - operator: logical::LogicalOperator::Sort { + operator: residual::ResidualQueryOperator::Sort { descending: !keys[0].ascending, }, inputs: vec![self.lower(child)?], @@ -599,7 +602,7 @@ where k: u64::try_from(*k).map_err(|_| { QueryPlanError::Invalid("CandidateTopK k exceeds u64".into()) })?, - grouping: logical::Grouping { + grouping: residual::Grouping { labels, without: grouping.is_without(), }, @@ -612,7 +615,7 @@ where operator, timing: planner_types::post_asap::ExecutionTiming::ReadTime, } if self.logical_source.is_some() => { - let operator = logical::binary_operator(operator)?; + let operator = residual::binary_operator(operator)?; QueryPlanNode::Logical { operator, inputs: vec![self.lower(lhs)?, self.lower(rhs)?], @@ -632,7 +635,7 @@ where planner_types::post_asap::ExactKind::Sum | planner_types::post_asap::ExactKind::Count ) { - let operator = logical::selected_aggregate_operator( + let operator = residual::selected_aggregate_operator( self.logical_source.as_deref().unwrap(), node, )?; @@ -647,8 +650,8 @@ where return Ok(id); } let operation = match kind { - planner_types::post_asap::ExactKind::Sum => logical::Aggregation::Sum, - planner_types::post_asap::ExactKind::Count => logical::Aggregation::Count, + planner_types::post_asap::ExactKind::Sum => residual::Aggregation::Sum, + planner_types::post_asap::ExactKind::Count => residual::Aggregation::Count, _ => { return Err(QueryPlanError::Invalid( "unsupported aggregation over selected summary values".into(), @@ -675,9 +678,9 @@ where }) .collect::, _>>()?; QueryPlanNode::Logical { - operator: logical::LogicalOperator::Aggregate { + operator: residual::ResidualQueryOperator::Aggregate { operation, - grouping: logical::Grouping { + grouping: residual::Grouping { labels, without: keys.is_without(), }, @@ -813,7 +816,7 @@ where Err(error) => { if let Some(original) = &self.logical_source { let (root, nodes) = - logical::selected_residual_nodes(original, node)?; + residual::selected_residual_nodes(original, node)?; return self.graft(id, root, nodes); } return Err(error); @@ -1049,6 +1052,178 @@ fn physical_grouping( Ok(PhysicalGrouping::Reduce(names)) } +#[cfg(test)] +mod catalog_binding_tests { + use super::*; + use crate::physical::summary_catalog::SummaryCatalog; + use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; + + fn fixture() -> (QueryPlan, SummaryCatalog) { + let mut config = PrecomputeMaterialization::new( + AggregationType::Sum, + String::new(), + Default::default(), + KeyByLabelNames::new(vec!["job".into()]), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + String::new(), + 10, + 10, + WindowKind::Tumbling, + String::new(), + "m".into(), + None, + None, + None, + ); + config.pane_origin_ms = Some(0); + let catalog = SummaryCatalog::from_materializations(7, 2, &[config.clone()]).unwrap(); + let entry = QueryPlanEntry { + language: crate::query_plan::QueryLanguage::PromQl, + query_id: "q".into(), + canonical_query: "sum_over_time(m[1m])".into(), + fixed_evaluation: None, + root: QueryNodeId(1), + nodes: BTreeMap::from([( + QueryNodeId(1), + QueryPlanNode::ReadMaterialization { + binding: MaterializationBinding { + item_labels: Vec::new(), + materialization: config.policy_fingerprint().into(), + output_grouping: PhysicalGrouping::PerEntity, + window_ms: 10_000, + pane_origin_ms: Some(0), + readout_lookback_ms: Some(60_000), + }, + }, + )]), + instant: InstantExecution { + lookback_ms: 60_000, + full_history: false, + cumulative_readout: true, + }, + fallback: FallbackPolicy::ExactBackend, + }; + ( + QueryPlan { + plan_id: 7, + plan_version: 2, + clickhouse_context: None, + entries: BTreeMap::from([(entry.canonical_query.clone(), entry)]), + }, + catalog, + ) + } + fn binding(plan: &mut QueryPlan) -> &mut MaterializationBinding { + let QueryPlanNode::ReadMaterialization { binding } = plan + .entries + .values_mut() + .next() + .unwrap() + .nodes + .values_mut() + .next() + .unwrap() + else { + panic!("fixture") + }; + binding + } + + // One pane ID is compatible with a longer semantic readout window. + #[test] + fn catalog_binding_round_trip_preserves_pane_and_readout_windows() { + let (plan, catalog) = fixture(); + let wire = serde_json::to_vec(&plan).unwrap(); + let mut decoded: QueryPlan = serde_json::from_slice(&wire).unwrap(); + decoded.validate_against_catalog(&catalog).unwrap(); + assert_eq!( + decoded + .lookup("sum_over_time(m[1m])") + .unwrap() + .canonical_query, + "sum_over_time(m[1m])" + ); + assert!(String::from_utf8(wire).unwrap().contains("canonical_query")); + assert_eq!(binding(&mut decoded).window_ms, 10_000); + assert_eq!(binding(&mut decoded).readout_lookback_ms, Some(60_000)); + } + + // The catalog owns source and grouping; the binding owns only its stable ID. + #[test] + fn catalog_binding_rejects_source_grouping_and_identity_drift() { + let (plan, catalog) = fixture(); + let mut broken = plan.clone(); + binding(&mut broken).materialization = PolicyFingerprint(123).into(); + assert!(broken.validate_against_catalog(&catalog).is_err()); + let mut broken = plan.clone(); + broken.plan_version += 1; + assert!(broken.validate_against_catalog(&catalog).is_err()); + let mut broken = plan; + binding(&mut broken).window_ms = 0; + assert!(broken.validate_against_catalog(&catalog).is_err()); + } + + // Catalog descriptor corruption must fail even if the materialization exists. + #[test] + fn catalog_binding_rejects_broken_descriptor_reference() { + let (plan, mut catalog) = fixture(); + catalog.summary_descriptors.clear(); + assert!(plan.validate_against_catalog(&catalog).is_err()); + } + + #[test] + fn counter_readout_requires_counter_sds_fidelity() { + fn as_rate_plan(mut plan: QueryPlan) -> QueryPlan { + let entry = plan.entries.values_mut().next().unwrap(); + let read = entry.root; + let root = QueryNodeId(2); + entry.root = root; + entry.nodes.insert( + root, + QueryPlanNode::ExactReadout { + input: read, + readout: ExactReadout::Rate, + }, + ); + plan + } + + let (sum_plan, sum_catalog) = fixture(); + assert!(as_rate_plan(sum_plan) + .validate_against_catalog(&sum_catalog) + .unwrap_err() + .to_string() + .contains("exact counter SDS")); + + let mut counter = PrecomputeMaterialization::new( + AggregationType::Increase, + String::new(), + Default::default(), + KeyByLabelNames::new(vec!["job".into()]), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + String::new(), + 10, + 10, + WindowKind::Tumbling, + String::new(), + "m".into(), + None, + None, + None, + ); + counter.pane_origin_ms = Some(0); + let counter_catalog = + SummaryCatalog::from_materializations(7, 2, &[counter.clone()]).unwrap(); + let (mut counter_plan, _) = fixture(); + binding(&mut counter_plan).materialization = counter.policy_fingerprint().into(); + as_rate_plan(counter_plan) + .validate_against_catalog(&counter_catalog) + .unwrap(); + } +} + #[cfg(test)] mod tests { use super::*; @@ -1199,7 +1374,7 @@ mod tests { QueryPlanNode::CandidateTopK { inputs: [QueryNodeId(0), QueryNodeId(1)], k: 2, - grouping: logical::Grouping { + grouping: residual::Grouping { labels: vec![], without: false, }, @@ -1229,175 +1404,3 @@ mod tests { assert!(entry.validate(&BTreeSet::new()).is_err()); } } - -#[cfg(test)] -mod catalog_binding_tests { - use super::*; - use crate::physical::summary_catalog::SummaryCatalog; - use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; - - fn fixture() -> (QueryPlan, SummaryCatalog) { - let mut config = PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - Default::default(), - KeyByLabelNames::new(vec!["job".into()]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 10, - 10, - WindowKind::Tumbling, - String::new(), - "m".into(), - None, - None, - None, - ); - config.pane_origin_ms = Some(0); - let catalog = SummaryCatalog::from_materializations(7, 2, &[config.clone()]).unwrap(); - let entry = QueryPlanEntry { - language: crate::query_plan::QueryLanguage::PromQl, - query_id: "q".into(), - canonical_query: "sum_over_time(m[1m])".into(), - fixed_evaluation: None, - root: QueryNodeId(1), - nodes: BTreeMap::from([( - QueryNodeId(1), - QueryPlanNode::ReadMaterialization { - binding: MaterializationBinding { - item_labels: Vec::new(), - materialization: config.policy_fingerprint().into(), - output_grouping: PhysicalGrouping::PerEntity, - window_ms: 10_000, - pane_origin_ms: Some(0), - readout_lookback_ms: Some(60_000), - }, - }, - )]), - instant: InstantExecution { - lookback_ms: 60_000, - full_history: false, - cumulative_readout: true, - }, - fallback: FallbackPolicy::ExactBackend, - }; - ( - QueryPlan { - plan_id: 7, - plan_version: 2, - clickhouse_context: None, - entries: BTreeMap::from([(entry.canonical_query.clone(), entry)]), - }, - catalog, - ) - } - fn binding(plan: &mut QueryPlan) -> &mut MaterializationBinding { - let QueryPlanNode::ReadMaterialization { binding } = plan - .entries - .values_mut() - .next() - .unwrap() - .nodes - .values_mut() - .next() - .unwrap() - else { - panic!("fixture") - }; - binding - } - - // One pane ID is compatible with a longer semantic readout window. - #[test] - fn catalog_binding_round_trip_preserves_pane_and_readout_windows() { - let (plan, catalog) = fixture(); - let wire = serde_json::to_vec(&plan).unwrap(); - let mut decoded: QueryPlan = serde_json::from_slice(&wire).unwrap(); - decoded.validate_against_catalog(&catalog).unwrap(); - assert_eq!( - decoded - .lookup("sum_over_time(m[1m])") - .unwrap() - .canonical_query, - "sum_over_time(m[1m])" - ); - assert!(String::from_utf8(wire).unwrap().contains("canonical_query")); - assert_eq!(binding(&mut decoded).window_ms, 10_000); - assert_eq!(binding(&mut decoded).readout_lookback_ms, Some(60_000)); - } - - // The catalog owns source and grouping; the binding owns only its stable ID. - #[test] - fn catalog_binding_rejects_source_grouping_and_identity_drift() { - let (plan, catalog) = fixture(); - let mut broken = plan.clone(); - binding(&mut broken).materialization = PolicyFingerprint(123).into(); - assert!(broken.validate_against_catalog(&catalog).is_err()); - let mut broken = plan.clone(); - broken.plan_version += 1; - assert!(broken.validate_against_catalog(&catalog).is_err()); - let mut broken = plan; - binding(&mut broken).window_ms = 0; - assert!(broken.validate_against_catalog(&catalog).is_err()); - } - - // Catalog descriptor corruption must fail even if the materialization exists. - #[test] - fn catalog_binding_rejects_broken_descriptor_reference() { - let (plan, mut catalog) = fixture(); - catalog.summary_descriptors.clear(); - assert!(plan.validate_against_catalog(&catalog).is_err()); - } - - #[test] - fn counter_readout_requires_counter_sds_fidelity() { - fn as_rate_plan(mut plan: QueryPlan) -> QueryPlan { - let entry = plan.entries.values_mut().next().unwrap(); - let read = entry.root; - let root = QueryNodeId(2); - entry.root = root; - entry.nodes.insert( - root, - QueryPlanNode::ExactReadout { - input: read, - readout: ExactReadout::Rate, - }, - ); - plan - } - - let (sum_plan, sum_catalog) = fixture(); - assert!(as_rate_plan(sum_plan) - .validate_against_catalog(&sum_catalog) - .unwrap_err() - .to_string() - .contains("exact counter SDS")); - - let mut counter = PrecomputeMaterialization::new( - AggregationType::Increase, - String::new(), - Default::default(), - KeyByLabelNames::new(vec!["job".into()]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 10, - 10, - WindowKind::Tumbling, - String::new(), - "m".into(), - None, - None, - None, - ); - counter.pane_origin_ms = Some(0); - let counter_catalog = - SummaryCatalog::from_materializations(7, 2, &[counter.clone()]).unwrap(); - let (mut counter_plan, _) = fixture(); - binding(&mut counter_plan).materialization = counter.policy_fingerprint().into(); - as_rate_plan(counter_plan) - .validate_against_catalog(&counter_catalog) - .unwrap(); - } -} diff --git a/control_plane/src/query_plan/clickhouse_exact.rs b/control_plane/src/query_plan/clickhouse_exact.rs index fa227841f..b5afab685 100644 --- a/control_plane/src/query_plan/clickhouse_exact.rs +++ b/control_plane/src/query_plan/clickhouse_exact.rs @@ -267,99 +267,6 @@ pub(super) fn render(expr: &QueryExpr) -> Result { } } -#[cfg(test)] -mod tests { - use super::*; - use planner_types::pre_asap::{Column, DataType, Predicate, ProjectItem}; - use std::rc::Rc; - #[test] - fn composite_cut_preserves_branch_time_and_positional_projection() { - let scan = Rc::new(QueryExpr::Scan { - source: Source::Table { - table_ref: "db.samples".into(), - }, - schema: Schema::new(vec![ - Column::new("ts", DataType::Int64, false), - Column::new("v", DataType::Float64, false), - ]), - predicates: vec![Predicate(Rc::new(QueryExpr::Compare { - left: Rc::new(QueryExpr::Column(0)), - op: CompareOpKind::Lt, - right: Rc::new(QueryExpr::Literal(ScalarValue::Int64(-100))), - }))], - }); - let project = QueryExpr::Project { - cols: vec![ProjectItem { - alias: Some("result".into()), - expr: QueryExpr::Column(1), - }], - qualifier: None, - child: scan, - }; - let sql = render(&project).unwrap(); - assert!(sql.contains("`ts` < -100")); - assert!(sql.starts_with("SELECT `v` AS `result`")); - assert!(!sql.contains("{from:")); - assert!(!sql.contains("{to:")); - } - #[test] - fn typed_list_access_renders_native_element_lookup() { - let schema = Schema::new(vec![Column::new( - "samples", - DataType::List { - element: Box::new(Column::new("item", DataType::Float64, false)), - }, - false, - )]); - let expr = QueryExpr::FunctionCall { - name: "asap_element_access".into(), - args: vec![ - QueryExpr::Column(0), - QueryExpr::Literal(ScalarValue::Int64(-1)), - ], - }; - assert_eq!( - scalar(&expr, &schema).unwrap(), - "arrayElement(`samples`, -1)" - ); - } - - #[test] - fn typed_struct_field_renders_native_lookup() { - let schema = Schema::new(vec![Column::new( - "sample", - DataType::Struct { - fields: vec![ - Column::new("ts", DataType::Int64, false), - Column::new("value", DataType::Float64, true), - ], - }, - false, - )]); - let expr = QueryExpr::FunctionCall { - name: "asap_struct_field".into(), - args: vec![ - QueryExpr::Column(0), - QueryExpr::Literal(ScalarValue::Utf8("value".into())), - ], - }; - assert_eq!( - scalar(&expr, &schema).unwrap(), - "tupleElement(`sample`, 'value')" - ); - } - - #[test] - fn unsupported_scalar_is_not_forwarded_as_arbitrary_native_code() { - let schema = Schema::new(vec![]); - let expr = QueryExpr::FunctionCall { - name: "unreviewedFunction".into(), - args: vec![], - }; - assert!(scalar(&expr, &schema).is_err()); - } -} - #[cfg(test)] mod original_tests { use super::*; @@ -469,3 +376,96 @@ mod original_tests { assert_eq!(rendered, "'a\\\\\\'b\n'"); } } + +#[cfg(test)] +mod tests { + use super::*; + use planner_types::pre_asap::{Column, DataType, Predicate, ProjectItem}; + use std::rc::Rc; + #[test] + fn composite_cut_preserves_branch_time_and_positional_projection() { + let scan = Rc::new(QueryExpr::Scan { + source: Source::Table { + table_ref: "db.samples".into(), + }, + schema: Schema::new(vec![ + Column::new("ts", DataType::Int64, false), + Column::new("v", DataType::Float64, false), + ]), + predicates: vec![Predicate(Rc::new(QueryExpr::Compare { + left: Rc::new(QueryExpr::Column(0)), + op: CompareOpKind::Lt, + right: Rc::new(QueryExpr::Literal(ScalarValue::Int64(-100))), + }))], + }); + let project = QueryExpr::Project { + cols: vec![ProjectItem { + alias: Some("result".into()), + expr: QueryExpr::Column(1), + }], + qualifier: None, + child: scan, + }; + let sql = render(&project).unwrap(); + assert!(sql.contains("`ts` < -100")); + assert!(sql.starts_with("SELECT `v` AS `result`")); + assert!(!sql.contains("{from:")); + assert!(!sql.contains("{to:")); + } + #[test] + fn typed_list_access_renders_native_element_lookup() { + let schema = Schema::new(vec![Column::new( + "samples", + DataType::List { + element: Box::new(Column::new("item", DataType::Float64, false)), + }, + false, + )]); + let expr = QueryExpr::FunctionCall { + name: "asap_element_access".into(), + args: vec![ + QueryExpr::Column(0), + QueryExpr::Literal(ScalarValue::Int64(-1)), + ], + }; + assert_eq!( + scalar(&expr, &schema).unwrap(), + "arrayElement(`samples`, -1)" + ); + } + + #[test] + fn typed_struct_field_renders_native_lookup() { + let schema = Schema::new(vec![Column::new( + "sample", + DataType::Struct { + fields: vec![ + Column::new("ts", DataType::Int64, false), + Column::new("value", DataType::Float64, true), + ], + }, + false, + )]); + let expr = QueryExpr::FunctionCall { + name: "asap_struct_field".into(), + args: vec![ + QueryExpr::Column(0), + QueryExpr::Literal(ScalarValue::Utf8("value".into())), + ], + }; + assert_eq!( + scalar(&expr, &schema).unwrap(), + "tupleElement(`sample`, 'value')" + ); + } + + #[test] + fn unsupported_scalar_is_not_forwarded_as_arbitrary_native_code() { + let schema = Schema::new(vec![]); + let expr = QueryExpr::FunctionCall { + name: "unreviewedFunction".into(), + args: vec![], + }; + assert!(scalar(&expr, &schema).is_err()); + } +} diff --git a/control_plane/src/query_plan/logical.rs b/control_plane/src/query_plan/residual.rs similarity index 91% rename from control_plane/src/query_plan/logical.rs rename to control_plane/src/query_plan/residual.rs index e802ea4ba..3de22a684 100644 --- a/control_plane/src/query_plan/logical.rs +++ b/control_plane/src/query_plan/residual.rs @@ -8,7 +8,7 @@ use promql_parser::{ }; use std::collections::BTreeMap; -pub use asap_types::query_plan::logical::*; +pub use asap_types::query_plan::residual::*; /// Stable identity of a Planner-authorized materializable DAG leaf. This is a /// workload-selection key, not another physical materialization definition. @@ -54,7 +54,7 @@ impl Lower { } fn operation( &mut self, - operator: LogicalOperator, + operator: ResidualQueryOperator, inputs: Vec, ) -> Result { operator.validate(inputs.len())?; @@ -84,7 +84,7 @@ impl Lower { }) .collect(); self.operation( - LogicalOperator::Scan { + ResidualQueryOperator::Scan { metric: s.name.clone(), matchers, range_ms, @@ -101,7 +101,7 @@ impl Lower { Expr::Paren(p) => self.lower(&p.expr), Expr::Unary(u) => { let input = self.lower(&u.expr)?; - self.operation(LogicalOperator::UnaryNegate, vec![input]) + self.operation(ResidualQueryOperator::UnaryNegate, vec![input]) } Expr::VectorSelector(s) => self.scan(s, None), Expr::MatrixSelector(s) => self.scan(&s.vs, Some(millis(s.range)?)), @@ -111,7 +111,7 @@ impl Lower { } let input = self.lower(&s.expr)?; self.operation( - LogicalOperator::Subquery { + ResidualQueryOperator::Subquery { range_ms: millis(s.range)?, // Prometheus uses its configured default evaluation // interval when `[range:]` omits the resolution. The @@ -163,7 +163,7 @@ impl Lower { self.nodes = nodes_before; self.seen = seen_before; self.operation( - LogicalOperator::ExactSubquery { + ResidualQueryOperator::ExactSubquery { query: a.expr.to_string(), }, vec![], @@ -171,7 +171,7 @@ impl Lower { } }; return self.operation( - LogicalOperator::TopKSelection { + ResidualQueryOperator::TopKSelection { k: u64::try_from(k).unwrap_or(0), grouping, }, @@ -191,7 +191,7 @@ impl Lower { }; let input = self.lower(&a.expr)?; self.operation( - LogicalOperator::Aggregate { + ResidualQueryOperator::Aggregate { operation, grouping, }, @@ -200,11 +200,11 @@ impl Lower { } Expr::Call(c) => { let operator = match c.func.name { - "scalar" => LogicalOperator::VectorToScalar, - "histogram_quantile" => LogicalOperator::HistogramQuantile, - "sort" => LogicalOperator::Sort { descending: false }, - "sort_desc" => LogicalOperator::Sort { descending: true }, - name => LogicalOperator::Temporal { + "scalar" => ResidualQueryOperator::VectorToScalar, + "histogram_quantile" => ResidualQueryOperator::HistogramQuantile, + "sort" => ResidualQueryOperator::Sort { descending: false }, + "sort_desc" => ResidualQueryOperator::Sort { descending: true }, + name => ResidualQueryOperator::Temporal { operation: match name { "rate" => TemporalOperation::Rate, "increase" => TemporalOperation::Increase, @@ -251,7 +251,7 @@ impl Lower { }; let inputs = vec![self.lower(&b.lhs)?, self.lower(&b.rhs)?]; self.operation( - LogicalOperator::Binary { + ResidualQueryOperator::Binary { operation, return_bool: b.return_bool(), }, @@ -341,7 +341,7 @@ pub(super) fn residual_nodes( pub(super) fn binary_operator( operator: &planner_types::post_asap::BinaryOperator, -) -> Result { +) -> Result { if operator.vector_match.is_some() { return Err(invalid("explicit residual vector matching unsupported")); } @@ -364,174 +364,12 @@ pub(super) fn binary_operator( ))) } }; - Ok(LogicalOperator::Binary { + Ok(ResidualQueryOperator::Binary { operation, return_bool: false, }) } -#[cfg(test)] -mod tests { - use super::*; - fn instant() -> InstantExecution { - InstantExecution { - lookback_ms: 300_000, - full_history: false, - cumulative_readout: false, - } - } - - #[test] - fn complete_o11y_corpus_lowers_to_serialized_operations() { - // Every original workload occurrence must compile to an executable typed graph. - let corpus: serde_json::Value = - serde_json::from_str(include_str!("../../tests/fixtures/o11y_queries.json")).unwrap(); - for row in corpus["queries"].as_array().unwrap() { - let query = row["query"].as_str().unwrap(); - let entry = crate::query_plan::logical::compile_logical( - row["id"].as_str().unwrap().into(), - query.into(), - instant(), - FallbackPolicy::Reject, - ) - .unwrap_or_else(|error| panic!("{query}: {error}")); - let encoded = serde_json::to_string(&entry).unwrap(); - let restored: QueryPlanEntry = serde_json::from_str(&encoded).unwrap(); - restored.validate(&Default::default()).unwrap(); - assert!(!restored - .nodes - .values() - .any(|node| matches!(node, QueryPlanNode::ExactFallback { .. }))); - } - } - #[test] - fn residual_mapping_preserves_filters_and_rejects_different_sources() { - // Physical lowering must prove correspondence with the Planner-kept semantic subtree. - let query = "sum(rate(requests_total{job=\"api\"}[5m]))"; - let residual = crate::query_parser::parse_query_expr_canonical( - query, - planner_types::types::AccuracyTarget::Exact, - ) - .unwrap(); - let (_, nodes) = residual_nodes(query, &residual).unwrap(); - assert!(nodes.values().any(|node| matches!(node, QueryPlanNode::Logical { operator: LogicalOperator::Scan { matchers, .. }, .. } if matchers.iter().any(|m| m.name == "job" && m.value == "api")))); - assert!(residual_nodes("sum(rate(other_total[5m]))", &residual).is_err()); - } - #[test] - fn repeated_subexpressions_share_node_identity() { - // Serialized edges must retain CSE rather than duplicating raw work. - let entry = crate::query_plan::logical::compile_logical( - "q".into(), - "sum(up) / sum(up)".into(), - instant(), - FallbackPolicy::Reject, - ) - .unwrap(); - let QueryPlanNode::Logical { inputs, .. } = &entry.nodes[&entry.root] else { - panic!("binary expected") - }; - assert_eq!(inputs[0], inputs[1]); - } - #[test] - fn malformed_operator_arity_is_rejected_at_installation() { - // A serialized graph cannot bypass the operation's input contract. - assert!(LogicalOperator::HistogramQuantile.validate(1).is_err()); - assert!(LogicalOperator::Subquery { - range_ms: 60_000, - step_ms: 0, - offset_ms: 0 - } - .validate(1) - .is_err()); - } - - #[test] - fn real_topk_queries_lower_to_value_selection() { - for (query, k) in [ - ( - "topk(2, sum by (job) (rate(backend_process_cpu_seconds_total[1h])))", - 2, - ), - ( - "topk(2, sum by (job) (backend_process_resident_memory_bytes))", - 2, - ), - ("topk(2, max_over_time(backend_retry_backlog_depth[6h]))", 2), - ( - "topk(1, sum by (job) (increase(backend_http_5xx_total[6h])) / sum by (job) (increase(backend_http_requests_total[6h])))", - 1, - ), - ( - "topk(3, avg_over_time((sum by (job) (backend_process_resident_memory_bytes))[6h:]))", - 3, - ), - ] { - let entry = crate::query_plan::logical::compile_logical( - "topk".into(), - query.into(), - instant(), - FallbackPolicy::Reject, - ) - .unwrap_or_else(|error| panic!("{query}: {error}")); - assert!(matches!( - entry.nodes[&entry.root], - QueryPlanNode::Logical { - operator: LogicalOperator::TopKSelection { k: actual, .. }, - .. - } if actual == k - )); - } - } - - #[test] - fn topk_keeps_unsupported_child_as_exact_leaf() { - let entry = crate::query_plan::logical::compile_logical( - "topk-subquery".into(), - "topk(3, label_replace(memory_bytes, \"dst\", \"$1\", \"src\", \"(.*)\"))".into(), - instant(), - FallbackPolicy::Reject, - ) - .unwrap(); - assert!(matches!( - entry.nodes[&entry.root], - QueryPlanNode::Logical { - operator: LogicalOperator::TopKSelection { k: 3, .. }, - .. - } - )); - assert!(entry.nodes.values().any(|node| matches!( - node, - QueryPlanNode::Logical { - operator: LogicalOperator::ExactSubquery { .. }, - .. - } - ))); - } - - #[test] - fn topk_preserves_by_and_without_partitioning() { - for (query, labels, without) in [ - ("topk by (cluster) (2, m)", vec!["cluster"], false), - ("topk without (pod) (2, m)", vec!["pod"], true), - ] { - let entry = crate::query_plan::logical::compile_logical( - "topk-group".into(), - query.into(), - instant(), - FallbackPolicy::Reject, - ) - .unwrap(); - assert!(matches!( - &entry.nodes[&entry.root], - QueryPlanNode::Logical { - operator: LogicalOperator::TopKSelection { grouping, .. }, - .. - } if grouping.labels == labels && grouping.without == without - )); - } - } -} - /// Prove a physical-native substitute represents exactly the selected summary leaf. /// A second Planner invocation is an equality witness, not a replacement selection. pub(crate) fn selected_residual_nodes( @@ -604,11 +442,11 @@ pub(crate) fn selected_residual_nodes( pub(super) fn selected_aggregate_operator( original: &str, selected: &planner_types::post_asap::SummaryNode, -) -> Result { +) -> Result { let (root, nodes) = selected_residual_nodes(original, selected)?; match nodes.get(&root) { Some(QueryPlanNode::Logical { - operator: operator @ LogicalOperator::Aggregate { .. }, + operator: operator @ ResidualQueryOperator::Aggregate { .. }, .. }) => Ok(operator.clone()), _ => Err(invalid( @@ -644,7 +482,7 @@ mod hybrid_tests { FallbackPolicy::Reject, |node, _| { let (_, _, spatial_filter) = - crate::physical::compiler::materialization_leaf_contract(node) + crate::physical::compiler::raw_materialization_input_contract(node) .map_err(QueryPlanError::Invalid)?; Ok(MaterializationBinding { item_labels: Vec::new(), @@ -664,21 +502,21 @@ mod hybrid_tests { assert!(!entry.nodes.values().any(|node| matches!( node, QueryPlanNode::Logical { - operator: LogicalOperator::ExactSubquery { .. }, + operator: ResidualQueryOperator::ExactSubquery { .. }, .. } ))); assert!(!entry.nodes.values().any(|node| matches!( node, QueryPlanNode::Logical { - operator: LogicalOperator::Scan { .. }, + operator: ResidualQueryOperator::Scan { .. }, .. } ))); assert!(matches!( entry.nodes[&entry.root], QueryPlanNode::Logical { - operator: LogicalOperator::Binary { .. }, + operator: ResidualQueryOperator::Binary { .. }, .. } )); @@ -712,7 +550,7 @@ mod hybrid_tests { #[cfg(test)] mod planner_workload_tests { use super::*; - use crate::physical::compiler::{BackendLocalPlanningSnapshot, PhysicalCompiler}; + use crate::physical::compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}; fn lookback(expr: &Expr) -> u64 { match expr { @@ -727,7 +565,7 @@ mod planner_workload_tests { } } - fn compile_one(query: &str) -> crate::physical::compiler::PhysicalPlan { + fn compile_one(query: &str) -> crate::physical::compiler::CompiledPhysicalPlan { let mut fixture: serde_json::Value = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) @@ -738,12 +576,12 @@ mod planner_workload_tests { let window = lookback(&parser::parse(query).unwrap()); entry["time_selection"]["lookback"] = (if window == 0 { 300_000 } else { window }).into(); fixture["query_workload"]["repeating_queries"] = vec![entry].into(); - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_value(fixture).unwrap(); + let snapshot: BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); let (request, environment) = snapshot - .planning_request() + .into_physical_compilation_request() .unwrap_or_else(|error| panic!("{query}: {error}")); - PhysicalCompiler - .compile(request, environment) + PhysicalPlanCompiler + .compile_promql(request, environment) .unwrap_or_else(|error| panic!("{query}: {error}")) } @@ -762,7 +600,7 @@ mod planner_workload_tests { matches!( entry.nodes[&entry.root], QueryPlanNode::Logical { - operator: LogicalOperator::TopKSelection { .. }, + operator: ResidualQueryOperator::TopKSelection { .. }, .. } ), @@ -790,7 +628,7 @@ mod planner_workload_tests { assert!(matches!( entry.nodes[&entry.root], QueryPlanNode::Logical { - operator: LogicalOperator::TopKSelection { .. }, + operator: ResidualQueryOperator::TopKSelection { .. }, .. } )); @@ -828,10 +666,12 @@ mod planner_workload_tests { entries.push(entry); } fixture["query_workload"]["repeating_queries"] = entries.into(); - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_value(fixture).unwrap(); - let (request, environment) = snapshot.planning_request().unwrap(); - assert!(request.hybrid_execution); - let plan = PhysicalCompiler.compile(request, environment).unwrap(); + let snapshot: BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); + let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); + assert!(request.allow_mixed_summary_and_exact_execution); + let plan = PhysicalPlanCompiler + .compile_promql(request, environment) + .unwrap(); assert_eq!(plan.query_plan.entries.len(), 24); assert!(plan.query_plan.entries.values().all(|entry| !entry .nodes @@ -852,7 +692,7 @@ mod planner_workload_tests { let operator = selected_aggregate_operator(query, &selected).unwrap(); assert!(matches!( operator, - LogicalOperator::Aggregate { + ResidualQueryOperator::Aggregate { operation: Aggregation::Max, .. } @@ -882,7 +722,7 @@ mod planner_workload_tests { assert!(matches!( nodes[&root], QueryPlanNode::Logical { - operator: LogicalOperator::Aggregate { + operator: ResidualQueryOperator::Aggregate { operation: Aggregation::Min, .. }, @@ -913,7 +753,7 @@ pub(crate) fn selected_range_max_materialization( let (root, nodes) = selected_residual_nodes(original, node)?; let Some(QueryPlanNode::Logical { operator: - LogicalOperator::Temporal { + ResidualQueryOperator::Temporal { operation: TemporalOperation::Max, }, inputs, @@ -926,7 +766,7 @@ pub(crate) fn selected_range_max_materialization( } let Some(QueryPlanNode::Logical { operator: - LogicalOperator::Scan { + ResidualQueryOperator::Scan { metric: Some(metric), matchers, range_ms: Some(range_ms), @@ -1027,7 +867,7 @@ fn counter_contract( nodes: &BTreeMap, ) -> Option { let QueryPlanNode::Logical { - operator: LogicalOperator::Temporal { operation }, + operator: ResidualQueryOperator::Temporal { operation }, inputs, } = nodes.get(&root)? else { @@ -1042,7 +882,7 @@ fn counter_contract( } let QueryPlanNode::Logical { operator: - LogicalOperator::Scan { + ResidualQueryOperator::Scan { metric: Some(metric), matchers, range_ms: Some(range_ms), @@ -1102,7 +942,7 @@ pub fn finalize_residuals(entry: &mut QueryPlanEntry) -> Result<(), QueryPlanErr assign_retention(entry) } -pub fn materialization_candidate_keys( +pub fn eligible_materialization_keys( original: &str, selected: &std::rc::Rc, ) -> Result, QueryPlanError> { @@ -1168,7 +1008,7 @@ fn expression_shape( .get(&id) .ok_or_else(|| invalid("missing expression node"))?; if let QueryPlanNode::Logical { - operator: LogicalOperator::ExactSubquery { query }, + operator: ResidualQueryOperator::ExactSubquery { query }, .. } = node { @@ -1245,7 +1085,7 @@ pub fn externalize_residuals(entry: &mut QueryPlanEntry) -> Result<(), QueryPlan ) || matches!( node, QueryPlanNode::Logical { - operator: LogicalOperator::TopKSelection { .. }, + operator: ResidualQueryOperator::TopKSelection { .. }, .. } ); @@ -1253,9 +1093,9 @@ pub fn externalize_residuals(entry: &mut QueryPlanEntry) -> Result<(), QueryPlan if let QueryPlanNode::Logical { operator, .. } = node { exact = matches!( operator, - LogicalOperator::Scan { .. } - | LogicalOperator::ExactSubquery { .. } - | LogicalOperator::CandidateExactSubquery { .. } + ResidualQueryOperator::Scan { .. } + | ResidualQueryOperator::ExactSubquery { .. } + | ResidualQueryOperator::CandidateExactSubquery { .. } ); } for child in node.inputs() { @@ -1270,8 +1110,8 @@ pub fn externalize_residuals(entry: &mut QueryPlanEntry) -> Result<(), QueryPlan if matches!( entry.nodes.get(&id), Some(QueryPlanNode::Logical { - operator: LogicalOperator::ExactSubquery { .. } - | LogicalOperator::CandidateExactSubquery { .. }, + operator: ResidualQueryOperator::ExactSubquery { .. } + | ResidualQueryOperator::CandidateExactSubquery { .. }, .. }) ) { @@ -1284,7 +1124,7 @@ pub fn externalize_residuals(entry: &mut QueryPlanEntry) -> Result<(), QueryPlan entry.nodes.insert( id, QueryPlanNode::Logical { - operator: LogicalOperator::ExactSubquery { + operator: ResidualQueryOperator::ExactSubquery { query: query.clone(), }, inputs: vec![], @@ -1301,7 +1141,7 @@ pub fn externalize_residuals(entry: &mut QueryPlanEntry) -> Result<(), QueryPlan matches!( node, QueryPlanNode::Logical { - operator: LogicalOperator::Scan { .. }, + operator: ResidualQueryOperator::Scan { .. }, .. } ) @@ -1327,7 +1167,7 @@ fn assign_retention(entry: &mut QueryPlanEntry) -> Result<(), QueryPlanError> { .ok_or_else(|| invalid("missing index ancestor"))?; let mut child_depth = depth; if let QueryPlanNode::Logical { operator, .. } = node { - if let LogicalOperator::Subquery { + if let ResidualQueryOperator::Subquery { range_ms, offset_ms, .. @@ -1390,10 +1230,177 @@ mod remote_boundary_regressions { .unwrap(); let selected = crate::planner_selection::select_summary_default(&parsed).unwrap(); assert_eq!( - materialization_candidate_keys(query, &selected) + eligible_materialization_keys(query, &selected) .unwrap() .len(), 2 ); } } + +#[deprecated(note = "Use eligible_materialization_keys")] +pub use eligible_materialization_keys as materialization_candidate_keys; + +#[cfg(test)] +mod tests { + use super::*; + fn instant() -> InstantExecution { + InstantExecution { + lookback_ms: 300_000, + full_history: false, + cumulative_readout: false, + } + } + + #[test] + fn complete_o11y_corpus_lowers_to_serialized_operations() { + // Every original workload occurrence must compile to an executable typed graph. + let corpus: serde_json::Value = + serde_json::from_str(include_str!("../../tests/fixtures/o11y_queries.json")).unwrap(); + for row in corpus["queries"].as_array().unwrap() { + let query = row["query"].as_str().unwrap(); + let entry = crate::query_plan::residual::compile_logical( + row["id"].as_str().unwrap().into(), + query.into(), + instant(), + FallbackPolicy::Reject, + ) + .unwrap_or_else(|error| panic!("{query}: {error}")); + let encoded = serde_json::to_string(&entry).unwrap(); + let restored: QueryPlanEntry = serde_json::from_str(&encoded).unwrap(); + restored.validate(&Default::default()).unwrap(); + assert!(!restored + .nodes + .values() + .any(|node| matches!(node, QueryPlanNode::ExactFallback { .. }))); + } + } + #[test] + fn residual_mapping_preserves_filters_and_rejects_different_sources() { + // Physical lowering must prove correspondence with the Planner-kept semantic subtree. + let query = "sum(rate(requests_total{job=\"api\"}[5m]))"; + let residual = crate::query_parser::parse_query_expr_canonical( + query, + planner_types::types::AccuracyTarget::Exact, + ) + .unwrap(); + let (_, nodes) = residual_nodes(query, &residual).unwrap(); + assert!(nodes.values().any(|node| matches!(node, QueryPlanNode::Logical { operator: ResidualQueryOperator::Scan { matchers, .. }, .. } if matchers.iter().any(|m| m.name == "job" && m.value == "api")))); + assert!(residual_nodes("sum(rate(other_total[5m]))", &residual).is_err()); + } + #[test] + fn repeated_subexpressions_share_node_identity() { + // Serialized edges must retain CSE rather than duplicating raw work. + let entry = crate::query_plan::residual::compile_logical( + "q".into(), + "sum(up) / sum(up)".into(), + instant(), + FallbackPolicy::Reject, + ) + .unwrap(); + let QueryPlanNode::Logical { inputs, .. } = &entry.nodes[&entry.root] else { + panic!("binary expected") + }; + assert_eq!(inputs[0], inputs[1]); + } + #[test] + fn malformed_operator_arity_is_rejected_at_installation() { + // A serialized graph cannot bypass the operation's input contract. + assert!(ResidualQueryOperator::HistogramQuantile + .validate(1) + .is_err()); + assert!(ResidualQueryOperator::Subquery { + range_ms: 60_000, + step_ms: 0, + offset_ms: 0 + } + .validate(1) + .is_err()); + } + + #[test] + fn real_topk_queries_lower_to_value_selection() { + for (query, k) in [ + ( + "topk(2, sum by (job) (rate(backend_process_cpu_seconds_total[1h])))", + 2, + ), + ( + "topk(2, sum by (job) (backend_process_resident_memory_bytes))", + 2, + ), + ("topk(2, max_over_time(backend_retry_backlog_depth[6h]))", 2), + ( + "topk(1, sum by (job) (increase(backend_http_5xx_total[6h])) / sum by (job) (increase(backend_http_requests_total[6h])))", + 1, + ), + ( + "topk(3, avg_over_time((sum by (job) (backend_process_resident_memory_bytes))[6h:]))", + 3, + ), + ] { + let entry = crate::query_plan::residual::compile_logical( + "topk".into(), + query.into(), + instant(), + FallbackPolicy::Reject, + ) + .unwrap_or_else(|error| panic!("{query}: {error}")); + assert!(matches!( + entry.nodes[&entry.root], + QueryPlanNode::Logical { + operator: ResidualQueryOperator::TopKSelection { k: actual, .. }, + .. + } if actual == k + )); + } + } + + #[test] + fn topk_keeps_unsupported_child_as_exact_leaf() { + let entry = crate::query_plan::residual::compile_logical( + "topk-subquery".into(), + "topk(3, label_replace(memory_bytes, \"dst\", \"$1\", \"src\", \"(.*)\"))".into(), + instant(), + FallbackPolicy::Reject, + ) + .unwrap(); + assert!(matches!( + entry.nodes[&entry.root], + QueryPlanNode::Logical { + operator: ResidualQueryOperator::TopKSelection { k: 3, .. }, + .. + } + )); + assert!(entry.nodes.values().any(|node| matches!( + node, + QueryPlanNode::Logical { + operator: ResidualQueryOperator::ExactSubquery { .. }, + .. + } + ))); + } + + #[test] + fn topk_preserves_by_and_without_partitioning() { + for (query, labels, without) in [ + ("topk by (cluster) (2, m)", vec!["cluster"], false), + ("topk without (pod) (2, m)", vec!["pod"], true), + ] { + let entry = crate::query_plan::residual::compile_logical( + "topk-group".into(), + query.into(), + instant(), + FallbackPolicy::Reject, + ) + .unwrap(); + assert!(matches!( + &entry.nodes[&entry.root], + QueryPlanNode::Logical { + operator: ResidualQueryOperator::TopKSelection { grouping, .. }, + .. + } if grouping.labels == labels && grouping.without == without + )); + } + } +} diff --git a/control_plane/src/replan.rs b/control_plane/src/replan.rs index 4ee23c58a..49e72d9c9 100644 --- a/control_plane/src/replan.rs +++ b/control_plane/src/replan.rs @@ -33,7 +33,7 @@ use crate::physical::plan_cache::CachedDeploymentPlanner; use crate::physical::stage_split; use crate::physical::workload_planner as rules; use crate::store::{PlanStore, WorkloadStore}; -use crate::types::QueryWorkload; +use crate::types::LegacyMetricWorkload; use crate::workload::AggRole; fn short_hash(s: &str) -> String { @@ -231,7 +231,7 @@ impl Replanner { } /// Same as [`try_emit_typed_edge_yaml`] but takes the - /// `QueryWorkload` directly. Used by `replan_metric` which already + /// `LegacyMetricWorkload` directly. Used by `replan_metric` which already /// has the workload in scope. /// /// `agent_id` is threaded into the emitted opamp `X-Agent-ID` header @@ -239,7 +239,7 @@ impl Replanner { /// pass `"$AGENT_ID"` and rely on the agent container's env. fn try_emit_typed_edge_yaml_for_workload( &self, - workload: &QueryWorkload, + workload: &LegacyMetricWorkload, agent_id: &str, ) -> Option { let deployment_expr = rules::bind_workload_typed(workload)?; @@ -595,7 +595,7 @@ impl Replanner { /// return `No result for query`. fn build_backend_stage_config( &self, - workload: &QueryWorkload, + workload: &LegacyMetricWorkload, role: AggRole, ) -> Option { // ── Typed sketch path (Quantile / Cardinality / TopK / Frequency) ── @@ -610,7 +610,7 @@ impl Replanner { // through `extract_edge_facts` fails, and always leaves // `grouping` empty (`QueryExpr::Aggregate.by` is positional // `ColumnId`s with no label-name resolution today). The - // `QueryWorkload` carries both unambiguously, and every + // `LegacyMetricWorkload` carries both unambiguously, and every // aggregation under one workload shares them. // Per-metric item_label (the high-card dimension a CMS/CountSketch // hashes): threaded into the policy params so the data-plane ingest @@ -899,8 +899,8 @@ mod tests { )) } - fn test_workload(metric: &str) -> (QueryWorkload, WorkloadCharacteristics) { - let wl = QueryWorkload { + fn test_workload(metric: &str) -> (LegacyMetricWorkload, WorkloadCharacteristics) { + let wl = LegacyMetricWorkload { metric_name: metric.into(), label_filters: HashMap::new(), group_by_labels: vec![], diff --git a/control_plane/src/store/workload.rs b/control_plane/src/store/workload.rs index 14f71f83c..62a40f800 100644 --- a/control_plane/src/store/workload.rs +++ b/control_plane/src/store/workload.rs @@ -1,4 +1,4 @@ -//! Persists the `QueryWorkload` + `WorkloadCharacteristics` associated with +//! Persists the `LegacyMetricWorkload` + `WorkloadCharacteristics` associated with //! each planned `(metric, AggRole)` pair so that the re-planner can re-run //! `plan()` without needing the original `QuerySpec` HTTP payload. //! @@ -11,14 +11,14 @@ use std::collections::HashMap; use std::sync::RwLock; -use crate::types::{QueryWorkload, WorkloadCharacteristics}; +use crate::types::{LegacyMetricWorkload, WorkloadCharacteristics}; use crate::workload::AggRole; /// Composite key `(metric_name, role)` for the store. pub type WorkloadKey = (String, AggRole); pub struct WorkloadStore { - inner: RwLock>, + inner: RwLock>, } impl Default for WorkloadStore { @@ -45,7 +45,7 @@ impl WorkloadStore { &self, metric: impl Into, role: AggRole, - wl: QueryWorkload, + wl: LegacyMetricWorkload, wc: WorkloadCharacteristics, ) { self.inner @@ -60,7 +60,7 @@ impl WorkloadStore { &self, metric: &str, role: AggRole, - ) -> Option<(QueryWorkload, WorkloadCharacteristics)> { + ) -> Option<(LegacyMetricWorkload, WorkloadCharacteristics)> { self.inner .read() .unwrap() @@ -74,7 +74,7 @@ impl WorkloadStore { pub fn get_all_for_metric( &self, metric: &str, - ) -> Vec<(AggRole, QueryWorkload, WorkloadCharacteristics)> { + ) -> Vec<(AggRole, LegacyMetricWorkload, WorkloadCharacteristics)> { self.inner .read() .unwrap() @@ -106,8 +106,8 @@ mod tests { use std::collections::HashMap; use std::time::Duration; - fn wl(name: &str) -> QueryWorkload { - QueryWorkload { + fn wl(name: &str) -> LegacyMetricWorkload { + LegacyMetricWorkload { metric_name: name.into(), label_filters: HashMap::new(), group_by_labels: vec![], diff --git a/control_plane/src/types.rs b/control_plane/src/types.rs index 93763b678..ecb2cd063 100644 --- a/control_plane/src/types.rs +++ b/control_plane/src/types.rs @@ -25,7 +25,7 @@ pub enum DataDistribution { /// Observable characteristics of the incoming data stream. /// -/// Callers supply these alongside a [`QueryWorkload`] so the planner can +/// Callers supply these alongside a [`LegacyMetricWorkload`] so the planner can /// compare raw vs. sketch-full vs. sketch-delta transmission costs and /// estimate the CPU / memory overhead at the SDK or agent collector. /// @@ -263,7 +263,7 @@ impl std::fmt::Display for ProcessorMode { // ── Core types ──────────────────────────────────────────────────────────────── #[derive(Debug, Clone)] -pub struct QueryWorkload { +pub struct LegacyMetricWorkload { pub metric_name: String, pub label_filters: HashMap, pub group_by_labels: Vec, @@ -287,7 +287,7 @@ pub struct QueryWorkload { pub quantiles: Vec, } -impl QueryWorkload { +impl LegacyMetricWorkload { /// Scalar sizing input for legacy cost formulas, not a confidence guarantee. pub fn error_bound(&self) -> f64 { match self.accuracy { @@ -656,3 +656,6 @@ pub struct CollectionPlan { /// Bandwidth and overhead estimates for all three transmission strategies. pub transmission_cost_summary: TransmissionCostSummary, } + +#[deprecated(note = "Use LegacyMetricWorkload; canonical workloads are Planner-owned")] +pub use LegacyMetricWorkload as QueryWorkload; diff --git a/control_plane/src/workload.rs b/control_plane/src/workload.rs index 1c76f0181..a99381b70 100644 --- a/control_plane/src/workload.rs +++ b/control_plane/src/workload.rs @@ -237,7 +237,7 @@ pub struct WorkloadEntry { /// Optional explicit sketch family override. When set, the planner pins /// this family for the metric (modulo `(sketch, statistic)` validity /// by ASAPPlanner's legal candidate enumeration). Threaded - /// into `QueryWorkload::sketch_type_override` by the registry pre-pop + /// into `LegacyMetricWorkload::sketch_type_override` by the registry pre-pop /// path so the typed L4 binding (`bind_workload_typed`) honours it. /// /// MVP-§46 contract entries 5–8 in `deploy/configs/mvp-workload.yaml` @@ -264,12 +264,12 @@ pub struct WorkloadEntry { /// http_requests_total_latency_ms[30s])` carries no `by (...)` /// clause, so the PromQL parser surfaces an EMPTY group_by_labels. /// Without a declarative field the analyzer ends up with an empty - /// `QueryWorkload.group_by_labels` → an empty `keep_keys` list → + /// `LegacyMetricWorkload.group_by_labels` → an empty `keep_keys` list → /// the agent strips ALL attrs and mints a single sid per metric /// (instead of one per `(metric, zone)`), defeating the streaming- /// config contract. /// - /// Threaded into `QueryWorkload::group_by_labels` by the registry + /// Threaded into `LegacyMetricWorkload::group_by_labels` by the registry /// pre-pop loop in `main`, so it merges with any `by (...)` keys /// the PromQL parser surfaces. Empty / missing ⇒ same behaviour as /// pre-B3 (no allowlist injected). @@ -363,7 +363,7 @@ pub struct WorkloadEntry { /// to reach the planner with *different* flush periods — the startup path /// hardcoded `None`. Threaded into `QuerySpec::repeat_every` by the /// registry pre-pop loop in `main`, which the analyzer parses into - /// [`crate::types::QueryWorkload::repeat_every`]. + /// [`crate::types::LegacyMetricWorkload::repeat_every`]. /// /// `None` / missing ⇒ unchanged behaviour (the cost model falls back to /// its window-derived flush rate). @@ -380,7 +380,7 @@ pub struct WorkloadEntry { /// [`WorkloadEntry`] then had to be re-threaded in each copy, and /// `repeat_every` is the field that proves the cost: the HTTP /// `POST /api/v1/plan` path carried the declared cadence into -/// [`crate::types::QueryWorkload::repeat_every`] while the startup path pinned +/// [`crate::types::LegacyMetricWorkload::repeat_every`] while the startup path pinned /// `None`, so the same declaration was costed with two different batch-mode /// flush periods depending on which entry point registered it. /// @@ -397,7 +397,7 @@ pub struct WorkloadEntry { /// `keep_keys(datapoint.attributes, [...])` OTTL processor strips wire attrs /// down to this list BEFORE sketching. /// * `sketch_family_override` is threaded into `QuerySpec::sketch_type`, which -/// populates `QueryWorkload::sketch_type_override` — what `bind_workload_typed` +/// populates `LegacyMetricWorkload::sketch_type_override` — what `bind_workload_typed` /// reads to honour the MVP §46 HLL / CountSketch / CountMinSketch pins. pub fn query_spec_for_entry(entry: &WorkloadEntry) -> crate::pipeline::QuerySpec { crate::pipeline::QuerySpec { diff --git a/crates/asap_types/src/aggregation_config.rs b/crates/asap_types/src/aggregation_config.rs index c18fb4822..19c77c1f8 100644 --- a/crates/asap_types/src/aggregation_config.rs +++ b/crates/asap_types/src/aggregation_config.rs @@ -198,24 +198,24 @@ pub struct PrecomputeMaterialization { #[derive(Debug, Clone)] pub struct AggregationIdInfo { /// `PolicyFingerprint::as_u64()` of the key aggregation's config. - pub aggregation_id_for_key: u64, + pub key_policy_fingerprint: u64, /// `PolicyFingerprint::as_u64()` of the value aggregation's config. - pub aggregation_id_for_value: u64, + pub value_policy_fingerprint: u64, pub aggregation_type_for_key: AggregationType, pub aggregation_type_for_value: AggregationType, } impl AggregationIdInfo { pub fn policy_fp_for_key(&self) -> PolicyFingerprint { - PolicyFingerprint(self.aggregation_id_for_key) + PolicyFingerprint(self.key_policy_fingerprint) } pub fn policy_fp_for_value(&self) -> PolicyFingerprint { - PolicyFingerprint(self.aggregation_id_for_value) + PolicyFingerprint(self.value_policy_fingerprint) } } /// Compatibility name for legacy streaming-config and precompute call sites. -/// New PhysicalPlan code should use [`PrecomputeMaterialization`]. +/// New CompiledPhysicalPlan code should use [`PrecomputeMaterialization`]. pub type AggregationConfig = PrecomputeMaterialization; impl PrecomputeMaterialization { diff --git a/crates/asap_types/src/executable_plan.rs b/crates/asap_types/src/executable_plan.rs index fe59854ae..04647cd44 100644 --- a/crates/asap_types/src/executable_plan.rs +++ b/crates/asap_types/src/executable_plan.rs @@ -281,7 +281,7 @@ mod tests { use super::*; // Shared installation metadata must be safe to retain in cross-thread - // ActivePhysicalPlan snapshots without importing the compiler crate. + // RuntimePhysicalPlan snapshots without importing the compiler crate. #[test] fn installed_contract_is_send_sync_and_preserves_wire_identity() { fn send_sync() {} diff --git a/crates/asap_types/src/grouping_projection.rs b/crates/asap_types/src/grouping_projection.rs index 8bb955cb3..91dbbed2f 100644 --- a/crates/asap_types/src/grouping_projection.rs +++ b/crates/asap_types/src/grouping_projection.rs @@ -143,57 +143,6 @@ impl<'de> Deserialize<'de> for GroupingProjection { } } -#[cfg(test)] -mod tests { - use super::*; - /// Legacy label lists become one non-null string column per name. - #[test] - fn legacy_and_typed_groups_share_one_projection() { - let legacy: GroupingProjection = - serde_json::from_value(serde_json::json!({"labels":["job"]})).unwrap(); - assert_eq!( - legacy.columns(), - &[Column::new("job", DataType::Utf8, false)] - ); - let typed: GroupingProjection = serde_json::from_value( - serde_json::json!([{"name":"job","dtype":"utf8","nullable":false}]), - ) - .unwrap(); - assert_eq!(typed, legacy); - assert_eq!(typed.serialize_to_json(), serde_json::json!(["job"])); - assert_eq!( - serde_json::to_value(&typed).unwrap(), - serde_json::json!(["job"]) - ); - #[derive(Serialize)] - struct LegacyConfig { - #[serde(serialize_with = "serialize_config_grouping")] - grouping_labels: GroupingProjection, - } - assert_eq!( - serde_json::to_value(LegacyConfig { - grouping_labels: typed - }) - .unwrap(), - serde_json::json!({"grouping_labels":{"labels":["job"]}}) - ); - } - /// Numeric grouping types must survive wire transport rather than become labels. - #[test] - fn typed_group_retains_type_and_rejects_duplicate_columns() { - let typed = GroupingProjection::new(vec![Column::new("tenant", DataType::Int64, true)]); - let decoded: GroupingProjection = - serde_json::from_value(typed.serialize_to_json()).unwrap(); - assert_eq!(decoded, typed); - assert!(!decoded.is_legacy_labels()); - let duplicate = GroupingProjection::new(vec![ - Column::new("tenant", DataType::Int64, true), - Column::new("tenant", DataType::Utf8, false), - ]); - assert!(duplicate.validate().is_err()); - } -} - #[cfg(test)] mod identity_tests { use super::*; @@ -452,3 +401,54 @@ mod population_key_tests { } } } + +#[cfg(test)] +mod tests { + use super::*; + /// Legacy label lists become one non-null string column per name. + #[test] + fn legacy_and_typed_groups_share_one_projection() { + let legacy: GroupingProjection = + serde_json::from_value(serde_json::json!({"labels":["job"]})).unwrap(); + assert_eq!( + legacy.columns(), + &[Column::new("job", DataType::Utf8, false)] + ); + let typed: GroupingProjection = serde_json::from_value( + serde_json::json!([{"name":"job","dtype":"utf8","nullable":false}]), + ) + .unwrap(); + assert_eq!(typed, legacy); + assert_eq!(typed.serialize_to_json(), serde_json::json!(["job"])); + assert_eq!( + serde_json::to_value(&typed).unwrap(), + serde_json::json!(["job"]) + ); + #[derive(Serialize)] + struct LegacyConfig { + #[serde(serialize_with = "serialize_config_grouping")] + grouping_labels: GroupingProjection, + } + assert_eq!( + serde_json::to_value(LegacyConfig { + grouping_labels: typed + }) + .unwrap(), + serde_json::json!({"grouping_labels":{"labels":["job"]}}) + ); + } + /// Numeric grouping types must survive wire transport rather than become labels. + #[test] + fn typed_group_retains_type_and_rejects_duplicate_columns() { + let typed = GroupingProjection::new(vec![Column::new("tenant", DataType::Int64, true)]); + let decoded: GroupingProjection = + serde_json::from_value(typed.serialize_to_json()).unwrap(); + assert_eq!(decoded, typed); + assert!(!decoded.is_legacy_labels()); + let duplicate = GroupingProjection::new(vec![ + Column::new("tenant", DataType::Int64, true), + Column::new("tenant", DataType::Utf8, false), + ]); + assert!(duplicate.validate().is_err()); + } +} diff --git a/crates/asap_types/src/policy_registry.rs b/crates/asap_types/src/policy_registry.rs index 945556d22..ef27475c5 100644 --- a/crates/asap_types/src/policy_registry.rs +++ b/crates/asap_types/src/policy_registry.rs @@ -22,7 +22,7 @@ //! //! Two `AggregationConfig`s that produce the same `PolicyFingerprint` //! ARE the same policy. The registry treats this as a *deduplication* -//! invariant — if two distinct entries in the source `aggregation_configs` +//! invariant — if two distinct entries in the source `materializations_by_policy_fingerprint` //! map produce the same fingerprint, the later one wins (last-write //! semantics). In practice the source should never contain duplicates; //! if it does, that's a control-plane bug worth surfacing in telemetry diff --git a/crates/asap_types/src/producer_plan.rs b/crates/asap_types/src/producer_plan.rs index 57cfa7e3a..0d9e58457 100644 --- a/crates/asap_types/src/producer_plan.rs +++ b/crates/asap_types/src/producer_plan.rs @@ -17,7 +17,8 @@ pub struct CollectorMaterialization { pub group_by: Vec, pub window_secs: u64, pub abstract_window_framework: SummaryWindowFramework, - pub window_implementation_id: String, + #[serde(rename = "window_implementation_id", alias = "window_realization_id")] + pub window_realization_id: String, pub slide_secs: u64, #[serde( default, @@ -161,7 +162,7 @@ pub struct AdaptiveU64Bounds { /// Guardrails for telemetry-driven runtime adaptation. This is an /// authorization contract, not an instruction to mutate the active plan. -/// Every accepted change becomes a staged successor PhysicalPlan. +/// Every accepted change becomes a staged successor CompiledPhysicalPlan. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] #[serde(deny_unknown_fields)] pub struct RuntimeAdaptationPolicy { diff --git a/crates/asap_types/src/query_plan.rs b/crates/asap_types/src/query_plan.rs index 8b0c48e8c..eac8b0b55 100644 --- a/crates/asap_types/src/query_plan.rs +++ b/crates/asap_types/src/query_plan.rs @@ -5,7 +5,10 @@ //! node IDs. Serving executes this graph without reconstructing Planner IR or //! searching for compatible materializations. -pub mod logical; +pub mod residual; + +#[deprecated(note = "Use query_plan::residual")] +pub use residual as logical; use std::collections::{BTreeMap, BTreeSet}; @@ -551,7 +554,7 @@ pub enum QueryPlanNode { output_schema: planner_types::post_asap::SummarySchema, }, Logical { - operator: logical::LogicalOperator, + operator: residual::ResidualQueryOperator, inputs: Vec, }, Scalar { @@ -585,7 +588,7 @@ pub enum QueryPlanNode { CandidateTopK { inputs: [QueryNodeId; 2], k: u64, - grouping: logical::Grouping, + grouping: residual::Grouping, completeness: CandidateCompleteness, }, /// An exact subtree evaluated outside ASAP. Its results enter the query DAG diff --git a/crates/asap_types/src/query_plan/logical.rs b/crates/asap_types/src/query_plan/residual.rs similarity index 94% rename from crates/asap_types/src/query_plan/logical.rs rename to crates/asap_types/src/query_plan/residual.rs index 5c3f6e138..8d76d645c 100644 --- a/crates/asap_types/src/query_plan/logical.rs +++ b/crates/asap_types/src/query_plan/residual.rs @@ -8,7 +8,7 @@ fn invalid(message: impl Into) -> QueryPlanError { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum LogicalOperator { +pub enum ResidualQueryOperator { /// A maximal exact scalar/vector subtree evaluated by Prometheus. ExactSubquery { query: String, @@ -114,7 +114,7 @@ pub enum TemporalOperation { Count, } -impl LogicalOperator { +impl ResidualQueryOperator { pub fn validate(&self, inputs: usize) -> Result<(), QueryPlanError> { let expected = match self { Self::Scan { .. } | Self::ExactSubquery { .. } => 0, @@ -153,3 +153,7 @@ impl LogicalOperator { Ok(()) } } + +// Compatibility imports; new callers use the domain names above. +#[deprecated(note = "Use ResidualQueryOperator")] +pub use ResidualQueryOperator as LogicalOperator; diff --git a/crates/asap_types/src/routing_index.rs b/crates/asap_types/src/routing_index.rs index 86bd497cf..4e0dd5153 100644 --- a/crates/asap_types/src/routing_index.rs +++ b/crates/asap_types/src/routing_index.rs @@ -27,7 +27,7 @@ //! "cheap, but call at swap time not per query, if it shows up in //! profiles" note `StreamingConfig::policy_registry`'s own doc comment //! already flags — is a further, larger change (it means threading a -//! cached derived value through `HotReloadStreamingConfig`'s swap path) +//! cached derived value through `StreamingConfigHandle`'s swap path) //! and is not done by this type on its own. use std::collections::{BTreeSet, HashMap}; diff --git a/crates/asap_types/src/sds.rs b/crates/asap_types/src/sds.rs index a44fff11d..f24f27798 100644 --- a/crates/asap_types/src/sds.rs +++ b/crates/asap_types/src/sds.rs @@ -1072,6 +1072,32 @@ fn data_descriptor_id( DataDescriptorId(key) } +#[cfg(test)] +mod partition_identity_tests { + use super::*; + #[test] + fn entity_and_global_population_have_distinct_identity() { + let legacy = DataDescriptor::new_typed( + DataSourceIdentity::TimeSeries { metric: "m".into() }, + ValueProjectionIdentity::SampleValue, + "", + Vec::::new(), + "v1", + ); + let entity = legacy + .clone() + .with_partitioning(Some(PopulationPartitioning::PerEntity)); + let grouped = legacy + .clone() + .with_partitioning(Some(PopulationPartitioning::Grouped)); + assert_ne!(entity.id, grouped.id); + assert_ne!(entity.id, legacy.id); + assert_eq!(legacy.clone().with_partitioning(None).id, legacy.id); + entity.validate().unwrap(); + grouped.validate().unwrap(); + } +} + #[cfg(test)] mod tests { use super::*; @@ -1622,29 +1648,3 @@ mod tests { ]))); } } - -#[cfg(test)] -mod partition_identity_tests { - use super::*; - #[test] - fn entity_and_global_population_have_distinct_identity() { - let legacy = DataDescriptor::new_typed( - DataSourceIdentity::TimeSeries { metric: "m".into() }, - ValueProjectionIdentity::SampleValue, - "", - Vec::::new(), - "v1", - ); - let entity = legacy - .clone() - .with_partitioning(Some(PopulationPartitioning::PerEntity)); - let grouped = legacy - .clone() - .with_partitioning(Some(PopulationPartitioning::Grouped)); - assert_ne!(entity.id, grouped.id); - assert_ne!(entity.id, legacy.id); - assert_eq!(legacy.clone().with_partitioning(None).id, legacy.id); - entity.validate().unwrap(); - grouped.validate().unwrap(); - } -} diff --git a/data_plane/benches/sketch_db.rs b/data_plane/benches/sketch_db.rs index 136257868..01162475e 100644 --- a/data_plane/benches/sketch_db.rs +++ b/data_plane/benches/sketch_db.rs @@ -15,7 +15,7 @@ //! `iter_batched` (criterion-0.5.1 `src/bencher.rs:264-272`) consumes its //! input by value, so the per-iteration `Drop` of the setup `SketchStore` //! runs inside the timed region. The `SketchStore` Drop scales linearly -//! with `num_sids` (each `SketchInstanceMetadata` owns a `String`, +//! with `num_sids` (each `SummarySeriesMetadata` owns a `String`, //! `BTreeSet`, `SketchConfig`, `Option` — roughly ~100 ns //! to drop apiece). At 10k sids that's ~1 ms of pure drop work per //! "append" — i.e. ~1000× the actual `append_sample` cost. See @@ -38,7 +38,7 @@ use data_plane::storage_engines::sketch_db::data::{ AccuracyBound, AggKind, AggregationType, Capability, SketchAlgorithm, SketchConfig, SketchEncoding, }; -use data_plane::storage_engines::sketch_db::index::{SketchInstanceMetadata, SketchStore}; +use data_plane::storage_engines::sketch_db::index::{SketchStore, SummarySeriesMetadata}; use data_plane::storage_engines::SketchSampleState; // ── Payload builders ──────────────────────────────────────────────────────── @@ -109,8 +109,8 @@ fn sketch_meta( sid: u64, algorithm: SketchAlgorithm, config: SketchConfig, -) -> SketchInstanceMetadata { - SketchInstanceMetadata { +) -> SummarySeriesMetadata { + SummarySeriesMetadata { sid, metric_name: "bench_metric".into(), group_by_keys: BTreeSet::new(), @@ -131,8 +131,8 @@ fn sketch_meta( } } -fn precompute_meta(sid: u64, metric: &str, agg_type: AggregationType) -> SketchInstanceMetadata { - SketchInstanceMetadata { +fn precompute_meta(sid: u64, metric: &str, agg_type: AggregationType) -> SummarySeriesMetadata { + SummarySeriesMetadata { sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), @@ -424,7 +424,7 @@ fn matching_streaming_config(metric: &str) -> data_plane::storage_engines::types } /// `reconcile_from_streaming_config` ran on EVERY ingest batch and, in -/// the pre-optimization code, deep-cloned every `SketchInstanceMetadata` +/// the pre-optimization code, deep-cloned every `SummarySeriesMetadata` /// in the catalog (`snapshot_instances()`) — the dominant ingest-path /// CPU cost in live `perf` profiling (BTreeMap/String clone + malloc /// churn). This bench measures one un-gated reconcile against a diff --git a/data_plane/examples/audit_clickhouse_fallback.rs b/data_plane/examples/audit_clickhouse_fallback.rs index cd04319f0..81781851b 100644 --- a/data_plane/examples/audit_clickhouse_fallback.rs +++ b/data_plane/examples/audit_clickhouse_fallback.rs @@ -7,14 +7,14 @@ use control_plane::{ query_plan::{ClickHousePlanningContext, QueryPlan}, }; use data_plane::{ - drivers::query::servers::http::{build_active_physical_plan, PhysicalPlanInstallRequest}, + drivers::query::servers::http::{validate_and_build_runtime_plan, PhysicalPlanInstallRequest}, query_engines::asap_clickhouse_query_engine::{ accelerator::CatalogClickHouseAccelerator, ClickHouseAccelerationOutcome, ClickHouseAccelerator, ClickHouseHttpFallback, ClickHouseHttpServer, }, storage_engines::{ sketch_db::index::SketchStore, - types::{BackendStorageRouting, HotReloadActivePhysicalPlan}, + types::{ActivePhysicalPlanHandle, BackendStorageRouting}, }, }; use planner_types::{ @@ -74,7 +74,7 @@ async fn main() { let mut precompute_plan = PrecomputePlan::build_backend_local(envelope.clone(), vec![]).unwrap(); precompute_plan.summary_catalog = Some(reference.clone()); - let mut transmission_plan = control_plane::physical::compiler::compile_transmission_plan( + let mut transmission_plan = control_plane::physical::compiler::build_transmission_plan( envelope, &precompute_plan, &BTreeMap::new(), @@ -91,7 +91,7 @@ async fn main() { }), entries: BTreeMap::new(), }; - let active = build_active_physical_plan( + let active = validate_and_build_runtime_plan( PhysicalPlanInstallRequest { summary_catalog: catalog, collector_plans: vec![], @@ -112,7 +112,7 @@ async fn main() { )); let accelerator = Arc::new(CatalogClickHouseAccelerator::with_active_physical_plan( Arc::new(SketchStore::new()), - HotReloadActivePhysicalPlan::new(active), + ActivePhysicalPlanHandle::new(active), )); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); diff --git a/data_plane/examples/sketch_db_diag.rs b/data_plane/examples/sketch_db_diag.rs index fdf27f6bb..325d72b0c 100644 --- a/data_plane/examples/sketch_db_diag.rs +++ b/data_plane/examples/sketch_db_diag.rs @@ -4,7 +4,7 @@ //! Criterion's `iter_batched` consumes the input by value and the input's //! `Drop` runs *inside* the timed region (see criterion-0.5.1 //! `src/bencher.rs:264-272`). Our `setup` builds a `SketchStore` with N -//! `SketchInstanceMetadata` entries registered; dropping that store at N=10_000 +//! `SummarySeriesMetadata` entries registered; dropping that store at N=10_000 //! pays for 10k Drops of `metric_name: String`, `BTreeSet`, `SketchConfig`, //! `AccuracyBound`, etc. That drop, not `append_sample`, is what makes the //! 10k case look ~90× slower than the 100-sid case. @@ -27,7 +27,7 @@ use data_plane::storage_engines::sketch_db::data::{ AccuracyBound, AggKind, Capability, SketchAlgorithm, SketchConfig, SketchEncoding, SketchSampleState, }; -use data_plane::storage_engines::sketch_db::index::{SketchInstanceMetadata, SketchStore}; +use data_plane::storage_engines::sketch_db::index::{SketchStore, SummarySeriesMetadata}; fn ddsketch_payload() -> Vec { let mut sk = DdSketch::new(0.01); @@ -46,11 +46,11 @@ fn ddsketch_payload() -> Vec { .encode_to_vec() } -fn dd_meta(sid: u64) -> SketchInstanceMetadata { +fn dd_meta(sid: u64) -> SummarySeriesMetadata { let cfg = SketchConfig::DDSketch { relative_accuracy: 0.01, }; - SketchInstanceMetadata { + SummarySeriesMetadata { sid, metric_name: "bench_metric".into(), group_by_keys: BTreeSet::new(), diff --git a/data_plane/src/bin/monitor_coordinator_harness.rs b/data_plane/src/bin/monitor_coordinator_harness.rs index 5ae094be2..3f993ed2a 100644 --- a/data_plane/src/bin/monitor_coordinator_harness.rs +++ b/data_plane/src/bin/monitor_coordinator_harness.rs @@ -18,7 +18,7 @@ //! carry the same key or registrations are rejected as unconfigured. Empty //! (default) = ungrouped. -use data_plane::monitor::{MonitorConfig, MonitorCoordinator, MonitorServiceImpl}; +use data_plane::update_sampling::{MonitorConfig, MonitorCoordinator, MonitorServiceImpl}; /// The edge_id the Go e2edriver connects as (asap-precompute-go/monitor/ /// grpcclient/cmd/e2edriver/main.go: `monitor.NewEngine("e2e-edge", ...)`). diff --git a/data_plane/src/drivers/ingest/otel.rs b/data_plane/src/drivers/ingest/otel.rs index df3396763..b7e235b7b 100644 --- a/data_plane/src/drivers/ingest/otel.rs +++ b/data_plane/src/drivers/ingest/otel.rs @@ -629,10 +629,10 @@ fn resolve_bucket_sid_for_agg_config( &agg_kind_canonical, |sid| { ingest_state - .sketch_index + .summary_store .validate_routed_catalog_generation(captured_generation)?; let activation = ingest_state - .sketch_index + .summary_store .authorize_series_reactivation(sid, config.policy_fingerprint().into())?; if activation .as_deref() @@ -656,16 +656,16 @@ async fn route_otlp_to_precompute( // Snapshot the latest agg_configs from the hot-reload handle so // new aggregations are visible without restart. - let physical_plan_snapshot = ingest_state.physical_plan_snapshot(); - let catalog_generation = physical_plan_snapshot + let active_physical_plan_snapshot = ingest_state.active_physical_plan_snapshot(); + let catalog_generation = active_physical_plan_snapshot .as_ref() .and_then(|plan| plan.precompute_plan.summary_catalog.clone()) .map(Arc::new); - let snap = physical_plan_snapshot + let snap = active_physical_plan_snapshot .as_ref() - .map(|plan| plan.runtime_config.clone()) + .map(|plan| plan.streaming_config.clone()) .unwrap_or_else(|| ingest_state.config_snapshot()); - let agg_configs = snap.get_all_aggregation_configs(); + let agg_configs = snap.materializations(); // Schema retirement #5 — the agg_id-keyed `SchemaRegistry` is // gone; sid-level lifecycle now lives on `SketchStore`. Reconcile // against the current streaming config so newly-added / @@ -678,7 +678,7 @@ async fn route_otlp_to_precompute( // collapses to a single relaxed atomic load and skips the full // catalog scan — the dominant ingest-path CPU cost in profiling. let _ = crate::storage_engines::sketch_db::lifecycle::reconcile_if_config_changed( - ingest_state.sketch_index.as_ref(), + ingest_state.summary_store.as_ref(), &snap, crate::storage_engines::sketch_db::DEFAULT_RETIREMENT_RETENTION, ); @@ -925,16 +925,16 @@ async fn route_modified_otlp_sketches_to_precompute( // Load the generation exactly once. Deriving both the runtime config and // transmission contract from this Arc prevents an activation between two // independent ArcSwap loads from producing a torn ingest view. - let physical_plan_snapshot = ingest_state.physical_plan_snapshot(); - let snap = physical_plan_snapshot + let active_physical_plan_snapshot = ingest_state.active_physical_plan_snapshot(); + let snap = active_physical_plan_snapshot .as_ref() - .map(|plan| plan.runtime_config.clone()) + .map(|plan| plan.streaming_config.clone()) .unwrap_or_else(|| ingest_state.config_snapshot()); - let catalog_generation = physical_plan_snapshot + let catalog_generation = active_physical_plan_snapshot .as_ref() .and_then(|plan| plan.precompute_plan.summary_catalog.clone()) .map(Arc::new); - let active_physical_plan = physical_plan_snapshot.filter(|plan| plan.plan_id() != 0); + let active_physical_plan = active_physical_plan_snapshot.filter(|plan| plan.plan_id() != 0); let lineage_batch_guard = active_physical_plan .as_ref() .map(|_| ingest_state.observability.frame_lineage.lock_batch()); @@ -945,13 +945,13 @@ async fn route_modified_otlp_sketches_to_precompute( // the batch was accepted, while a contract error applies none of it. preflight_summary_frames(request, ingest_state, active)?; } - let agg_configs = snap.get_all_aggregation_configs(); + let agg_configs = snap.materializations(); // Schema retirement #5 — agg_id-keyed registry retired; sid-level // reconcile is the only path going forward. See the raw-OTLP // routine above for the full rationale (incl. the config-Arc gate // that skips the catalog scan when the config is unchanged). let _ = crate::storage_engines::sketch_db::lifecycle::reconcile_if_config_changed( - ingest_state.sketch_index.as_ref(), + ingest_state.summary_store.as_ref(), &snap, crate::storage_engines::sketch_db::DEFAULT_RETIREMENT_RETENTION, ); @@ -1119,7 +1119,7 @@ async fn route_modified_otlp_sketches_to_precompute( // Canonicalize the metric name once per metric: strip the // agent-side sketch-family suffix (`_kll`, `_hll`, …) so // the whole ingest pipeline — sid resolution, series-key - // snapshot cache, `SketchInstanceMetadata.metric_name`, + // snapshot cache, `SummarySeriesMetadata.metric_name`, // `derive_sketch_policy_fp`, and the legacy precompute // router match below — keys on the RAW metric name that // the controller's streaming-config and the query @@ -1238,7 +1238,7 @@ async fn route_modified_otlp_sketches_to_precompute( // because `fp=""` is a perfectly good mint/lookup key. let resolved_sid: Option = if dp.series_id != 0 && attrs_pairs.is_empty() { let sid = dp.series_id; - if ingest_state.sketch_index.instance(sid).is_some() { + if ingest_state.summary_store.instance(sid).is_some() { Some(sid) } else { unknown_sids.push(sid); @@ -1288,12 +1288,12 @@ async fn route_modified_otlp_sketches_to_precompute( &agg_kind_canonical, |sid| { ingest_state - .sketch_index + .summary_store .validate_routed_catalog_generation( catalog_generation.as_deref(), )?; let activation = ingest_state - .sketch_index + .summary_store .authorize_series_reactivation(sid, definition)?; if activation.as_deref().is_some_and(|generation| { Some(generation) != catalog_generation.as_deref() @@ -1347,7 +1347,7 @@ async fn route_modified_otlp_sketches_to_precompute( if let Some(frame) = frame_identity.as_ref() { let observed_policy = ingest_state - .sketch_index + .summary_store .instance(sid) .map(|metadata| metadata.policy_fp) .unwrap_or_else(|| { @@ -1366,7 +1366,7 @@ async fn route_modified_otlp_sketches_to_precompute( } } - // Phase 5 — register a `SketchInstanceMetadata` on + // Phase 5 — register a `SummarySeriesMetadata` on // first sight of `sid` and append this DP's sketch // state to the per-sid columnar storage. The instance // is keyed by sid, so subsequent DPs on the same sid @@ -1376,11 +1376,11 @@ async fn route_modified_otlp_sketches_to_precompute( // and its key set IS the group-by KEY set. { use crate::storage_engines::sketch_db::index::{ - AccuracyBound, Capability, SketchAlgorithm, SketchInstanceMetadata, + AccuracyBound, Capability, SketchAlgorithm, SummarySeriesMetadata, }; use std::collections::BTreeSet; - if ingest_state.sketch_index.instance(sid).is_none() { + if ingest_state.summary_store.instance(sid).is_none() { let algorithm = sketch_algorithm_for(&dp); let cap = match algorithm { SketchAlgorithm::DDSketch | SketchAlgorithm::Kll => { @@ -1443,7 +1443,7 @@ async fn route_modified_otlp_sketches_to_precompute( let item_label_for_sid: Option = { snap.get_aggregation_config(policy_fp.as_u64()) .or_else(|| { - snap.get_all_aggregation_configs() + snap.materializations() .values() .find(|c| c.metric == canonical_name) }) @@ -1452,7 +1452,7 @@ async fn route_modified_otlp_sketches_to_precompute( .filter(|s| !s.is_empty()) .map(|s| s.to_string()) }; - ingest_state.sketch_index.register(SketchInstanceMetadata { + ingest_state.summary_store.register(SummarySeriesMetadata { sid, metric_name: canonical_name.clone(), group_by_keys, @@ -1470,9 +1470,9 @@ async fn route_modified_otlp_sketches_to_precompute( policy_fp, }); if let Some(label) = &item_label_for_sid { - ingest_state.sketch_index.set_item_label(sid, label); + ingest_state.summary_store.set_item_label(sid, label); } - } else if let Some(existing) = ingest_state.sketch_index.instance(sid) { + } else if let Some(existing) = ingest_state.summary_store.instance(sid) { // P1-4 (a) — one-way capability UPGRADE. The sid // was first registered from a non-heap frame // (PROTO, a delta, or a heap-LESS MSGPACK), so it @@ -1512,7 +1512,7 @@ async fn route_modified_otlp_sketches_to_precompute( // in place (same sid → same policy/metric // index slots), so this is an atomic swap to // the stronger capability. - ingest_state.sketch_index.register(upgraded); + ingest_state.summary_store.register(upgraded); debug!( "OTLP sketch sid {} upgraded {:?} -> FrequencyTopk({:?}) \ on heap-bearing frame (metric={}, encoding={})", @@ -1723,7 +1723,7 @@ async fn route_modified_otlp_sketches_to_precompute( == asap_types::producer_plan::SummaryFrameKind::Full { ingest_state - .sketch_index + .summary_store .clear_summary_lineage_incomplete(sid, frame); } } @@ -1743,7 +1743,7 @@ async fn route_modified_otlp_sketches_to_precompute( } Err(error) => { ingest_state - .sketch_index + .summary_store .mark_summary_lineage_incomplete(sid, frame); warn!( plan_id = frame.plan_id, @@ -1775,7 +1775,7 @@ async fn route_modified_otlp_sketches_to_precompute( ); let encoding = encoding_to_handle(dp.encoding).unwrap_or(SketchEncoding::ProtoFull); - if !ingest_state.sketch_index.append_sample( + if !ingest_state.summary_store.append_sample( sid, label_values, window, @@ -2075,7 +2075,7 @@ fn derive_sketch_policy_fp( /// keyed on the *bare* metric `request_size_bytes`) and the query /// analyzer (`control_plane::asap_tier_analysis`, which lifts the bare /// metric name out of the PromQL selector) speak the bare name. With -/// the suffix left on, `SketchInstanceMetadata.metric_name` is the +/// the suffix left on, `SummarySeriesMetadata.metric_name` is the /// suffixed form, so `SketchIndex::instances_matching(bare, …)` and /// `find_matching_policies` / `find_policy_by_content` (all of which /// compare `metric_name` for equality) never match — every warm sketch @@ -2238,7 +2238,7 @@ struct ModifiedOtlpSketchDp { /// the SketchStore's columnar storage keys on. start_time_unix_nano: u64, /// Phase 5 — sketch-instance configuration lifted off the parent - /// container. Drives `SketchInstanceMetadata.sketch_config` and the + /// container. Drives `SummarySeriesMetadata.sketch_config` and the /// derived `AccuracyBound`. container_config: crate::storage_engines::sketch_db::index::SketchConfig, } @@ -2250,7 +2250,7 @@ struct ModifiedOtlpSketchDp { fn preflight_summary_frames( request: &ExportMetricsServiceRequest, ingest_state: &IngestState, - active: &crate::storage_engines::types::ActivePhysicalPlan, + active: &crate::storage_engines::types::RuntimePhysicalPlan, ) -> Result<(), String> { use asap_otel_proto::tonic::metrics::v1::metric::Data; @@ -2258,7 +2258,7 @@ fn preflight_summary_frames( metric_name: &str, mut dp: ModifiedOtlpSketchDp, ingest_state: &IngestState, - active: &crate::storage_engines::types::ActivePhysicalPlan, + active: &crate::storage_engines::types::RuntimePhysicalPlan, ) -> Result { let canonical_name = canonical_sketch_metric_name(metric_name, dp.algorithm.clone()); let frame = @@ -2353,7 +2353,7 @@ fn preflight_summary_frames( // schema. Either route must agree with the declared materialization. let observed = if dp.series_id != 0 && dp.attrs.is_empty() { ingest_state - .sketch_index + .summary_store .instance(dp.series_id) .map(|metadata| metadata.policy_fp) .ok_or_else(|| { @@ -3249,7 +3249,7 @@ mod canonical_metric_name_tests { //! strip of the agent's `_` suffix (ASAPCollector //! `asapedgeprocessor` sets `MetricSuffix: "_" + family`). Without //! it, warm-tier sketch queries against the raw metric name - //! capability-miss because `SketchInstanceMetadata.metric_name` and + //! capability-miss because `SummarySeriesMetadata.metric_name` and //! the controller's streaming-config policy `metric` never line up. use super::*; @@ -3642,7 +3642,7 @@ mod sid_resolution_tests { use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::precompute_engine::series_router::SeriesRouter; use crate::storage_engines::sketch_db::index::SketchStore; - use crate::storage_engines::types::{HotReloadStreamingConfig, StreamingConfig}; + use crate::storage_engines::types::{StreamingConfig, StreamingConfigHandle}; use asap_otel_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; use asap_otel_proto::tonic::common::v1::{any_value::Value as AnyVal, AnyValue, KeyValue}; use asap_otel_proto::tonic::metrics::v1::{ @@ -3656,7 +3656,7 @@ mod sid_resolution_tests { let (tx, mut rx) = mpsc::channel(1024); let router = SeriesRouter::new(vec![tx]); let streaming = StreamingConfig::new(std::collections::HashMap::new()); - let hot_reload = HotReloadStreamingConfig::new(streaming.clone()); + let hot_reload = StreamingConfigHandle::new(streaming.clone()); let state = Arc::new(IngestState { router, samples_ingested: std::sync::atomic::AtomicU64::new(0), @@ -3665,7 +3665,7 @@ mod sid_resolution_tests { pass_raw_samples: false, sketch_snapshots: dashmap::DashMap::new(), series_resolver: Arc::new(SeriesIdResolver::new()), - sketch_index: Arc::new(SketchStore::new()), + summary_store: Arc::new(SketchStore::new()), observability: crate::precompute_engine::ingest_handler::IngestObservability::default(), }); let drain = tokio::spawn(async move { while rx.recv().await.is_some() {} }); @@ -3741,11 +3741,11 @@ mod sid_resolution_tests { "resolver mints a non-zero sid (zero is reserved on the wire)" ); assert_eq!( - state.sketch_index.instance_count(), + state.summary_store.instance_count(), 1, "SketchStore registered one instance under the resolver-minted sid" ); - assert!(state.sketch_index.instance(assigned.series_id).is_some()); + assert!(state.summary_store.instance(assigned.series_id).is_some()); drop(state); let _ = drain.await; @@ -3777,7 +3777,7 @@ mod sid_resolution_tests { outcome.series_assignments.is_empty(), "no assignment when attrs are missing" ); - assert_eq!(state.sketch_index.instance_count(), 0); + assert_eq!(state.summary_store.instance_count(), 0); drop(state); let _ = drain.await; @@ -3806,7 +3806,7 @@ mod sid_resolution_tests { .expect("seed ingest succeeds"); let assigned_sid = seed_outcome.series_assignments[0].series_id; assert!( - state.sketch_index.instance(assigned_sid).is_some(), + state.summary_store.instance(assigned_sid).is_some(), "seed registers the resolver-minted sid" ); @@ -3846,7 +3846,7 @@ mod sid_resolution_tests { ); // The assigned sid stays registered — the second DP routed to // it via the resolver's cache hit. - assert!(state.sketch_index.instance(assigned_sid).is_some()); + assert!(state.summary_store.instance(assigned_sid).is_some()); drop(state); let _ = drain.await; @@ -4087,7 +4087,7 @@ mod sid_resolution_tests { "no fresh assignment when sender already had a valid binding" ); // Both DPs landed against the same sid — no proliferation. - assert_eq!(state.sketch_index.instance_count(), 1); + assert_eq!(state.summary_store.instance_count(), 1); drop(state); let _ = drain.await; @@ -4427,7 +4427,7 @@ mod sid_resolution_tests { .await .expect("ingest succeeds"); let sid = out1.series_assignments[0].series_id; - let meta1 = state.sketch_index.instance(sid).expect("sid registered"); + let meta1 = state.summary_store.instance(sid).expect("sid registered"); assert_eq!( meta1.capability, Some(Capability::FrequencyEstimate(Some(SketchAlgorithm::Cms))), @@ -4458,7 +4458,7 @@ mod sid_resolution_tests { .expect("ingest succeeds"); let meta2 = state - .sketch_index + .summary_store .instance(sid) .expect("sid still registered"); assert_eq!( @@ -4470,7 +4470,7 @@ mod sid_resolution_tests { ); // Still one instance — the upgrade is an in-place overwrite, not a // new sid. - assert_eq!(state.sketch_index.instance_count(), 1); + assert_eq!(state.summary_store.instance_count(), 1); // ── Frame 3: a later heap-LESS frame must NOT downgrade. ── let plain2 = asap_sketchlib::CountMinSketch::new(ROWS as usize, COLS as usize); @@ -4488,7 +4488,7 @@ mod sid_resolution_tests { .await .expect("ingest succeeds"); let meta3 = state - .sketch_index + .summary_store .instance(sid) .expect("sid still registered"); assert_eq!( @@ -4549,11 +4549,11 @@ mod sid_resolution_tests { "no unknown sids — the DP was ingestable, not dropped" ); assert_eq!( - state.sketch_index.instance_count(), + state.summary_store.instance_count(), 1, "SketchStore registered the global-aggregation instance" ); - assert!(state.sketch_index.instance(assigned.series_id).is_some()); + assert!(state.summary_store.instance(assigned.series_id).is_some()); drop(state); let _ = drain.await; @@ -4574,7 +4574,7 @@ mod sid_bucketing_tests { use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::precompute_engine::series_router::{SeriesRouter, WorkerMessage}; use crate::storage_engines::sketch_db::index::SketchStore; - use crate::storage_engines::types::{HotReloadStreamingConfig, StreamingConfig}; + use crate::storage_engines::types::{StreamingConfig, StreamingConfigHandle}; use asap_otel_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; use asap_otel_proto::tonic::common::v1::{any_value::Value as AnyVal, AnyValue, KeyValue}; use asap_otel_proto::tonic::metrics::v1::{ @@ -4694,7 +4694,7 @@ mod sid_bucketing_tests { let mut configs = HashMap::new(); configs.insert(cfg.policy_fp_u64(), cfg.clone()); let streaming = StreamingConfig::new(configs); - let hot_reload = HotReloadStreamingConfig::new(streaming); + let hot_reload = StreamingConfigHandle::new(streaming); let resolver = Arc::new(SeriesIdResolver::new()); let state = Arc::new(IngestState { @@ -4705,7 +4705,7 @@ mod sid_bucketing_tests { pass_raw_samples: false, sketch_snapshots: dashmap::DashMap::new(), series_resolver: resolver.clone(), - sketch_index: Arc::new(SketchStore::new()), + summary_store: Arc::new(SketchStore::new()), observability: crate::precompute_engine::ingest_handler::IngestObservability::default(), }); diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index b37d4c798..b70425013 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -204,7 +204,7 @@ impl PrometheusRemoteWriteReceiver { let generation = self .inner .ingest - .physical_plan_snapshot() + .active_physical_plan_snapshot() .and_then(|plan| plan.precompute_plan.summary_catalog.clone()) .ok_or("finite completion requires a catalog generation")?; self.inner.ingest.router.drain().await?; @@ -213,7 +213,7 @@ impl PrometheusRemoteWriteReceiver { if self .inner .ingest - .sketch_index + .summary_store .seal_finite_summary_input(&generation)? { break; @@ -228,13 +228,13 @@ impl PrometheusRemoteWriteReceiver { let plan = self .inner .ingest - .physical_plan_snapshot() + .active_physical_plan_snapshot() .ok_or("finite maintenance requires an installed physical plan")?; if plan.precompute_plan.summary_catalog.as_ref() != Some(&generation) { return Err("finite maintenance generation changed during drain".into()); } crate::precompute_engine::maintenance_runtime::execute_finite_maintenance( - &self.inner.ingest.sketch_index, + &self.inner.ingest.summary_store, &self.inner.ingest.series_resolver, &plan.precompute_plan, )?; @@ -291,7 +291,7 @@ impl PrometheusRemoteWriteReceiver { let physical_plan = self .inner .ingest - .physical_plan_snapshot() + .active_physical_plan_snapshot() .ok_or(RemoteWriteError::InactivePhysicalPlan)?; if physical_plan.precompute_plan.envelope.plan_id == 0 || !matches!( @@ -452,7 +452,7 @@ impl PrometheusRemoteWriteReceiver { let revision = self .inner .ingest - .sketch_index + .summary_store .admit_summary_updates(&generation, coordinates)?; Ok(Some(Arc::new( crate::storage_engines::types::SummaryInputRevision { @@ -660,7 +660,7 @@ fn canonicalize_labels( fn route_messages( samples: &[CanonicalSample], ingest: &Arc, - physical_plan: &crate::storage_engines::types::ActivePhysicalPlan, + physical_plan: &crate::storage_engines::types::RuntimePhysicalPlan, ) -> Result, RemoteWriteError> { type Bucket = ( u64, @@ -668,14 +668,14 @@ fn route_messages( Arc, ); type RoutedSample = (String, i64, f64); - let snapshot = physical_plan.runtime_config.clone(); + let snapshot = physical_plan.streaming_config.clone(); let _ = crate::storage_engines::sketch_db::lifecycle::reconcile_if_config_changed( - ingest.sketch_index.as_ref(), + ingest.summary_store.as_ref(), &snapshot, crate::storage_engines::sketch_db::DEFAULT_RETIREMENT_RETENTION, ); let configs = snapshot - .get_all_aggregation_configs() + .materializations() .values() .filter(|config| config.derived_input.is_none()) .filter_map(|config| { @@ -767,11 +767,11 @@ fn route_messages( let sid = ingest .series_resolver .resolve_with_reactivation(&config.metric, attrs_fp, &materialization_kind, |sid| { - ingest.sketch_index.validate_routed_catalog_generation( + ingest.summary_store.validate_routed_catalog_generation( physical_plan.precompute_plan.summary_catalog.as_ref(), )?; let activation = ingest - .sketch_index + .summary_store .authorize_series_reactivation(sid, policy_fp.into())?; if let Some(generation) = &activation { if physical_plan.precompute_plan.summary_catalog.as_ref() @@ -928,8 +928,8 @@ mod tests { use crate::precompute_engine::ingest_handler::IngestObservability; use crate::precompute_engine::series_router::SeriesRouter; use crate::storage_engines::types::{ - ActivePhysicalPlan, BackendStorageRouting, HotReloadActivePhysicalPlan, - HotReloadStreamingConfig, StreamingConfig, + ActivePhysicalPlanHandle, BackendStorageRouting, RuntimePhysicalPlan, StreamingConfig, + StreamingConfigHandle, }; use tokio::sync::mpsc; @@ -939,7 +939,7 @@ mod tests { .unwrap() } - fn physical_config(streaming: StreamingConfig) -> HotReloadStreamingConfig { + fn physical_config(streaming: StreamingConfig) -> StreamingConfigHandle { use asap_types::producer_plan::{FrameIdentityContract, SequenceScope, TransmissionPlan}; use control_plane::physical::compiler::{ IngestContract, IngestProtocol, PlanEnvelope, PrecomputePlan, TimestampUnit, @@ -956,7 +956,7 @@ mod tests { capability_snapshot_id: "test".into(), }; let configs = streaming - .aggregation_configs + .materializations_by_policy_fingerprint .values() .cloned() .collect::>(); @@ -971,7 +971,7 @@ mod tests { plan_version: reference.plan_version, snapshot_sha256: reference.snapshot_sha256, }; - let active = ActivePhysicalPlan { + let active = RuntimePhysicalPlan { envelope: envelope.clone(), summary_catalog: Some(catalog), precompute_plan: PrecomputePlan { @@ -988,7 +988,11 @@ mod tests { schemas: Vec::new(), producers: Vec::new(), executable_dags: Default::default(), - materializations: streaming.aggregation_configs.values().cloned().collect(), + materializations: streaming + .materializations_by_policy_fingerprint + .values() + .cloned() + .collect(), }, transmission_plan: TransmissionPlan { summary_catalog: None, @@ -1001,11 +1005,11 @@ mod tests { }, rules: Vec::new(), }, - runtime_config: Arc::new(streaming), + streaming_config: Arc::new(streaming), query_plan: Arc::new(asap_types::query_plan::QueryPlan::empty()), storage_routing: Arc::new(BackendStorageRouting::empty()), }; - HotReloadStreamingConfig::from_active(HotReloadActivePhysicalPlan::new(active)) + StreamingConfigHandle::from_active_physical_plan(ActivePhysicalPlanHandle::new(active)) } fn receiver(config: PrometheusRemoteWriteConfig) -> PrometheusRemoteWriteReceiver { @@ -1018,7 +1022,7 @@ mod tests { pass_raw_samples: false, sketch_snapshots: dashmap::DashMap::new(), series_resolver: Arc::new(super::super::SeriesIdResolver::new()), - sketch_index: Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()), + summary_store: Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()), observability: IngestObservability::default(), }); PrometheusRemoteWriteReceiver::new(config, ingest) @@ -1064,14 +1068,14 @@ mod tests { pass_raw_samples: false, sketch_snapshots: dashmap::DashMap::new(), series_resolver: Arc::new(super::super::SeriesIdResolver::new()), - sketch_index: Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()), + summary_store: Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()), observability: IngestObservability::default(), }); ingest - .sketch_index + .summary_store .install_summary_catalog( ingest - .physical_plan_snapshot() + .active_physical_plan_snapshot() .unwrap() .summary_catalog .as_ref() @@ -1088,7 +1092,7 @@ mod tests { #[test] fn canonical_routing_preserves_group_labels_without_series_text_roundtrip() { let (base, _worker) = configured_receiver(); - let snapshot = base.inner.ingest.physical_plan_snapshot().unwrap(); + let snapshot = base.inner.ingest.active_physical_plan_snapshot().unwrap(); let mut config = snapshot.precompute_plan.materializations[0].clone(); config.population_key_encoding = asap_types::PopulationKeyEncoding::CanonicalLabelsV1; config.partitioning = Some(asap_types::sds::PopulationPartitioning::Grouped); @@ -1096,7 +1100,7 @@ mod tests { config.policy_fp_u64(), config.clone(), )]))); - let physical = hot.physical_plan_snapshot().unwrap(); + let physical = hot.active_physical_plan_snapshot().unwrap(); // A direct routing fixture; public installation remains intentionally gated. let (sender, _worker) = mpsc::channel(8); let ingest = Arc::new(IngestState { @@ -1107,11 +1111,11 @@ mod tests { pass_raw_samples: false, sketch_snapshots: dashmap::DashMap::new(), series_resolver: Arc::new(super::super::SeriesIdResolver::new()), - sketch_index: Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()), + summary_store: Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()), observability: IngestObservability::default(), }); ingest - .sketch_index + .summary_store .install_summary_catalog(physical.summary_catalog.as_ref().unwrap().clone()) .unwrap(); let value = "a;z=\"x,y\\q"; @@ -1212,7 +1216,7 @@ mod tests { (pooled_kll_fp.0, pooled_kll), ])); let hot_reload = physical_config(streaming); - let physical_plan = hot_reload.physical_plan_snapshot().unwrap(); + let physical_plan = hot_reload.active_physical_plan_snapshot().unwrap(); let (sender, _worker) = mpsc::channel(8); let ingest = Arc::new(IngestState { router: SeriesRouter::new(vec![sender]), @@ -1222,11 +1226,11 @@ mod tests { pass_raw_samples: false, sketch_snapshots: dashmap::DashMap::new(), series_resolver: Arc::new(super::super::SeriesIdResolver::new()), - sketch_index: Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()), + summary_store: Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()), observability: IngestObservability::default(), }); ingest - .sketch_index + .summary_store .install_summary_catalog(physical_plan.summary_catalog.as_ref().unwrap().clone()) .unwrap(); let request = WriteRequest { @@ -1371,11 +1375,11 @@ mod tests { router: SeriesRouter::new(vec![sender]), samples_ingested: AtomicU64::new(0), samples_blocked_by_schema_barrier: AtomicU64::new(0), - hot_reload_config: HotReloadStreamingConfig::new(StreamingConfig::default()), + hot_reload_config: StreamingConfigHandle::new(StreamingConfig::default()), pass_raw_samples: false, sketch_snapshots: dashmap::DashMap::new(), series_resolver: Arc::new(super::super::SeriesIdResolver::new()), - sketch_index: Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()), + summary_store: Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()), observability: IngestObservability::default(), }); let receiver = PrometheusRemoteWriteReceiver::new(Default::default(), ingest); @@ -1573,7 +1577,7 @@ mod tests { let second = queued.recv().await.unwrap(); let ingest = &receiver.inner.ingest; let sink = Arc::new(SketchStoreSink::new( - ingest.sketch_index.clone(), + ingest.summary_store.clone(), ingest.hot_reload_config.clone(), ingest.series_resolver.clone(), )); @@ -1607,7 +1611,7 @@ mod tests { let policy = *ingest .hot_reload_config .snapshot() - .aggregation_configs + .materializations_by_policy_fingerprint .keys() .next() .unwrap(); @@ -1620,7 +1624,7 @@ mod tests { readout_lookback_ms: None, }; let context = QueryExecutionContext { - index: &ingest.sketch_index, + index: &ingest.summary_store, t0_ms: 0, t1_ms: 60_000, is_cumulative: true, diff --git a/data_plane/src/drivers/ingest/series_resolver.rs b/data_plane/src/drivers/ingest/series_resolver.rs index 5bfb68d28..abab29e2b 100644 --- a/data_plane/src/drivers/ingest/series_resolver.rs +++ b/data_plane/src/drivers/ingest/series_resolver.rs @@ -865,104 +865,6 @@ pub fn canonical_attrs_fingerprint(attrs: &[(&str, &str)]) -> String { buf } -#[cfg(test)] -mod tests { - use super::*; - - /// Stand-in canonical `AggKind` string. Tests don't care about the - /// specific encoding — the resolver only uses the value for key - /// equality. Production callers compute this via - /// `AggKind::canonical_string()`. - const TEST_AGG: &str = "sketch:DDSketch:D:0.01"; - - #[test] - fn versioned_population_routing_separates_delimiter_collisions() { - use asap_types::PopulationKeyEncoding::{CanonicalLabelsV1, LegacyDelimited}; - let a = [("a", "b;c=d")]; - let b = [("a", "b"), ("c", "d")]; - assert_eq!( - population_attrs_fingerprint(LegacyDelimited, &a).unwrap(), - "a=b;c=d;" - ); - assert_eq!( - population_attrs_fingerprint(LegacyDelimited, &a), - population_attrs_fingerprint(LegacyDelimited, &b) - ); - let ka = population_attrs_fingerprint(CanonicalLabelsV1, &a).unwrap(); - let kb = population_attrs_fingerprint(CanonicalLabelsV1, &b).unwrap(); - assert_ne!(ka, kb); - assert_eq!( - kb, - population_attrs_fingerprint(CanonicalLabelsV1, &[("c", "d"), ("a", "b")]).unwrap() - ); - let resolver = SeriesIdResolver::new(); - assert_ne!( - resolver.resolve("m", &ka, TEST_AGG), - resolver.resolve("m", &kb, TEST_AGG) - ); - assert!( - population_attrs_fingerprint(CanonicalLabelsV1, &[("a", "1"), ("a", "2")]).is_err() - ); - } - - #[test] - fn idempotent_same_input_same_sid() { - let r = SeriesIdResolver::new(); - let sid1 = r.resolve("http_requests_total", "zone=z0;", TEST_AGG); - let sid2 = r.resolve("http_requests_total", "zone=z0;", TEST_AGG); - assert_eq!(sid1, sid2, "same input must produce same sid"); - } - - #[test] - fn distinct_inputs_distinct_sids() { - let r = SeriesIdResolver::new(); - let s_z0 = r.resolve("metric_a", "zone=z0;", TEST_AGG); - let s_z1 = r.resolve("metric_a", "zone=z1;", TEST_AGG); - assert_ne!(s_z0, s_z1); - } - - #[test] - fn distinct_metrics_same_attrs_distinct_sids() { - let r = SeriesIdResolver::new(); - let s_a = r.resolve("metric_a", "zone=z0;", TEST_AGG); - let s_b = r.resolve("metric_b", "zone=z0;", TEST_AGG); - assert_ne!(s_a, s_b); - } - - #[test] - fn distinct_agg_kinds_same_series_distinct_sids() { - // Two aggregations over the same (metric, attrs) tuple — e.g. - // a DDSketch and a Sum on `http_latency_ms{zone=z0}` — get - // SEPARATE sids. This is the core property of Interpretation B: - // sid identity is `(metric, attrs, agg_kind)`. - let r = SeriesIdResolver::new(); - let s_dd = r.resolve("http_latency_ms", "zone=z0;", "sketch:DDSketch:D:0.01"); - let s_sum = r.resolve("http_latency_ms", "zone=z0;", "precompute:Sum:"); - assert_ne!( - s_dd, s_sum, - "different agg_kinds over the same series must mint distinct sids", - ); - } - - #[test] - fn fingerprint_sorts_keys() { - let f1 = canonical_attrs_fingerprint(&[("zone", "z0"), ("rack", "r00")]); - let f2 = canonical_attrs_fingerprint(&[("rack", "r00"), ("zone", "z0")]); - assert_eq!(f1, f2, "fingerprint must be order-independent"); - assert_eq!(f1, "rack=r00;zone=z0;"); - } - - #[test] - fn lookup_returns_existing_without_mint() { - let r = SeriesIdResolver::new(); - let sid = r.resolve("m", "k=v;", TEST_AGG); - assert_eq!(r.lookup("m", "k=v;", TEST_AGG), Some(sid)); - assert_eq!(r.lookup("m", "k=v2;", TEST_AGG), None); - // Same (metric, attrs) but different agg_kind is a miss. - assert_eq!(r.lookup("m", "k=v;", "precompute:Sum:"), None); - } -} - #[cfg(test)] mod persistence_tests { use super::*; @@ -1306,3 +1208,101 @@ mod persistence_tests { assert_ne!(sid, sid2); } } + +#[cfg(test)] +mod tests { + use super::*; + + /// Stand-in canonical `AggKind` string. Tests don't care about the + /// specific encoding — the resolver only uses the value for key + /// equality. Production callers compute this via + /// `AggKind::canonical_string()`. + const TEST_AGG: &str = "sketch:DDSketch:D:0.01"; + + #[test] + fn versioned_population_routing_separates_delimiter_collisions() { + use asap_types::PopulationKeyEncoding::{CanonicalLabelsV1, LegacyDelimited}; + let a = [("a", "b;c=d")]; + let b = [("a", "b"), ("c", "d")]; + assert_eq!( + population_attrs_fingerprint(LegacyDelimited, &a).unwrap(), + "a=b;c=d;" + ); + assert_eq!( + population_attrs_fingerprint(LegacyDelimited, &a), + population_attrs_fingerprint(LegacyDelimited, &b) + ); + let ka = population_attrs_fingerprint(CanonicalLabelsV1, &a).unwrap(); + let kb = population_attrs_fingerprint(CanonicalLabelsV1, &b).unwrap(); + assert_ne!(ka, kb); + assert_eq!( + kb, + population_attrs_fingerprint(CanonicalLabelsV1, &[("c", "d"), ("a", "b")]).unwrap() + ); + let resolver = SeriesIdResolver::new(); + assert_ne!( + resolver.resolve("m", &ka, TEST_AGG), + resolver.resolve("m", &kb, TEST_AGG) + ); + assert!( + population_attrs_fingerprint(CanonicalLabelsV1, &[("a", "1"), ("a", "2")]).is_err() + ); + } + + #[test] + fn idempotent_same_input_same_sid() { + let r = SeriesIdResolver::new(); + let sid1 = r.resolve("http_requests_total", "zone=z0;", TEST_AGG); + let sid2 = r.resolve("http_requests_total", "zone=z0;", TEST_AGG); + assert_eq!(sid1, sid2, "same input must produce same sid"); + } + + #[test] + fn distinct_inputs_distinct_sids() { + let r = SeriesIdResolver::new(); + let s_z0 = r.resolve("metric_a", "zone=z0;", TEST_AGG); + let s_z1 = r.resolve("metric_a", "zone=z1;", TEST_AGG); + assert_ne!(s_z0, s_z1); + } + + #[test] + fn distinct_metrics_same_attrs_distinct_sids() { + let r = SeriesIdResolver::new(); + let s_a = r.resolve("metric_a", "zone=z0;", TEST_AGG); + let s_b = r.resolve("metric_b", "zone=z0;", TEST_AGG); + assert_ne!(s_a, s_b); + } + + #[test] + fn distinct_agg_kinds_same_series_distinct_sids() { + // Two aggregations over the same (metric, attrs) tuple — e.g. + // a DDSketch and a Sum on `http_latency_ms{zone=z0}` — get + // SEPARATE sids. This is the core property of Interpretation B: + // sid identity is `(metric, attrs, agg_kind)`. + let r = SeriesIdResolver::new(); + let s_dd = r.resolve("http_latency_ms", "zone=z0;", "sketch:DDSketch:D:0.01"); + let s_sum = r.resolve("http_latency_ms", "zone=z0;", "precompute:Sum:"); + assert_ne!( + s_dd, s_sum, + "different agg_kinds over the same series must mint distinct sids", + ); + } + + #[test] + fn fingerprint_sorts_keys() { + let f1 = canonical_attrs_fingerprint(&[("zone", "z0"), ("rack", "r00")]); + let f2 = canonical_attrs_fingerprint(&[("rack", "r00"), ("zone", "z0")]); + assert_eq!(f1, f2, "fingerprint must be order-independent"); + assert_eq!(f1, "rack=r00;zone=z0;"); + } + + #[test] + fn lookup_returns_existing_without_mint() { + let r = SeriesIdResolver::new(); + let sid = r.resolve("m", "k=v;", TEST_AGG); + assert_eq!(r.lookup("m", "k=v;", TEST_AGG), Some(sid)); + assert_eq!(r.lookup("m", "k=v2;", TEST_AGG), None); + // Same (metric, attrs) but different agg_kind is a miss. + assert_eq!(r.lookup("m", "k=v;", "precompute:Sum:"), None); + } +} diff --git a/data_plane/src/drivers/query/adapters/prometheus_http.rs b/data_plane/src/drivers/query/adapters/prometheus_http.rs index 76b298bd7..4a986b39a 100644 --- a/data_plane/src/drivers/query/adapters/prometheus_http.rs +++ b/data_plane/src/drivers/query/adapters/prometheus_http.rs @@ -372,14 +372,14 @@ impl HttpProtocolAdapter for PrometheusHttpAdapter { async fn handle_runtime_info( &self, - sketch_index: Arc, + summary_store: Arc, ) -> Result, StatusCode> { debug!("Handling runtime info request in Prometheus adapter"); // M2.3.6g — earliest timestamps now come from SketchStore's // per-sid `first_seen_unix_ms` metadata. Wire field renamed // accordingly below. - let earliest_timestamps = sketch_index.earliest_timestamps_per_series_id(); + let earliest_timestamps = summary_store.earliest_timestamps_per_series_id(); // Get runtime info from fallback if available let mut runtime_data = if let Some(fallback) = &self.config.fallback { diff --git a/data_plane/src/drivers/query/adapters/traits.rs b/data_plane/src/drivers/query/adapters/traits.rs index b96058a48..66b78d20c 100644 --- a/data_plane/src/drivers/query/adapters/traits.rs +++ b/data_plane/src/drivers/query/adapters/traits.rs @@ -163,16 +163,16 @@ pub trait HttpProtocolAdapter: QueryRequestAdapter + QueryResponseAdapter + Send /// backend. async fn handle_runtime_info( &self, - sketch_index: std::sync::Arc, + summary_store: std::sync::Arc, ) -> Result, StatusCode>; async fn handle_runtime_info_with_headers( &self, - sketch_index: std::sync::Arc, + summary_store: std::sync::Arc, headers: HashMap, ) -> Result, StatusCode> { // Default implementation ignores headers and calls the old method let _ = headers; - self.handle_runtime_info(sketch_index).await + self.handle_runtime_info(summary_store).await } } diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index bd4037af5..665a250f9 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -146,10 +146,10 @@ pub struct HttpServer { /// map. See `docs/design-gorilla-s3-cold-engine.md` §8. query_router: Arc, /// M2.3.6g — SketchStore replaces `Arc`. - sketch_index: Arc, + summary_store: Arc, /// Hot-reloadable `StreamingConfig` source. `None` when hot-reload /// is not wired up by the caller (unit tests, legacy binaries). - hot_reload_config: Option, + hot_reload_config: Option, /// Per-metric storage-backend routing table consulted by the HTTP /// instant-query handler at request time. When `Some(..)` and the /// query parses, the handler extracts the metric name from the @@ -194,7 +194,7 @@ pub struct HttpServer { /// Serializes multi-document physical-plan publication so two control /// plane generations cannot interleave their plan projections and catalog. physical_plan_lock: Arc>, - active_physical_plan: Option, + active_physical_plan: Option, physical_plan_lifecycle: Option, remote_write: Option, } @@ -208,10 +208,10 @@ struct AppState { /// Phase 5 M2.3.6g — SketchStore replaces `Arc` as the /// only data backend HTTP-side endpoints consult. Today the only /// consumer is the runtime-info handler. - sketch_index: Arc, + summary_store: Arc, adapter: Arc, fallback: Option>, - hot_reload_config: Option, + hot_reload_config: Option, /// See [`HttpServer::backend_storage_routing`]. backend_storage_routing: Option, /// Backfill registry (sketch DB §10). See `HttpServer::backfill`. @@ -221,7 +221,7 @@ struct AppState { /// See [`HttpServer::probe_cache`]. probe_cache: Option>, physical_plan_lock: Arc>, - active_physical_plan: Option, + active_physical_plan: Option, physical_plan_lifecycle: Option, remote_write: Option, } @@ -230,7 +230,7 @@ impl HttpServer { pub fn new( config: HttpServerConfig, query_engine: Arc, - sketch_index: Arc, + summary_store: Arc, ) -> Self { // Bootstrap the capability router with `ASAPQueryEngine` // registered under its canonical query-engine id. @@ -242,7 +242,7 @@ impl HttpServer { adapter_override: None, query_engine, query_router, - sketch_index, + summary_store, hot_reload_config: None, backend_storage_routing: None, backfill: None, @@ -308,13 +308,13 @@ impl HttpServer { self } - /// Attach a `HotReloadStreamingConfig` handle so the + /// Attach a `StreamingConfigHandle` handle so the /// `GET/POST /api/v1/streaming-config` endpoints can read and /// swap the currently active config. Without this handle the /// endpoints return `503 Service Unavailable`. pub fn with_hot_reload_config( mut self, - handle: crate::storage_engines::types::HotReloadStreamingConfig, + handle: crate::storage_engines::types::StreamingConfigHandle, ) -> Self { self.hot_reload_config = Some(handle); self @@ -322,7 +322,7 @@ impl HttpServer { pub fn with_active_physical_plan( mut self, - handle: crate::storage_engines::types::HotReloadActivePhysicalPlan, + handle: crate::storage_engines::types::ActivePhysicalPlanHandle, ) -> Self { self.physical_plan_lifecycle = Some( crate::storage_engines::types::PhysicalPlanLifecycle::new(handle.clone()), @@ -436,7 +436,7 @@ impl HttpServer { config: self.config.clone(), query_engine: self.query_engine, query_router: self.query_router, - sketch_index: self.sketch_index, + summary_store: self.summary_store, adapter: adapter.clone(), fallback: self.config.adapter_config.fallback.clone(), hot_reload_config: self.hot_reload_config.clone(), @@ -564,7 +564,7 @@ impl HttpServer { config: self.config.clone(), query_engine: self.query_engine.clone(), query_router: self.query_router.clone(), - sketch_index: self.sketch_index.clone(), + summary_store: self.summary_store.clone(), adapter: adapter.clone(), fallback: self.config.adapter_config.fallback.clone(), hot_reload_config: self.hot_reload_config.clone(), @@ -866,13 +866,13 @@ async fn process_query_request( /// single-target metrics keep their original semantics — every shape /// resolves to the one configured backend. fn resolve_metric_storage(state: &AppState, query: &str, tenant: &str) -> StorageBackend { - // A non-bootstrap atomic PhysicalPlan owns routing. Every request first + // A non-bootstrap atomic CompiledPhysicalPlan owns routing. Every request first // enters the ASAP engine, where QueryPlan lookup either executes its // compiler-bound DAG or returns an explicit fallback reason. Consulting // the legacy shape/SID candidate heuristics here would bypass QueryPlan // (and can also discard the request's explicit evaluation timestamp). if state.active_physical_plan.as_ref().is_some_and(|active| { - let snapshot = active.snapshot(); + let snapshot = active.active_snapshot(); snapshot.query_plan.plan_id != 0 && !snapshot.query_plan.entries.is_empty() }) { debug!( @@ -918,7 +918,7 @@ fn resolve_metric_storage(state: &AppState, query: &str, tenant: &str) -> Storag crate::storage_engines::types::QueryOperatorShape::RatePostHoc | crate::storage_engines::types::QueryOperatorShape::Topk ) - && metric_has_exact_agg_sum_sid(&state.sketch_index, &metric_name) + && metric_has_exact_agg_sum_sid(&state.summary_store, &metric_name) { debug!( "resolve_metric_storage: overriding {:?} → SketchStore \ @@ -960,8 +960,8 @@ fn resolve_metric_storage(state: &AppState, query: &str, tenant: &str) -> Storag shape, crate::storage_engines::types::QueryOperatorShape::Topk ) - && !metric_has_frequency_topk_sid(&state.sketch_index, &metric_name) - && !metric_has_exact_agg_sum_sid(&state.sketch_index, &metric_name) + && !metric_has_frequency_topk_sid(&state.summary_store, &metric_name) + && !metric_has_exact_agg_sum_sid(&state.summary_store, &metric_name) { debug!( "resolve_metric_storage: overriding SketchStore → \ @@ -998,7 +998,7 @@ fn resolve_metric_storage(state: &AppState, query: &str, tenant: &str) -> Storag crate::storage_engines::types::QueryOperatorShape::Sum ) && query_is_sum_over_time(&expr) - && metric_has_exact_agg_sum_sid(&state.sketch_index, &metric_name) + && metric_has_exact_agg_sum_sid(&state.summary_store, &metric_name) { debug!( "resolve_metric_storage: overriding SketchStore → \ @@ -2132,7 +2132,7 @@ async fn handle_runtime_info( // Delegate to adapter for protocol-specific handling state .adapter - .handle_runtime_info_with_headers(state.sketch_index.clone(), forwarding_headers) + .handle_runtime_info_with_headers(state.summary_store.clone(), forwarding_headers) .await } @@ -2640,7 +2640,7 @@ mod tests { use crate::precompute_engine::ingest_handler::{IngestObservability, IngestState}; use crate::precompute_engine::series_router::SeriesRouter; use crate::query_engines::ASAPQueryEngine; - use crate::storage_engines::types::{HotReloadStreamingConfig, StreamingConfig}; + use crate::storage_engines::types::{StreamingConfig, StreamingConfigHandle}; use prost::Message; use reqwest::Client; use std::sync::atomic::AtomicU64; @@ -2740,12 +2740,12 @@ mod tests { asap_types::summary_catalog::SummaryCatalog::from_materializations(7, 1, &[]).unwrap(), ); let generation = catalog.reference().unwrap(); - let sketch_index = Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); - sketch_index + let summary_store = Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); + summary_store .install_summary_catalog(Arc::clone(&catalog)) .unwrap(); - let active = crate::storage_engines::types::HotReloadActivePhysicalPlan::new( - crate::storage_engines::types::ActivePhysicalPlan { + let active = crate::storage_engines::types::ActivePhysicalPlanHandle::new( + crate::storage_engines::types::RuntimePhysicalPlan { envelope: envelope.clone(), summary_catalog: Some(Arc::clone(&catalog)), precompute_plan: PrecomputePlan { @@ -2775,7 +2775,7 @@ mod tests { }, rules: Vec::new(), }, - runtime_config: streaming_config.clone(), + streaming_config: streaming_config.clone(), query_plan: Arc::new(asap_types::query_plan::QueryPlan { plan_id: 7, plan_version: 1, @@ -2787,7 +2787,7 @@ mod tests { ), }, ); - let hot_reload = HotReloadStreamingConfig::from_active(active.clone()); + let hot_reload = StreamingConfigHandle::from_active_physical_plan(active.clone()); let (sender, _worker) = mpsc::channel(8); let ingest = Arc::new(IngestState { router: SeriesRouter::new(vec![sender]), @@ -2797,7 +2797,7 @@ mod tests { pass_raw_samples: false, sketch_snapshots: dashmap::DashMap::new(), series_resolver: Arc::new(crate::drivers::ingest::SeriesIdResolver::new()), - sketch_index: Arc::clone(&sketch_index), + summary_store: Arc::clone(&summary_store), observability: IngestObservability::default(), }); let receiver = @@ -2810,7 +2810,7 @@ mod tests { adapter_config, }, Arc::new(ASAPQueryEngine::new(15_000)), - sketch_index, + summary_store, ) .with_active_physical_plan(active) .with_remote_write(receiver.clone()); @@ -2876,9 +2876,7 @@ mod tests { ); } - async fn setup_test_server_with_hot_reload( - hot_reload: Option, - ) -> u16 { + async fn setup_test_server_with_hot_reload(hot_reload: Option) -> u16 { let adapter_config = AdapterConfig::prometheus_promql( "http://127.0.0.1:9999".to_string(), // Unused for this test false, // forward_unsupported_queries @@ -3009,7 +3007,7 @@ mod tests { /// the GET snapshot emission. #[tokio::test] async fn test_streaming_config_hot_reload_round_trip() { - let hot_reload = HotReloadStreamingConfig::new(StreamingConfig::default()); + let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); let server_port = setup_test_server_with_hot_reload(Some(hot_reload.clone())).await; let client = Client::new(); @@ -3095,13 +3093,13 @@ aggregations: let after_body: serde_json::Value = after.json().await.unwrap(); assert_eq!(after_body["aggregation_count"], 2); - // The underlying HotReloadStreamingConfig handle (cloned into + // The underlying StreamingConfigHandle handle (cloned into // the server at setup) also reflects the swap — proving that // downstream consumers that re-snapshot would see the new // state. PR 5: the map is keyed on fingerprints, so just // assert the entry count. let direct_snap = hot_reload.snapshot(); - assert_eq!(direct_snap.aggregation_configs.len(), 2); + assert_eq!(direct_snap.materializations_by_policy_fingerprint.len(), 2); } #[tokio::test] @@ -3132,7 +3130,7 @@ aggregations: #[tokio::test] async fn test_streaming_config_hot_reload_rejects_bad_yaml() { - let hot_reload = HotReloadStreamingConfig::new(StreamingConfig::default()); + let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); let server_port = setup_test_server_with_hot_reload(Some(hot_reload)).await; let client = Client::new(); @@ -3157,8 +3155,8 @@ aggregations: /// legacy `SchemaRegistry` is gone, so there is no longer a /// `schemas` parameter — every reconcile decision is sid-level. async fn setup_test_server_with_hot_reload_and_sketch_index( - hot_reload: HotReloadStreamingConfig, - sketch_index: Arc, + hot_reload: StreamingConfigHandle, + summary_store: Arc, ) -> u16 { let adapter_config = AdapterConfig::prometheus_promql("http://127.0.0.1:9999".to_string(), false); @@ -3170,7 +3168,7 @@ aggregations: let streaming_config = Arc::new(StreamingConfig::default()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let server = - HttpServer::new(config, query_engine, sketch_index).with_hot_reload_config(hot_reload); + HttpServer::new(config, query_engine, summary_store).with_hot_reload_config(hot_reload); server .start_test_server() .await @@ -3188,10 +3186,10 @@ aggregations: group_by: &[&str], ) { use crate::storage_engines::sketch_db::data::AggKind; - use crate::storage_engines::sketch_db::index::SketchInstanceMetadata; + use crate::storage_engines::sketch_db::index::SummarySeriesMetadata; use std::collections::BTreeSet; let group_by_keys: BTreeSet = group_by.iter().map(|s| s.to_string()).collect(); - store.register(SketchInstanceMetadata { + store.register(SummarySeriesMetadata { sid, metric_name: metric.to_string(), group_by_keys, @@ -3221,16 +3219,16 @@ aggregations: use crate::storage_engines::sketch_db::index::SketchStore; use crate::storage_engines::sketch_db::AggStatus; - let hot_reload = HotReloadStreamingConfig::new(StreamingConfig::default()); - let sketch_index = Arc::new(SketchStore::new()); + let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); + let summary_store = Arc::new(SketchStore::new()); // Pre-register two Active sids whose signatures match the // first config below; only sid 1 will survive the second // swap. - register_precompute_sid(&sketch_index, 1, "cpu_usage", &["host"]); - register_precompute_sid(&sketch_index, 2, "mem_usage", &["host"]); + register_precompute_sid(&summary_store, 1, "cpu_usage", &["host"]); + register_precompute_sid(&summary_store, 2, "mem_usage", &["host"]); let server_port = setup_test_server_with_hot_reload_and_sketch_index( hot_reload.clone(), - sketch_index.clone(), + summary_store.clone(), ) .await; let client = Client::new(); @@ -3287,11 +3285,11 @@ aggregations: "no sid should retire when every signature still appears in the new config; got {retired_ids:?}", ); assert_eq!( - sketch_index.instance(1).unwrap().status(), + summary_store.instance(1).unwrap().status(), AggStatus::Active ); assert_eq!( - sketch_index.instance(2).unwrap().status(), + summary_store.instance(2).unwrap().status(), AggStatus::Active ); @@ -3330,11 +3328,11 @@ aggregations: .collect::>(); assert_eq!(retired, vec![2u64]); assert_eq!( - sketch_index.instance(1).unwrap().status(), + summary_store.instance(1).unwrap().status(), AggStatus::Active ); assert_eq!( - sketch_index.instance(2).unwrap().status(), + summary_store.instance(2).unwrap().status(), AggStatus::Retired ); } @@ -3352,7 +3350,7 @@ aggregations: // `PolicyFingerprint::from_config`. The `agg_ids_added` u64 // in the HTTP response is the fingerprint's `as_u64()` form, // NOT the literal `42` the YAML once spelled out. - let hot_reload = HotReloadStreamingConfig::new(StreamingConfig::default()); + let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); let server_port = setup_test_server_with_hot_reload(Some(hot_reload)).await; let client = Client::new(); @@ -3403,13 +3401,13 @@ aggregations: // handler force-retires it. use crate::storage_engines::sketch_db::index::SketchStore; - let hot_reload = HotReloadStreamingConfig::new(StreamingConfig::default()); - let sketch_index = Arc::new(SketchStore::new()); - register_precompute_sid(&sketch_index, 1, "m1", &[]); - register_precompute_sid(&sketch_index, 2, "m2", &[]); + let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); + let summary_store = Arc::new(SketchStore::new()); + register_precompute_sid(&summary_store, 1, "m1", &[]); + register_precompute_sid(&summary_store, 2, "m2", &[]); let server_port = setup_test_server_with_hot_reload_and_sketch_index( hot_reload.clone(), - sketch_index.clone(), + summary_store.clone(), ) .await; let client = Client::new(); @@ -3499,7 +3497,7 @@ aggregations: // attached (every `HttpServer` carries one). With no // registered sids the endpoint reports an empty array, not // a 503. - let hot_reload = HotReloadStreamingConfig::new(StreamingConfig::default()); + let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); let server_port = setup_test_server_with_hot_reload(Some(hot_reload)).await; let client = Client::new(); @@ -3525,13 +3523,13 @@ aggregations: use crate::storage_engines::sketch_db::index::SketchStore; use crate::storage_engines::sketch_db::AggStatus; - let hot_reload = HotReloadStreamingConfig::new(StreamingConfig::default()); - let sketch_index = Arc::new(SketchStore::new()); - register_precompute_sid(&sketch_index, 11, "cpu", &["host"]); - register_precompute_sid(&sketch_index, 22, "mem", &["host"]); + let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); + let summary_store = Arc::new(SketchStore::new()); + register_precompute_sid(&summary_store, 11, "cpu", &["host"]); + register_precompute_sid(&summary_store, 22, "mem", &["host"]); let server_port = setup_test_server_with_hot_reload_and_sketch_index( hot_reload.clone(), - sketch_index.clone(), + summary_store.clone(), ) .await; let client = Client::new(); @@ -3550,7 +3548,7 @@ aggregations: assert_eq!(body["schema"]["sid"], 11); assert_eq!(body["schema"]["status"], "retired"); assert_eq!( - sketch_index.instance(11).unwrap().status(), + summary_store.instance(11).unwrap().status(), AggStatus::Retired ); @@ -3568,7 +3566,7 @@ aggregations: assert_eq!(body["schema"]["sid"], 22); assert_eq!(body["schema"]["status"], "expired"); assert_eq!( - sketch_index.instance(22).unwrap().status(), + summary_store.instance(22).unwrap().status(), AggStatus::Expired ); @@ -3590,11 +3588,11 @@ aggregations: async fn test_get_timeline_missing_param_returns_400() { use crate::storage_engines::sketch_db::index::SketchStore; - let hot_reload = HotReloadStreamingConfig::new(StreamingConfig::default()); - let sketch_index = Arc::new(SketchStore::new()); + let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); + let summary_store = Arc::new(SketchStore::new()); let server_port = setup_test_server_with_hot_reload_and_sketch_index( hot_reload.clone(), - sketch_index.clone(), + summary_store.clone(), ) .await; let client = Client::new(); @@ -3636,7 +3634,7 @@ aggregations: // reads from the sid catalog (always attached) instead of the // optional `SchemaRegistry`. Empty catalog → empty segments, // not a 503. - let hot_reload = HotReloadStreamingConfig::new(StreamingConfig::default()); + let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); let server_port = setup_test_server_with_hot_reload(Some(hot_reload)).await; let client = Client::new(); let resp = client @@ -3737,14 +3735,14 @@ aggregations: agg_map.insert(fp, cfg); } let streaming_config = Arc::new(StreamingConfig::new(agg_map)); - let hot_reload = HotReloadStreamingConfig::from_arc(streaming_config.clone()); + let hot_reload = StreamingConfigHandle::from_arc(streaming_config.clone()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); - let sketch_index = Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); + let summary_store = Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); for marker in active_agg_ids { let fp = marker_to_fp[marker]; - register_precompute_sid(&sketch_index, fp, &format!("metric_{marker}"), &[]); + register_precompute_sid(&summary_store, fp, &format!("metric_{marker}"), &[]); } - let server = HttpServer::new(config, query_engine, sketch_index) + let server = HttpServer::new(config, query_engine, summary_store) .with_backfill_registry(registry) .with_hot_reload_config(hot_reload); let port = server @@ -3896,7 +3894,7 @@ aggregations: async fn test_backfill_endpoints_503_without_registry() { // Build a server with NO backfill registry attached — every // backfill endpoint should 503. - let hot_reload = HotReloadStreamingConfig::new(StreamingConfig::default()); + let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); let server_port = setup_test_server_with_hot_reload(Some(hot_reload)).await; let client = Client::new(); @@ -4095,7 +4093,7 @@ aggregations: /// `QueryEngine`s. The hot-reload `StreamingConfig` is pinned at /// `metric_storage_backend` so query dispatch follows the /// requested capability axis. Returns the bound port + the - /// `HotReloadStreamingConfig` handle so tests can swap the + /// `StreamingConfigHandle` handle so tests can swap the /// `storage_backend` mid-flight if they need to. async fn setup_test_server_with_router( metric_storage_backend: StorageBackend, @@ -4113,7 +4111,7 @@ aggregations: let streaming_cfg = StreamingConfig::with_storage_backend(Default::default(), metric_storage_backend); let streaming_arc = Arc::new(streaming_cfg); - let hot_reload = HotReloadStreamingConfig::from_arc(streaming_arc.clone()); + let hot_reload = StreamingConfigHandle::from_arc(streaming_arc.clone()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let mut server = HttpServer::new( config, @@ -4157,7 +4155,7 @@ aggregations: // decisions must come from the per-metric routing table. let streaming_cfg = StreamingConfig::default(); let streaming_arc = Arc::new(streaming_cfg); - let hot_reload = HotReloadStreamingConfig::from_arc(streaming_arc.clone()); + let hot_reload = StreamingConfigHandle::from_arc(streaming_arc.clone()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let mut server = HttpServer::new( config, @@ -5081,7 +5079,7 @@ aggregations: crate::query_engines::routing::HotReloadBackendStorageRouting, ) { use crate::query_engines::routing::HotReloadBackendStorageRouting; - use crate::storage_engines::types::{HotReloadStreamingConfig, StreamingConfig}; + use crate::storage_engines::types::{StreamingConfig, StreamingConfigHandle}; let adapter_config = AdapterConfig::prometheus_promql("http://127.0.0.1:9999".to_string(), false); @@ -5092,7 +5090,7 @@ aggregations: }; let streaming_cfg = StreamingConfig::default(); let streaming_arc = Arc::new(streaming_cfg); - let hot_reload = HotReloadStreamingConfig::from_arc(streaming_arc.clone()); + let hot_reload = StreamingConfigHandle::from_arc(streaming_arc.clone()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let routing_handle = HotReloadBackendStorageRouting::empty(); let server = HttpServer::new( @@ -5629,7 +5627,7 @@ aggregations: let streaming_cfg = StreamingConfig::with_storage_backend(Default::default(), metric_storage_backend); let streaming_arc = Arc::new(streaming_cfg); - let hot_reload = HotReloadStreamingConfig::from_arc(streaming_arc.clone()); + let hot_reload = StreamingConfigHandle::from_arc(streaming_arc.clone()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let mut server = HttpServer::new( config, @@ -5934,7 +5932,7 @@ async fn handle_health(State(state): State) -> axum::response::Respons let Some(active) = state .active_physical_plan .as_ref() - .map(|handle| handle.snapshot()) + .map(|handle| handle.active_snapshot()) else { return (StatusCode::SERVICE_UNAVAILABLE, "no active PhysicalPlan").into_response(); }; @@ -5987,11 +5985,11 @@ async fn handle_store_metrics(State(state): State) -> axum::response:: // M2.3.6g — earliest timestamps come from SketchStore's per-sid // `first_seen_unix_ms` metadata. Always succeeds (no I/O). - let timestamps = state.sketch_index.earliest_timestamps_per_series_id(); + let timestamps = state.summary_store.earliest_timestamps_per_series_id(); let body = serde_json::json!({ "status": "success", "sid_count": timestamps.len(), - "approx_resident_bytes": state.sketch_index.approx_resident_bytes(), + "approx_resident_bytes": state.summary_store.approx_resident_bytes(), "earliest_timestamps_per_series_id": timestamps}); (StatusCode::OK, axum::Json(body)).into_response() } @@ -6007,7 +6005,7 @@ async fn handle_store_metrics(State(state): State) -> axum::response:: // snapshot after the swap. `ASAPQueryEngine`, the ingest router, and // in-flight precompute workers all hold startup snapshots today and // ignore the swap until they are rebuilt — see the module doc on -// `HotReloadStreamingConfig` for the full contract. Tests POST a new +// `StreamingConfigHandle` for the full contract. Tests POST a new // config and verify it via the GET endpoint; control plane integration // and per-query re-snapshot are phase 2. @@ -6024,8 +6022,8 @@ async fn handle_get_streaming_config(State(state): State) -> axum::res let snap = handle.snapshot(); let body = serde_json::json!({ "status": "success", - "aggregation_count": snap.aggregation_configs.len(), - "aggregation_ids": snap.aggregation_configs.keys().copied().collect::>(), + "aggregation_count": snap.materializations_by_policy_fingerprint.len(), + "aggregation_ids": snap.materializations_by_policy_fingerprint.keys().copied().collect::>(), "streaming_config": &*snap}); (StatusCode::OK, axum::Json(body)).into_response() } @@ -6074,9 +6072,17 @@ async fn handle_post_streaming_config( } }; - let new_ids: HashSet = new_config.aggregation_configs.keys().copied().collect(); + let new_ids: HashSet = new_config + .materializations_by_policy_fingerprint + .keys() + .copied() + .collect(); let old_arc = handle.swap(new_config); - let old_ids: HashSet = old_arc.aggregation_configs.keys().copied().collect(); + let old_ids: HashSet = old_arc + .materializations_by_policy_fingerprint + .keys() + .copied() + .collect(); let added: Vec = new_ids.difference(&old_ids).copied().collect(); let removed: Vec = old_ids.difference(&new_ids).copied().collect(); @@ -6085,7 +6091,7 @@ async fn handle_post_streaming_config( "streaming-config hot-reload removed agg_ids {:?} — any in-flight \ precompute worker groups for these ids will continue with their \ construction-time config until they close naturally (phase 1 \ - limitation; see HotReloadStreamingConfig module doc)", + limitation; see StreamingConfigHandle module doc)", removed ); } @@ -6100,7 +6106,7 @@ async fn handle_post_streaming_config( // `SketchStore::ingest_precompute_for_agg_config`). let snap = handle.snapshot(); let sid_summary = crate::storage_engines::sketch_db::lifecycle::reconcile_from_streaming_config( - state.sketch_index.as_ref(), + state.summary_store.as_ref(), snap.as_ref(), crate::storage_engines::sketch_db::DEFAULT_RETIREMENT_RETENTION, ); @@ -6118,10 +6124,10 @@ pub use asap_types::plan_publication::PhysicalPlanInstallRequest; /// Decode and cross-validate every backend view before it can become visible. /// Used by both startup artifact loading and the staged HTTP install path. -pub fn build_active_physical_plan( +pub fn validate_and_build_runtime_plan( request: PhysicalPlanInstallRequest, default_routing: Arc, -) -> Result { +) -> Result { use std::collections::BTreeSet; request .precompute_plan @@ -6177,10 +6183,10 @@ pub fn build_active_physical_plan( { return Err("physical subplans have different plan identity/version".into()); } - let runtime_config = + let streaming_config = crate::storage_engines::types::StreamingConfig::new(runtime_materializations); - let typed_fps: BTreeSet<_> = runtime_config - .aggregation_configs + let typed_fps: BTreeSet<_> = streaming_config + .materializations_by_policy_fingerprint .keys() .copied() .map(asap_types::PolicyFingerprint) @@ -6196,12 +6202,12 @@ pub fn build_active_physical_plan( ), None => default_routing, }; - Ok(crate::storage_engines::types::ActivePhysicalPlan { + Ok(crate::storage_engines::types::RuntimePhysicalPlan { envelope: envelope.clone(), summary_catalog: Some(Arc::new(request.summary_catalog)), precompute_plan: request.precompute_plan, transmission_plan: request.transmission_plan, - runtime_config: Arc::new(runtime_config), + streaming_config: Arc::new(streaming_config), query_plan: Arc::new(request.query_plan), storage_routing, }) @@ -6234,7 +6240,7 @@ async fn handle_post_physical_plan( ) .into_response(); }; - let current = active_handle.snapshot(); + let current = active_handle.active_snapshot(); if current.transmission_plan.envelope.plan_id != 0 { if let Err(error) = current.transmission_plan.authorize_successor( &request.transmission_plan, @@ -6252,7 +6258,7 @@ async fn handle_post_physical_plan( } } let _guard = state.physical_plan_lock.lock().await; - let active = match build_active_physical_plan(request, current.storage_routing.clone()) { + let active = match validate_and_build_runtime_plan(request, current.storage_routing.clone()) { Ok(active) => active, Err(error) => { return ( @@ -6339,7 +6345,7 @@ async fn handle_activate_physical_plan( .into_response(); }; let _guard = state.physical_plan_lock.lock().await; - let store = Arc::clone(&state.sketch_index); + let store = Arc::clone(&state.summary_store); let remote_write = state.remote_write.clone(); let old = match lifecycle.activate_with_prepare( request.plan_id, @@ -6371,7 +6377,7 @@ async fn handle_activate_physical_plan( .into_response() } }; - let activated = active_handle.snapshot(); + let activated = active_handle.active_snapshot(); let clickhouse_plan_count = activated .query_plan .entries @@ -6385,16 +6391,16 @@ async fn handle_activate_physical_plan( tokio::spawn(async move { loop { if Arc::strong_count(&old) == 1 { - lifecycle.retire_drained(draining_id, draining_version); + lifecycle.mark_drained_plan_retired(draining_id, draining_version); break; } tokio::time::sleep(Duration::from_millis(10)).await; } }); } - let snap = activated.runtime_config.clone(); + let snap = activated.streaming_config.clone(); let retired = crate::storage_engines::sketch_db::lifecycle::reconcile_from_streaming_config( - state.sketch_index.as_ref(), + state.summary_store.as_ref(), snap.as_ref(), crate::storage_engines::sketch_db::DEFAULT_RETIREMENT_RETENTION, ); @@ -6414,7 +6420,7 @@ async fn handle_summary_inventory(State(state): State) -> axum::respon let Some(active) = state .active_physical_plan .as_ref() - .map(|handle| handle.snapshot()) + .map(|handle| handle.active_snapshot()) else { return ( StatusCode::SERVICE_UNAVAILABLE, @@ -6443,7 +6449,7 @@ async fn handle_summary_inventory(State(state): State) -> axum::respon }) .collect(); let reporter = std::env::var("HOSTNAME").unwrap_or_else(|_| "asapquery-backend".into()); - match state.sketch_index.observed_summary_inventory( + match state.summary_store.observed_summary_inventory( &reporter, &reporter, &producers, @@ -6652,7 +6658,7 @@ async fn handle_post_storage_routing( /// /// Route is kept at the historical `/api/v1/db/schemas` path so /// external callers don't break; the response now surfaces the -/// sid-level [`SketchInstanceMetadata`] entries (with field `sid` +/// sid-level [`SummarySeriesMetadata`] entries (with field `sid` /// instead of `agg_id`) since the per-agg_id `SchemaRegistry` has /// been retired. async fn handle_get_schemas( @@ -6680,12 +6686,12 @@ async fn handle_get_schemas( }; let mut entries: Vec = state - .sketch_index + .summary_store .snapshot_instances() .iter() .filter(|m| allowed.contains(&m.status())) .map(|metadata| { - let descriptors = state.sketch_index.descriptors_for_series_id(metadata.sid); + let descriptors = state.summary_store.descriptors_for_series_id(metadata.sid); sid_instance_to_json(metadata, descriptors.as_ref()) }) .collect(); @@ -6713,7 +6719,7 @@ fn status_str(s: crate::storage_engines::sketch_db::AggStatus) -> &'static str { /// `expires_at_ms`, `metric_name` — and adds the sid-native fields /// (`sid`, `group_by_keys`, `agg_kind`, `first_seen_unix_ms`). fn sid_instance_to_json( - m: &crate::storage_engines::sketch_db::index::SketchInstanceMetadata, + m: &crate::storage_engines::sketch_db::index::SummarySeriesMetadata, descriptors: Option<&( std::sync::Arc, std::sync::Arc, @@ -6744,12 +6750,12 @@ async fn handle_post_schema_retire( axum::extract::Path(sid): axum::extract::Path, ) -> axum::response::Response { use axum::response::IntoResponse; - match state.sketch_index.force_retire( + match state.summary_store.force_retire( sid, crate::storage_engines::sketch_db::DEFAULT_RETIREMENT_RETENTION, ) { Some(meta) => { - let descriptors = state.sketch_index.descriptors_for_series_id(sid); + let descriptors = state.summary_store.descriptors_for_series_id(sid); let body = serde_json::json!({ "status": "success", "schema": sid_instance_to_json(&meta, descriptors.as_ref())}); @@ -6773,9 +6779,9 @@ async fn handle_post_schema_expire( axum::extract::Path(sid): axum::extract::Path, ) -> axum::response::Response { use axum::response::IntoResponse; - match state.sketch_index.force_expire(sid) { + match state.summary_store.force_expire(sid) { Some(meta) => { - let descriptors = state.sketch_index.descriptors_for_series_id(sid); + let descriptors = state.summary_store.descriptors_for_series_id(sid); let body = serde_json::json!({ "status": "success", "schema": sid_instance_to_json(&meta, descriptors.as_ref())}); @@ -6854,7 +6860,7 @@ async fn handle_get_timeline( // content-derived signature id (xxh64 of metric + agg_kind + // group_by_keys), stable across restarts. let segments = crate::storage_engines::sketch_db::query::timeline::timeline_for_metric( - &state.sketch_index, + &state.summary_store, metric, start_ms, end_ms, @@ -7007,7 +7013,7 @@ async fn handle_post_backfill_job( // "ingest started at the unix epoch" — backfill can then // cover up to wall-clock-now. let earliest = state - .sketch_index + .summary_store .snapshot_instances() .into_iter() .filter(|m| m.metric_name == agg_cfg.metric) @@ -7221,17 +7227,17 @@ mod logical_provenance_tests { #[cfg(test)] mod catalog_install_tests { - use super::{build_active_physical_plan, PhysicalPlanInstallRequest}; + use super::{validate_and_build_runtime_plan, PhysicalPlanInstallRequest}; use std::sync::Arc; fn request() -> PhysicalPlanInstallRequest { - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" )) .unwrap(); let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); PhysicalPlanInstallRequest { summary_catalog: plan.summary_catalog, @@ -7245,8 +7251,8 @@ mod catalog_install_tests { } fn install( request: PhysicalPlanInstallRequest, - ) -> Result { - build_active_physical_plan( + ) -> Result { + validate_and_build_runtime_plan( request, Arc::new(crate::storage_engines::types::BackendStorageRouting::empty()), ) @@ -7255,8 +7261,8 @@ mod catalog_install_tests { #[test] fn invalid_clickhouse_entry_cannot_change_active_generation() { let active = install(request()).expect("baseline plan installs"); - let handle = crate::storage_engines::types::HotReloadActivePhysicalPlan::new(active); - let before = handle.snapshot(); + let handle = crate::storage_engines::types::ActivePhysicalPlanHandle::new(active); + let before = handle.active_snapshot(); let mut candidate = request(); candidate.query_plan.clickhouse_context = Some(asap_types::query_plan::ClickHousePlanningContext { @@ -7294,7 +7300,7 @@ mod catalog_install_tests { let error = install(candidate).expect_err("invalid SQL binding must fail staging"); assert!(error.contains("pane origin"), "{error}"); - let after = handle.snapshot(); + let after = handle.active_snapshot(); assert_eq!(after.plan_id(), before.plan_id()); assert_eq!(after.plan_version(), before.plan_version()); } @@ -7329,18 +7335,18 @@ mod catalog_install_tests { // A same-version snapshot replacement fails before the active generation changes. #[test] fn catalog_install_rejects_drift_without_replacing_active_snapshot() { - let active = crate::storage_engines::types::HotReloadActivePhysicalPlan::new( + let active = crate::storage_engines::types::ActivePhysicalPlanHandle::new( install(request()).unwrap(), ); - let before = active.snapshot(); + let before = active.active_snapshot(); let mut changed = request(); changed.summary_catalog.plan_version += 1; assert!(install(changed).is_err()); - assert!(Arc::ptr_eq(&before, &active.snapshot())); + assert!(Arc::ptr_eq(&before, &active.active_snapshot())); let mut changed = request(); changed.summary_catalog.summary_descriptors.clear(); assert!(install(changed).is_err()); - assert!(Arc::ptr_eq(&before, &active.snapshot())); + assert!(Arc::ptr_eq(&before, &active.active_snapshot())); } // Physical pane width cannot be replaced by the semantic lookback at install. @@ -7387,3 +7393,6 @@ mod catalog_install_tests { assert!(error.contains("pane origin"), "{error}"); } } + +#[deprecated(note = "Use validate_and_build_runtime_plan")] +pub use validate_and_build_runtime_plan as build_active_physical_plan; diff --git a/data_plane/src/lib.rs b/data_plane/src/lib.rs index 1273bbc0a..4be3c0dfb 100644 --- a/data_plane/src/lib.rs +++ b/data_plane/src/lib.rs @@ -25,13 +25,14 @@ )] pub mod drivers; -pub mod monitor; +pub mod update_sampling; + +#[deprecated(note = "Use update_sampling")] +pub use update_sampling as monitor; pub mod precompute_engine; pub mod query_engines; pub mod storage_engines; -#[cfg(test)] -pub mod tests; pub mod utils; // Re-export commonly used types to avoid glob import conflicts @@ -62,3 +63,6 @@ pub use precompute_engine::PrecomputeEngine; pub use utils::read_streaming_config; pub type Result = std::result::Result>; + +#[cfg(test)] +pub mod tests; diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index eb19b6dd1..787f22b43 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -63,7 +63,7 @@ struct Args { /// Versioned canonical QueryWorkload + DataWorkload and backend-local /// implementation evidence. The ASAPQuery profile invokes the pinned - /// Planner and PhysicalCompiler at startup when this is supplied. + /// Planner and PhysicalPlanCompiler at startup when this is supplied. #[arg(long)] planning_snapshot: Option, @@ -539,7 +539,7 @@ async fn main() -> Result<()> { let startup_artifact = if let Some(path) = args.planning_snapshot.as_ref() { let bytes = fs::read(path)?; - let mut snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let mut snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_slice(&bytes).map_err(|error| { format!( "failed to decode planning snapshot {}: {error}", @@ -549,12 +549,14 @@ async fn main() -> Result<()> { let runtime_memory_budget = u64::try_from(args.persistence_memory_limit_mb) .unwrap_or(u64::MAX) .saturating_mul(1024 * 1024); - snapshot.implementation.max_retained_summary_bytes = snapshot - .implementation - .max_retained_summary_bytes + snapshot + .physical_inputs + .retained_summary_memory_budget_bytes = snapshot + .physical_inputs + .retained_summary_memory_budget_bytes .min(runtime_memory_budget); let plan = snapshot - .compile() + .compile_promql() .map_err(|error| format!("startup planning failed for {}: {error}", path.display()))?; if let Some(comparison) = &plan.cost_comparison { info!( @@ -585,7 +587,7 @@ async fn main() -> Result<()> { None }; let startup_physical_plan = if let Some(artifact) = startup_artifact { - let active = data_plane::drivers::query::servers::http::build_active_physical_plan( + let active = data_plane::drivers::query::servers::http::validate_and_build_runtime_plan( artifact, Arc::new(data_plane::storage_engines::types::BackendStorageRouting::empty()), ) @@ -616,7 +618,7 @@ async fn main() -> Result<()> { None }; let streaming_config = match startup_physical_plan.as_ref() { - Some(active) => active.runtime_config.clone(), + Some(active) => active.streaming_config.clone(), None => Arc::new(read_streaming_config( args.streaming_config .as_deref() @@ -625,7 +627,7 @@ async fn main() -> Result<()> { }; info!( "Loaded streaming config with {} entries", - streaming_config.get_all_aggregation_configs().len() + streaming_config.materializations().len() ); info!("Streaming config: {:?}", streaming_config); @@ -674,12 +676,12 @@ async fn main() -> Result<()> { } else { Arc::new(data_plane::drivers::ingest::series_resolver::SeriesIdResolver::new()) }; - let sketch_index = Arc::new(data_plane::storage_engines::sketch_db::index::SketchStore::new()); + let summary_store = Arc::new(data_plane::storage_engines::sketch_db::index::SketchStore::new()); if let Some(catalog) = startup_physical_plan .as_ref() .and_then(|plan| plan.summary_catalog.as_ref()) { - sketch_index + summary_store .install_summary_catalog(Arc::clone(catalog)) .map_err(std::io::Error::other)?; } @@ -733,7 +735,7 @@ async fn main() -> Result<()> { index_persistence_dir ); Some( - sketch_index + summary_store .start_persistence(cfg) .expect("SketchStore::start_persistence failed"), ) @@ -765,7 +767,7 @@ async fn main() -> Result<()> { schemas: Vec::new(), producers: Vec::new(), materializations: streaming_config - .aggregation_configs + .materializations_by_policy_fingerprint .values() .cloned() .collect(), @@ -784,12 +786,12 @@ async fn main() -> Result<()> { rules: Vec::new(), }; let initial_active_plan = startup_physical_plan.unwrap_or_else(|| { - data_plane::storage_engines::types::ActivePhysicalPlan { + data_plane::storage_engines::types::RuntimePhysicalPlan { envelope: initial_precompute_plan.envelope.clone(), summary_catalog: None, precompute_plan: initial_precompute_plan, transmission_plan: initial_transmission_plan, - runtime_config: streaming_config.clone(), + streaming_config: streaming_config.clone(), query_plan: Arc::new(asap_types::query_plan::QueryPlan::empty()), storage_routing: Arc::new( data_plane::storage_engines::types::BackendStorageRouting::empty(), @@ -797,14 +799,14 @@ async fn main() -> Result<()> { } }); let active_physical_plan = - data_plane::storage_engines::types::HotReloadActivePhysicalPlan::new(initial_active_plan); + data_plane::storage_engines::types::ActivePhysicalPlanHandle::new(initial_active_plan); let hot_reload_config = - data_plane::storage_engines::types::HotReloadStreamingConfig::from_active( + data_plane::storage_engines::types::StreamingConfigHandle::from_active_physical_plan( active_physical_plan.clone(), ); // Query execution reads generation-consistent runtime configuration from - // the ActivePhysicalPlan installed below. + // the RuntimePhysicalPlan installed below. let engine = { let mut engine = ASAPQueryEngine::new(args.prometheus_scrape_interval) // Phase 5 wire-in (refactor 2026-05): hand the ASAP-tier @@ -812,7 +814,7 @@ async fn main() -> Result<()> { // drives the Phase 6 archive failover via // EngineError::CapabilityMiss when the ASAP tier is empty // / ghost / unknown. - .with_sketch_index(sketch_index.clone()) + .with_sketch_index(summary_store.clone()) .with_active_physical_plan(active_physical_plan.clone()) .with_exact_subquery_endpoint(args.prometheus_server.clone()) .with_metricsql_exact_subquery_endpoint(args.victoriametrics_url.clone()); @@ -866,7 +868,7 @@ async fn main() -> Result<()> { // below is for the eviction service + diagnostic plumbing // until subsequent M2.3.6 sub-PRs delete those too. let output_sink = Arc::new(SketchStoreSink::new( - sketch_index.clone(), + summary_store.clone(), hot_reload_config.clone(), series_resolver.clone(), )); @@ -875,12 +877,12 @@ async fn main() -> Result<()> { hot_reload_config.clone(), output_sink, series_resolver.clone(), - sketch_index.clone(), + summary_store.clone(), ); if let Some(endpoint) = args.erp_runtime_samples_endpoint.clone() { let generation = engine .ingest_state() - .physical_plan_snapshot() + .active_physical_plan_snapshot() .and_then(|plan| plan.precompute_plan.summary_catalog.clone()) .ok_or_else(|| { std::io::Error::other("ERP observation requires an installed catalog") @@ -898,7 +900,7 @@ async fn main() -> Result<()> { // Spawn periodic memory diagnostics logger — M2.3.6g routes // through SketchStore now that SketchStore no longer holds // production data. - let diag_index = sketch_index.clone(); + let diag_index = summary_store.clone(); tokio::spawn(async move { spawn_memory_diagnostics(diag_index, Some(worker_diagnostics)).await; }); @@ -921,7 +923,7 @@ async fn main() -> Result<()> { // write-idle, fully-flushed sketch sids while keeping their queryable // metadata, bounding resident registry memory under series churn. if args.idle_sid_evict_secs > 0 { - let evict_index = sketch_index.clone(); + let evict_index = summary_store.clone(); let idle_ms = args.idle_sid_evict_secs.saturating_mul(1000); // Sweep a few times per idle horizon, clamped to a sane cadence. let sweep = std::time::Duration::from_secs(args.idle_sid_evict_secs.clamp(10, 60)); @@ -995,7 +997,7 @@ async fn main() -> Result<()> { // sample_p grant. Global-threshold alerting is retired (see // data_plane::monitor module docs) — this coordinator never fires one. let monitor_handle = if args.enable_monitor_coordinator { - use data_plane::monitor::{ + use data_plane::update_sampling::{ Functional, MonitorConfig, MonitorCoordinator, MonitorServiceImpl, }; let specs: Vec = streaming_config @@ -1110,13 +1112,13 @@ async fn main() -> Result<()> { // `SchemaRegistry`. `POST /api/v1/streaming-config` drives // lifecycle transitions at the sid level via the shared // `SketchStore` (already passed in below). - let mut server = HttpServer::new(http_config, engine, sketch_index.clone()) + let mut server = HttpServer::new(http_config, engine, summary_store.clone()) .with_active_physical_plan(active_physical_plan.clone()) .with_probe_cache(probe_cache.clone()); if args.profile == RuntimeProfile::Distributed { // Legacy partial-document endpoints remain available to distributed // deployments. The compatibility profile deliberately exposes only - // the atomic PhysicalPlan stage/activate lifecycle. + // the atomic CompiledPhysicalPlan stage/activate lifecycle. server = server.with_hot_reload_config(hot_reload_config.clone()); } @@ -1178,14 +1180,14 @@ async fn main() -> Result<()> { ); data_plane::storage_engines::types::BackendStorageRouting::empty() }; - if active_physical_plan.snapshot().plan_id() == 0 { - let current = active_physical_plan.snapshot(); - active_physical_plan.swap(data_plane::storage_engines::types::ActivePhysicalPlan { + if active_physical_plan.active_snapshot().plan_id() == 0 { + let current = active_physical_plan.active_snapshot(); + active_physical_plan.swap(data_plane::storage_engines::types::RuntimePhysicalPlan { envelope: current.envelope.clone(), summary_catalog: current.summary_catalog.clone(), precompute_plan: current.precompute_plan.clone(), transmission_plan: current.transmission_plan.clone(), - runtime_config: current.runtime_config.clone(), + streaming_config: current.streaming_config.clone(), query_plan: current.query_plan.clone(), storage_routing: Arc::new(bootstrap_routing), }); @@ -1326,7 +1328,7 @@ async fn main() -> Result<()> { // destination after the M2.3.6g store retirement). Resolver // is the same shared mint authority as live ingest, so // backfilled precompute sids share the OTel namespace. - .with_sketch_index(sketch_index.clone()) + .with_sketch_index(summary_store.clone()) .with_series_resolver(series_resolver.clone()); info!( "Spawning BackfillService drain loop (reader factory: default — Prometheus sources wired, S3/OtherSketch fail fast)" @@ -1366,7 +1368,7 @@ async fn main() -> Result<()> { data_plane::storage_engines::sketch_db::DEFAULT_RETIREMENT_RETENTION, ); let svc = data_plane::storage_engines::sketch_db::SchemaEvictionService::new( - sketch_index.clone(), + summary_store.clone(), backfill_registry.clone(), data_plane::storage_engines::sketch_db::SchemaEvictionConfig { poll_interval: std::time::Duration::from_secs(args.schema_eviction_poll_secs), @@ -1411,7 +1413,7 @@ async fn main() -> Result<()> { ); let accelerator = Arc::new( data_plane::query_engines::asap_clickhouse_query_engine::accelerator::CatalogClickHouseAccelerator::with_active_physical_plan_and_exact_backend( - sketch_index.clone(), + summary_store.clone(), active_physical_plan.clone(), fallback.clone(), ), @@ -1501,7 +1503,7 @@ fn process_resident_bytes() -> usize { /// Periodic memory diagnostics logger — runs every 30 seconds. async fn spawn_memory_diagnostics( - sketch_index: Arc, + summary_store: Arc, worker_diagnostics: Option>, ) { use data_plane::storage_engines::sketch_db::index::persistence::EpochSource; @@ -1513,8 +1515,8 @@ async fn spawn_memory_diagnostics( // 1. SketchStore diagnostics (M2.3.6g — replaces the // pre-M2.3 per-agg_id SketchStore::diagnostic_info). - let instance_count = sketch_index.instance_count(); - let series_count = sketch_index.series_len(); + let instance_count = summary_store.instance_count(); + let series_count = summary_store.series_len(); // `approx_memory_bytes` is the flusher's EVICTABLE-payload gauge: // it counts only live sketch payloads (current_epoch + sealed), so // it correctly reads ~0 once everything has been flushed to disk. @@ -1522,8 +1524,8 @@ async fn spawn_memory_diagnostics( // per-sid registry + intern caches stay resident and are not // flushable. Report evictable payload, structural overhead, their // total store estimate, and process RSS ground truth separately. - let payload_bytes = sketch_index.approx_memory_bytes(); - let resident_bytes = sketch_index.approx_resident_bytes(); + let payload_bytes = summary_store.approx_memory_bytes(); + let resident_bytes = summary_store.approx_resident_bytes(); let structural_bytes = resident_bytes.saturating_sub(payload_bytes); let rss_bytes = process_resident_bytes(); info!( diff --git a/data_plane/src/precompute_engine/engine.rs b/data_plane/src/precompute_engine/engine.rs index 8d59efe94..7db38db48 100644 --- a/data_plane/src/precompute_engine/engine.rs +++ b/data_plane/src/precompute_engine/engine.rs @@ -3,7 +3,7 @@ use crate::precompute_engine::ingest_handler::IngestState; use crate::precompute_engine::output_sink::OutputSink; use crate::precompute_engine::series_router::{SeriesRouter, WorkerMessage}; use crate::precompute_engine::worker::{Worker, WorkerRuntimeConfig}; -use crate::storage_engines::types::HotReloadStreamingConfig; +use crate::storage_engines::types::StreamingConfigHandle; use std::sync::atomic::{AtomicI64, AtomicUsize}; use std::sync::Arc; use tokio::sync::mpsc; @@ -20,15 +20,14 @@ pub struct PrecomputeWorkerDiagnostics { /// Creates worker threads and the series router. The ingest state /// (router + hot-reload handle) is built eagerly in `new()` so that /// ingest sources (currently OTLP) can hold a handle and push data -/// into the same worker pool. The legacy Prometheus / VictoriaMetrics -/// remote-write HTTP listener was deleted alongside the rest of the -/// remote-write ingest path — backend ingest is OTLP-only now. +/// into the same worker pool. OTLP ingestion and the backend-local Remote Write +/// profile both use the installed materialization view. pub struct PrecomputeEngine { config: PrecomputeEngineConfig, output_sink: Arc, diagnostics: Arc, ingest_state: Arc, - hot_reload_config: HotReloadStreamingConfig, + hot_reload_config: StreamingConfigHandle, /// Worker receivers, one per worker. Taken by `run()` when spawning workers. receivers: Vec>, } @@ -36,10 +35,10 @@ pub struct PrecomputeEngine { impl PrecomputeEngine { pub fn new( config: PrecomputeEngineConfig, - hot_reload_config: HotReloadStreamingConfig, + hot_reload_config: StreamingConfigHandle, output_sink: Arc, series_resolver: Arc, - sketch_index: Arc, + summary_store: Arc, ) -> Self { let worker_group_counts = (0..config.num_workers) .map(|_| Arc::new(AtomicUsize::new(0))) @@ -81,7 +80,7 @@ impl PrecomputeEngine { pass_raw_samples: config.pass_raw_samples, sketch_snapshots: dashmap::DashMap::new(), series_resolver, - sketch_index, + summary_store, observability: crate::precompute_engine::ingest_handler::IngestObservability::new(), }); @@ -108,10 +107,8 @@ impl PrecomputeEngine { } /// Start the precompute engine. This spawns worker tasks and the - /// periodic flush timer, then blocks until shutdown. The legacy - /// Prometheus / VictoriaMetrics HTTP ingest listener has been - /// removed; ingest now flows in via the OTLP receiver, which holds - /// the same `IngestState` handle returned by `ingest_state()`. + /// periodic flush timer, then blocks until shutdown. Protocol receivers + /// submit through the shared `IngestState` returned by `ingest_state()`. pub async fn run(mut self) -> Result<(), Box> { let num_workers = self.config.num_workers; let output_sink: Arc = Arc::new( diff --git a/data_plane/src/precompute_engine/ingest_handler.rs b/data_plane/src/precompute_engine/ingest_handler.rs index e50323b37..e047cc9e8 100644 --- a/data_plane/src/precompute_engine/ingest_handler.rs +++ b/data_plane/src/precompute_engine/ingest_handler.rs @@ -1,6 +1,6 @@ use crate::precompute_engine::series_router::SeriesRouter; use crate::precompute_engine::worker::parse_labels_from_series_key; -use crate::storage_engines::types::HotReloadStreamingConfig; +use crate::storage_engines::types::StreamingConfigHandle; use asap_types::aggregation_config::AggregationConfig; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; @@ -121,7 +121,7 @@ pub struct IngestState { /// Hot-reloadable streaming config. On each ingest batch, the /// router snapshots the latest config to derive agg_configs. /// This replaces the old frozen `Vec>`. - pub hot_reload_config: HotReloadStreamingConfig, + pub hot_reload_config: StreamingConfigHandle, /// When true, skip group-key extraction and pass raw samples through. pub pass_raw_samples: bool, /// Per-series snapshot cache for delta-sketch reconstitution @@ -164,7 +164,7 @@ pub struct IngestState { /// every modified-OTLP first-class sketch DataPoint; queried by /// the `ASAPQueryEngine` query path (ASAP-tier hit / ghost / unknown /// classification drives the Phase 6 archive failover). - pub sketch_index: Arc, + pub summary_store: Arc, /// CQ-6 / RES-1 — per-reason silent-drop counters plus the /// `sketch_snapshots` eviction configuration. Grouped into one /// `Default`-constructible field so the counters can live on @@ -185,10 +185,17 @@ impl IngestState { self.hot_reload_config.snapshot() } + pub fn active_physical_plan_snapshot( + &self, + ) -> Option> { + self.hot_reload_config.active_physical_plan_snapshot() + } + + #[deprecated(note = "use active_physical_plan_snapshot")] pub fn physical_plan_snapshot( &self, - ) -> Option> { - self.hot_reload_config.physical_plan_snapshot() + ) -> Option> { + self.active_physical_plan_snapshot() } /// RES-1 — record that a per-series snapshot base for `window_start` @@ -353,7 +360,7 @@ mod tests { map.insert(agg_id, make_config(agg_id, metric)); let streaming = StreamingConfig::new(map); let hot_reload = - crate::storage_engines::types::HotReloadStreamingConfig::new(streaming.clone()); + crate::storage_engines::types::StreamingConfigHandle::new(streaming.clone()); let state = Arc::new(IngestState { router, @@ -365,7 +372,7 @@ mod tests { series_resolver: Arc::new( crate::drivers::ingest::series_resolver::SeriesIdResolver::new(), ), - sketch_index: Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()), + summary_store: Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()), observability: IngestObservability::default(), }); diff --git a/data_plane/src/precompute_engine/maintenance_runtime.rs b/data_plane/src/precompute_engine/maintenance_runtime.rs index ab1f35d42..c29484007 100644 --- a/data_plane/src/precompute_engine/maintenance_runtime.rs +++ b/data_plane/src/precompute_engine/maintenance_runtime.rs @@ -5,7 +5,7 @@ use super::subdag_scheduler::{ execute_precompute_sink, IdempotentCommitSink, MaterializationCommitKey, PrecomputeOperatorRegistry, ScheduleError, }; -use crate::storage_engines::types::{AggregateCore, HotReloadStreamingConfig, PrecomputedOutput}; +use crate::storage_engines::types::{AggregateCore, PrecomputedOutput, StreamingConfigHandle}; use asap_types::executable_plan::{BackendExecutableBinding, BackendNodeBinding}; use planner_types::post_asap::{ExecutableDagNode, ExecutableOperatorPayload, PostAsapNodeId}; use sha2::{Digest, Sha256}; @@ -1610,12 +1610,12 @@ struct CommitRegistry(Mutex); impl CommitRegistry { fn plan_snapshot( &self, - plans: &HotReloadStreamingConfig, - ) -> Result>, String> { + plans: &StreamingConfigHandle, + ) -> Result>, String> { let mut state = self.0.lock().map_err(|_| "commit registry poisoned")?; // Read the authoritative generation while holding the registry lock, // so an old in-flight batch cannot restore an obsolete generation. - let plan = plans.physical_plan_snapshot(); + let plan = plans.active_physical_plan_snapshot(); let generation = plan .as_ref() .map(|plan| (plan.plan_id(), plan.plan_version())); @@ -1778,13 +1778,13 @@ impl IdempotentCommitSink for CommitRegistry { /// With no matching DAG, the source output is forwarded unchanged. pub struct MaintenanceDagSink { inner: Arc, - plans: HotReloadStreamingConfig, + plans: StreamingConfigHandle, commits: CommitRegistry, batch_guard: Mutex<()>, } impl MaintenanceDagSink { - pub fn new(inner: Arc, plans: HotReloadStreamingConfig) -> Self { + pub fn new(inner: Arc, plans: StreamingConfigHandle) -> Self { Self { inner, plans, @@ -1795,7 +1795,7 @@ impl MaintenanceDagSink { fn execute_one( &self, - plan: &crate::storage_engines::types::ActivePhysicalPlan, + plan: &crate::storage_engines::types::RuntimePhysicalPlan, output: PrecomputedOutput, state: Box, ) -> Result, String> { @@ -2105,6 +2105,44 @@ impl OutputSink for MaintenanceDagSink { } } +/// Materializations affected by an admitted source update, following installed +/// semantic dependencies rather than assuming source and output identities match. +pub(crate) fn affected_materializations( + plan: &asap_types::precompute_plan::PrecomputePlan, + source: asap_types::sds::SummaryDefinitionId, +) -> BTreeSet { + use asap_types::executable_plan::BackendNodeBinding; + let mut affected = BTreeSet::from([source]); + for installed in plan.executable_dags.values() { + let mut reachable = installed.binding.nodes.iter().filter_map(|(node, binding)| { + matches!(binding, BackendNodeBinding::Materialization { summary_definition } if *summary_definition == source).then_some(*node) + }).collect::>(); + let mut frontier = reachable.iter().copied().collect::>(); + while let Some(producer) = frontier.pop() { + for edge in &installed.document.edges { + let immutable = matches!(installed.binding.node(edge.consumer), + Some(BackendNodeBinding::Materialization { summary_definition }) + if plan.materializations.iter().any(|config| + config.policy_fingerprint() == summary_definition.fingerprint() + && config.derived_input.is_some())); + if edge.producer == producer && !immutable && reachable.insert(edge.consumer) { + frontier.push(edge.consumer); + } + } + } + for sink in &installed.binding.precompute_sinks { + if reachable.contains(sink) { + if let Some(BackendNodeBinding::Materialization { summary_definition }) = + installed.binding.nodes.get(sink) + { + affected.insert(*summary_definition); + } + } + } + } + affected +} + #[cfg(test)] mod tests { use super::*; @@ -2309,10 +2347,10 @@ mod tests { .unwrap(); snapshot["query_workload"]["repeating_queries"][0]["query"] = "sum(sum_over_time(m[1m]))".into(); - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_value(snapshot).unwrap(); let bundle = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let mut source_config = bundle.precompute_plan.materializations[0].clone(); // This operator fixture supplies global raw populations; its config @@ -3533,13 +3571,13 @@ mod tests { fn summary_update_rejects_multiple_output_populations_before_updating() { use planner_types::post_asap::{GroupingStrategy, SummaryUpdate}; use planner_types::pre_asap::{ColumnRef, Reduction}; - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); let mut config = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap() .precompute_plan .materializations[0] @@ -3594,13 +3632,13 @@ mod tests { fn dds_maintenance_rejects_nonpositive_population_before_returning_summary() { use planner_types::post_asap::{GroupingStrategy, SummaryUpdate}; use planner_types::pre_asap::{ColumnRef, Reduction}; - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); let mut config = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap() .precompute_plan .materializations[0] @@ -3757,7 +3795,7 @@ mod tests { #[test] fn downstream_failure_does_not_acknowledge_maintenance_publication() { use crate::storage_engines::types::{ - ActivePhysicalPlan, HotReloadActivePhysicalPlan, StreamingConfig, + ActivePhysicalPlanHandle, RuntimePhysicalPlan, StreamingConfig, }; use asap_types::executable_plan::{InstalledPostAsapDag, OwnedPostAsapDag}; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -3790,10 +3828,10 @@ mod tests { .unwrap(); snapshot["query_workload"]["repeating_queries"][0]["query"] = "sum(sum_over_time(m[1m]))".into(); - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_value(snapshot).unwrap(); let mut bundle = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let target_config = &bundle.precompute_plan.materializations[0]; let long_step = target_config.window_size.max( @@ -3841,12 +3879,12 @@ mod tests { binding, }, )]); - let active = ActivePhysicalPlan { + let active = RuntimePhysicalPlan { envelope: bundle.precompute_plan.envelope.clone(), summary_catalog: Some(Arc::new(bundle.summary_catalog)), precompute_plan: bundle.precompute_plan, transmission_plan: bundle.transmission_plan, - runtime_config: Arc::new(StreamingConfig::new(Default::default())), + streaming_config: Arc::new(StreamingConfig::new(Default::default())), query_plan: Arc::new(bundle.query_plan), storage_routing: Arc::new(Default::default()), }; @@ -3861,7 +3899,7 @@ mod tests { }); let sink = MaintenanceDagSink::new( downstream.clone(), - HotReloadStreamingConfig::from_active(HotReloadActivePhysicalPlan::new( + StreamingConfigHandle::from_active_physical_plan(ActivePhysicalPlanHandle::new( active.clone(), )), ); @@ -4070,41 +4108,3 @@ mod tests { assert!(commits.get(&key).unwrap().is_none()); } } - -/// Materializations affected by an admitted source update, following installed -/// semantic dependencies rather than assuming source and output identities match. -pub(crate) fn affected_materializations( - plan: &asap_types::precompute_plan::PrecomputePlan, - source: asap_types::sds::SummaryDefinitionId, -) -> BTreeSet { - use asap_types::executable_plan::BackendNodeBinding; - let mut affected = BTreeSet::from([source]); - for installed in plan.executable_dags.values() { - let mut reachable = installed.binding.nodes.iter().filter_map(|(node, binding)| { - matches!(binding, BackendNodeBinding::Materialization { summary_definition } if *summary_definition == source).then_some(*node) - }).collect::>(); - let mut frontier = reachable.iter().copied().collect::>(); - while let Some(producer) = frontier.pop() { - for edge in &installed.document.edges { - let immutable = matches!(installed.binding.node(edge.consumer), - Some(BackendNodeBinding::Materialization { summary_definition }) - if plan.materializations.iter().any(|config| - config.policy_fingerprint() == summary_definition.fingerprint() - && config.derived_input.is_some())); - if edge.producer == producer && !immutable && reachable.insert(edge.consumer) { - frontier.push(edge.consumer); - } - } - } - for sink in &installed.binding.precompute_sinks { - if reachable.contains(sink) { - if let Some(BackendNodeBinding::Materialization { summary_definition }) = - installed.binding.nodes.get(sink) - { - affected.insert(*summary_definition); - } - } - } - } - affected -} diff --git a/data_plane/src/precompute_engine/multisource_coordinator.rs b/data_plane/src/precompute_engine/multisource_coordinator.rs index d64cf6436..5404a08d9 100644 --- a/data_plane/src/precompute_engine/multisource_coordinator.rs +++ b/data_plane/src/precompute_engine/multisource_coordinator.rs @@ -639,10 +639,10 @@ mod tests { query["demand"]["fixed_interval_at"]["evaluation_phase"] = 0.into(); query["time_selection"]["lookback"] = 60000.into(); wire["query_workload"]["repeating_queries"] = serde_json::json!([query]); - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_value(wire).unwrap(); let mut plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap() .precompute_plan; let target = plan diff --git a/data_plane/src/precompute_engine/output_sink.rs b/data_plane/src/precompute_engine/output_sink.rs index c2e4ffc4c..555663d9b 100644 --- a/data_plane/src/precompute_engine/output_sink.rs +++ b/data_plane/src/precompute_engine/output_sink.rs @@ -1,7 +1,7 @@ use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::precompute_engine::ingest_handler::IngestObservability; use crate::storage_engines::sketch_db::index::SketchStore; -use crate::storage_engines::types::hot_reload_config::HotReloadStreamingConfig; +use crate::storage_engines::types::hot_reload_config::StreamingConfigHandle; use crate::storage_engines::types::{AggregateCore, PrecomputedOutput}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; @@ -67,8 +67,8 @@ fn consume_in_order(items: Vec, mut persist: impl FnMut(&T) -> bool) -> us /// agg-id lookup and sid hash. Completed accumulators are consumed in order, /// so a catch-up batch releases each pane as soon as it is serialized. pub struct SketchStoreSink { - sketch_index: Arc, - hot_reload: HotReloadStreamingConfig, + summary_store: Arc, + hot_reload: StreamingConfigHandle, /// Single shared resolver across the ingest + precompute paths. Under /// the registry-allocated sid model (PR-1..3), this is the canonical /// mint authority — precompute sids share the same `next_sid` counter @@ -88,12 +88,12 @@ pub struct SketchStoreSink { impl SketchStoreSink { pub fn new( - sketch_index: Arc, - hot_reload: HotReloadStreamingConfig, + summary_store: Arc, + hot_reload: StreamingConfigHandle, series_resolver: Arc, ) -> Self { Self { - sketch_index, + summary_store, hot_reload, series_resolver, observability: None, @@ -170,7 +170,7 @@ impl SketchStoreSink { crate::precompute_engine::metrics::record_materialized_outputs(1) }); } - self.sketch_index + self.summary_store .validate_routed_catalog_generation(output.catalog_generation.as_deref()) .ok()?; writer @@ -178,11 +178,11 @@ impl SketchStoreSink { |metric, fp, ak| { resolver .resolve_with_reactivation(metric, fp, ak, |sid| { - self.sketch_index.validate_routed_catalog_generation( + self.summary_store.validate_routed_catalog_generation( output.catalog_generation.as_deref(), )?; let activation = - self.sketch_index.authorize_series_reactivation( + self.summary_store.authorize_series_reactivation( sid, output.policy_fp.into(), )?; @@ -227,7 +227,7 @@ impl SketchStoreSink { time_range: asap_types::sds::HalfOpenTimeRange { start_ms, end_ms }, group_values, }; - if let Err(error) = self.sketch_index.publish_admitted_summary_update( + if let Err(error) = self.summary_store.publish_admitted_summary_update( &revision.generation, &coordinate, revision.first_revision, @@ -245,7 +245,7 @@ impl SketchStoreSink { } true } else { - self.sketch_index + self.summary_store .publish_unadmitted_summary_update(persist) .is_some() } @@ -441,11 +441,11 @@ mod tests { let mut configs = HashMap::new(); configs.insert(agg_id, cfg); let streaming = StreamingConfig::new(configs); - let hot_reload = HotReloadStreamingConfig::new(streaming.clone()); + let hot_reload = StreamingConfigHandle::new(streaming.clone()); - let sketch_index = Arc::new(SketchStore::new()); + let summary_store = Arc::new(SketchStore::new()); let sink = SketchStoreSink::new( - sketch_index.clone(), + summary_store.clone(), hot_reload, Arc::new(SeriesIdResolver::new()), ); @@ -458,16 +458,16 @@ mod tests { sink.emit_batch(vec![(output, acc)]).expect("emit ok"); assert_eq!( - sketch_index.instance_count(), + summary_store.instance_count(), 1, "SketchStore should have one precompute instance" ); - let instances = sketch_index + let instances = summary_store .list_by_status(crate::storage_engines::sketch_db::lifecycle::AggStatus::Active); assert_eq!(instances.len(), 1); let meta = instances[0].clone(); let sid = meta.sid; - assert_eq!(sketch_index.classify(sid), SeriesLookup::Hit); + assert_eq!(summary_store.classify(sid), SeriesLookup::Hit); assert!( matches!( meta.agg_kind, @@ -509,10 +509,7 @@ mod tests { Arc::new(SeriesIdResolver::open(temporary.path().join("resolver.wal")).unwrap()); let sink = SketchStoreSink::new( store.clone(), - HotReloadStreamingConfig::new(StreamingConfig::new(HashMap::from([( - fingerprint.0, - cfg, - )]))), + StreamingConfigHandle::new(StreamingConfig::new(HashMap::from([(fingerprint.0, cfg)]))), resolver, ); let original_generation = Arc::new(catalog.reference().unwrap()); @@ -592,10 +589,10 @@ mod tests { .insert("alpha".into(), serde_json::json!(0.01)); let policy_fp = cfg.policy_fp_u64(); let hot_reload = - HotReloadStreamingConfig::new(StreamingConfig::new(HashMap::from([(policy_fp, cfg)]))); - let sketch_index = Arc::new(SketchStore::new()); + StreamingConfigHandle::new(StreamingConfig::new(HashMap::from([(policy_fp, cfg)]))); + let summary_store = Arc::new(SketchStore::new()); let sink = SketchStoreSink::new( - sketch_index.clone(), + summary_store.clone(), hot_reload, Arc::new(SeriesIdResolver::new()), ); @@ -608,7 +605,7 @@ mod tests { )]) .expect("emit sketch"); - let meta = sketch_index + let meta = summary_store .list_by_status(crate::storage_engines::sketch_db::lifecycle::AggStatus::Active) .into_iter() .next() @@ -620,8 +617,8 @@ mod tests { .. } )); - assert_eq!(sketch_index.query_range(meta.sid, 1_000, 2_000).len(), 1); - assert!(sketch_index + assert_eq!(summary_store.query_range(meta.sid, 1_000, 2_000).len(), 1); + assert!(summary_store .query_exact_agg_range(meta.sid, 1_000, 2_000) .is_empty()); } @@ -631,10 +628,10 @@ mod tests { // Streaming config does NOT contain agg_id=99 — the sink // reports a recoverable error rather than acknowledging a lost write. let streaming = StreamingConfig::new(HashMap::new()); - let hot_reload = HotReloadStreamingConfig::new(streaming.clone()); - let sketch_index = Arc::new(SketchStore::new()); + let hot_reload = StreamingConfigHandle::new(streaming.clone()); + let summary_store = Arc::new(SketchStore::new()); let sink = SketchStoreSink::new( - sketch_index.clone(), + summary_store.clone(), hot_reload, Arc::new(SeriesIdResolver::new()), ); @@ -643,7 +640,7 @@ mod tests { let acc: Box = Box::new(SumAccumulator::with_sum(1.0)); sink.emit_batch(vec![(output, acc)]) .expect_err("unpersisted output must not be acknowledged"); - assert_eq!(sketch_index.instance_count(), 0); + assert_eq!(summary_store.instance_count(), 0); } /// CQ-6 — a registry-miss (policy_fp not in the running streaming @@ -652,11 +649,11 @@ mod tests { #[test] fn sink_increments_policy_miss_counter_on_registry_miss() { let streaming = StreamingConfig::new(HashMap::new()); - let hot_reload = HotReloadStreamingConfig::new(streaming.clone()); - let sketch_index = Arc::new(SketchStore::new()); + let hot_reload = StreamingConfigHandle::new(streaming.clone()); + let summary_store = Arc::new(SketchStore::new()); let obs = Arc::new(IngestObservability::new()); let sink = SketchStoreSink::new( - sketch_index.clone(), + summary_store.clone(), hot_reload, Arc::new(SeriesIdResolver::new()), ) @@ -669,7 +666,7 @@ mod tests { .expect_err("unpersisted output must not be acknowledged"); assert_eq!( - sketch_index.instance_count(), + summary_store.instance_count(), 0, "no write on registry miss" ); diff --git a/data_plane/src/precompute_engine/worker.rs b/data_plane/src/precompute_engine/worker.rs index fcc73edaf..bb7a4fdb8 100644 --- a/data_plane/src/precompute_engine/worker.rs +++ b/data_plane/src/precompute_engine/worker.rs @@ -9,7 +9,7 @@ use crate::precompute_engine::output_sink::OutputSink; use crate::precompute_engine::series_router::WorkerMessage; use crate::precompute_engine::window_manager::WindowManager; use crate::storage_engines::types::{ - AggregateCore, HotReloadStreamingConfig, KeyByLabelValues, PrecomputedOutput, + AggregateCore, KeyByLabelValues, PrecomputedOutput, StreamingConfigHandle, }; use asap_types::aggregation_config::AggregationConfig; use asap_types::PolicyFingerprint; @@ -162,7 +162,7 @@ pub struct Worker { /// Hot-reload handle — workers read config directly from ArcSwap /// instead of holding a local copy. All components see the same /// config at the same time. - hot_reload: HotReloadStreamingConfig, + hot_reload: StreamingConfigHandle, /// Allowed lateness in ms. allowed_lateness_ms: i64, /// When true, skip aggregation and pass raw samples through. @@ -199,7 +199,7 @@ impl Worker { id: usize, receiver: mpsc::Receiver, output_sink: Arc, - hot_reload: HotReloadStreamingConfig, + hot_reload: StreamingConfigHandle, runtime_config: WorkerRuntimeConfig, group_count: Arc, worker_watermark: Arc, @@ -415,7 +415,7 @@ impl Worker { /// hot-reload snapshot the first time we see this sid; `group_key` is /// remembered on the `GroupState` for emit-time label rendering. /// - /// Reads config directly from the `HotReloadStreamingConfig` + /// Reads config directly from the `StreamingConfigHandle` /// ArcSwap handle, so new policies from a config swap are visible /// immediately — no message passing, no delay. /// Returns None if `policy_fp` has no matching config (e.g. arrived @@ -1904,15 +1904,15 @@ mod tests { ) } - /// Build a fresh `HotReloadStreamingConfig` from a map of agg_id + /// Build a fresh `StreamingConfigHandle` from a map of agg_id /// → AggregationConfig. Worker::new takes this handle instead of /// the old `HashMap>`. Tests use this /// helper instead of constructing the handle inline at every /// callsite. fn make_hot_reload( configs: HashMap, - ) -> crate::storage_engines::types::HotReloadStreamingConfig { - crate::storage_engines::types::HotReloadStreamingConfig::new( + ) -> crate::storage_engines::types::StreamingConfigHandle { + crate::storage_engines::types::StreamingConfigHandle::new( crate::storage_engines::types::StreamingConfig::new(configs), ) } @@ -2709,13 +2709,13 @@ aggregations: // PR 5: the streaming-config key is the policy fingerprint. let agg_id = *streaming_config - .get_all_aggregation_configs() + .materializations() .keys() .next() .expect("one agg"); assert!(streaming_config.contains(agg_id)); - let agg_configs = streaming_config.get_all_aggregation_configs().clone(); + let agg_configs = streaming_config.materializations().clone(); let sink = Arc::new(CapturingOutputSink::new()); let mut worker = make_worker(agg_configs, sink.clone(), false, 0, LateDataPolicy::Drop); diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs index c47a9c3fe..2405bdf89 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs @@ -23,7 +23,7 @@ use crate::storage_engines::sketch_db::index::SketchStore; pub struct CatalogClickHouseAccelerator { pub store: Arc, - active_physical_plan: Option, + active_physical_plan: Option, exact_backend: Option>, } @@ -38,7 +38,7 @@ impl CatalogClickHouseAccelerator { pub fn with_active_physical_plan( store: Arc, - active: crate::storage_engines::types::HotReloadActivePhysicalPlan, + active: crate::storage_engines::types::ActivePhysicalPlanHandle, ) -> Self { let mut accelerator = Self::empty(store); accelerator.active_physical_plan = Some(active); @@ -51,7 +51,7 @@ impl CatalogClickHouseAccelerator { /// nodes unexecutable. pub fn with_active_physical_plan_and_exact_backend( store: Arc, - active: crate::storage_engines::types::HotReloadActivePhysicalPlan, + active: crate::storage_engines::types::ActivePhysicalPlanHandle, exact_backend: Arc, ) -> Self { Self::with_active_physical_plan(store, active).with_exact_backend(exact_backend) @@ -168,7 +168,11 @@ fn requested_format(request: &ClickHouseQueryRequest) -> Result ClickHouseAccelerationOutcome { - let Some(physical) = self.active_physical_plan.as_ref().map(|h| h.snapshot()) else { + let Some(physical) = self + .active_physical_plan + .as_ref() + .map(|h| h.active_snapshot()) + else { return ClickHouseAccelerationOutcome::Fallback( ClickHouseAccelerationFallback::CatalogMiss, ); @@ -228,7 +232,7 @@ impl CatalogClickHouseAccelerator { async fn execute_bound( &self, request: &ClickHouseQueryRequest, - physical: &crate::storage_engines::types::ActivePhysicalPlan, + physical: &crate::storage_engines::types::RuntimePhysicalPlan, entry: &asap_types::query_plan::QueryPlanEntry, runtime_range: Option<(u64, u64)>, ) -> ClickHouseAccelerationOutcome { @@ -359,7 +363,7 @@ mod tests { use crate::{ precompute_engine::operators::SumAccumulator, - storage_engines::sketch_db::index::{AggKind, Capability, SketchInstanceMetadata}, + storage_engines::sketch_db::index::{AggKind, Capability, SummarySeriesMetadata}, }; use asap_types::query_plan::{ ClickHousePlanningContext, ExactReadout, ExternalExactOutput, ExternalExactRequest, @@ -732,7 +736,7 @@ mod tests { .install_summary_catalog(Arc::new(sds.clone())) .unwrap(); if seed { - store.register(SketchInstanceMetadata { + store.register(SummarySeriesMetadata { sid: 7, metric_name: "requests".into(), group_by_keys: BTreeSet::new(), @@ -778,14 +782,14 @@ mod tests { ) .unwrap(); precompute.summary_catalog = Some(sds.reference().unwrap()); - let mut transmission = control_plane::physical::compiler::compile_transmission_plan( + let mut transmission = control_plane::physical::compiler::build_transmission_plan( envelope.clone(), &precompute, &BTreeMap::new(), ) .unwrap(); transmission.summary_catalog = Some(sds.reference().unwrap()); - let active = crate::drivers::query::servers::http::build_active_physical_plan( + let active = crate::drivers::query::servers::http::validate_and_build_runtime_plan( crate::drivers::query::servers::http::PhysicalPlanInstallRequest { summary_catalog: sds, collector_plans: vec![], @@ -800,7 +804,7 @@ mod tests { .unwrap(); let accelerator = CatalogClickHouseAccelerator::with_active_physical_plan( store, - crate::storage_engines::types::HotReloadActivePhysicalPlan::new(active), + crate::storage_engines::types::ActivePhysicalPlanHandle::new(active), ); let request = ClickHouseQueryRequest { method: Method::GET, @@ -882,7 +886,7 @@ mod tests { let (accelerator, mut request) = fixture_with_sql(1_000, Arc::new(SketchStore::new()), true, true).await; let active = accelerator.active_physical_plan.as_ref().unwrap(); - let mut snapshot = active.snapshot().as_ref().clone(); + let mut snapshot = active.active_snapshot().as_ref().clone(); let context = snapshot.query_plan.clickhouse_context.as_ref().unwrap(); let second_sql = "SELECT sum(value) FROM requests WHERE timestamp >= 1000 AND timestamp < 2000"; @@ -949,7 +953,7 @@ mod tests { let (accelerator, mut request) = fixture_with_sql(1_000, Arc::new(SketchStore::new()), true, true).await; let active = accelerator.active_physical_plan.as_ref().unwrap(); - let mut snapshot = active.snapshot().as_ref().clone(); + let mut snapshot = active.active_snapshot().as_ref().clone(); let context = snapshot.query_plan.clickhouse_context.as_ref().unwrap(); let (fixed, _, _) = control_plane::clickhouse::bind_clickhouse_sql( &request.sql, @@ -1039,7 +1043,7 @@ mod tests { cfg.value_projection = Some(asap_types::sds::ValueProjectionIdentity::Column { name: "value".into(), }); - let hot = crate::storage_engines::types::HotReloadStreamingConfig::from_arc(Arc::new( + let hot = crate::storage_engines::types::StreamingConfigHandle::from_arc(Arc::new( crate::storage_engines::types::StreamingConfig::new(HashMap::from([( cfg.policy_fp_u64(), cfg.clone(), @@ -1140,7 +1144,7 @@ mod tests { .active_physical_plan .as_ref() .unwrap() - .snapshot(); + .active_snapshot(); let entry = mixed_summary_external_entry( physical.query_plan.entries.values().next().unwrap().clone(), ); @@ -1186,7 +1190,7 @@ mod tests { .active_physical_plan .as_ref() .unwrap() - .snapshot(); + .active_snapshot(); let mut query_plan = physical.query_plan.as_ref().clone(); let key = query_plan.entries.keys().next().unwrap().clone(); let entry = mixed_summary_external_entry(query_plan.entries[&key].clone()); @@ -1199,7 +1203,7 @@ mod tests { )); let accelerator = CatalogClickHouseAccelerator::with_active_physical_plan_and_exact_backend( accelerator.store.clone(), - crate::storage_engines::types::HotReloadActivePhysicalPlan::new(active), + crate::storage_engines::types::ActivePhysicalPlanHandle::new(active), exact_backend.clone(), ); let ClickHouseAccelerationOutcome::Accelerated(response) = diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/clickhouse_result_adapter.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/clickhouse_result_adapter.rs index cdc46d262..4541d3d28 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/clickhouse_result_adapter.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/clickhouse_result_adapter.rs @@ -308,113 +308,6 @@ pub fn raw_response(response: ClickHouseRawResponse) -> Response { output } -#[cfg(test)] -mod tests { - use super::*; - use arrow::{ - array::{Int64Array, StringArray}, - datatypes::{DataType, Field, Schema}, - }; - use std::sync::Arc; - - #[test] - fn nullable_fields_remain_explicit_in_json_and_tsv() { - let batch = RecordBatch::try_new( - Arc::new(Schema::new(vec![ - Field::new("value", DataType::Float64, true), - Field::new("label", DataType::Utf8, true), - ])), - vec![ - Arc::new(Float64Array::from(vec![Some(1.25), None, Some(2.5)])), - Arc::new(StringArray::from(vec![Some(""), None, Some("\\N")])), - ], - ) - .unwrap(); - let result = ClickHouseQueryResult { - batches: vec![batch], - }; - let json: serde_json::Value = - serde_json::from_slice(&result.encode(ClickHouseFormat::Json).unwrap()).unwrap(); - assert_eq!( - json["data"], - serde_json::json!([{ "value":1.25,"label":"" }, { "value":null,"label":null }, {"value":2.5,"label":"\\N"}]) - ); - let lines = result.encode(ClickHouseFormat::JsonEachRow).unwrap(); - let null_row: serde_json::Value = - serde_json::from_slice(lines.split(|byte| *byte == b'\n').nth(1).unwrap()).unwrap(); - assert_eq!(null_row, serde_json::json!({"value":null,"label":null})); - assert_eq!( - result.encode(ClickHouseFormat::TabSeparated).unwrap(), - b"1.25\t\n\\N\t\\N\n2.5\t\\\\N\n" - ); - } - - #[test] - fn empty_map_bottom_type_uses_clickhouse_nothing() { - let entries = DataType::Struct( - vec![ - Field::new("key", DataType::Null, false), - Field::new("value", DataType::Null, false), - ] - .into(), - ); - let dtype = DataType::Map(Arc::new(Field::new("entries", entries, false)), false); - assert_eq!(clickhouse_type(&dtype, false), "Map(Nothing, Nothing)"); - assert_eq!(clickhouse_type(&DataType::Null, true), "Nullable(Nothing)"); - } - - #[test] - fn map_timestamp_transport_is_not_assumed_to_match_native_formatting() { - let entries = DataType::Struct( - vec![ - Field::new("key", DataType::Utf8, false), - Field::new( - "value", - DataType::Timestamp(arrow::datatypes::TimeUnit::Millisecond, None), - false, - ), - ] - .into(), - ); - let dtype = DataType::Map(Arc::new(Field::new("entries", entries, false)), false); - let batch = RecordBatch::try_new( - Arc::new(Schema::new(vec![Field::new("m", dtype.clone(), false)])), - vec![arrow::array::new_empty_array(&dtype)], - ) - .unwrap(); - let result = ClickHouseQueryResult { - batches: vec![batch], - }; - assert!(result.encode(ClickHouseFormat::Json).is_err()); - assert!(result.encode(ClickHouseFormat::TabSeparated).is_err()); - } - - #[test] - fn encodes_table_without_using_promql_query_result() { - let schema = Arc::new(Schema::new(vec![ - Field::new("zone", DataType::Utf8, false), - Field::new("count", DataType::Int64, false), - ])); - let batch = RecordBatch::try_new( - schema, - vec![ - Arc::new(StringArray::from(vec!["a\tb"])), - Arc::new(Int64Array::from(vec![7])), - ], - ) - .unwrap(); - let result = ClickHouseQueryResult { - batches: vec![batch], - }; - assert_eq!( - result.encode(ClickHouseFormat::TabSeparated).unwrap(), - b"a\\tb\t7\n" - ); - assert!(result.encode(ClickHouseFormat::JsonEachRow).is_err()); - assert!(result.encode(ClickHouseFormat::Json).is_err()); - } -} - struct JsonArrowRows<'a>(&'a [RecordBatch]); impl Serialize for JsonArrowRows<'_> { fn serialize(&self, serializer: S) -> Result { @@ -586,3 +479,110 @@ fn map_literal(array: &dyn Array, row: usize) -> Result) -> QueryExpr { + QueryExpr::FunctionCall { + name: name.into(), + args, + } + } + fn text(value: &str) -> QueryExpr { + QueryExpr::Literal(ScalarValue::Utf8(value.into())) + } + + #[test] + fn map_access_uses_declared_default_and_first_duplicate() { + let dtype = DataType::Map { + key: Box::new(DataType::Utf8), + value: Box::new(DataType::Int64), + value_nullable: false, + }; + let schema = Schema::new(vec![Column::new("m", dtype, false)]); + let access = function("asap_map_access", vec![QueryExpr::Column(0), text("a")]); + assert_eq!( + eval(&access, &[Cell::Map(vec![])], &schema).unwrap(), + Cell::Int64(0) + ); + assert_eq!( + eval( + &access, + &[Cell::Map(vec![ + (Cell::Utf8("a".into()), Cell::Int64(7)), + (Cell::Utf8("a".into()), Cell::Int64(9)) + ])], + &schema + ) + .unwrap(), + Cell::Int64(7) + ); + let nullable = Schema::new(vec![Column::new( + "m", + DataType::Map { + key: Box::new(DataType::Utf8), + value: Box::new(DataType::Int64), + value_nullable: true, + }, + false, + )]); + assert_eq!( + eval(&access, &[Cell::Map(vec![])], &nullable).unwrap(), + Cell::Null + ); + let null_key = function( + "asap_map_access", + vec![QueryExpr::Column(0), QueryExpr::Literal(ScalarValue::Null)], + ); + assert_eq!( + eval(&null_key, &[Cell::Map(vec![])], &schema).unwrap(), + Cell::Null + ); + } + + #[test] + fn map_concat_preserves_duplicates_and_empty_map() { + let map = |value| { + function( + "map", + vec![text("a"), QueryExpr::Literal(ScalarValue::Int64(value))], + ) + }; + let concat = function("mapConcat", vec![function("map", vec![]), map(7), map(9)]); + let schema = Schema::new(vec![]); + assert_eq!( + eval(&concat, &[], &schema).unwrap(), + Cell::Map(vec![ + (Cell::Utf8("a".into()), Cell::Int64(7)), + (Cell::Utf8("a".into()), Cell::Int64(9)) + ]) + ); + let mixed = function( + "map", + vec![ + text("a"), + QueryExpr::Literal(ScalarValue::Int64(1)), + text("b"), + QueryExpr::Literal(ScalarValue::Float64(2.5)), + ], + ); + assert!(eval(&mixed, &[], &schema).is_err()); + } + + #[test] + fn sorting_nested_nan_fails_before_comparator_can_treat_it_as_equal() { + let dtype = DataType::Map { + key: Box::new(DataType::Utf8), + value: Box::new(DataType::Float64), + value_nullable: false, + }; + let input = ClickHouseRelation { + rows: vec![vec![Cell::Map(vec![( + Cell::Utf8("a".into()), + Cell::Float64(f64::NAN), + )])]], + fields: vec![("m".into(), dtype.clone(), false)], + coverage: None, + }; + let schema = SummarySchema { + fields: vec![planner_types::post_asap::SummaryField { + name: "m".into(), + dtype: SummaryFamilyType::Plain(dtype), + nullable: false, + }], + time_index: None, + }; + let operation = ValueOperation::Sort { + keys: vec![SortKey { + expr: QueryExpr::Column(0), + ascending: true, + nulls_first: false, + }], + partition_by: planner_types::pre_asap::GroupKeys::none(), + }; + assert!(ClickHouseRelationalAdapter + .apply_operation(&operation, &schema, input) + .is_err()); + } + + #[test] + fn mixed_comparison_preserves_integer_precision_and_boundaries() { + assert_eq!( + integer_float_cmp(9_007_199_254_740_993, 9_007_199_254_740_992.0), + Some(Ordering::Greater) + ); + assert_eq!( + integer_float_cmp(i64::MAX, 9_223_372_036_854_775_808.0), + Some(Ordering::Less) + ); + assert_eq!( + integer_float_cmp(i64::MIN, -9_223_372_036_854_775_808.0), + Some(Ordering::Equal) + ); + assert_eq!(integer_float_cmp(-1, -1.5), Some(Ordering::Greater)); + assert_eq!(integer_float_cmp(1, 1.5), Some(Ordering::Less)); + assert_eq!(integer_float_cmp(0, f64::INFINITY), Some(Ordering::Less)); + assert_eq!( + integer_float_cmp(0, f64::NEG_INFINITY), + Some(Ordering::Greater) + ); + assert_eq!(integer_float_cmp(0, f64::NAN), None); + } + + #[test] + fn integer_modulo_never_rounds_through_float() { + assert_eq!( + arithmetic( + &ArithmeticOpKind::Mod, + Cell::Int64(9_007_199_254_740_993), + Cell::Int64(2) + ) + .unwrap(), + Cell::Int64(1) + ); + assert_eq!( + arithmetic(&ArithmeticOpKind::Mod, Cell::Int64(-7), Cell::Int64(3)).unwrap(), + Cell::Int64(-1) + ); + assert!(arithmetic(&ArithmeticOpKind::Mod, Cell::Int64(7), Cell::Int64(0)).is_err()); + assert!(arithmetic( + &ArithmeticOpKind::Mod, + Cell::Int64(i64::MIN), + Cell::Int64(-1) + ) + .is_err()); + } +} + #[cfg(test)] mod tests { use super::*; @@ -1515,180 +1692,3 @@ mod tests { ); } } - -#[cfg(test)] -mod scalar_contract_tests { - use super::*; - use planner_types::pre_asap::{Column, Schema}; - - fn function(name: &str, args: Vec) -> QueryExpr { - QueryExpr::FunctionCall { - name: name.into(), - args, - } - } - fn text(value: &str) -> QueryExpr { - QueryExpr::Literal(ScalarValue::Utf8(value.into())) - } - - #[test] - fn map_access_uses_declared_default_and_first_duplicate() { - let dtype = DataType::Map { - key: Box::new(DataType::Utf8), - value: Box::new(DataType::Int64), - value_nullable: false, - }; - let schema = Schema::new(vec![Column::new("m", dtype, false)]); - let access = function("asap_map_access", vec![QueryExpr::Column(0), text("a")]); - assert_eq!( - eval(&access, &[Cell::Map(vec![])], &schema).unwrap(), - Cell::Int64(0) - ); - assert_eq!( - eval( - &access, - &[Cell::Map(vec![ - (Cell::Utf8("a".into()), Cell::Int64(7)), - (Cell::Utf8("a".into()), Cell::Int64(9)) - ])], - &schema - ) - .unwrap(), - Cell::Int64(7) - ); - let nullable = Schema::new(vec![Column::new( - "m", - DataType::Map { - key: Box::new(DataType::Utf8), - value: Box::new(DataType::Int64), - value_nullable: true, - }, - false, - )]); - assert_eq!( - eval(&access, &[Cell::Map(vec![])], &nullable).unwrap(), - Cell::Null - ); - let null_key = function( - "asap_map_access", - vec![QueryExpr::Column(0), QueryExpr::Literal(ScalarValue::Null)], - ); - assert_eq!( - eval(&null_key, &[Cell::Map(vec![])], &schema).unwrap(), - Cell::Null - ); - } - - #[test] - fn map_concat_preserves_duplicates_and_empty_map() { - let map = |value| { - function( - "map", - vec![text("a"), QueryExpr::Literal(ScalarValue::Int64(value))], - ) - }; - let concat = function("mapConcat", vec![function("map", vec![]), map(7), map(9)]); - let schema = Schema::new(vec![]); - assert_eq!( - eval(&concat, &[], &schema).unwrap(), - Cell::Map(vec![ - (Cell::Utf8("a".into()), Cell::Int64(7)), - (Cell::Utf8("a".into()), Cell::Int64(9)) - ]) - ); - let mixed = function( - "map", - vec![ - text("a"), - QueryExpr::Literal(ScalarValue::Int64(1)), - text("b"), - QueryExpr::Literal(ScalarValue::Float64(2.5)), - ], - ); - assert!(eval(&mixed, &[], &schema).is_err()); - } - - #[test] - fn sorting_nested_nan_fails_before_comparator_can_treat_it_as_equal() { - let dtype = DataType::Map { - key: Box::new(DataType::Utf8), - value: Box::new(DataType::Float64), - value_nullable: false, - }; - let input = ClickHouseRelation { - rows: vec![vec![Cell::Map(vec![( - Cell::Utf8("a".into()), - Cell::Float64(f64::NAN), - )])]], - fields: vec![("m".into(), dtype.clone(), false)], - coverage: None, - }; - let schema = SummarySchema { - fields: vec![planner_types::post_asap::SummaryField { - name: "m".into(), - dtype: SummaryFamilyType::Plain(dtype), - nullable: false, - }], - time_index: None, - }; - let operation = ValueOperation::Sort { - keys: vec![SortKey { - expr: QueryExpr::Column(0), - ascending: true, - nulls_first: false, - }], - partition_by: planner_types::pre_asap::GroupKeys::none(), - }; - assert!(ClickHouseRelationalAdapter - .apply_operation(&operation, &schema, input) - .is_err()); - } - - #[test] - fn mixed_comparison_preserves_integer_precision_and_boundaries() { - assert_eq!( - integer_float_cmp(9_007_199_254_740_993, 9_007_199_254_740_992.0), - Some(Ordering::Greater) - ); - assert_eq!( - integer_float_cmp(i64::MAX, 9_223_372_036_854_775_808.0), - Some(Ordering::Less) - ); - assert_eq!( - integer_float_cmp(i64::MIN, -9_223_372_036_854_775_808.0), - Some(Ordering::Equal) - ); - assert_eq!(integer_float_cmp(-1, -1.5), Some(Ordering::Greater)); - assert_eq!(integer_float_cmp(1, 1.5), Some(Ordering::Less)); - assert_eq!(integer_float_cmp(0, f64::INFINITY), Some(Ordering::Less)); - assert_eq!( - integer_float_cmp(0, f64::NEG_INFINITY), - Some(Ordering::Greater) - ); - assert_eq!(integer_float_cmp(0, f64::NAN), None); - } - - #[test] - fn integer_modulo_never_rounds_through_float() { - assert_eq!( - arithmetic( - &ArithmeticOpKind::Mod, - Cell::Int64(9_007_199_254_740_993), - Cell::Int64(2) - ) - .unwrap(), - Cell::Int64(1) - ); - assert_eq!( - arithmetic(&ArithmeticOpKind::Mod, Cell::Int64(-7), Cell::Int64(3)).unwrap(), - Cell::Int64(-1) - ); - assert!(arithmetic(&ArithmeticOpKind::Mod, Cell::Int64(7), Cell::Int64(0)).is_err()); - assert!(arithmetic( - &ArithmeticOpKind::Mod, - Cell::Int64(i64::MIN), - Cell::Int64(-1) - ) - .is_err()); - } -} diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/server.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/server.rs index aedbac3c6..967d314c2 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/server.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/server.rs @@ -124,6 +124,107 @@ impl ClickHouseHttpServer { } } +fn request( + method: Method, + params: HashMap, + headers: HeaderMap, + body: Bytes, +) -> Result { + let parameters: BTreeMap<_, _> = params.into_iter().collect(); + let sql = parameters + .get("query") + .cloned() + .or_else(|| String::from_utf8(body.to_vec()).ok()) + .unwrap_or_default(); + if sql.trim().is_empty() { + return Err((StatusCode::BAD_REQUEST, "missing query").into_response()); + } + Ok(ClickHouseQueryRequest { + method, + sql, + body, + parameters, + headers, + }) +} + +async fn query_get( + State(state): State, + Query(params): Query>, + headers: HeaderMap, +) -> Response { + let request = match request(Method::GET, params, headers, Bytes::new()) { + Ok(v) => v, + Err(e) => return e, + }; + execute_or_fallback(&state, &request).await +} + +async fn query_post( + State(state): State, + Query(params): Query>, + headers: HeaderMap, + body: Bytes, +) -> Response { + let request = match request(Method::POST, params, headers, body) { + Ok(v) => v, + Err(e) => return e, + }; + execute_or_fallback(&state, &request).await +} + +async fn execute_or_fallback(state: &ServerState, request: &ClickHouseQueryRequest) -> Response { + match state.accelerator.execute(request).await { + ClickHouseAccelerationOutcome::Accelerated(response) => { + let mut response = raw_response(response); + response + .headers_mut() + .entry("x-asap-execution") + .or_insert(axum::http::HeaderValue::from_static("warm")); + response + .headers_mut() + .entry("x-asap-execution-detail") + .or_insert(axum::http::HeaderValue::from_static("asap")); + response + } + ClickHouseAccelerationOutcome::Fallback(reason) => { + tracing::info!( + failure_stage = reason.stage(), + failure_reason = reason.reason_code(), + failure_detail = ?reason, + "ClickHouse acceleration routed to exact fallback" + ); + let stage = reason.stage(); + let reason = reason.reason_code(); + return match state.fallback.execute(request).await { + Ok(v) => { + let mut response = raw_response(v); + for (name, value) in [ + ("x-asap-execution", "exact_fallback"), + ("x-asap-execution-detail", reason), + ("x-asap-failure-stage", stage), + ("x-asap-failure-reason", reason), + ] { + response.headers_mut().insert( + axum::http::HeaderName::from_static(name), + axum::http::HeaderValue::from_static(value), + ); + } + response + } + Err(e) => (StatusCode::BAD_GATEWAY, e.to_string()).into_response(), + }; + } + } +} + +async fn ping(State(state): State) -> Response { + match state.fallback.ping().await { + Ok(v) => raw_response(v), + Err(e) => (StatusCode::BAD_GATEWAY, e.to_string()).into_response(), + } +} + #[cfg(test)] mod tests { use super::*; @@ -307,104 +408,3 @@ mod tests { } } } - -fn request( - method: Method, - params: HashMap, - headers: HeaderMap, - body: Bytes, -) -> Result { - let parameters: BTreeMap<_, _> = params.into_iter().collect(); - let sql = parameters - .get("query") - .cloned() - .or_else(|| String::from_utf8(body.to_vec()).ok()) - .unwrap_or_default(); - if sql.trim().is_empty() { - return Err((StatusCode::BAD_REQUEST, "missing query").into_response()); - } - Ok(ClickHouseQueryRequest { - method, - sql, - body, - parameters, - headers, - }) -} - -async fn query_get( - State(state): State, - Query(params): Query>, - headers: HeaderMap, -) -> Response { - let request = match request(Method::GET, params, headers, Bytes::new()) { - Ok(v) => v, - Err(e) => return e, - }; - execute_or_fallback(&state, &request).await -} - -async fn query_post( - State(state): State, - Query(params): Query>, - headers: HeaderMap, - body: Bytes, -) -> Response { - let request = match request(Method::POST, params, headers, body) { - Ok(v) => v, - Err(e) => return e, - }; - execute_or_fallback(&state, &request).await -} - -async fn execute_or_fallback(state: &ServerState, request: &ClickHouseQueryRequest) -> Response { - match state.accelerator.execute(request).await { - ClickHouseAccelerationOutcome::Accelerated(response) => { - let mut response = raw_response(response); - response - .headers_mut() - .entry("x-asap-execution") - .or_insert(axum::http::HeaderValue::from_static("warm")); - response - .headers_mut() - .entry("x-asap-execution-detail") - .or_insert(axum::http::HeaderValue::from_static("asap")); - response - } - ClickHouseAccelerationOutcome::Fallback(reason) => { - tracing::info!( - failure_stage = reason.stage(), - failure_reason = reason.reason_code(), - failure_detail = ?reason, - "ClickHouse acceleration routed to exact fallback" - ); - let stage = reason.stage(); - let reason = reason.reason_code(); - return match state.fallback.execute(request).await { - Ok(v) => { - let mut response = raw_response(v); - for (name, value) in [ - ("x-asap-execution", "exact_fallback"), - ("x-asap-execution-detail", reason), - ("x-asap-failure-stage", stage), - ("x-asap-failure-reason", reason), - ] { - response.headers_mut().insert( - axum::http::HeaderName::from_static(name), - axum::http::HeaderValue::from_static(value), - ); - } - response - } - Err(e) => (StatusCode::BAD_GATEWAY, e.to_string()).into_response(), - }; - } - } -} - -async fn ping(State(state): State) -> Response { - match state.fallback.ping().await { - Ok(v) => raw_response(v), - Err(e) => (StatusCode::BAD_GATEWAY, e.to_string()).into_response(), - } -} diff --git a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs index 3009c68cc..394453757 100644 --- a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs +++ b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs @@ -184,18 +184,18 @@ mod tests { use super::*; use asap_types::summary_catalog::SummaryCatalog; use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; - use control_plane::physical::compiler::BackendLocalPlanningSnapshot; + use control_plane::physical::compiler::BackendLocalPlanningInput; - fn fixture() -> control_plane::physical::compiler::PhysicalPlan { + fn fixture() -> control_plane::physical::compiler::CompiledPhysicalPlan { let mut value: serde_json::Value = serde_json::from_str(include_str!( "../../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); value["query_workload"]["repeating_queries"][0]["query"] = "sum(sum_over_time(m[1m]))".into(); - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_value(value).unwrap(); + let snapshot: BackendLocalPlanningInput = serde_json::from_value(value).unwrap(); crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap() } diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index 516b518d7..f28691335 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -92,7 +92,7 @@ pub struct ASAPQueryEngine { /// EngineRouter's archive failover (Phase 6). When `None`, the /// engine behaves as it did before Phase 5 wire-in (every query /// goes through `handle_query`'s legacy path). - sketch_index: Option>, + summary_store: Option>, /// Phase-5 hybrid-stitch hook — set by `with_archive_engine` from /// `main.rs`'s engine builder. When the ASAP-tier reducer reports a /// `ASAPTierResult.coverage` narrower than the requested @@ -107,7 +107,7 @@ pub struct ASAPQueryEngine { Option>, /// Generation-consistent physical snapshot used by the production query /// path. The QueryPlan and SummaryCatalog must come from the same snapshot. - active_physical_plan: Option, + active_physical_plan: Option, exact_subquery_endpoint: Option, metricsql_exact_subquery_endpoint: Option, exact_subquery_client: reqwest::Client, @@ -120,7 +120,7 @@ impl ASAPQueryEngine { now_ms: u64, ) -> Result { - let physical = self.physical_plan_snapshot().ok_or_else(|| { + let physical = self.active_physical_plan_snapshot().ok_or_else(|| { crate::query_engines::EngineError::capability_miss( "query_plan", "no active physical plan", @@ -132,7 +132,9 @@ impl ASAPQueryEngine { .map_err(|error| { crate::query_engines::EngineError::capability_miss("query_plan", error.to_string()) })?; - let leaves = self.prepare_logical(&physical, planned, &[now_ms]).await?; + let leaves = self + .prepare_query_inputs(&physical, planned, &[now_ms]) + .await?; let (mut result, mut stats) = self.execute_logical_entry(&physical, planned, &leaves, now_ms)?; stats.remote_evaluations = leaves.values().map(|leaf| leaf.remote_evaluations).sum(); @@ -149,7 +151,7 @@ impl ASAPQueryEngine { step_ms: u64, ) -> Result { - let physical = self.physical_plan_snapshot().ok_or_else(|| { + let physical = self.active_physical_plan_snapshot().ok_or_else(|| { crate::query_engines::EngineError::capability_miss( "query_plan", "no active physical plan", @@ -166,12 +168,12 @@ impl ASAPQueryEngine { } /// Construct the query executor. Runtime configuration is read only from - /// the generation-consistent `ActivePhysicalPlan` installed separately. + /// the generation-consistent `RuntimePhysicalPlan` installed separately. pub fn new(prometheus_scrape_interval: u64) -> Self { Self { prometheus_scrape_interval, control_plane_client: None, - sketch_index: None, + summary_store: None, archive_engine: None, active_physical_plan: None, exact_subquery_endpoint: None, @@ -192,9 +194,9 @@ impl ASAPQueryEngine { self.metricsql_exact_subquery_endpoint = Some(endpoint); self } - async fn prepare_logical( + async fn prepare_query_inputs( &self, - physical: &crate::storage_engines::types::ActivePhysicalPlan, + physical: &crate::storage_engines::types::RuntimePhysicalPlan, entry: &asap_types::query_plan::QueryPlanEntry, times: &[u64], ) -> Result { @@ -259,7 +261,7 @@ impl ASAPQueryEngine { fn execute_logical_entry( &self, - physical: &crate::storage_engines::types::ActivePhysicalPlan, + physical: &crate::storage_engines::types::RuntimePhysicalPlan, entry: &asap_types::query_plan::QueryPlanEntry, leaves: &super::logical_dag::PreparedLeaves, at: u64, @@ -272,7 +274,7 @@ impl ASAPQueryEngine { > { use crate::query_engines::EngineError; let revision = self - .sketch_index + .summary_store .as_ref() .map(|index| index.summary_update_revision()); let result = @@ -304,7 +306,7 @@ impl ASAPQueryEngine { subtree.instant.full_history = false; subtree.instant.cumulative_readout = true; let requirement = readiness_requirement(&subtree); - let index = self.sketch_index.as_ref().ok_or_else(|| { + let index = self.summary_store.as_ref().ok_or_else(|| { EngineError::capability_miss( "installed_logical_dag", "summary store unavailable", @@ -385,7 +387,7 @@ impl ASAPQueryEngine { )) }); let current = self - .sketch_index + .summary_store .as_ref() .map(|index| index.summary_update_revision()); if match (revision, current) { @@ -403,7 +405,7 @@ impl ASAPQueryEngine { async fn execute_logical_range( &self, - physical: &crate::storage_engines::types::ActivePhysicalPlan, + physical: &crate::storage_engines::types::RuntimePhysicalPlan, entry: &asap_types::query_plan::QueryPlanEntry, start: u64, end: u64, @@ -423,7 +425,7 @@ impl ASAPQueryEngine { let times: Vec = (0..=(end - start) / step) .map(|n| start + n * step) .collect(); - let leaves = self.prepare_logical(physical, entry, ×).await?; + let leaves = self.prepare_query_inputs(physical, entry, ×).await?; let mut series = std::collections::BTreeMap::, RangeVectorElement>::new(); let mut total = super::logical_dag::ExecutionStats::default(); @@ -478,18 +480,18 @@ impl ASAPQueryEngine { pub fn with_active_physical_plan( mut self, - handle: crate::storage_engines::types::HotReloadActivePhysicalPlan, + handle: crate::storage_engines::types::ActivePhysicalPlanHandle, ) -> Self { self.active_physical_plan = Some(handle); self } - fn physical_plan_snapshot( + fn active_physical_plan_snapshot( &self, - ) -> Option> { + ) -> Option> { self.active_physical_plan .as_ref() - .map(|handle| handle.snapshot()) + .map(|handle| handle.active_snapshot()) .filter(|plan| plan.plan_id() != 0) } @@ -513,7 +515,7 @@ impl ASAPQueryEngine { mut self, index: Arc, ) -> Self { - self.sketch_index = Some(index); + self.summary_store = Some(index); self } @@ -548,7 +550,7 @@ impl ASAPQueryEngine { t1_ms: u64, t2_ms: u64, ) -> Vec { - let Some(idx) = self.sketch_index.as_ref() else { + let Some(idx) = self.summary_store.as_ref() else { return Vec::new(); }; crate::storage_engines::sketch_db::query::timeline::timeline_for_metric( @@ -658,7 +660,7 @@ impl ASAPQueryEngine { step_ms: u64, ) -> Result { - if let Some(physical) = self.physical_plan_snapshot() { + if let Some(physical) = self.active_physical_plan_snapshot() { if let Ok(entry) = physical.query_plan.lookup(query) { if entry.nodes.values().any(|node| { matches!(node, asap_types::query_plan::QueryPlanNode::Logical { .. }) @@ -669,14 +671,14 @@ impl ASAPQueryEngine { } } } - let Some(idx) = self.sketch_index.as_ref() else { + let Some(idx) = self.summary_store.as_ref() else { return Err(crate::query_engines::EngineError::capability_miss( crate::storage_engines::types::StorageBackend::SketchStore.data_source_id(), format!("ASAPQueryEngine: no sketch index for `{query}` — failing over"), )); }; - let physical_plan = self.physical_plan_snapshot(); + let physical_plan = self.active_physical_plan_snapshot(); let mut readiness = None; let planned = match physical_plan.as_ref() { Some(physical_plan) => match physical_plan.query_plan.lookup(query) { @@ -1011,10 +1013,10 @@ impl crate::query_engines::routing::query_engine_routing::QueryEngine for ASAPQu now_ms: u64, ) -> Result { - if let Some(physical) = self.physical_plan_snapshot() { + if let Some(physical) = self.active_physical_plan_snapshot() { if let Ok(entry) = physical.query_plan.lookup(query) { let leaves = self - .prepare_logical(&physical, entry, &[now_ms]) + .prepare_query_inputs(&physical, entry, &[now_ms]) .await .map_err(|error| { tracing::warn!(query, error = %error, "installed query DAG preparation failed"); @@ -1037,8 +1039,8 @@ impl crate::query_engines::routing::query_engine_routing::QueryEngine for ASAPQu // SummaryCatalog/materialization resolver → SID lookup → DAG executor. // A typed resolver/executor error becomes CapabilityMiss, which lets // EngineRouter continue to the archive backend. - if let Some(idx) = self.sketch_index.as_ref() { - let physical_plan = self.physical_plan_snapshot(); + if let Some(idx) = self.summary_store.as_ref() { + let physical_plan = self.active_physical_plan_snapshot(); let mut readiness = None; let planned = match physical_plan.as_ref() { Some(physical_plan) => match physical_plan.query_plan.lookup(query) { @@ -1522,11 +1524,11 @@ mod aux_pushdown_tests { fn make_engine() -> ASAPQueryEngine { use crate::storage_engines::types::{ - CleanupPolicy, HotReloadStreamingConfig, StreamingConfig, + CleanupPolicy, StreamingConfig, StreamingConfigHandle, }; let sc = Arc::new(StreamingConfig::new(HashMap::new())); - let hr = HotReloadStreamingConfig::from_arc(sc.clone()); + let hr = StreamingConfigHandle::from_arc(sc.clone()); let _ = sc; ASAPQueryEngine::new(60) } @@ -1662,23 +1664,23 @@ mod asap_tier_classify_tests { use crate::query_engines::routing::query_engine_routing::QueryEngine as _; use crate::query_engines::EngineError; use crate::storage_engines::sketch_db::index::{ - AccuracyBound, Capability, SketchAlgorithm, SketchConfig, SketchInstanceMetadata, - SketchSampleState, SketchStore, + AccuracyBound, Capability, SketchAlgorithm, SketchConfig, SketchSampleState, SketchStore, + SummarySeriesMetadata, }; - use crate::storage_engines::types::{CleanupPolicy, HotReloadStreamingConfig}; + use crate::storage_engines::types::{CleanupPolicy, StreamingConfigHandle}; use std::collections::{BTreeMap, BTreeSet}; fn build_engine_with_index(idx: Arc) -> ASAPQueryEngine { let streaming_config = Arc::new(crate::storage_engines::types::StreamingConfig::default()); - let hot_reload = HotReloadStreamingConfig::from_arc(streaming_config); + let hot_reload = StreamingConfigHandle::from_arc(streaming_config); ASAPQueryEngine::new(15000).with_sketch_index(idx) } - fn dd_meta(sid: u64, metric: &str, group_by: &[&str]) -> SketchInstanceMetadata { + fn dd_meta(sid: u64, metric: &str, group_by: &[&str]) -> SummarySeriesMetadata { let cfg = SketchConfig::DDSketch { relative_accuracy: 0.01, }; - SketchInstanceMetadata { + SummarySeriesMetadata { sid, metric_name: metric.to_string(), group_by_keys: group_by @@ -1884,7 +1886,7 @@ mod asap_tier_classify_tests { for (i, zone) in zones.iter().enumerate() { let sid = 9000 + i as u64; - idx.register(SketchInstanceMetadata { + idx.register(SummarySeriesMetadata { sid, metric_name: "http_requests_total".to_string(), group_by_keys: ["zone".to_string()].into_iter().collect(), @@ -1948,14 +1950,14 @@ mod asap_tier_classify_tests { assert_eq!(by_zone.get("z3").copied(), Some(400.0)); } - // Build a now-anchored KLL `SketchInstanceMetadata` + sample so the + // Build a now-anchored KLL `SummarySeriesMetadata` + sample so the // engine's instant/range default lookbacks reach it. Mirrors the // live MVP workload: the agent emits a bare-named KLL sketch // (`http_requests_total_latency_ms`) into the SketchStore. - fn kll_meta(sid: u64, metric: &str) -> SketchInstanceMetadata { + fn kll_meta(sid: u64, metric: &str) -> SummarySeriesMetadata { // Latest ASAPPlanner sizes an epsilon=0.01 KLL at k=269. let cfg = SketchConfig::Kll { k: 269 }; - SketchInstanceMetadata { + SummarySeriesMetadata { sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), @@ -1990,10 +1992,10 @@ mod asap_tier_classify_tests { env.encode_to_vec() } - fn hll_meta(sid: u64, metric: &str) -> SketchInstanceMetadata { + fn hll_meta(sid: u64, metric: &str) -> SummarySeriesMetadata { // Latest ASAPPlanner requires p=14 for a 1% HLL error target. let cfg = SketchConfig::Hll { precision: 14 }; - SketchInstanceMetadata { + SummarySeriesMetadata { sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), @@ -2414,7 +2416,7 @@ mod asap_tier_classify_tests { for (i, (zone, per_window)) in [("z0", 600.0_f64), ("z1", 900.0)].iter().enumerate() { let sid = 11_000 + i as u64; - idx.register(SketchInstanceMetadata { + idx.register(SummarySeriesMetadata { sid, metric_name: "http_requests_total".to_string(), group_by_keys: ["zone".to_string()].into_iter().collect(), @@ -2476,7 +2478,7 @@ mod asap_tier_classify_tests { for (i, (zone, per_window)) in [("z0", 600.0_f64), ("z1", 900.0)].iter().enumerate() { let sid = 13_000 + i as u64; - idx.register(SketchInstanceMetadata { + idx.register(SummarySeriesMetadata { sid, metric_name: "http_requests_total".to_string(), group_by_keys: ["zone".to_string()].into_iter().collect(), @@ -2546,7 +2548,7 @@ mod asap_tier_classify_tests { for (i, (zone, per_window)) in [("z0", 600.0_f64), ("z1", 900.0)].iter().enumerate() { let sid = 14_000 + i as u64; - idx.register(SketchInstanceMetadata { + idx.register(SummarySeriesMetadata { sid, metric_name: "http_requests_total".to_string(), group_by_keys: ["zone".to_string()].into_iter().collect(), @@ -2631,7 +2633,7 @@ mod asap_tier_classify_tests { for (i, (zone, per_window)) in [("z0", 600.0_f64), ("z1", 900.0)].iter().enumerate() { let sid = 15_000 + i as u64; - idx.register(SketchInstanceMetadata { + idx.register(SummarySeriesMetadata { sid, metric_name: "http_requests_total".to_string(), group_by_keys: ["zone".to_string()].into_iter().collect(), @@ -2697,7 +2699,7 @@ mod asap_tier_classify_tests { let zones = ["z0", "z1", "z2", "z3"]; for (i, zone) in zones.iter().enumerate() { let sid = 12_000 + i as u64; - idx.register(SketchInstanceMetadata { + idx.register(SummarySeriesMetadata { sid, metric_name: "http_requests_total".to_string(), group_by_keys: ["zone".to_string()].into_iter().collect(), @@ -2767,7 +2769,7 @@ mod asap_tier_classify_tests { let zones = ["z0", "z1", "z2", "z3"]; for (i, zone) in zones.iter().enumerate() { let sid = 13_000 + i as u64; - idx.register(SketchInstanceMetadata { + idx.register(SummarySeriesMetadata { sid, metric_name: "http_requests_total".to_string(), group_by_keys: ["zone".to_string()].into_iter().collect(), @@ -2833,7 +2835,7 @@ mod asap_tier_classify_tests { ) { // Matches ControlPlaneCostModel's epsilon=0.01 CMS sizing. let cfg = SketchConfig::CountMin { rows: 5, cols: 512 }; - idx.register(SketchInstanceMetadata { + idx.register(SummarySeriesMetadata { sid, metric_name: metric.to_string(), group_by_keys: group_by @@ -3014,16 +3016,16 @@ mod outer_agg_integration_tests { use crate::query_engines::EngineError; use crate::storage_engines::sketch_db::index::{ AccuracyBound, Capability, SketchAlgorithm, SketchConfig, SketchEncoding, - SketchInstanceMetadata, SketchSampleState, SketchStore, + SketchSampleState, SketchStore, SummarySeriesMetadata, }; - use crate::storage_engines::types::HotReloadStreamingConfig; + use crate::storage_engines::types::StreamingConfigHandle; use asap_sketchlib::DdSketch; use asap_sketchlib::MessagePackCodec; use std::collections::{BTreeMap, BTreeSet}; fn build_engine_with_index(idx: Arc) -> ASAPQueryEngine { let streaming_config = Arc::new(crate::storage_engines::types::StreamingConfig::default()); - let hot_reload = HotReloadStreamingConfig::from_arc(streaming_config); + let hot_reload = StreamingConfigHandle::from_arc(streaming_config); ASAPQueryEngine::new(15000).with_sketch_index(idx) } @@ -3038,11 +3040,11 @@ mod outer_agg_integration_tests { sk.to_msgpack().expect("ddsketch msgpack serialization") } - fn dd_meta_for(sid: u64, metric: &str, group_by: &[&str]) -> SketchInstanceMetadata { + fn dd_meta_for(sid: u64, metric: &str, group_by: &[&str]) -> SummarySeriesMetadata { let cfg = SketchConfig::DDSketch { relative_accuracy: 0.01, }; - SketchInstanceMetadata { + SummarySeriesMetadata { sid, metric_name: metric.to_string(), group_by_keys: group_by @@ -3367,9 +3369,9 @@ mod range_stitch_tests { use crate::query_engines::EngineError; use crate::storage_engines::sketch_db::index::{ AccuracyBound, Capability, SketchAlgorithm, SketchConfig, SketchEncoding, - SketchInstanceMetadata, SketchSampleState, SketchStore, + SketchSampleState, SketchStore, SummarySeriesMetadata, }; - use crate::storage_engines::types::{HotReloadStreamingConfig, KeyByLabelValues}; + use crate::storage_engines::types::{KeyByLabelValues, StreamingConfigHandle}; use async_trait::async_trait; use std::collections::{BTreeMap, BTreeSet}; @@ -3430,9 +3432,9 @@ mod range_stitch_tests { /// A CountMin FrequencyEstimate sid — `count_over_time` over it emits one /// PER-WINDOW sample (not a single cumulative scalar), which is what the /// range stitch needs so warm contributes one value per covered window. - fn cms_meta(sid: u64, metric: &str) -> SketchInstanceMetadata { + fn cms_meta(sid: u64, metric: &str) -> SummarySeriesMetadata { let cfg = SketchConfig::CountMin { rows: 5, cols: 512 }; - SketchInstanceMetadata { + SummarySeriesMetadata { sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), @@ -3503,7 +3505,7 @@ mod range_stitch_tests { }); let streaming_config = Arc::new(crate::storage_engines::types::StreamingConfig::default()); - let hot_reload = HotReloadStreamingConfig::from_arc(streaming_config); + let hot_reload = StreamingConfigHandle::from_arc(streaming_config); let engine = ASAPQueryEngine::new(15000) .with_sketch_index(idx) .with_archive_engine(archive); @@ -3586,7 +3588,7 @@ mod range_stitch_tests { // No `.with_archive_engine(...)` — stitch must NOT fire. let streaming_config = Arc::new(crate::storage_engines::types::StreamingConfig::default()); - let hot_reload = HotReloadStreamingConfig::from_arc(streaming_config); + let hot_reload = StreamingConfigHandle::from_arc(streaming_config); let engine = ASAPQueryEngine::new(15000).with_sketch_index(idx); let result = engine @@ -3617,13 +3619,13 @@ mod range_stitch_tests { FallbackPolicy, InstantExecution, QueryLanguage, QueryNodeId, QueryPlanEntry, QueryPlanNode, }; - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" )) .unwrap(); let mut plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let identity = asap_types::query_plan::canonical_promql("1 + 2").unwrap(); plan.query_plan.entries.insert( @@ -3653,7 +3655,7 @@ mod range_stitch_tests { fallback: FallbackPolicy::ExactBackend, }, ); - let mut active = crate::drivers::query::servers::http::build_active_physical_plan( + let mut active = crate::drivers::query::servers::http::validate_and_build_runtime_plan( crate::drivers::query::servers::http::PhysicalPlanInstallRequest { summary_catalog: plan.summary_catalog, collector_plans: plan.collector_plans, @@ -3667,8 +3669,8 @@ mod range_stitch_tests { ) .unwrap(); active.envelope.expiry_unix_ms = None; - let active = crate::storage_engines::types::HotReloadActivePhysicalPlan::new(active); - let hot = HotReloadStreamingConfig::from_active(active.clone()); + let active = crate::storage_engines::types::ActivePhysicalPlanHandle::new(active); + let hot = StreamingConfigHandle::from_active_physical_plan(active.clone()); let engine = ASAPQueryEngine::new(15).with_active_physical_plan(active); let error = engine .execute_metricsql_at(&identity, 1_000) diff --git a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs index aa1537b3e..cc2b75d3d 100644 --- a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs +++ b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs @@ -2,8 +2,8 @@ use super::logical_dag::{PreparedLeaf, PreparedLeaves, Value}; use crate::query_engines::EngineError; use asap_types::query_plan::{ - logical::LogicalOperator, ExternalExactInput, ExternalExactRequest, QueryLanguage, QueryNodeId, - QueryPlanEntry, QueryPlanNode, + logical::ResidualQueryOperator, ExternalExactInput, ExternalExactRequest, QueryLanguage, + QueryNodeId, QueryPlanEntry, QueryPlanNode, }; use std::collections::{BTreeMap, BTreeSet, HashMap}; @@ -17,7 +17,7 @@ fn miss(message: impl Into) -> EngineError { /// Traverse only the installed graph, including epoch-aligned nested subquery grids. #[derive(Debug, Clone)] enum ExactLeaf { - Legacy(LogicalOperator), + Legacy(ResidualQueryOperator), External(ExternalExactRequest), } @@ -47,14 +47,14 @@ fn leaves( .ok_or_else(|| miss("missing installed node"))?; match node { QueryPlanNode::Logical { operator, inputs } => match operator { - LogicalOperator::Scan { .. } => { + ResidualQueryOperator::Scan { .. } => { return Err(miss("local raw Scan is forbidden in deployed plans")) } - LogicalOperator::ExactSubquery { .. } - | LogicalOperator::CandidateExactSubquery { .. } => { + ResidualQueryOperator::ExactSubquery { .. } + | ResidualQueryOperator::CandidateExactSubquery { .. } => { result.insert((id, at), ExactLeaf::Legacy(operator.clone())); } - LogicalOperator::Subquery { + ResidualQueryOperator::Subquery { range_ms, step_ms, offset_ms, @@ -116,7 +116,7 @@ pub(super) fn external_dependencies( ); } else if matches!( leaf, - ExactLeaf::Legacy(LogicalOperator::CandidateExactSubquery { .. }) + ExactLeaf::Legacy(ResidualQueryOperator::CandidateExactSubquery { .. }) ) { let input = *entry.nodes[&id] .inputs() @@ -298,10 +298,13 @@ pub(super) async fn prepare_external( for ((id, at), leaf) in leaves(entry, times)? { u64::try_from(at).map_err(|_| miss("subquery predates epoch"))?; let (language, query, candidate_input) = match &leaf { - ExactLeaf::Legacy(LogicalOperator::ExactSubquery { query }) => { + ExactLeaf::Legacy(ResidualQueryOperator::ExactSubquery { query }) => { (QueryLanguage::PromQl, query.clone(), None) } - ExactLeaf::Legacy(LogicalOperator::CandidateExactSubquery { query, item_label }) => ( + ExactLeaf::Legacy(ResidualQueryOperator::CandidateExactSubquery { + query, + item_label, + }) => ( QueryLanguage::PromQl, query.clone(), Some((entry.nodes[&id].inputs()[0], item_label.as_str())), @@ -751,7 +754,7 @@ mod tests { #[tokio::test] async fn exact_leaf_calls_prometheus_and_combines_with_prepared_summary() { // A successful exact branch remains an intermediate, not a whole-root fallback. - use asap_types::query_plan::logical::BinaryOperation; + use asap_types::query_plan::residual::BinaryOperation; use std::sync::{ atomic::{AtomicUsize, Ordering}, Arc, @@ -771,7 +774,7 @@ mod tests { ( QueryNodeId(0), QueryPlanNode::Logical { - operator: LogicalOperator::Binary { + operator: ResidualQueryOperator::Binary { operation: BinaryOperation::Div, return_bool: false, }, @@ -785,7 +788,7 @@ mod tests { ( QueryNodeId(2), QueryPlanNode::Logical { - operator: LogicalOperator::ExactSubquery { query: "b".into() }, + operator: ResidualQueryOperator::ExactSubquery { query: "b".into() }, inputs: vec![], }, ), @@ -827,7 +830,7 @@ mod tests { repeated.nodes.insert( QueryNodeId(1), QueryPlanNode::Logical { - operator: LogicalOperator::ExactSubquery { query: "b".into() }, + operator: ResidualQueryOperator::ExactSubquery { query: "b".into() }, inputs: vec![], }, ); @@ -863,7 +866,7 @@ mod tests { use crate::query_engines::query_result::{InstantVectorElement, QueryResult}; use crate::storage_engines::sketch_db::{ data::AggKind, - index::{Capability, SketchInstanceMetadata}, + index::{Capability, SummarySeriesMetadata}, }; use crate::storage_engines::types::{KeyByLabelValues, Measurement}; use asap_types::query_plan::{ @@ -877,7 +880,7 @@ mod tests { const AT: u64 = 300_000; const MATERIALIZATION: asap_types::PolicyFingerprint = asap_types::PolicyFingerprint(9001); let store = crate::storage_engines::sketch_db::index::SketchStore::new(); - store.register(SketchInstanceMetadata { + store.register(SummarySeriesMetadata { sid: 41, metric_name: "http_requests_total".into(), group_by_keys: std::collections::BTreeSet::from(["job".into()]), @@ -909,7 +912,7 @@ mod tests { ( QueryNodeId(0), QueryPlanNode::Logical { - operator: LogicalOperator::Binary { + operator: ResidualQueryOperator::Binary { operation: BinaryOperation::Div, return_bool: false, }, @@ -919,7 +922,7 @@ mod tests { ( QueryNodeId(1), QueryPlanNode::Logical { - operator: LogicalOperator::ExactSubquery { + operator: ResidualQueryOperator::ExactSubquery { query: exact_query.into(), }, inputs: vec![], @@ -1034,7 +1037,7 @@ mod tests { let entry = entry(BTreeMap::from([( QueryNodeId(0), QueryPlanNode::Logical { - operator: LogicalOperator::Scan { + operator: ResidualQueryOperator::Scan { metric: Some("m".into()), matchers: vec![], range_ms: None, diff --git a/data_plane/src/query_engines/asap_query_engine/live_serve.rs b/data_plane/src/query_engines/asap_query_engine/live_serve.rs index ccfb7333a..2e6ca5373 100644 --- a/data_plane/src/query_engines/asap_query_engine/live_serve.rs +++ b/data_plane/src/query_engines/asap_query_engine/live_serve.rs @@ -286,7 +286,7 @@ mod tests { use crate::storage_engines::sketch_db::data::{AggKind, SketchConfig}; use crate::storage_engines::sketch_db::index::{ - AccuracyBound, Capability, SketchAlgorithm, SketchInstanceMetadata, SketchSampleState, + AccuracyBound, Capability, SketchAlgorithm, SketchSampleState, SummarySeriesMetadata, }; fn ddsketch_fixture() -> SketchStore { @@ -294,7 +294,7 @@ mod tests { let cfg = SketchConfig::DDSketch { relative_accuracy: 0.01, }; - idx.register(SketchInstanceMetadata { + idx.register(SummarySeriesMetadata { sid: 1, metric_name: "latency_ms".to_string(), group_by_keys: std::collections::BTreeSet::new(), @@ -331,7 +331,7 @@ mod tests { let cfg = SketchConfig::Hll { precision: 14 }; let mut group_by_keys = std::collections::BTreeSet::new(); group_by_keys.insert("service".to_string()); - idx.register(SketchInstanceMetadata { + idx.register(SummarySeriesMetadata { sid, metric_name: "unique_users".to_string(), group_by_keys, @@ -423,7 +423,7 @@ mod tests { fn formal_range_plan_returns_exact_requested_steps() { let idx = SketchStore::new(); let policy = asap_types::PolicyFingerprint(901); - idx.register(SketchInstanceMetadata { + idx.register(SummarySeriesMetadata { sid: 9, metric_name: "bytes".into(), group_by_keys: std::collections::BTreeSet::new(), diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs index 93c124072..201966b01 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs @@ -4,8 +4,8 @@ use crate::query_engines::{ EngineError, }; use crate::storage_engines::types::KeyByLabelValues; -use asap_types::query_plan::logical::{ - Aggregation, BinaryOperation, Grouping, LogicalOperator, TemporalOperation, +use asap_types::query_plan::residual::{ + Aggregation, BinaryOperation, Grouping, ResidualQueryOperator, TemporalOperation, }; use asap_types::query_plan::{CandidateCompleteness, QueryNodeId, QueryPlanEntry, QueryPlanNode}; use std::collections::{BTreeMap, BTreeSet}; @@ -185,9 +185,9 @@ impl Result> Evaluator<' QueryPlanNode::Logical { operator, inputs } => { if matches!( operator, - LogicalOperator::Scan { .. } - | LogicalOperator::ExactSubquery { .. } - | LogicalOperator::CandidateExactSubquery { .. } + ResidualQueryOperator::Scan { .. } + | ResidualQueryOperator::ExactSubquery { .. } + | ResidualQueryOperator::CandidateExactSubquery { .. } ) { return Err(miss( "installed Prometheus leaf was not prepared; backend raw execution is forbidden", @@ -224,7 +224,7 @@ impl Result> Evaluator<' } fn logical( &mut self, - operator: LogicalOperator, + operator: ResidualQueryOperator, inputs: &[QueryNodeId], at: i64, ) -> Result { @@ -235,14 +235,14 @@ impl Result> Evaluator<' .ok_or_else(|| miss("missing logical input")) }; match operator { - LogicalOperator::ExactSubquery { .. } - | LogicalOperator::CandidateExactSubquery { .. } => { + ResidualQueryOperator::ExactSubquery { .. } + | ResidualQueryOperator::CandidateExactSubquery { .. } => { Err(miss("Prometheus exact leaf was not prepared")) } - LogicalOperator::Scan { .. } => { + ResidualQueryOperator::Scan { .. } => { Err(miss("local raw Scan is forbidden in deployed plans")) } - LogicalOperator::UnaryNegate => match self.eval(input(0)?, at)? { + ResidualQueryOperator::UnaryNegate => match self.eval(input(0)?, at)? { Value::Scalar(value) => Ok(Value::Scalar(-value)), Value::Vector(values) => Ok(Value::Vector( values @@ -252,7 +252,7 @@ impl Result> Evaluator<' )), _ => Err(miss("cannot negate range vector")), }, - LogicalOperator::VectorToScalar => { + ResidualQueryOperator::VectorToScalar => { let values = vector(self.eval(input(0)?, at)?)?; Ok(Value::Scalar(if values.len() == 1 { values[0].1 @@ -260,18 +260,18 @@ impl Result> Evaluator<' f64::NAN })) } - LogicalOperator::Aggregate { + ResidualQueryOperator::Aggregate { operation, grouping, } => { let values = vector(self.eval(input(0)?, at)?)?; Ok(Value::Vector(aggregate(operation, &grouping, values))) } - LogicalOperator::TopKSelection { k, grouping } => { + ResidualQueryOperator::TopKSelection { k, grouping } => { let values = vector(self.eval(input(0)?, at)?)?; Ok(Value::Vector(topk_selection(k, &grouping, values))) } - LogicalOperator::Binary { + ResidualQueryOperator::Binary { operation, return_bool, } => { @@ -279,7 +279,7 @@ impl Result> Evaluator<' let right = self.eval(input(1)?, at)?; binary(operation, return_bool, left, right) } - LogicalOperator::Temporal { operation } => { + ResidualQueryOperator::Temporal { operation } => { let Value::Matrix(values, start, end) = self.eval(input(0)?, at)? else { return Err(miss("temporal operator requires range vector")); }; @@ -340,7 +340,7 @@ impl Result> Evaluator<' .collect(), )) } - LogicalOperator::Sort { descending } => { + ResidualQueryOperator::Sort { descending } => { let mut values = vector(self.eval(input(0)?, at)?)?; values.sort_by(|a, b| { if a.1.is_nan() && b.1.is_nan() { @@ -357,7 +357,7 @@ impl Result> Evaluator<' }); Ok(Value::Vector(values)) } - LogicalOperator::HistogramQuantile => { + ResidualQueryOperator::HistogramQuantile => { let Value::Scalar(quantile) = self.eval(input(0)?, at)? else { return Err(miss("quantile requires scalar")); }; @@ -374,7 +374,7 @@ impl Result> Evaluator<' .collect(), )) } - LogicalOperator::Subquery { + ResidualQueryOperator::Subquery { range_ms, step_ms, offset_ms, @@ -804,7 +804,7 @@ mod topk_tests { #[test] fn installed_topk_combines_with_prometheus_exact_child() { - let mut entry = control_plane::query_plan::logical::compile_logical( + let mut entry = control_plane::query_plan::residual::compile_logical( "hybrid-topk".into(), "topk(2, m)".into(), InstantExecution { @@ -815,7 +815,7 @@ mod topk_tests { FallbackPolicy::ExactBackend, ) .unwrap(); - control_plane::query_plan::logical::finalize_residuals(&mut entry).unwrap(); + control_plane::query_plan::residual::finalize_residuals(&mut entry).unwrap(); let leaf = entry .nodes .iter() @@ -823,7 +823,7 @@ mod topk_tests { matches!( node, QueryPlanNode::Logical { - operator: LogicalOperator::ExactSubquery { .. }, + operator: ResidualQueryOperator::ExactSubquery { .. }, .. } ) @@ -890,7 +890,7 @@ mod topk_tests { ( QueryNodeId(0), QueryPlanNode::Logical { - operator: LogicalOperator::ExactSubquery { + operator: ResidualQueryOperator::ExactSubquery { query: "m[1s]".into(), }, inputs: vec![], @@ -899,7 +899,7 @@ mod topk_tests { ( QueryNodeId(1), QueryPlanNode::Logical { - operator: LogicalOperator::Temporal { operation }, + operator: ResidualQueryOperator::Temporal { operation }, inputs: vec![QueryNodeId(0)], }, ), @@ -974,7 +974,7 @@ mod topk_tests { ( root, QueryPlanNode::Logical { - operator: LogicalOperator::TopKSelection { + operator: ResidualQueryOperator::TopKSelection { k: 2, grouping: Grouping { labels: vec![], diff --git a/data_plane/src/query_engines/asap_query_engine/mod.rs b/data_plane/src/query_engines/asap_query_engine/mod.rs index 1fd50b3bf..911869d1c 100644 --- a/data_plane/src/query_engines/asap_query_engine/mod.rs +++ b/data_plane/src/query_engines/asap_query_engine/mod.rs @@ -24,7 +24,7 @@ pub mod summary_executor; // engine still consumes it via that canonical path. pub use crate::storage_engines::sketch_db::query as asap_tier; +pub use engine::ASAPQueryEngine; + #[cfg(test)] pub mod tests; - -pub use engine::ASAPQueryEngine; diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index b91738c34..e8d0c4b35 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -860,7 +860,7 @@ mod tests { } use crate::storage_engines::sketch_db::data::{AggKind, SketchConfig}; use crate::storage_engines::sketch_db::index::{ - AccuracyBound, Capability, SketchAlgorithm, SketchInstanceMetadata, SketchSampleState, + AccuracyBound, Capability, SketchAlgorithm, SketchSampleState, SummarySeriesMetadata, }; fn accuracy() -> AccuracyTarget { @@ -877,7 +877,7 @@ mod tests { let cfg = SketchConfig::Hll { precision: 14 }; let mut group_by_keys = std::collections::BTreeSet::new(); group_by_keys.insert("service".to_string()); - idx.register(SketchInstanceMetadata { + idx.register(SummarySeriesMetadata { sid, metric_name: "unique_users".to_string(), group_by_keys, @@ -919,7 +919,7 @@ mod tests { let cfg = SketchConfig::DDSketch { relative_accuracy: 0.01, }; - idx.register(SketchInstanceMetadata { + idx.register(SummarySeriesMetadata { sid: 1, metric_name: "latency_ms".to_string(), group_by_keys: std::collections::BTreeSet::new(), @@ -1127,7 +1127,7 @@ mod tests { fn exact_agg_outcome_reports_window_end_coverage() { let idx = SketchStore::new(); idx.register( - crate::storage_engines::sketch_db::index::SketchInstanceMetadata { + crate::storage_engines::sketch_db::index::SummarySeriesMetadata { sid: 1, metric_name: "bytes_total".to_string(), group_by_keys: std::collections::BTreeSet::new(), @@ -1163,7 +1163,7 @@ mod tests { // Compile the two readouts, store one pane series, and execute the actual ratio. #[test] fn compiled_shared_sum_panes_preserve_each_lookback() { - use control_plane::physical::compiler::{BackendLocalPlanningSnapshot, PhysicalCompiler}; + use control_plane::physical::compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}; let mut snapshot: serde_json::Value = serde_json::from_str(include_str!( "../../../../docs/examples/asapquery-planning-snapshot.json" )) @@ -1172,14 +1172,14 @@ mod tests { entry["query"] = serde_json::json!("sum_over_time(a[1m]) / sum_over_time(a[10m])"); entry["requirements"]["accuracy"]["explicit"] = serde_json::json!("Exact"); entry["demand"]["fixed_interval_at"]["interval"] = serde_json::json!(60_000); - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_value(snapshot).unwrap(); - let (request, env) = snapshot.planning_request().unwrap(); - let plan = PhysicalCompiler.compile(request, env).unwrap(); + let snapshot: BackendLocalPlanningInput = serde_json::from_value(snapshot).unwrap(); + let (request, env) = snapshot.into_physical_compilation_request().unwrap(); + let plan = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); let config = &plan.precompute_plan.materializations[0]; let policy = config.policy_fingerprint(); let idx = SketchStore::new(); - idx.register(SketchInstanceMetadata { + idx.register(SummarySeriesMetadata { sid: 7, metric_name: "a".into(), group_by_keys: Default::default(), @@ -1253,7 +1253,7 @@ mod tests { fn repeated_multi_pane_reads_exclude_expired_state_and_reject_gaps() { let idx = SketchStore::new(); let policy = asap_types::PolicyFingerprint(777); - idx.register(SketchInstanceMetadata { + idx.register(SummarySeriesMetadata { sid: 7, metric_name: "requests_total".into(), group_by_keys: std::collections::BTreeSet::new(), @@ -1351,7 +1351,7 @@ mod tests { fn exact_query_plan_rate_uses_reset_aware_readout() { let idx = SketchStore::new(); let policy = asap_types::PolicyFingerprint(777); - idx.register(SketchInstanceMetadata { + idx.register(SummarySeriesMetadata { sid: 7, metric_name: "requests_total".into(), group_by_keys: std::collections::BTreeSet::new(), diff --git a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs index 0b3583e42..0757b8acb 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs @@ -1413,7 +1413,7 @@ mod tests { use super::*; use crate::query_engines::asap_query_engine::summary_exec::{execute, ExecOutcome}; use crate::storage_engines::sketch_db::index::{ - AccuracyBound, Capability, SketchInstanceMetadata, SketchSampleState, SketchStore, + AccuracyBound, Capability, SketchSampleState, SketchStore, SummarySeriesMetadata, }; use planner_types::post_asap::{SummaryField, SummarySchema}; use planner_types::pre_asap::{Column, DataType, Schema}; @@ -1557,9 +1557,9 @@ mod tests { }) } - fn kll_meta(sid: u64, metric: &str, group_by: &[&str]) -> SketchInstanceMetadata { + fn kll_meta(sid: u64, metric: &str, group_by: &[&str]) -> SummarySeriesMetadata { let cfg = SketchConfig::Kll { k: 200 }; - SketchInstanceMetadata { + SummarySeriesMetadata { sid, metric_name: metric.to_string(), group_by_keys: group_by @@ -1580,9 +1580,9 @@ mod tests { } } - fn hll_meta(sid: u64, metric: &str) -> SketchInstanceMetadata { + fn hll_meta(sid: u64, metric: &str) -> SummarySeriesMetadata { let cfg = SketchConfig::Hll { precision: 10 }; - SketchInstanceMetadata { + SummarySeriesMetadata { sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), @@ -1626,9 +1626,9 @@ mod tests { sk.to_msgpack().expect("encode HLL msgpack") } - fn cms_meta(sid: u64, metric: &str) -> SketchInstanceMetadata { + fn cms_meta(sid: u64, metric: &str) -> SummarySeriesMetadata { let cfg = SketchConfig::CountMin { rows: 4, cols: 256 }; - SketchInstanceMetadata { + SummarySeriesMetadata { sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), @@ -1699,9 +1699,9 @@ mod tests { }) } - fn cms_with_heap_meta(sid: u64, metric: &str) -> SketchInstanceMetadata { + fn cms_with_heap_meta(sid: u64, metric: &str) -> SummarySeriesMetadata { let cfg = SketchConfig::CountMin { rows: 4, cols: 256 }; - SketchInstanceMetadata { + SummarySeriesMetadata { sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), @@ -1770,8 +1770,8 @@ mod tests { // ── ExactAgg (Sum) fixtures ──────────────────────────────────────── - fn sum_exact_agg_meta(sid: u64, metric: &str, group_by: &[&str]) -> SketchInstanceMetadata { - SketchInstanceMetadata { + fn sum_exact_agg_meta(sid: u64, metric: &str, group_by: &[&str]) -> SummarySeriesMetadata { + SummarySeriesMetadata { sid, metric_name: metric.to_string(), group_by_keys: group_by diff --git a/data_plane/src/query_engines/routing/backend_storage_routing.rs b/data_plane/src/query_engines/routing/backend_storage_routing.rs index 100c25216..d42970e78 100644 --- a/data_plane/src/query_engines/routing/backend_storage_routing.rs +++ b/data_plane/src/query_engines/routing/backend_storage_routing.rs @@ -899,7 +899,7 @@ pub fn routing_table_hash(table: &BackendStorageRouting) -> String { // --------------------------------------------------------------------------- /// Per-tenant atomic-swap wrapper around `BackendStorageRouting`, -/// mirroring [`crate::storage_engines::types::HotReloadStreamingConfig`]. Lets the +/// mirroring [`crate::storage_engines::types::StreamingConfigHandle`]. Lets the /// `POST /api/v1/storage_routing` HTTP handler swap one tenant's table /// at runtime without restarting the backend or touching any other /// tenant's table. Cloneable; clones share the underlying `ArcSwap` so @@ -948,7 +948,7 @@ pub struct HotReloadBackendStorageRouting { /// once and pick the tenant's `Arc`. inner: std::sync::Arc>>>, - active: Option, + active: Option, } impl HotReloadBackendStorageRouting { @@ -989,8 +989,8 @@ impl HotReloadBackendStorageRouting { } } - pub fn from_active(active: crate::storage_engines::types::HotReloadActivePhysicalPlan) -> Self { - let initial = active.snapshot().storage_routing.clone(); + pub fn from_active(active: crate::storage_engines::types::ActivePhysicalPlanHandle) -> Self { + let initial = active.active_snapshot().storage_routing.clone(); let mut map = HashMap::new(); map.insert(initial.tenant().to_string(), initial); Self { @@ -1016,7 +1016,7 @@ impl HotReloadBackendStorageRouting { /// caller's lifetime; concurrent swaps don't invalidate it. pub fn snapshot_for_tenant(&self, tenant: &str) -> std::sync::Arc { if let Some(active) = &self.active { - let routing = active.snapshot().storage_routing.clone(); + let routing = active.active_snapshot().storage_routing.clone(); if routing.tenant() == tenant || tenant == DEFAULT_TENANT { return routing; } diff --git a/data_plane/src/storage_engines/mod.rs b/data_plane/src/storage_engines/mod.rs index 9a0ffcb72..72e910c8d 100644 --- a/data_plane/src/storage_engines/mod.rs +++ b/data_plane/src/storage_engines/mod.rs @@ -24,7 +24,7 @@ pub mod types; pub use sketch_db::index::{ AccuracyBound, Capability, SeriesLookup, SketchAlgorithm, SketchConfig, SketchEncoding, - SketchInstanceMetadata, SketchSampleState, SketchStore, SketchTimeSeries, + SketchSampleState, SketchStore, SketchTimeSeries, SummarySeriesMetadata, }; pub use sketch_db::AggStatus; pub use traits::*; diff --git a/data_plane/src/storage_engines/sketch_db/backfill/mod.rs b/data_plane/src/storage_engines/sketch_db/backfill/mod.rs index d8d15b012..90b5f1e29 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/mod.rs @@ -905,6 +905,29 @@ fn now_ms() -> u64 { .unwrap_or(0) } +// 2026-05 reorg: backfill-* and the two BackfillSource impls moved +// into this folder as submodules. +pub mod clickhouse_reader; +pub mod processor; +pub mod prometheus_reader; +pub mod raw_sample_reader; +pub mod service; +pub mod window_builder; +pub mod worker; + +pub use clickhouse_reader::{clickhouse_reader_factory, ClickHouseReader, ClickHouseReaderConfig}; +pub use processor::BackfillWindowProcessor; +pub use prometheus_reader::PrometheusReader; +pub use raw_sample_reader::{ + LabelFilter, MockRawSampleReader, RawSample, RawSampleReader, RawSampleReaderError, +}; +pub use service::{ + default_reader_factory, noop_reader_factory, BackfillService, BackfillServiceConfig, + BackfillServiceHandle, ReaderFactory, +}; +pub use window_builder::build_backfilled_accumulator; +pub use worker::{BackfillWorker, BackfillWorkerError, WindowProcessor}; + #[cfg(test)] mod tests { use super::*; @@ -1447,26 +1470,3 @@ mod tests { assert_eq!(r.coverage(1, (50, 10)), Coverage::Complete); } } - -// 2026-05 reorg: backfill-* and the two BackfillSource impls moved -// into this folder as submodules. -pub mod clickhouse_reader; -pub mod processor; -pub mod prometheus_reader; -pub mod raw_sample_reader; -pub mod service; -pub mod window_builder; -pub mod worker; - -pub use clickhouse_reader::{clickhouse_reader_factory, ClickHouseReader, ClickHouseReaderConfig}; -pub use processor::BackfillWindowProcessor; -pub use prometheus_reader::PrometheusReader; -pub use raw_sample_reader::{ - LabelFilter, MockRawSampleReader, RawSample, RawSampleReader, RawSampleReaderError, -}; -pub use service::{ - default_reader_factory, noop_reader_factory, BackfillService, BackfillServiceConfig, - BackfillServiceHandle, ReaderFactory, -}; -pub use window_builder::build_backfilled_accumulator; -pub use worker::{BackfillWorker, BackfillWorkerError, WindowProcessor}; diff --git a/data_plane/src/storage_engines/sketch_db/backfill/processor.rs b/data_plane/src/storage_engines/sketch_db/backfill/processor.rs index b791bf827..7b6fee4d5 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/processor.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/processor.rs @@ -72,7 +72,7 @@ use tracing::debug; use crate::drivers::ingest::population_attrs_fingerprint; use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::precompute_engine::worker::parse_labels_from_series_key; -use crate::storage_engines::types::{AggregateCore, HotReloadStreamingConfig, KeyByLabelValues}; +use crate::storage_engines::types::{AggregateCore, KeyByLabelValues, StreamingConfigHandle}; use asap_types::aggregation_config::AggregationConfig; use asap_types::PolicyFingerprint; @@ -177,13 +177,13 @@ pub struct BackfillWindowProcessor { /// `StreamingConfig` at each window to find the /// `AggregationConfig` for `agg_id`. The snapshot is cheap /// (Arc refcount bump) so we don't optimise further. - config: HotReloadStreamingConfig, + config: StreamingConfigHandle, /// Phase 5 M2.3.6g — replayed batches land here. The legacy /// `Arc` field is gone; SketchStore is the only /// destination. Optional so tests that don't observe write /// effects can skip attaching one (the processor becomes a /// registry-only logger in that case). - sketch_index: Option>, + summary_store: Option>, /// Shared sid mint authority. Same `SeriesIdResolver` the OTel /// ingest path uses, so backfilled precompute sids land in the /// same unified namespace as live precompute / sketch sids. Only @@ -202,13 +202,13 @@ pub struct BackfillWindowProcessor { impl BackfillWindowProcessor { pub fn new( - config: HotReloadStreamingConfig, + config: StreamingConfigHandle, registry: Arc, job_id: u64, ) -> Self { Self { config, - sketch_index: None, + summary_store: None, series_resolver: None, registry, job_id, @@ -220,10 +220,10 @@ impl BackfillWindowProcessor { /// so existing call sites opt in with one chained call. pub fn with_sketch_index( mut self, - sketch_index: Arc, + summary_store: Arc, ) -> Self { - self.catalog_generation = sketch_index.active_catalog_generation(); - self.sketch_index = Some(sketch_index); + self.catalog_generation = summary_store.active_catalog_generation(); + self.summary_store = Some(summary_store); self } @@ -303,7 +303,7 @@ impl WindowProcessor for BackfillWindowProcessor { r.as_ref(), &config, &sample.labels, - self.sketch_index.as_deref(), + self.summary_store.as_deref(), self.catalog_generation.as_deref(), )?, None => fallback_bucket_id(&group_key), @@ -373,7 +373,7 @@ impl WindowProcessor for BackfillWindowProcessor { // provenance below. When sketch_index is attached but no // resolver was provided, the precompute write is skipped // with a warn — sid minting requires the shared resolver. - if let Some(idx) = self.sketch_index.as_ref() { + if let Some(idx) = self.summary_store.as_ref() { match self.series_resolver.as_ref() { Some(_resolver) => { // B7.7 — sid is pre-resolved per bucket above; hand @@ -471,7 +471,7 @@ mod tests { let cfg = sum_config(1, "latency", vec!["svc"]); let fp = cfg.policy_fp_u64(); let streaming = streaming_config_with(cfg.clone()); - let hot = HotReloadStreamingConfig::from_arc(streaming.clone()); + let hot = StreamingConfigHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); let job_id = registry.create( fp, @@ -515,7 +515,7 @@ mod tests { async fn unknown_agg_id_fails_cleanly() { let cfg = sum_config(1, "m", vec![]); let streaming = streaming_config_with(cfg); - let hot = HotReloadStreamingConfig::from_arc(streaming.clone()); + let hot = StreamingConfigHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); let job_id = registry.create( 999, @@ -538,7 +538,7 @@ mod tests { let cfg = sum_config(1, "m", vec![]); let fp = cfg.policy_fp_u64(); let streaming = streaming_config_with(cfg); - let hot = HotReloadStreamingConfig::from_arc(streaming.clone()); + let hot = StreamingConfigHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); let job_id = registry.create( fp, @@ -559,7 +559,7 @@ mod tests { let cfg = sum_config(1, "latency", vec!["svc"]); let fp = cfg.policy_fp_u64(); let streaming = streaming_config_with(cfg); - let hot = HotReloadStreamingConfig::from_arc(streaming.clone()); + let hot = StreamingConfigHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); let job_id = registry.create( fp, @@ -851,7 +851,7 @@ mod tests { /// /// Drives `process_window` end-to-end with samples spanning two /// distinct `svc` values × two samples each. Asserts: - /// - exactly two `SketchInstanceMetadata` entries land in the + /// - exactly two `SummarySeriesMetadata` entries land in the /// `SketchStore` (one per distinct sid bucket) /// - their sids equal what the shared `SeriesIdResolver` would /// mint for the same `(metric, grouping-values, agg_kind)` @@ -868,16 +868,16 @@ mod tests { let cfg = sum_config(1, "latency", vec!["svc"]); let fp = cfg.policy_fp_u64(); let streaming = streaming_config_with(cfg.clone()); - let hot = HotReloadStreamingConfig::from_arc(streaming.clone()); + let hot = StreamingConfigHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); - let sketch_index = Arc::new(SketchStore::new()); + let summary_store = Arc::new(SketchStore::new()); let catalog = asap_types::summary_catalog::SummaryCatalog::from_materializations( 1, 1, &[cfg.clone()], ) .unwrap(); - sketch_index + summary_store .install_summary_catalog(Arc::new(catalog)) .unwrap(); let resolver = Arc::new(SeriesIdResolver::new()); @@ -890,7 +890,7 @@ mod tests { ); let processor = BackfillWindowProcessor::new(hot, registry.clone(), job_id) - .with_sketch_index(sketch_index.clone()) + .with_sketch_index(summary_store.clone()) .with_series_resolver(resolver.clone()); // Two distinct svc values × two samples each. Same window @@ -925,7 +925,7 @@ mod tests { // Two distinct sids landed in the index. assert_eq!( - sketch_index.instance_count(), + summary_store.instance_count(), 2, "one sid per distinct svc bucket" ); @@ -939,8 +939,8 @@ mod tests { let sid_b = resolve_backfill_bucket_sid(&resolver, &cfg, "latency{svc=\"b\"}", None, None).unwrap(); assert_ne!(sid_a, sid_b, "distinct svc values mint distinct sids"); - assert_eq!(sketch_index.classify(sid_a), SeriesLookup::Hit); - assert_eq!(sketch_index.classify(sid_b), SeriesLookup::Hit); + assert_eq!(summary_store.classify(sid_a), SeriesLookup::Hit); + assert_eq!(summary_store.classify(sid_b), SeriesLookup::Hit); // Provenance was recorded once per window (not once per // bucket) — same shape as the pre-rekey path. diff --git a/data_plane/src/storage_engines/sketch_db/backfill/prometheus_reader.rs b/data_plane/src/storage_engines/sketch_db/backfill/prometheus_reader.rs index aa5ab6524..c53481eed 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/prometheus_reader.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/prometheus_reader.rs @@ -334,79 +334,6 @@ impl<'de> Deserialize<'de> for MatrixPoint { } } -#[cfg(test)] -mod tests { - use super::*; - use std::collections::HashMap; - - #[test] - fn build_promql_bare_metric_when_no_filters() { - let f = LabelFilter::for_metric("latency"); - assert_eq!(build_promql(&f), "latency"); - } - - #[test] - fn build_promql_stable_ordering() { - let f = LabelFilter::for_metric("m") - .with_label("b", "2") - .with_label("a", "1"); - // Sorted by key regardless of insertion order. - assert_eq!(build_promql(&f), r#"m{a="1",b="2"}"#); - } - - #[test] - fn build_promql_escapes_quotes_and_backslashes() { - let f = LabelFilter::for_metric("m").with_label("k", r#"val"with\slash"#); - assert_eq!(build_promql(&f), r#"m{k="val\"with\\slash"}"#); - } - - #[test] - fn render_series_key_uses_name_and_sorts_labels() { - let mut metric = HashMap::new(); - metric.insert("__name__".to_string(), "latency".to_string()); - metric.insert("svc".to_string(), "a".to_string()); - metric.insert("env".to_string(), "prod".to_string()); - assert_eq!( - render_series_key("fallback", &metric), - r#"latency{env="prod",svc="a"}"# - ); - } - - #[test] - fn render_series_key_handles_bare_metric() { - let mut metric = HashMap::new(); - metric.insert("__name__".to_string(), "lone".to_string()); - assert_eq!(render_series_key("fallback", &metric), "lone"); - } - - #[test] - fn render_series_key_falls_back_to_default_when_no_name() { - let metric: HashMap = HashMap::new(); - assert_eq!(render_series_key("fallback", &metric), "fallback"); - } - - #[test] - fn fractional_seconds_round_trip() { - assert_eq!(ms_to_fractional_seconds(0), "0.000"); - assert_eq!(ms_to_fractional_seconds(1_234_567), "1234.567"); - assert_eq!(fractional_seconds_to_ms(1.5), 1500); - assert_eq!(fractional_seconds_to_ms(0.0), 0); - } - - #[tokio::test] - async fn read_samples_inverted_range_returns_invalid_range() { - let r = PrometheusReader::new("http://unused"); - let err = r - .read_samples(100, 50, &LabelFilter::for_metric("m")) - .await - .unwrap_err(); - match err { - RawSampleReaderError::InvalidRange { .. } => {} - other => panic!("expected InvalidRange, got {other}"), - } - } -} - // ── Integration tests against a mock Prometheus HTTP server ───────── #[cfg(test)] @@ -688,3 +615,76 @@ mod integration_tests { handle.abort(); } } + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + #[test] + fn build_promql_bare_metric_when_no_filters() { + let f = LabelFilter::for_metric("latency"); + assert_eq!(build_promql(&f), "latency"); + } + + #[test] + fn build_promql_stable_ordering() { + let f = LabelFilter::for_metric("m") + .with_label("b", "2") + .with_label("a", "1"); + // Sorted by key regardless of insertion order. + assert_eq!(build_promql(&f), r#"m{a="1",b="2"}"#); + } + + #[test] + fn build_promql_escapes_quotes_and_backslashes() { + let f = LabelFilter::for_metric("m").with_label("k", r#"val"with\slash"#); + assert_eq!(build_promql(&f), r#"m{k="val\"with\\slash"}"#); + } + + #[test] + fn render_series_key_uses_name_and_sorts_labels() { + let mut metric = HashMap::new(); + metric.insert("__name__".to_string(), "latency".to_string()); + metric.insert("svc".to_string(), "a".to_string()); + metric.insert("env".to_string(), "prod".to_string()); + assert_eq!( + render_series_key("fallback", &metric), + r#"latency{env="prod",svc="a"}"# + ); + } + + #[test] + fn render_series_key_handles_bare_metric() { + let mut metric = HashMap::new(); + metric.insert("__name__".to_string(), "lone".to_string()); + assert_eq!(render_series_key("fallback", &metric), "lone"); + } + + #[test] + fn render_series_key_falls_back_to_default_when_no_name() { + let metric: HashMap = HashMap::new(); + assert_eq!(render_series_key("fallback", &metric), "fallback"); + } + + #[test] + fn fractional_seconds_round_trip() { + assert_eq!(ms_to_fractional_seconds(0), "0.000"); + assert_eq!(ms_to_fractional_seconds(1_234_567), "1234.567"); + assert_eq!(fractional_seconds_to_ms(1.5), 1500); + assert_eq!(fractional_seconds_to_ms(0.0), 0); + } + + #[tokio::test] + async fn read_samples_inverted_range_returns_invalid_range() { + let r = PrometheusReader::new("http://unused"); + let err = r + .read_samples(100, 50, &LabelFilter::for_metric("m")) + .await + .unwrap_err(); + match err { + RawSampleReaderError::InvalidRange { .. } => {} + other => panic!("expected InvalidRange, got {other}"), + } + } +} diff --git a/data_plane/src/storage_engines/sketch_db/backfill/service.rs b/data_plane/src/storage_engines/sketch_db/backfill/service.rs index 3b10f79b8..c3b45704b 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/service.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/service.rs @@ -58,7 +58,7 @@ use crate::storage_engines::sketch_db::backfill::worker::BackfillWorker; use crate::storage_engines::sketch_db::backfill::{ BackfillRegistry, BackfillSource, BackfillStatus, }; -use crate::storage_engines::types::HotReloadStreamingConfig; +use crate::storage_engines::types::StreamingConfigHandle; /// Given a `BackfillSource`, return a reader that can read raw /// samples from it. Used by the service to pick a concrete reader @@ -100,11 +100,11 @@ pub struct BackfillService { registry: Arc, /// Phase 5 M2.3.6g — replayed batches land in `SketchStore` only; /// the legacy `Arc` field is gone. - sketch_index: Option>, + summary_store: Option>, /// Shared sid mint authority — wired alongside `sketch_index` so /// backfilled precompute sids share the namespace with live ingest. series_resolver: Option>, - config_source: HotReloadStreamingConfig, + config_source: StreamingConfigHandle, reader_factory: ReaderFactory, service_config: BackfillServiceConfig, } @@ -112,13 +112,13 @@ pub struct BackfillService { impl BackfillService { pub fn new( registry: Arc, - config_source: HotReloadStreamingConfig, + config_source: StreamingConfigHandle, reader_factory: ReaderFactory, service_config: BackfillServiceConfig, ) -> Self { Self { registry, - sketch_index: None, + summary_store: None, series_resolver: None, config_source, reader_factory, @@ -130,9 +130,9 @@ impl BackfillService { /// there. Builder-style; safe to omit (legacy tests). pub fn with_sketch_index( mut self, - sketch_index: Arc, + summary_store: Arc, ) -> Self { - self.sketch_index = Some(sketch_index); + self.summary_store = Some(summary_store); self } @@ -246,7 +246,7 @@ impl BackfillService { self.registry.clone(), job.job_id, ); - if let Some(idx) = self.sketch_index.as_ref() { + if let Some(idx) = self.summary_store.as_ref() { processor = processor.with_sketch_index(idx.clone()); } if let Some(resolver) = self.series_resolver.as_ref() { @@ -413,7 +413,7 @@ mod tests { let mut cfg = sum_config(1, "latency"); cfg.table_name = Some("expected_table".into()); let agg_fp = cfg.policy_fp_u64(); - let hot = HotReloadStreamingConfig::from_arc(streaming_with(cfg)); + let hot = StreamingConfigHandle::from_arc(streaming_with(cfg)); let registry = Arc::new(BackfillRegistry::new()); let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); let called = calls.clone(); @@ -449,7 +449,7 @@ mod tests { let cfg = sum_config(1, "latency"); let agg_fp = cfg.policy_fp_u64(); let streaming = streaming_with(cfg); - let hot = HotReloadStreamingConfig::from_arc(streaming.clone()); + let hot = StreamingConfigHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); // Factory returns a fresh mock reader per call — seeded with a @@ -496,7 +496,7 @@ mod tests { let cfg = sum_config(1, "latency"); let agg_fp = cfg.policy_fp_u64(); let streaming = streaming_with(cfg); - let hot = HotReloadStreamingConfig::from_arc(streaming.clone()); + let hot = StreamingConfigHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); let service = BackfillService::new( @@ -530,7 +530,7 @@ mod tests { let cfg = sum_config(1, "latency"); let agg_fp = cfg.policy_fp_u64(); let streaming = streaming_with(cfg); - let hot = HotReloadStreamingConfig::from_arc(streaming.clone()); + let hot = StreamingConfigHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); // Factory records the order in which it's invoked. @@ -592,7 +592,7 @@ mod tests { async fn service_shutdown_stops_the_loop() { let cfg = sum_config(1, "m"); let streaming = streaming_with(cfg); - let hot = HotReloadStreamingConfig::from_arc(streaming.clone()); + let hot = StreamingConfigHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); let service = BackfillService::new( diff --git a/data_plane/src/storage_engines/sketch_db/data/mod.rs b/data_plane/src/storage_engines/sketch_db/data/mod.rs index d5c423311..9454c624d 100644 --- a/data_plane/src/storage_engines/sketch_db/data/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/data/mod.rs @@ -67,7 +67,7 @@ pub use asap_types::AggregationType; /// Sketch-instance configuration carried per-Metric on the OTLP wire /// (Phase 2 lifted these from per-DP up to the parent sketch container). /// Backend reads the relevant variant at ingest time and stores it in -/// `SketchInstanceMetadata.agg_kind`. +/// `SummarySeriesMetadata.agg_kind`. #[derive(Debug, Clone)] pub enum SketchConfig { UnivMon { diff --git a/data_plane/src/storage_engines/sketch_db/index/epoch_columnar.rs b/data_plane/src/storage_engines/sketch_db/index/epoch_columnar.rs index 695c75052..86c6ff39e 100644 --- a/data_plane/src/storage_engines/sketch_db/index/epoch_columnar.rs +++ b/data_plane/src/storage_engines/sketch_db/index/epoch_columnar.rs @@ -842,7 +842,7 @@ pub struct SidStoreData { /// `0` means "never written" / freshly (re)hydrated. Drives idle-sid /// eviction: a sid with no writes for the idle threshold whose state /// is fully durable on disk can have this whole `SidStoreData` dropped - /// from memory while its queryable `SketchInstanceMetadata` is kept + /// from memory while its queryable `SummarySeriesMetadata` is kept /// (the series stays answerable from the disk tier and rehydrates on /// the next write). Updated under the per-sid write lock the append /// path already holds, so it costs nothing extra on the hot path. diff --git a/data_plane/src/storage_engines/sketch_db/index/maintenance.rs b/data_plane/src/storage_engines/sketch_db/index/maintenance.rs index 20cedea52..65c5907bc 100644 --- a/data_plane/src/storage_engines/sketch_db/index/maintenance.rs +++ b/data_plane/src/storage_engines/sketch_db/index/maintenance.rs @@ -737,13 +737,13 @@ mod tests { fn complete_population_keeps_every_sid_and_rejects_missing_live_binding() { // Multiple SIDs are inventory entries, never an implicit singleton; // removing a live binding cannot hide its retained durable population. - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let mut first = plan.precompute_plan.materializations[0].clone(); first.aggregation_type = asap_types::AggregationType::Sum; @@ -848,13 +848,13 @@ mod tests { fn cohort_requires_every_durable_source_in_one_catalog_generation() { // Neither a missing second window nor a new catalog may yield a // partially acquired cohort, even when the first source is complete. - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let mut first = plan.precompute_plan.materializations[0].clone(); first.aggregation_type = asap_types::AggregationType::Sum; @@ -1084,10 +1084,10 @@ mod tests { entry["demand"]["fixed_interval_at"]["evaluation_phase"] = 0.into(); entry["time_selection"]["lookback"] = 60_000.into(); fixture["query_workload"]["repeating_queries"] = serde_json::json!([entry]); - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let source = plan .precompute_plan @@ -1400,13 +1400,13 @@ mod tests { fn committed_recovery_restores_live_seal_and_rejects_additive_derived_writes() { // A durable sidecar may survive an I/O error before the caller updates // its live seal. Recovery must repair admission, not only return a hit. - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../../../docs/examples/asapquery-planning-snapshot.json" )) .unwrap(); let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let mut source_config = plan.precompute_plan.materializations[0].clone(); source_config.aggregation_type = asap_types::AggregationType::Sum; diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index 80114880d..d63d0ec2e 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -1,7 +1,7 @@ //! Sketch index — Phase 5 of the controller-into-backend refactor (2026-05). //! //! Two-level index: -//! - `instances`: sid → SketchInstanceMetadata (one entry per logical +//! - `instances`: sid → SummarySeriesMetadata (one entry per logical //! sketch instance — its metric name, group-by KEY set, capability, //! sketch_type, sketch_config, accuracy bound). //! - `series`: sid → per-sid storage (`SidStoreData`) carrying the @@ -206,7 +206,7 @@ fn build_attrs_fp_and_label_map( /// `SketchStore::is_writable(sid)`; until then both registries run /// side by side. #[derive(Debug, Clone)] -pub struct SketchInstanceMetadata { +pub struct SummarySeriesMetadata { pub sid: u64, pub metric_name: String, /// The group-by KEY set — `dp.attributes.keys()` after the agent's @@ -250,7 +250,7 @@ pub struct SketchInstanceMetadata { pub policy_fp: PolicyFingerprint, } -impl SketchInstanceMetadata { +impl SummarySeriesMetadata { /// Compute the current `AggStatus` against the wall clock. /// Mirrors `AggSchema::status` — purely a function of timestamps. pub fn status(&self) -> AggStatus { @@ -653,7 +653,7 @@ pub struct SketchStore { /// per-attribute-set CMS (only the bucket total is meaningful). /// Kept as a decoupled side-table so recording item_label does not /// change sid identity (`AggKind` canonical string) or churn the many - /// `SketchInstanceMetadata` / `AggKind::Sketch` literals. + /// `SummarySeriesMetadata` / `AggKind::Sketch` literals. item_labels: RwLock>, /// sid → per-sid columnar storage. Empty `SidStoreData` (or absent /// key) for ghost sids — query path detects this and falls through @@ -843,14 +843,14 @@ impl SketchStore { /// absent from `policy_to_series_ids` / `metric_to_series_ids` (the pre-fix race /// where the two indexes were written under separate sequential /// locks). See the index-field doc comments for the full invariant. - pub fn register(&self, meta: SketchInstanceMetadata) { + pub fn register(&self, meta: SummarySeriesMetadata) { let mut instances = self.instances.write().unwrap(); self.register_with_instances(meta, &mut instances); } fn register_with_instances( &self, - meta: SketchInstanceMetadata, + meta: SummarySeriesMetadata, instances: &mut HashMap, ) -> bool { let sid = meta.sid; @@ -1175,7 +1175,7 @@ impl SketchStore { /// Look up the metadata for a sid (cloned because callers usually /// release the index lock before working with it). - pub fn instance(&self, sid: u64) -> Option> { + pub fn instance(&self, sid: u64) -> Option> { self.instances .read() .unwrap() @@ -1387,7 +1387,7 @@ impl SketchStore { /// (which would deadlock) and should stay allocation-light — extract /// the small data you need (a `Capability` clone, a `bool`) and act /// after this returns. - pub fn with_instance R>( + pub fn with_instance R>( &self, sid: u64, f: F, @@ -1397,7 +1397,7 @@ impl SketchStore { } /// Append a window's sketch state under `sid`. Caller is responsible - /// for ensuring the corresponding `SketchInstanceMetadata` was + /// for ensuring the corresponding `SummarySeriesMetadata` was /// registered (or the sketch arrives orphan and the caller chooses /// to drop / reject / register-on-the-fly). /// @@ -2473,7 +2473,7 @@ impl SketchStore { /// `SidStoreData` (epoch columns + intern-table label cache + the /// `series` slot) for every sketch sid that has gone write-idle past /// `idle_threshold_ms` AND whose state is fully durable on disk, while - /// KEEPING its [`SketchInstanceMetadata`] in `instances`. + /// KEEPING its [`SummarySeriesMetadata`] in `instances`. /// /// Why keep the metadata: the query path's disk union /// ([`Self::query_range`] → `union_disk_parts_into`) needs @@ -2556,7 +2556,7 @@ impl SketchStore { // 1. SeriesId bindings, compatibility metadata, and shared SDS descriptors. if let Ok(insts) = self.instances.read() { for m in insts.values() { - total += std::mem::size_of::(); + total += std::mem::size_of::(); total += m.metric_name.len(); for k in &m.group_by_keys { total += k.len() + std::mem::size_of::(); @@ -2596,7 +2596,7 @@ impl SketchStore { /// Snapshot shared metadata handles without holding the registry lock /// across user code. This is O(N) pointer cloning and does not copy /// descriptor strings, label sets, or aggregation configuration. - pub fn snapshot_instances(&self) -> Vec> { + pub fn snapshot_instances(&self) -> Vec> { match self.instances.read() { Ok(map) => map .values() @@ -2654,7 +2654,7 @@ impl SketchStore { /// allocation-light. Callers that need to mutate or call user code /// should collect the cheap data they need (e.g. `Vec` of /// sids) here, then act after this returns. - pub fn for_each_instance(&self, mut f: F) { + pub fn for_each_instance(&self, mut f: F) { if let Ok(map) = self.instances.read() { for (sid, meta) in map.iter() { f(*sid, meta); @@ -2665,7 +2665,7 @@ impl SketchStore { /// Iterate (clones) all instance metadata matching `status`. /// Used by the eviction service to enumerate `Expired` sids /// without holding a long read lock. - pub fn list_by_status(&self, status: AggStatus) -> Vec> { + pub fn list_by_status(&self, status: AggStatus) -> Vec> { let map = match self.instances.read() { Ok(m) => m, Err(_) => return Vec::new(), @@ -2736,7 +2736,7 @@ impl SketchStore { &self, sid: u64, retention: Duration, - ) -> Option> { + ) -> Option> { let _mutation = self.begin_state_mutation(); let mut map = self.instances.write().ok()?; let instance = map.get_mut(&sid)?; @@ -2755,7 +2755,7 @@ impl SketchStore { /// state, or `None` if the sid is unknown. Intended for /// operator / debug-endpoint use so eviction can be observed in /// e2e tests without waiting out retirement retention. - pub fn force_expire(&self, sid: u64) -> Option> { + pub fn force_expire(&self, sid: u64) -> Option> { let _mutation = self.begin_state_mutation(); let mut map = self.instances.write().ok()?; let instance = map.get_mut(&sid)?; @@ -2864,7 +2864,7 @@ impl SketchStore { /// `series` DashMap is touched after the index guards are released /// (it is independently keyed and not part of the metadata-index /// invariant). - pub fn remove_instance(&self, sid: u64) -> Option> { + pub fn remove_instance(&self, sid: u64) -> Option> { let _mutation = self.begin_state_mutation(); let removed = { // Fixed lock order: instances → policy_to_series_ids → metric_to_series_ids. @@ -3056,7 +3056,7 @@ impl SketchStore { // state was reachable only through the legacy precompute // query path; capability-matching couldn't see it. if !self.register_with_instances( - SketchInstanceMetadata { + SummarySeriesMetadata { sid, metric_name: agg_cfg.metric.clone(), group_by_keys, @@ -3447,7 +3447,7 @@ impl SketchStore { }; let capability = rec.capability(); let accuracy = rec.accuracy(); - self.register(SketchInstanceMetadata { + self.register(SummarySeriesMetadata { sid: rec.sid, metric_name: rec.metric_name, group_by_keys: rec.group_by_keys.into_iter().collect(), @@ -3680,6 +3680,22 @@ impl crate::storage_engines::sketch_db::index::persistence::EpochSource for Sket } } +// Compatibility imports; new callers use the domain names above. +#[deprecated(note = "Use SummarySeriesMetadata")] +pub use SummarySeriesMetadata as SketchInstanceMetadata; + +// 2026-05 reorg: generic epoch-partitioned columnar storage lives +// alongside the store that uses it. +mod admission; +mod maintenance; +pub(crate) use maintenance::{CompleteRawMaintenanceCohort, FrozenExactWindows}; +pub mod epoch_columnar; + +// `persistence` moved up to `sketch_db::persistence`. Re-exported here +// so legacy `crate::storage_engines::sketch_db::index::persistence::*` +// paths continue working without consumer changes. +pub use crate::storage_engines::sketch_db::persistence; + #[cfg(test)] mod tests { use super::*; @@ -3707,18 +3723,18 @@ mod tests { assert_eq!(gapped.query(0, 90_000), None); } - fn meta(sid: u64) -> SketchInstanceMetadata { + fn meta(sid: u64) -> SummarySeriesMetadata { meta_with_policy(sid, asap_types::PolicyFingerprint::UNSET) } fn meta_with_policy( sid: u64, policy_fp: asap_types::PolicyFingerprint, - ) -> SketchInstanceMetadata { + ) -> SummarySeriesMetadata { let cfg = SketchConfig::DDSketch { relative_accuracy: 0.01, }; - SketchInstanceMetadata { + SummarySeriesMetadata { sid, metric_name: "m".into(), group_by_keys: BTreeSet::new(), @@ -3787,13 +3803,13 @@ mod tests { #[test] fn observed_inventory_uses_installed_catalog_and_real_store_entries() { - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" )) .unwrap(); let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); let store = SketchStore::new(); @@ -3862,13 +3878,13 @@ mod tests { #[test] fn registered_series_without_payload_is_not_a_summary_instance() { - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" )) .unwrap(); let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); let store = SketchStore::new(); @@ -4647,7 +4663,7 @@ mod tests { /// Metadata with a chosen metric name + group-by key set, so the /// secondary-index tests can register several metrics/keys. - fn meta_metric_keys(sid: u64, metric: &str, keys: &[&str]) -> SketchInstanceMetadata { + fn meta_metric_keys(sid: u64, metric: &str, keys: &[&str]) -> SummarySeriesMetadata { let mut m = meta(sid); m.metric_name = metric.to_string(); m.group_by_keys = keys.iter().map(|k| k.to_string()).collect(); @@ -4766,7 +4782,7 @@ mod tests { /// Metadata with a single group-by key `host`, so the disk read-back /// path can rebuild the `{host: }` label map from the stored /// values vector. - fn meta_with_host_key(sid: u64) -> SketchInstanceMetadata { + fn meta_with_host_key(sid: u64) -> SummarySeriesMetadata { let mut m = meta(sid); m.group_by_keys = ["host".to_string()].into_iter().collect(); m @@ -5022,13 +5038,13 @@ mod tests { use crate::storage_engines::sketch_db::index::persistence::metadata::{ SidMetaRecord, SidMetadataStore, }; - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" )) .unwrap(); let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); let metadata = meta_with_policy(507, fingerprint); @@ -5065,13 +5081,13 @@ mod tests { #[test] fn catalog_reactivation_uses_new_physical_series_without_old_disk_payload() { use crate::drivers::ingest::series_resolver::SeriesIdResolver; - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" )) .unwrap(); let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); let definition = fingerprint.into(); @@ -5166,13 +5182,13 @@ mod tests { fn completed_windows_reject_late_updates_after_restart() { // Completion is a storage admission rule, including legacy producers, // and survives restart without allowing a correction into consumed state. - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" )) .unwrap(); let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); let directory = tempfile::tempdir().unwrap(); @@ -5254,13 +5270,13 @@ mod tests { fn finite_completion_flushes_payload_before_persisting_immutability() { // With neither memory pressure nor a hot-tier deadline, completion must // explicitly flush its payload before persisting a non-replayable window. - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" )) .unwrap(); let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); let directory = tempfile::tempdir().unwrap(); @@ -5364,13 +5380,13 @@ mod tests { #[test] fn durable_lifecycle_is_not_resurrected_by_restart_or_a_stale_flush() { - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" )) .unwrap(); let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); let directory = tempfile::tempdir().unwrap(); @@ -5448,13 +5464,13 @@ mod tests { #[test] fn observed_inventory_includes_durable_instances_after_restart() { - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" )) .unwrap(); let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) - .compile() + .compile_promql() .unwrap(); let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); let definition_id = SummaryDefinitionId::from(fingerprint); @@ -5528,9 +5544,9 @@ mod tests { // they FAIL ("No result"); with the metadata-sidecar fix they pass // because recovery re-registers the disk-resident sids. - fn meta_kll_host(sid: u64) -> SketchInstanceMetadata { + fn meta_kll_host(sid: u64) -> SummarySeriesMetadata { let cfg = SketchConfig::Kll { k: 200 }; - SketchInstanceMetadata { + SummarySeriesMetadata { sid, metric_name: "http_latency".into(), group_by_keys: ["host".to_string()].into_iter().collect(), @@ -6128,15 +6144,3 @@ mod tests { assert_eq!(idx.series.len(), 2); } } - -// 2026-05 reorg: generic epoch-partitioned columnar storage lives -// alongside the store that uses it. -mod admission; -mod maintenance; -pub(crate) use maintenance::{CompleteRawMaintenanceCohort, FrozenExactWindows}; -pub mod epoch_columnar; - -// `persistence` moved up to `sketch_db::persistence`. Re-exported here -// so legacy `crate::storage_engines::sketch_db::index::persistence::*` -// paths continue working without consumer changes. -pub use crate::storage_engines::sketch_db::persistence; diff --git a/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs b/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs index dbe81fbb2..e60ae9404 100644 --- a/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs +++ b/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs @@ -89,20 +89,20 @@ impl Default for SchemaEvictionConfig { /// rekey them on sid and restore the cancel step). pub struct SchemaEvictionService { backfill: Arc, - sketch_index: Arc, + summary_store: Arc, retirement_retention: Duration, config: SchemaEvictionConfig, } impl SchemaEvictionService { pub fn new( - sketch_index: Arc, + summary_store: Arc, backfill: Arc, config: SchemaEvictionConfig, ) -> Self { Self { backfill, - sketch_index, + summary_store, retirement_retention: DEFAULT_RETIREMENT_RETENTION, config, } @@ -154,7 +154,7 @@ impl SchemaEvictionService { /// the service deterministically without spinning up a tokio /// runtime + polling loop. pub fn run_once(&self) { - let expired = self.sketch_index.list_by_status(AggStatus::Expired); + let expired = self.summary_store.list_by_status(AggStatus::Expired); if expired.is_empty() { return; } @@ -183,7 +183,7 @@ impl SchemaEvictionService { ); continue; } - let removed = self.sketch_index.remove_instance(sid).is_some(); + let removed = self.summary_store.remove_instance(sid).is_some(); info!( sid, %metric, @@ -301,7 +301,7 @@ mod tests { } fn write_one( - sketch_index: &SketchStore, + summary_store: &SketchStore, streaming_config: &StreamingConfig, agg_id: u64, ts: u64, @@ -325,7 +325,7 @@ mod tests { static RESOLVER: Arc = Arc::new(SeriesIdResolver::new()); } let resolver = RESOLVER.with(|r| r.clone()); - sketch_index + summary_store .ingest_precompute_for_agg_config( |m, fp, ak| resolver.resolve(m, fp, ak), agg_cfg, @@ -340,21 +340,21 @@ mod tests { /// sids stay `Active`. fn fixture_with_expired_metric_1() -> (Arc, Arc) { let (initial, id_to_fp) = make_streaming_config(&[1, 2]); - let sketch_index = Arc::new(SketchStore::new()); - let sid_a = write_one(&sketch_index, &initial, id_to_fp[&1], 100); - let _ = write_one(&sketch_index, &initial, id_to_fp[&1], 200); - let _ = write_one(&sketch_index, &initial, id_to_fp[&2], 300); + let summary_store = Arc::new(SketchStore::new()); + let sid_a = write_one(&summary_store, &initial, id_to_fp[&1], 100); + let _ = write_one(&summary_store, &initial, id_to_fp[&1], 200); + let _ = write_one(&summary_store, &initial, id_to_fp[&2], 300); // Both metric_1 writes share the same agg-signature → one // sid; mark it Expired directly. metric_2's sid stays Active. - sketch_index.force_expire(sid_a); + summary_store.force_expire(sid_a); - (Arc::new(BackfillRegistry::new()), sketch_index) + (Arc::new(BackfillRegistry::new()), summary_store) } #[tokio::test(flavor = "current_thread")] async fn run_once_drops_expired_sid() { - let (backfill, sketch_index) = fixture_with_expired_metric_1(); - let before_metric_2: usize = sketch_index + let (backfill, summary_store) = fixture_with_expired_metric_1(); + let before_metric_2: usize = summary_store .list_by_status(AggStatus::Active) .into_iter() .filter(|m| m.metric_name == "metric_2") @@ -362,7 +362,7 @@ mod tests { assert!(before_metric_2 >= 1, "fixture seeded metric_2 sid"); let svc = SchemaEvictionService::new( - sketch_index.clone(), + summary_store.clone(), backfill, SchemaEvictionConfig { poll_interval: Duration::from_secs(60), @@ -373,16 +373,16 @@ mod tests { // metric_1's expired sid is gone; metric_2's active sid // remains. - let metric_1_remaining = sketch_index + let metric_1_remaining = summary_store .list_by_status(AggStatus::Active) .into_iter() - .chain(sketch_index.list_by_status(AggStatus::Retired)) - .chain(sketch_index.list_by_status(AggStatus::Expired)) + .chain(summary_store.list_by_status(AggStatus::Retired)) + .chain(summary_store.list_by_status(AggStatus::Expired)) .filter(|m| m.metric_name == "metric_1") .count(); assert_eq!(metric_1_remaining, 0, "expired sid must be removed"); assert!( - sketch_index + summary_store .list_by_status(AggStatus::Active) .iter() .any(|m| m.metric_name == "metric_2"), @@ -394,19 +394,19 @@ mod tests { async fn run_once_is_noop_without_expired_sids() { let (initial, id_to_fp) = make_streaming_config(&[1]); let backfill = Arc::new(BackfillRegistry::new()); - let sketch_index = Arc::new(SketchStore::new()); - let _ = write_one(&sketch_index, &initial, id_to_fp[&1], 100); - let before = sketch_index.instance_count(); + let summary_store = Arc::new(SketchStore::new()); + let _ = write_one(&summary_store, &initial, id_to_fp[&1], 100); + let before = summary_store.instance_count(); let svc = SchemaEvictionService::new( - sketch_index.clone(), + summary_store.clone(), backfill, SchemaEvictionConfig::default(), ); svc.run_once(); assert_eq!( - sketch_index.instance_count(), + summary_store.instance_count(), before, "no expired sids — SketchStore untouched" ); @@ -414,10 +414,10 @@ mod tests { #[tokio::test(flavor = "current_thread")] async fn dry_run_logs_but_does_not_drop() { - let (backfill, sketch_index) = fixture_with_expired_metric_1(); - let before = sketch_index.instance_count(); + let (backfill, summary_store) = fixture_with_expired_metric_1(); + let before = summary_store.instance_count(); let svc = SchemaEvictionService::new( - sketch_index.clone(), + summary_store.clone(), backfill, SchemaEvictionConfig { poll_interval: Duration::from_secs(60), @@ -427,7 +427,7 @@ mod tests { svc.run_once(); // Dry-run: every sid stays. - assert_eq!(sketch_index.instance_count(), before); + assert_eq!(summary_store.instance_count(), before); } #[test] diff --git a/data_plane/src/storage_engines/sketch_db/lifecycle/mod.rs b/data_plane/src/storage_engines/sketch_db/lifecycle/mod.rs index 292ebefc9..8f098879b 100644 --- a/data_plane/src/storage_engines/sketch_db/lifecycle/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/lifecycle/mod.rs @@ -3,7 +3,7 @@ //! //! Post-M2.3 reorg #5. The sid-level lifecycle state itself (the //! `retired_at_ms` / `expires_at_ms` fields on -//! [`crate::storage_engines::sketch_db::index::SketchInstanceMetadata`], +//! [`crate::storage_engines::sketch_db::index::SummarySeriesMetadata`], //! plus `force_retire` / `force_expire` / `is_writable` / `list_by_status` //! / `remove_instance` / `remove_instances_for_agg_config` methods on //! `SketchStore`) lives in `index/` next to the data it gates. This diff --git a/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs b/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs index d3574d782..92f97e227 100644 --- a/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs +++ b/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs @@ -27,7 +27,7 @@ //! equal so they're never retired by this path. That matches the //! pre-retirement behavior: `SchemaRegistry::reconcile` only retired //! agg_ids in its own registry, which mirrored the -//! `StreamingConfig.aggregation_configs` map (also precompute-only). +//! `StreamingConfig.materializations_by_policy_fingerprint` map (also precompute-only). //! Sketch lifecycle stays driven by the control plane's eviction RPC //! until M3 unifies the two. @@ -96,7 +96,7 @@ pub fn reconcile_from_streaming_config( // First pass: find orphaned Active sids without cloning the // catalog. `reconcile_from_streaming_config` runs on every ingest // batch, and the old `snapshot_instances()` here deep-cloned every - // `SketchInstanceMetadata` (String + BTreeSet + AggKind) + // `SummarySeriesMetadata` (String + BTreeSet + AggKind) // on each call — the dominant ingest-path CPU cost in profiling // (BTreeMap/String clone + malloc churn). We only need to read each // instance's signature under the read lock; collect just the cheap @@ -203,7 +203,7 @@ fn signature_from_agg_config(cfg: &AggregationConfig) -> Vec { fn build_live_signature_set(config: &StreamingConfig) -> HashSet> { config - .get_all_aggregation_configs() + .materializations() .values() .map(signature_from_agg_config) .collect() @@ -295,7 +295,7 @@ mod tests { use asap_types::KeyByLabelNames; use crate::storage_engines::sketch_db::data::AggKind; - use crate::storage_engines::sketch_db::index::{SketchInstanceMetadata, SketchStore}; + use crate::storage_engines::sketch_db::index::{SketchStore, SummarySeriesMetadata}; fn agg_config( metric: &str, @@ -326,9 +326,9 @@ mod tests { metric: &str, agg_type: AggregationType, group_by: Vec<&str>, - ) -> SketchInstanceMetadata { + ) -> SummarySeriesMetadata { let group_by_keys: BTreeSet = group_by.into_iter().map(|s| s.to_string()).collect(); - SketchInstanceMetadata { + SummarySeriesMetadata { sid, metric_name: metric.to_string(), group_by_keys, @@ -432,9 +432,9 @@ mod tests { kind: crate::storage_engines::sketch_db::data::SketchAlgorithm, config: crate::storage_engines::sketch_db::data::SketchConfig, group_by: Vec<&str>, - ) -> SketchInstanceMetadata { + ) -> SummarySeriesMetadata { let group_by_keys: BTreeSet = group_by.into_iter().map(|s| s.to_string()).collect(); - SketchInstanceMetadata { + SummarySeriesMetadata { sid, metric_name: metric.to_string(), group_by_keys, diff --git a/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs b/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs index 388bc05b2..552883c1c 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs @@ -6,7 +6,7 @@ //! `agg_id`), the label *values* (`KeyByLabelValues`), the //! `sketch_type_name`, the encoding tag, the time bounds, and the //! opaque sketch bytes. They do NOT carry the pieces of -//! [`SketchInstanceMetadata`](crate::storage_engines::sketch_db::index::SketchInstanceMetadata) +//! [`SummarySeriesMetadata`](crate::storage_engines::sketch_db::index::SummarySeriesMetadata) //! that the QUERY path needs to find and serve a series: //! //! * `metric_name` — the analyzer's diff --git a/data_plane/src/storage_engines/sketch_db/persistence/source.rs b/data_plane/src/storage_engines/sketch_db/persistence/source.rs index 734cfaf6a..661b6a927 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/source.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/source.rs @@ -132,7 +132,7 @@ pub trait EpochSource: Send + Sync { fn approx_memory_bytes(&self) -> usize; /// Persistable instance metadata for `sid` — the pieces of the store's - /// `SketchInstanceMetadata` the QUERY path needs but the on-disk part + /// `SummarySeriesMetadata` the QUERY path needs but the on-disk part /// format does NOT carry (metric name, group-by KEYS, structured /// `AggKind`). Called by the flusher right before it makes a part /// durable so recovery can re-register the sid as a queryable instance diff --git a/data_plane/src/storage_engines/sketch_db/query/timeline.rs b/data_plane/src/storage_engines/sketch_db/query/timeline.rs index 51329ad48..77756c095 100644 --- a/data_plane/src/storage_engines/sketch_db/query/timeline.rs +++ b/data_plane/src/storage_engines/sketch_db/query/timeline.rs @@ -10,7 +10,7 @@ //! //! ## Algorithm //! -//! 1. Snapshot all `SketchInstanceMetadata` for `metric` from the +//! 1. Snapshot all `SummarySeriesMetadata` for `metric` from the //! sid catalog. //! 2. Group by content signature: `(agg_kind, group_by_keys)`. Each //! group corresponds to one logical agg-config (multiple sids of @@ -43,7 +43,7 @@ use std::collections::{BTreeMap, BTreeSet}; use xxhash_rust::xxh64::xxh64; use crate::storage_engines::sketch_db::data::{AggKind, SketchAlgorithm, SketchConfig}; -use crate::storage_engines::sketch_db::index::{SketchInstanceMetadata, SketchStore}; +use crate::storage_engines::sketch_db::index::{SketchStore, SummarySeriesMetadata}; use crate::storage_engines::sketch_db::lifecycle::AggStatus; /// A single `(agg_id, clipped_range)` segment returned by @@ -159,7 +159,7 @@ struct SignatureKey { encoded: Vec, } -fn signature_key(meta: &SketchInstanceMetadata) -> SignatureKey { +fn signature_key(meta: &SummarySeriesMetadata) -> SignatureKey { let mut buf: Vec = Vec::new(); buf.extend_from_slice(meta.metric_name.as_bytes()); buf.push(0); @@ -198,7 +198,7 @@ impl Default for AggSignatureGroup { } impl AggSignatureGroup { - fn fold_in(&mut self, meta: &SketchInstanceMetadata) { + fn fold_in(&mut self, meta: &SummarySeriesMetadata) { // Stable signature id: xxh64 over the same canonical encoding // `signature_key` uses. Computed lazily on first fold-in; // every sid in the group produces the same hash. @@ -327,8 +327,8 @@ mod tests { first_seen: i64, retired: Option, expires: Option, - ) -> SketchInstanceMetadata { - SketchInstanceMetadata { + ) -> SummarySeriesMetadata { + SummarySeriesMetadata { sid, metric_name: metric.into(), group_by_keys: BTreeSet::new(), diff --git a/data_plane/src/storage_engines/sketch_db/sds.rs b/data_plane/src/storage_engines/sketch_db/sds.rs index 38a3d98de..0848da10a 100644 --- a/data_plane/src/storage_engines/sketch_db/sds.rs +++ b/data_plane/src/storage_engines/sketch_db/sds.rs @@ -8,7 +8,7 @@ use std::collections::HashMap; use std::sync::{Arc, RwLock, Weak}; use super::data::{AggKind, SketchConfig}; -use super::index::SketchInstanceMetadata; +use super::index::SummarySeriesMetadata; #[cfg(test)] use crate::storage_engines::types::AggregationType; pub use asap_types::sds::{ @@ -81,7 +81,7 @@ fn legacy_summary(kind: &AggKind) -> SummaryDescriptor { /// copied into every pane row. #[derive(Debug, Clone)] pub struct SdsBinding { - pub metadata: Arc, + pub metadata: Arc, pub summary_descriptor: Arc, pub data_descriptor: Arc, /// Immutable provenance of this physical series lifetime, shared with @@ -90,7 +90,7 @@ pub struct SdsBinding { } impl std::ops::Deref for SdsBinding { - type Target = SketchInstanceMetadata; + type Target = SummarySeriesMetadata; fn deref(&self) -> &Self::Target { &self.metadata @@ -148,7 +148,7 @@ impl SummaryDescriptorRegistry { self.authoritative_catalog.read().unwrap().clone() } - pub fn bind(&self, metadata: SketchInstanceMetadata) -> Result { + pub fn bind(&self, metadata: SummarySeriesMetadata) -> Result { let authoritative = self.authoritative_snapshot(); let configured = if let Some((catalog, _)) = authoritative.as_ref() { if metadata.policy_fp.is_unset() { @@ -301,8 +301,8 @@ mod tests { filter: &str, agg_type: AggregationType, policy: u64, - ) -> SketchInstanceMetadata { - SketchInstanceMetadata { + ) -> SummarySeriesMetadata { + SummarySeriesMetadata { sid, metric_name: metric.into(), group_by_keys: BTreeSet::from(["job".into()]), diff --git a/data_plane/src/storage_engines/types/hot_reload_config.rs b/data_plane/src/storage_engines/types/hot_reload_config.rs index 88786343c..efb76f7e0 100644 --- a/data_plane/src/storage_engines/types/hot_reload_config.rs +++ b/data_plane/src/storage_engines/types/hot_reload_config.rs @@ -7,7 +7,7 @@ //! //! ## How the pieces see the swap //! -//! Runtime bootstrap readers share clones of the same `HotReloadStreamingConfig` +//! Runtime bootstrap readers share clones of the same `StreamingConfigHandle` //! handle (internally `Arc>`), so they //! observe the swap at the same instant: //! @@ -23,7 +23,7 @@ //! `GroupState` for them. //! //! Query execution obtains its generation-consistent runtime configuration, -//! query plan, and catalog from `ActivePhysicalPlan` instead of this handle. +//! query plan, and catalog from `RuntimePhysicalPlan` instead of this handle. //! //! ## Config-upgrade contract for the control plane //! @@ -83,9 +83,9 @@ use arc_swap::ArcSwap; use crate::storage_engines::types::StreamingConfig; /// One immutable, generation-consistent runtime snapshot. Every execution -/// subsystem must project its view from the same `Arc`. +/// subsystem must project its view from the same `Arc`. #[derive(Debug, Clone)] -pub struct ActivePhysicalPlan { +pub struct RuntimePhysicalPlan { /// Authoritative generation and lifecycle identity shared by every plan /// projection in this immutable snapshot. pub envelope: asap_types::precompute_plan::PlanEnvelope, @@ -93,12 +93,12 @@ pub struct ActivePhysicalPlan { pub summary_catalog: Option>, pub precompute_plan: asap_types::precompute_plan::PrecomputePlan, pub transmission_plan: asap_types::producer_plan::TransmissionPlan, - pub runtime_config: Arc, + pub streaming_config: Arc, pub query_plan: Arc, pub storage_routing: Arc, } -impl ActivePhysicalPlan { +impl RuntimePhysicalPlan { pub fn plan_id(&self) -> u64 { self.envelope.plan_id } @@ -123,8 +123,8 @@ impl ActivePhysicalPlan { } #[derive(Clone)] -pub struct HotReloadActivePhysicalPlan { - inner: Arc>, +pub struct ActivePhysicalPlanHandle { + inner: Arc>, readiness: Arc>, } @@ -153,7 +153,7 @@ struct MaterializationReadinessState { } impl MaterializationReadinessState { - fn for_plan(plan: &ActivePhysicalPlan) -> Self { + fn for_plan(plan: &RuntimePhysicalPlan) -> Self { let plan_id = plan.plan_id(); let plan_version = plan.plan_version(); let statuses = plan @@ -265,18 +265,18 @@ pub enum PhysicalPlanLifecycleError { #[derive(Clone)] pub struct PhysicalPlanLifecycle { - active: HotReloadActivePhysicalPlan, + active: ActivePhysicalPlanHandle, state: Arc>, } struct PhysicalPlanLifecycleState { - staged: BTreeMap<(u64, u64), ActivePhysicalPlan>, + staged: BTreeMap<(u64, u64), RuntimePhysicalPlan>, statuses: BTreeMap<(u64, u64), PhysicalPlanStatus>, } impl PhysicalPlanLifecycle { - pub fn new(active: HotReloadActivePhysicalPlan) -> Self { - let snapshot = active.snapshot(); + pub fn new(active: ActivePhysicalPlanHandle) -> Self { + let snapshot = active.active_snapshot(); let mut statuses = BTreeMap::new(); if snapshot.plan_id() != 0 { statuses.insert( @@ -295,7 +295,7 @@ impl PhysicalPlanLifecycle { pub fn stage( &self, - plan: ActivePhysicalPlan, + plan: RuntimePhysicalPlan, now: u64, ) -> Result<(), PhysicalPlanLifecycleError> { let key = (plan.plan_id(), plan.plan_version()); @@ -304,7 +304,7 @@ impl PhysicalPlanLifecycle { return Err(PhysicalPlanLifecycleError::Expired { expiry, now }); } } - let active = self.active.snapshot(); + let active = self.active.active_snapshot(); if active.plan_id() != 0 && key.1 <= active.plan_version() { return Err(PhysicalPlanLifecycleError::StaleVersion { plan_id: key.0, @@ -359,7 +359,7 @@ impl PhysicalPlanLifecycle { plan_id: u64, plan_version: u64, now: u64, - ) -> Result, PhysicalPlanLifecycleError> { + ) -> Result, PhysicalPlanLifecycleError> { self.activate_with_prepare(plan_id, plan_version, now, |_| Ok::<(), String>(())) } @@ -371,8 +371,8 @@ impl PhysicalPlanLifecycle { plan_id: u64, plan_version: u64, now: u64, - prepare: impl FnOnce(&ActivePhysicalPlan) -> Result<(), E>, - ) -> Result, PhysicalPlanLifecycleError> + prepare: impl FnOnce(&RuntimePhysicalPlan) -> Result<(), E>, + ) -> Result, PhysicalPlanLifecycleError> where E: std::fmt::Display, { @@ -399,7 +399,7 @@ impl PhysicalPlanLifecycle { return Err(PhysicalPlanLifecycleError::Expired { expiry, now }); } } - let current = self.active.snapshot(); + let current = self.active.active_snapshot(); if current.plan_id() != 0 && plan.plan_version() <= current.plan_version() { return Err(PhysicalPlanLifecycleError::StaleVersion { plan_id, @@ -433,7 +433,7 @@ impl PhysicalPlanLifecycle { /// Mark a superseded generation retired after all readers of the old /// immutable snapshot have drained. - pub fn retire_drained(&self, plan_id: u64, plan_version: u64) { + pub fn mark_drained_plan_retired(&self, plan_id: u64, plan_version: u64) { if let Some(status) = self .state .lock() @@ -448,7 +448,7 @@ impl PhysicalPlanLifecycle { } } -fn status_for(plan: &ActivePhysicalPlan, phase: PhysicalPlanPhase) -> PhysicalPlanStatus { +fn status_for(plan: &RuntimePhysicalPlan, phase: PhysicalPlanPhase) -> PhysicalPlanStatus { PhysicalPlanStatus { plan_id: plan.plan_id(), plan_version: plan.plan_version(), @@ -458,8 +458,8 @@ fn status_for(plan: &ActivePhysicalPlan, phase: PhysicalPlanPhase) -> PhysicalPl } } -impl HotReloadActivePhysicalPlan { - pub fn new(initial: ActivePhysicalPlan) -> Self { +impl ActivePhysicalPlanHandle { + pub fn new(initial: RuntimePhysicalPlan) -> Self { let readiness = MaterializationReadinessState::for_plan(&initial); Self { inner: Arc::new(ArcSwap::new(Arc::new(initial))), @@ -467,11 +467,11 @@ impl HotReloadActivePhysicalPlan { } } - pub fn snapshot(&self) -> Arc { + pub fn active_snapshot(&self) -> Arc { self.inner.load_full() } - pub fn swap(&self, next: ActivePhysicalPlan) -> Arc { + pub fn swap(&self, next: RuntimePhysicalPlan) -> Arc { let next_readiness = MaterializationReadinessState::for_plan(&next); let old = self.inner.swap(Arc::new(next)); *self @@ -554,10 +554,10 @@ impl HotReloadActivePhysicalPlan { } } -impl std::fmt::Debug for HotReloadActivePhysicalPlan { +impl std::fmt::Debug for ActivePhysicalPlanHandle { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let snapshot = self.snapshot(); - f.debug_struct("HotReloadActivePhysicalPlan") + let snapshot = self.active_snapshot(); + f.debug_struct("ActivePhysicalPlanHandle") .field("plan_id", &snapshot.plan_id()) .field("query_count", &snapshot.query_plan.entries.len()) .field( @@ -568,16 +568,18 @@ impl std::fmt::Debug for HotReloadActivePhysicalPlan { } } -/// Thin wrapper around `ArcSwap` with ergonomic -/// snapshot + swap helpers. Cloneable; clones share the same -/// underlying `ArcSwap` so all holders see the same swaps. +/// Streaming materialization view with two compatibility modes. Legacy mode +/// owns a swappable config; active-plan mode reads the config from the current +/// immutable runtime plan. In active-plan mode, `swap` only updates the legacy +/// backing slot and does not publish a new plan. Use plan activation to change +/// the authoritative configuration. #[derive(Clone)] -pub struct HotReloadStreamingConfig { +pub struct StreamingConfigHandle { inner: Arc>, - active: Option, + active: Option, } -impl HotReloadStreamingConfig { +impl StreamingConfigHandle { /// Construct with an initial `StreamingConfig`. Takes ownership — /// callers who need to keep their own handle should `.clone()` the /// `StreamingConfig` before calling `new`. @@ -598,8 +600,8 @@ impl HotReloadStreamingConfig { } } - pub fn from_active(active: HotReloadActivePhysicalPlan) -> Self { - let initial = active.snapshot().runtime_config.clone(); + pub fn from_active_physical_plan(active: ActivePhysicalPlanHandle) -> Self { + let initial = active.active_snapshot().streaming_config.clone(); Self { inner: Arc::new(ArcSwap::new(initial)), active: Some(active), @@ -612,12 +614,12 @@ impl HotReloadStreamingConfig { pub fn snapshot(&self) -> Arc { self.active .as_ref() - .map(|a| a.snapshot().runtime_config.clone()) + .map(|a| a.active_snapshot().streaming_config.clone()) .unwrap_or_else(|| self.inner.load_full()) } - pub fn physical_plan_snapshot(&self) -> Option> { - self.active.as_ref().map(|active| active.snapshot()) + pub fn active_physical_plan_snapshot(&self) -> Option> { + self.active.as_ref().map(|active| active.active_snapshot()) } /// Atomically replace the current config. The previous `Arc` is @@ -630,15 +632,51 @@ impl HotReloadStreamingConfig { } } -impl std::fmt::Debug for HotReloadStreamingConfig { +impl std::fmt::Debug for StreamingConfigHandle { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let snap = self.snapshot(); - f.debug_struct("HotReloadStreamingConfig") - .field("num_agg_configs", &snap.aggregation_configs.len()) + f.debug_struct("StreamingConfigHandle") + .field( + "num_agg_configs", + &snap.materializations_by_policy_fingerprint.len(), + ) .finish() } } +// Compatibility imports; new callers use the domain names above. +#[deprecated(note = "Use ActivePhysicalPlanHandle")] +pub use ActivePhysicalPlanHandle as HotReloadActivePhysicalPlan; +#[deprecated(note = "Use RuntimePhysicalPlan")] +pub use RuntimePhysicalPlan as ActivePhysicalPlan; + +#[deprecated(note = "Use StreamingConfigHandle")] +pub use StreamingConfigHandle as HotReloadStreamingConfig; + +impl StreamingConfigHandle { + #[deprecated(note = "Use from_active_physical_plan")] + pub fn from_active(active: ActivePhysicalPlanHandle) -> Self { + Self::from_active_physical_plan(active) + } + #[deprecated(note = "Use active_physical_plan_snapshot")] + pub fn physical_plan_snapshot(&self) -> Option> { + self.active_physical_plan_snapshot() + } +} +impl PhysicalPlanLifecycle { + #[deprecated(note = "Use mark_drained_plan_retired")] + pub fn retire_drained(&self, plan_id: u64, plan_version: u64) { + self.mark_drained_plan_retired(plan_id, plan_version) + } +} + +impl ActivePhysicalPlanHandle { + #[deprecated(note = "Use active_snapshot")] + pub fn snapshot(&self) -> Arc { + self.active_snapshot() + } +} + #[cfg(test)] mod tests { use super::*; @@ -654,7 +692,7 @@ mod tests { plan_version: u64, activation_unix_ms: u64, expiry_unix_ms: Option, - ) -> ActivePhysicalPlan { + ) -> RuntimePhysicalPlan { let envelope = asap_types::precompute_plan::PlanEnvelope { plan_id, plan_version, @@ -665,7 +703,7 @@ mod tests { planner_revision: control_plane::physical::compiler::PLANNER_REVISION.into(), capability_snapshot_id: "test".into(), }; - ActivePhysicalPlan { + RuntimePhysicalPlan { envelope: envelope.clone(), summary_catalog: None, precompute_plan: asap_types::precompute_plan::PrecomputePlan { @@ -705,7 +743,7 @@ mod tests { }, rules: Vec::new(), }, - runtime_config: Arc::new(StreamingConfig::new(HashMap::new())), + streaming_config: Arc::new(StreamingConfig::new(HashMap::new())), query_plan: Arc::new(asap_types::query_plan::QueryPlan { plan_id, plan_version, @@ -756,46 +794,56 @@ mod tests { #[test] fn snapshot_reflects_initial_config() { let (cfg, id_to_fp) = cfg_with_ids(&[1, 2, 3]); - let hr = HotReloadStreamingConfig::new(cfg); + let hr = StreamingConfigHandle::new(cfg); let snap = hr.snapshot(); - assert_eq!(snap.aggregation_configs.len(), 3); - assert!(snap.aggregation_configs.contains_key(&id_to_fp[&2])); + assert_eq!(snap.materializations_by_policy_fingerprint.len(), 3); + assert!(snap + .materializations_by_policy_fingerprint + .contains_key(&id_to_fp[&2])); } #[test] fn swap_replaces_config_atomically() { let (cfg1, id_to_fp1) = cfg_with_ids(&[1, 2]); let (cfg2, id_to_fp2) = cfg_with_ids(&[3, 4, 5]); - let hr = HotReloadStreamingConfig::new(cfg1); + let hr = StreamingConfigHandle::new(cfg1); let old = hr.swap(cfg2); // Old snapshot still reflects pre-swap contents. - assert_eq!(old.aggregation_configs.len(), 2); - assert!(old.aggregation_configs.contains_key(&id_to_fp1[&1])); + assert_eq!(old.materializations_by_policy_fingerprint.len(), 2); + assert!(old + .materializations_by_policy_fingerprint + .contains_key(&id_to_fp1[&1])); // New snapshot reflects post-swap contents. let new_snap = hr.snapshot(); - assert_eq!(new_snap.aggregation_configs.len(), 3); - assert!(new_snap.aggregation_configs.contains_key(&id_to_fp2[&5])); - assert!(!new_snap.aggregation_configs.contains_key(&id_to_fp1[&1])); + assert_eq!(new_snap.materializations_by_policy_fingerprint.len(), 3); + assert!(new_snap + .materializations_by_policy_fingerprint + .contains_key(&id_to_fp2[&5])); + assert!(!new_snap + .materializations_by_policy_fingerprint + .contains_key(&id_to_fp1[&1])); } #[test] fn clones_share_underlying_swap() { let (cfg1, _) = cfg_with_ids(&[1]); let (cfg2, id_to_fp2) = cfg_with_ids(&[2, 3]); - let hr = HotReloadStreamingConfig::new(cfg1); + let hr = StreamingConfigHandle::new(cfg1); let hr_clone = hr.clone(); hr.swap(cfg2); // The clone sees the swap because both handles share the // same ArcSwap inside. let snap = hr_clone.snapshot(); - assert_eq!(snap.aggregation_configs.len(), 2); - assert!(snap.aggregation_configs.contains_key(&id_to_fp2[&3])); + assert_eq!(snap.materializations_by_policy_fingerprint.len(), 2); + assert!(snap + .materializations_by_policy_fingerprint + .contains_key(&id_to_fp2[&3])); } #[test] fn concurrent_readers_see_consistent_snapshot() { let (cfg1, _) = cfg_with_ids(&[1, 2]); - let hr = HotReloadStreamingConfig::new(cfg1); + let hr = StreamingConfigHandle::new(cfg1); let hr_writer = hr.clone(); let writer = thread::spawn(move || { for i in 0..50 { @@ -810,7 +858,7 @@ mod tests { // Under race, the snapshot must be internally // consistent — either 2 entries (original) or 3 // (post-swap). Never a torn state. - let n = snap.aggregation_configs.len(); + let n = snap.materializations_by_policy_fingerprint.len(); assert!(n == 2 || n == 3, "torn snapshot: {n} entries"); } }); @@ -820,33 +868,33 @@ mod tests { #[test] fn physical_plan_stages_activates_drains_and_retires() { - let active = HotReloadActivePhysicalPlan::new(physical_plan(7, 1, 100, None)); + let active = ActivePhysicalPlanHandle::new(physical_plan(7, 1, 100, None)); let lifecycle = PhysicalPlanLifecycle::new(active.clone()); lifecycle .stage(physical_plan(7, 2, 200, Some(500)), 150) .unwrap(); - assert_eq!(active.snapshot().plan_version(), 1); + assert_eq!(active.active_snapshot().plan_version(), 1); assert!(matches!( lifecycle.activate(7, 2, 199), Err(PhysicalPlanLifecycleError::ActivationNotReached { .. }) )); let old = lifecycle.activate(7, 2, 200).unwrap(); assert_eq!(old.plan_version(), 1); - assert_eq!(active.snapshot().plan_version(), 2); + assert_eq!(active.active_snapshot().plan_version(), 2); let statuses = lifecycle.statuses(); assert_eq!(statuses.len(), 2); assert_eq!(statuses[0].phase, PhysicalPlanPhase::Draining); assert_eq!(statuses[1].phase, PhysicalPlanPhase::Active); - lifecycle.retire_drained(7, 1); + lifecycle.mark_drained_plan_retired(7, 1); assert_eq!(lifecycle.statuses()[0].phase, PhysicalPlanPhase::Retired); } #[test] fn failed_activation_preparation_keeps_active_and_staged_generations() { - let active = HotReloadActivePhysicalPlan::new(physical_plan(7, 1, 100, None)); - let held_reader = active.snapshot(); + let active = ActivePhysicalPlanHandle::new(physical_plan(7, 1, 100, None)); + let held_reader = active.active_snapshot(); let lifecycle = PhysicalPlanLifecycle::new(active.clone()); lifecycle .stage(physical_plan(7, 2, 200, None), 150) @@ -859,7 +907,7 @@ mod tests { error, PhysicalPlanLifecycleError::Prepare("catalog rejected".into()) ); - assert_eq!(active.snapshot().plan_version(), 1); + assert_eq!(active.active_snapshot().plan_version(), 1); assert_eq!(held_reader.plan_version(), 1); assert!(lifecycle.statuses().iter().any(|status| { status.plan_version == 2 && status.phase == PhysicalPlanPhase::Staged @@ -868,14 +916,14 @@ mod tests { lifecycle .activate_with_prepare(7, 2, 200, |_| Ok::<(), String>(())) .unwrap(); - assert_eq!(active.snapshot().plan_version(), 2); + assert_eq!(active.active_snapshot().plan_version(), 2); } // Failed publication releases only its staging slot; active readers remain valid. #[test] fn discard_staged_allows_retry_and_never_discards_active() { - let active = HotReloadActivePhysicalPlan::new(physical_plan(7, 1, 100, None)); - let held_reader = active.snapshot(); + let active = ActivePhysicalPlanHandle::new(physical_plan(7, 1, 100, None)); + let held_reader = active.active_snapshot(); let lifecycle = PhysicalPlanLifecycle::new(active.clone()); lifecycle .stage(physical_plan(7, 2, 200, None), 150) @@ -886,13 +934,13 @@ mod tests { .unwrap(); lifecycle.activate(7, 2, 300).unwrap(); assert!(lifecycle.discard_staged(7, 2).is_err()); - assert_eq!(active.snapshot().plan_version(), 2); + assert_eq!(active.active_snapshot().plan_version(), 2); assert_eq!(held_reader.plan_version(), 1); } #[test] fn materialization_readiness_is_generation_scoped_and_monotonic() { - let active = HotReloadActivePhysicalPlan::new(physical_plan(7, 1, 100, None)); + let active = ActivePhysicalPlanHandle::new(physical_plan(7, 1, 100, None)); let fingerprint = asap_types::PolicyFingerprint(41); active.readiness.lock().unwrap().statuses.insert( fingerprint, @@ -923,7 +971,7 @@ mod tests { #[test] fn physical_plan_rejects_stale_and_expired_generations() { - let active = HotReloadActivePhysicalPlan::new(physical_plan(7, 2, 100, None)); + let active = ActivePhysicalPlanHandle::new(physical_plan(7, 2, 100, None)); let lifecycle = PhysicalPlanLifecycle::new(active); assert!(matches!( lifecycle.stage(physical_plan(7, 1, 100, None), 150), @@ -937,7 +985,7 @@ mod tests { #[test] fn activation_rechecks_version_and_cannot_downgrade_across_plan_ids() { - let active = HotReloadActivePhysicalPlan::new(physical_plan(7, 1, 100, None)); + let active = ActivePhysicalPlanHandle::new(physical_plan(7, 1, 100, None)); let lifecycle = PhysicalPlanLifecycle::new(active.clone()); lifecycle .stage(physical_plan(8, 3, 100, None), 100) @@ -955,6 +1003,6 @@ mod tests { .. }) )); - assert_eq!(active.snapshot().plan_version(), 3); + assert_eq!(active.active_snapshot().plan_version(), 3); } } diff --git a/data_plane/src/storage_engines/types/streaming_config.rs b/data_plane/src/storage_engines/types/streaming_config.rs index 5c8b9ff42..c5d402c08 100644 --- a/data_plane/src/storage_engines/types/streaming_config.rs +++ b/data_plane/src/storage_engines/types/streaming_config.rs @@ -26,7 +26,11 @@ use super::storage_backend::StorageBackend; /// from_configs` primitive this method now calls directly. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct StreamingConfig { - pub aggregation_configs: HashMap, + #[serde( + rename = "aggregation_configs", + alias = "materializations_by_policy_fingerprint" + )] + pub materializations_by_policy_fingerprint: HashMap, /// Phase-5 capability-routing axis: which storage tier serves this /// per-metric runtime config. The controller pushes this when planning /// (see `docs/design-gorilla-s3-cold-engine.md` §8); pre-Phase-5 @@ -42,9 +46,9 @@ pub struct StreamingConfig { } impl StreamingConfig { - pub fn new(aggregation_configs: HashMap) -> Self { + pub fn new(materializations_by_policy_fingerprint: HashMap) -> Self { Self { - aggregation_configs, + materializations_by_policy_fingerprint, storage_backend: StorageBackend::default(), monitors: Vec::new(), } @@ -59,11 +63,11 @@ impl StreamingConfig { /// Used by the controller-driven plan-push path; tests typically /// stay on `Self::new(...)` and let the default land. pub fn with_storage_backend( - aggregation_configs: HashMap, + materializations_by_policy_fingerprint: HashMap, storage_backend: StorageBackend, ) -> Self { Self { - aggregation_configs, + materializations_by_policy_fingerprint, storage_backend, monitors: Vec::new(), } @@ -77,21 +81,23 @@ impl StreamingConfig { } pub fn get_aggregation_config(&self, aggregation_id: u64) -> Option<&AggregationConfig> { - self.aggregation_configs.get(&aggregation_id) + self.materializations_by_policy_fingerprint + .get(&aggregation_id) } - pub fn get_all_aggregation_configs(&self) -> &HashMap { - &self.aggregation_configs + pub fn materializations(&self) -> &HashMap { + &self.materializations_by_policy_fingerprint } pub fn contains(&self, aggregation_id: u64) -> bool { - self.aggregation_configs.contains_key(&aggregation_id) + self.materializations_by_policy_fingerprint + .contains_key(&aggregation_id) } /// Derived content-addressed view. Builds a [`PolicyRegistry`] keyed /// on [`asap_types::PolicyFingerprint`] — the merged-sid-identity-chain /// replacement for the `aggregation_id`-keyed lookup. Cheap (O(N) - /// over `aggregation_configs.len()`); call at swap time, not per + /// over `materializations_by_policy_fingerprint.len()`); call at swap time, not per /// query, if it shows up in hot-path profiles. /// /// Dual-keyed transition: this method exists alongside the legacy @@ -99,7 +105,11 @@ impl StreamingConfig { /// one at a time. The two views are derived from the same source — /// they can never disagree. pub fn policy_registry(&self) -> PolicyRegistry { - PolicyRegistry::from_configs(self.aggregation_configs.values().cloned()) + PolicyRegistry::from_configs( + self.materializations_by_policy_fingerprint + .values() + .cloned(), + ) } pub fn from_yaml_file(yaml_file: &str) -> Result { @@ -116,7 +126,8 @@ impl StreamingConfig { /// (operator-authored query→agg_ids YAML feeding a retention_map) /// is gone — the controller drives capability matching dynamically. pub fn from_yaml_data(data: &Value) -> Result { - let mut aggregation_configs: HashMap = HashMap::new(); + let mut materializations_by_policy_fingerprint: HashMap = + HashMap::new(); if let Some(aggregations) = data.get("aggregations").and_then(|v| v.as_sequence()) { for aggregation_data in aggregations { @@ -146,11 +157,11 @@ impl StreamingConfig { // PR 5: the map key IS the policy-fingerprint u64. // `AggregationConfig::policy_fp_u64()` is the canonical // accessor for this value. - aggregation_configs.insert(config.policy_fp_u64(), config); + materializations_by_policy_fingerprint.insert(config.policy_fp_u64(), config); } } - let mut config = Self::new(aggregation_configs); + let mut config = Self::new(materializations_by_policy_fingerprint); // Continuous-monitoring (CDM) specs: a top-level `monitors:` array, each // entry deserializing into a MonitorSpec. Absent → empty (the common // case). The data-plane monitor coordinator reads these. @@ -170,7 +181,7 @@ impl Index for StreamingConfig { type Output = AggregationConfig; fn index(&self, aggregation_id: u64) -> &Self::Output { - &self.aggregation_configs[&aggregation_id] + &self.materializations_by_policy_fingerprint[&aggregation_id] } } @@ -180,6 +191,13 @@ impl Default for StreamingConfig { } } +impl StreamingConfig { + #[deprecated(note = "Use materializations")] + pub fn get_all_aggregation_configs(&self) -> &HashMap { + self.materializations() + } +} + #[cfg(test)] mod tests { use super::*; @@ -250,8 +268,12 @@ aggregations:\n\ - aggregationType: DDSketch\n aggregationSubType: ''\n metric: cpu_seconds\n labels:\n grouping: [host]\n rollup: []\n aggregated: []\n parameters:\n relative_accuracy: 0.01\n windowSize: 30\n windowType: tumbling\n spatialFilter: ''\n"; let data: Value = serde_yaml::from_str(yaml).expect("yaml ok"); let cfg = StreamingConfig::from_yaml_data(&data).expect("decode without id"); - assert_eq!(cfg.aggregation_configs.len(), 1); - let (k, v) = cfg.aggregation_configs.iter().next().unwrap(); + assert_eq!(cfg.materializations_by_policy_fingerprint.len(), 1); + let (k, v) = cfg + .materializations_by_policy_fingerprint + .iter() + .next() + .unwrap(); assert_ne!(*k, 0, "derived id is not the 0 sentinel"); assert_eq!(*k, v.policy_fp_u64(), "map key equals fingerprint u64"); assert_eq!(v.metric, "cpu_seconds"); @@ -272,8 +294,16 @@ aggregations:\n\ let wo: Value = serde_yaml::from_str(without).expect("without yaml ok"); let cw = StreamingConfig::from_yaml_data(&w).expect("with"); let cwo = StreamingConfig::from_yaml_data(&wo).expect("without"); - let (kw, _) = cw.aggregation_configs.iter().next().unwrap(); - let (kwo, _) = cwo.aggregation_configs.iter().next().unwrap(); + let (kw, _) = cw + .materializations_by_policy_fingerprint + .iter() + .next() + .unwrap(); + let (kwo, _) = cwo + .materializations_by_policy_fingerprint + .iter() + .next() + .unwrap(); assert_eq!( kw, kwo, "explicit aggregationId in YAML must not change identity" diff --git a/data_plane/src/tests/capability_miss_http_e2e_tests.rs b/data_plane/src/tests/capability_miss_http_e2e_tests.rs index 2930ea74d..c313e8f7f 100644 --- a/data_plane/src/tests/capability_miss_http_e2e_tests.rs +++ b/data_plane/src/tests/capability_miss_http_e2e_tests.rs @@ -3,7 +3,7 @@ //! //! The in-process version of this loop already lives in //! `simple_engine.rs::e2e_feedback_loop_tests` — it swaps the -//! `HotReloadStreamingConfig` handle directly from a mock +//! `StreamingConfigHandle` handle directly from a mock //! `ControlPlaneClient`. What was missing, and what this file adds, //! is the **real HTTP round-trip**: //! @@ -18,7 +18,7 @@ //! │ 3. receive miss → craft config YAML //! ▼ //! POST backend:/api/v1/streaming-config (real HTTP) -//! │ 4. HotReloadStreamingConfig.swap +//! │ 4. StreamingConfigHandle.swap //! ▼ //! GET backend:/api/v1/streaming-config //! → aggregation_count ≥ 1 (loop closed) @@ -39,7 +39,7 @@ use crate::drivers::query::adapters::AdapterConfig; use crate::drivers::query::servers::http::{HttpServer, HttpServerConfig}; use crate::query_engines::ASAPQueryEngine; #[cfg(test)] -use crate::storage_engines::types::{HotReloadStreamingConfig, StreamingConfig}; +use crate::storage_engines::types::{StreamingConfig, StreamingConfigHandle}; use axum::{extract::State, routing::post, Router}; use reqwest::Client; use serde_json::Value; @@ -99,7 +99,7 @@ fn expected_fp_for(metric: &str) -> u64 { let data: serde_yaml::Value = serde_yaml::from_str(&yaml).expect("yaml parses"); let sc = crate::storage_engines::types::StreamingConfig::from_yaml_data(&data) .expect("yaml decodes"); - *sc.aggregation_configs + *sc.materializations_by_policy_fingerprint .keys() .next() .expect("one agg in the canned plan") @@ -162,7 +162,7 @@ async fn start_mock_control_plane(state: MockControlPlaneState) -> u16 { port } -async fn start_backend(control_plane_url: String, hot_reload: HotReloadStreamingConfig) -> u16 { +async fn start_backend(control_plane_url: String, hot_reload: StreamingConfigHandle) -> u16 { let _streaming_config = hot_reload.snapshot(); let engine = Arc::new( ASAPQueryEngine::new(15_000) @@ -226,7 +226,7 @@ async fn poll_until_plan_active( async fn spin_up_loop( metric: &str, expected_agg_id: u64, -) -> (String, MockControlPlaneState, HotReloadStreamingConfig) { +) -> (String, MockControlPlaneState, StreamingConfigHandle) { let control_plane_state = MockControlPlaneState { received_count: Arc::new(AtomicUsize::new(0)), pushed_plan_ts: Arc::new(Mutex::new(None)), @@ -240,7 +240,7 @@ async fn spin_up_loop( let control_plane_url = format!("http://127.0.0.1:{control_plane_port}/api/v1/plan"); // 2. backend up, with control-plane URL baked in - let hot_reload = HotReloadStreamingConfig::new(StreamingConfig::default()); + let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); let backend_port = start_backend(control_plane_url, hot_reload.clone()).await; let backend_url = format!("http://127.0.0.1:{backend_port}"); diff --git a/data_plane/src/tests/test_utilities/engine_factories.rs b/data_plane/src/tests/test_utilities/engine_factories.rs index b5daec83c..333df984b 100644 --- a/data_plane/src/tests/test_utilities/engine_factories.rs +++ b/data_plane/src/tests/test_utilities/engine_factories.rs @@ -21,14 +21,14 @@ use std::collections::HashMap; /// Each factory gets its own resolver instance; tests are isolated so /// the `next_sid = 1, 2, ...` counter doesn't bleed between fixtures. fn ingest_with_fresh_resolver( - sketch_index: &crate::storage_engines::sketch_db::index::SketchStore, + summary_store: &crate::storage_engines::sketch_db::index::SketchStore, resolver: &std::sync::Arc, agg_cfg: &AggregationConfig, output: &PrecomputedOutput, accumulator: &dyn AggregateCore, ) -> Option { let resolver = resolver.clone(); - sketch_index.ingest_precompute_for_agg_config( + summary_store.ingest_precompute_for_agg_config( |m, fp, ak| resolver.resolve(m, fp, ak), agg_cfg, output, @@ -88,7 +88,7 @@ pub fn create_engine_single_pop_with_aggregated( .cloned() .collect(); - let mut aggregation_configs = HashMap::new(); + let mut materializations_by_policy_fingerprint = HashMap::new(); let agg_config = AggregationConfig { population_key_encoding: Default::default(), aggregation_type, @@ -116,15 +116,15 @@ pub fn create_engine_single_pop_with_aggregated( value_source_column: None, }; let agg_id = agg_config.policy_fp_u64(); - aggregation_configs.insert(agg_id, agg_config); + materializations_by_policy_fingerprint.insert(agg_id, agg_config); let streaming_config = Arc::new(StreamingConfig { - aggregation_configs, + materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), }); - let sketch_index = + let summary_store = std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); let resolver = std::sync::Arc::new(SeriesIdResolver::new()); @@ -142,10 +142,10 @@ pub fn create_engine_single_pop_with_aggregated( key, asap_types::PolicyFingerprint(agg_id), ); - ingest_with_fresh_resolver(&sketch_index, &resolver, &agg_cfg, &output, acc.as_ref()); + ingest_with_fresh_resolver(&summary_store, &resolver, &agg_cfg, &output, acc.as_ref()); } - ASAPQueryEngine::new(1).with_sketch_index(sketch_index) + ASAPQueryEngine::new(1).with_sketch_index(summary_store) } /// Creates a ASAPQueryEngine with dual-input (separate value and keys @@ -186,7 +186,7 @@ pub fn create_engine_dual_input( .cloned() .collect(); - let mut aggregation_configs = HashMap::new(); + let mut materializations_by_policy_fingerprint = HashMap::new(); // Value aggregation let value_agg_config = AggregationConfig { @@ -216,7 +216,7 @@ pub fn create_engine_dual_input( value_source_column: None, }; let value_id = value_agg_config.policy_fp_u64(); - aggregation_configs.insert(value_id, value_agg_config); + materializations_by_policy_fingerprint.insert(value_id, value_agg_config); // Keys aggregation let keys_agg_config = AggregationConfig { @@ -246,15 +246,15 @@ pub fn create_engine_dual_input( value_source_column: None, }; let keys_id = keys_agg_config.policy_fp_u64(); - aggregation_configs.insert(keys_id, keys_agg_config); + materializations_by_policy_fingerprint.insert(keys_id, keys_agg_config); let streaming_config = Arc::new(StreamingConfig { - aggregation_configs, + materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), }); - let sketch_index = + let summary_store = std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); let resolver = std::sync::Arc::new(SeriesIdResolver::new()); @@ -275,7 +275,7 @@ pub fn create_engine_dual_input( key, asap_types::PolicyFingerprint(value_id), ); - ingest_with_fresh_resolver(&sketch_index, &resolver, &agg_cfg_1, &output, acc.as_ref()); + ingest_with_fresh_resolver(&summary_store, &resolver, &agg_cfg_1, &output, acc.as_ref()); } for (label_values_opt, acc) in keys_data { let key = label_values_opt.map(|labels| KeyByLabelValues { labels }); @@ -285,10 +285,10 @@ pub fn create_engine_dual_input( key, asap_types::PolicyFingerprint(keys_id), ); - ingest_with_fresh_resolver(&sketch_index, &resolver, &agg_cfg_2, &output, acc.as_ref()); + ingest_with_fresh_resolver(&summary_store, &resolver, &agg_cfg_2, &output, acc.as_ref()); } - ASAPQueryEngine::new(1).with_sketch_index(sketch_index) + ASAPQueryEngine::new(1).with_sketch_index(summary_store) } /// Creates a ASAPQueryEngine with two independent metrics, each with their own @@ -312,7 +312,7 @@ pub fn create_engine_two_metrics( let labels_a: Vec = grouping_labels_a.iter().map(|s| s.to_string()).collect(); let labels_b: Vec = grouping_labels_b.iter().map(|s| s.to_string()).collect(); - let mut aggregation_configs = HashMap::new(); + let mut materializations_by_policy_fingerprint = HashMap::new(); let agg_config_a = AggregationConfig { population_key_encoding: Default::default(), @@ -341,7 +341,7 @@ pub fn create_engine_two_metrics( value_source_column: None, }; let id_a = agg_config_a.policy_fp_u64(); - aggregation_configs.insert(id_a, agg_config_a); + materializations_by_policy_fingerprint.insert(id_a, agg_config_a); let agg_config_b = AggregationConfig { population_key_encoding: Default::default(), @@ -370,15 +370,15 @@ pub fn create_engine_two_metrics( value_source_column: None, }; let id_b = agg_config_b.policy_fp_u64(); - aggregation_configs.insert(id_b, agg_config_b); + materializations_by_policy_fingerprint.insert(id_b, agg_config_b); let streaming_config = Arc::new(StreamingConfig { - aggregation_configs, + materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), }); - let sketch_index = + let summary_store = std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); let resolver = std::sync::Arc::new(SeriesIdResolver::new()); let agg_cfg_1 = streaming_config @@ -398,7 +398,7 @@ pub fn create_engine_two_metrics( key, asap_types::PolicyFingerprint(id_a), ); - ingest_with_fresh_resolver(&sketch_index, &resolver, &agg_cfg_1, &output, acc.as_ref()); + ingest_with_fresh_resolver(&summary_store, &resolver, &agg_cfg_1, &output, acc.as_ref()); } for (label_values_opt, acc) in data_b { let key = label_values_opt.map(|labels| KeyByLabelValues { labels }); @@ -408,10 +408,10 @@ pub fn create_engine_two_metrics( key, asap_types::PolicyFingerprint(id_b), ); - ingest_with_fresh_resolver(&sketch_index, &resolver, &agg_cfg_2, &output, acc.as_ref()); + ingest_with_fresh_resolver(&summary_store, &resolver, &agg_cfg_2, &output, acc.as_ref()); } let _ = (query_a, query_b); - ASAPQueryEngine::new(1).with_sketch_index(sketch_index) + ASAPQueryEngine::new(1).with_sketch_index(summary_store) } /// Creates a ASAPQueryEngine with three independent metrics, each with their own @@ -440,7 +440,7 @@ pub fn create_engine_three_metrics( let labels_b: Vec = grouping_labels_b.iter().map(|s| s.to_string()).collect(); let labels_c: Vec = grouping_labels_c.iter().map(|s| s.to_string()).collect(); - let mut aggregation_configs = HashMap::new(); + let mut materializations_by_policy_fingerprint = HashMap::new(); let mut ids: Vec = Vec::new(); for (agg_type, labels, metric) in [ @@ -476,16 +476,16 @@ pub fn create_engine_three_metrics( }; let id = cfg.policy_fp_u64(); ids.push(id); - aggregation_configs.insert(id, cfg); + materializations_by_policy_fingerprint.insert(id, cfg); } let streaming_config = Arc::new(StreamingConfig { - aggregation_configs, + materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), }); - let sketch_index = + let summary_store = std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); let resolver = std::sync::Arc::new(SeriesIdResolver::new()); let agg_cfgs: Vec<_> = ids @@ -509,12 +509,12 @@ pub fn create_engine_three_metrics( key, asap_types::PolicyFingerprint(agg_id), ); - ingest_with_fresh_resolver(&sketch_index, &resolver, agg_cfg, &output, acc.as_ref()); + ingest_with_fresh_resolver(&summary_store, &resolver, agg_cfg, &output, acc.as_ref()); } } let _ = (labels_a, labels_b, labels_c, query_a, query_b, query_c); - ASAPQueryEngine::new(1).with_sketch_index(sketch_index) + ASAPQueryEngine::new(1).with_sketch_index(summary_store) } /// Creates a single-pop engine with data at multiple timestamps for testing merge. @@ -529,7 +529,7 @@ pub fn create_engine_multi_timestamp( let grouping_label_strings: Vec = grouping_labels.iter().map(|s| s.to_string()).collect(); - let mut aggregation_configs = HashMap::new(); + let mut materializations_by_policy_fingerprint = HashMap::new(); let agg_config = AggregationConfig { population_key_encoding: Default::default(), aggregation_type, @@ -557,15 +557,15 @@ pub fn create_engine_multi_timestamp( value_source_column: None, }; let agg_id = agg_config.policy_fp_u64(); - aggregation_configs.insert(agg_id, agg_config); + materializations_by_policy_fingerprint.insert(agg_id, agg_config); let streaming_config = Arc::new(StreamingConfig { - aggregation_configs, + materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), }); - let sketch_index = + let summary_store = std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); let resolver = std::sync::Arc::new(SeriesIdResolver::new()); let agg_cfg = streaming_config @@ -580,9 +580,9 @@ pub fn create_engine_multi_timestamp( key, asap_types::PolicyFingerprint(agg_id), ); - ingest_with_fresh_resolver(&sketch_index, &resolver, &agg_cfg, &output, acc.as_ref()); + ingest_with_fresh_resolver(&summary_store, &resolver, &agg_cfg, &output, acc.as_ref()); } - ASAPQueryEngine::new(1).with_sketch_index(sketch_index) + ASAPQueryEngine::new(1).with_sketch_index(summary_store) } /// Creates a single-pop engine with data at multiple timestamps and configurable window. @@ -603,7 +603,7 @@ pub fn create_engine_multi_timestamp_with_window( let grouping_label_strings: Vec = grouping_labels.iter().map(|s| s.to_string()).collect(); - let mut aggregation_configs = HashMap::new(); + let mut materializations_by_policy_fingerprint = HashMap::new(); let agg_config = AggregationConfig { population_key_encoding: Default::default(), aggregation_type, @@ -631,15 +631,15 @@ pub fn create_engine_multi_timestamp_with_window( value_source_column: None, }; let agg_id = agg_config.policy_fp_u64(); - aggregation_configs.insert(agg_id, agg_config); + materializations_by_policy_fingerprint.insert(agg_id, agg_config); let streaming_config = Arc::new(StreamingConfig { - aggregation_configs, + materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), }); - let sketch_index = + let summary_store = std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); let resolver = std::sync::Arc::new(SeriesIdResolver::new()); let agg_cfg = streaming_config @@ -654,7 +654,7 @@ pub fn create_engine_multi_timestamp_with_window( key, asap_types::PolicyFingerprint(agg_id), ); - ingest_with_fresh_resolver(&sketch_index, &resolver, &agg_cfg, &output, acc.as_ref()); + ingest_with_fresh_resolver(&summary_store, &resolver, &agg_cfg, &output, acc.as_ref()); } - ASAPQueryEngine::new(1).with_sketch_index(sketch_index) + ASAPQueryEngine::new(1).with_sketch_index(summary_store) } diff --git a/data_plane/src/tests/test_utilities/planning.rs b/data_plane/src/tests/test_utilities/planning.rs index 8e6bb709c..f05dac22e 100644 --- a/data_plane/src/tests/test_utilities/planning.rs +++ b/data_plane/src/tests/test_utilities/planning.rs @@ -1,25 +1,28 @@ //! Synthetic complete quotes for deployment fixtures; never used in production. use control_plane::physical::compiler::{ - BackendLocalPlanningSnapshot, PhysicalCompiler, BACKEND_REVISION, PLANNER_REVISION, + BackendLocalPlanningInput, PhysicalPlanCompiler, BACKEND_REVISION, PLANNER_REVISION, }; pub(crate) fn quoted_snapshot( - mut snapshot: BackendLocalPlanningSnapshot, + mut snapshot: BackendLocalPlanningInput, metricsql: bool, -) -> BackendLocalPlanningSnapshot { +) -> BackendLocalPlanningInput { use control_plane::physical::workload_cost::{ - manifest, with_exact_alternative, WorkloadCostEvidence, WorkloadQuote, + enumerate_exact_and_materialized_candidates, manifest, WorkloadCostEvidence, WorkloadQuote, }; - let (request, environment) = snapshot.clone().planning_request().unwrap(); - let quotes = with_exact_alternative(request) + let (request, environment) = snapshot + .clone() + .into_physical_compilation_request() + .unwrap(); + let quotes = enumerate_exact_and_materialized_candidates(request) .unwrap() .into_iter() .enumerate() .filter_map(|(index, candidate)| { let plan = if metricsql { - PhysicalCompiler.compile_metricsql(candidate.clone(), environment.clone()) + PhysicalPlanCompiler.compile_metricsql(candidate.clone(), environment.clone()) } else { - PhysicalCompiler.compile(candidate.clone(), environment.clone()) + PhysicalPlanCompiler.compile_promql(candidate.clone(), environment.clone()) } .ok()?; let manifest = manifest(&plan, &candidate.queries).unwrap(); diff --git a/data_plane/src/monitor/coordinator.rs b/data_plane/src/update_sampling/coordinator.rs similarity index 100% rename from data_plane/src/monitor/coordinator.rs rename to data_plane/src/update_sampling/coordinator.rs diff --git a/data_plane/src/monitor/epoch.rs b/data_plane/src/update_sampling/epoch.rs similarity index 100% rename from data_plane/src/monitor/epoch.rs rename to data_plane/src/update_sampling/epoch.rs diff --git a/data_plane/src/monitor/mod.rs b/data_plane/src/update_sampling/mod.rs similarity index 100% rename from data_plane/src/monitor/mod.rs rename to data_plane/src/update_sampling/mod.rs diff --git a/data_plane/src/monitor/sampling_alloc.rs b/data_plane/src/update_sampling/sampling_alloc.rs similarity index 100% rename from data_plane/src/monitor/sampling_alloc.rs rename to data_plane/src/update_sampling/sampling_alloc.rs diff --git a/data_plane/src/monitor/server.rs b/data_plane/src/update_sampling/server.rs similarity index 99% rename from data_plane/src/monitor/server.rs rename to data_plane/src/update_sampling/server.rs index f24b79626..f26ff9b7f 100644 --- a/data_plane/src/monitor/server.rs +++ b/data_plane/src/update_sampling/server.rs @@ -337,7 +337,7 @@ impl MonitorService for MonitorServiceImpl { #[cfg(test)] mod reconfigure_tests { use super::MonitorCoordinator; - use crate::monitor::coordinator::MonitorConfig; + use crate::update_sampling::coordinator::MonitorConfig; fn cfg(agg_id: u64, key: &str, tau: f64) -> MonitorConfig { MonitorConfig { diff --git a/data_plane/src/utils/file_io.rs b/data_plane/src/utils/file_io.rs index 7e90183d1..5150ddf0e 100644 --- a/data_plane/src/utils/file_io.rs +++ b/data_plane/src/utils/file_io.rs @@ -48,12 +48,8 @@ aggregations: write!(streaming_temp_file, "{streaming_yaml_content}").unwrap(); let config = read_streaming_config(streaming_temp_file.path().to_str().unwrap()).unwrap(); - assert!(!config.aggregation_configs.is_empty()); - let agg = config - .get_all_aggregation_configs() - .values() - .next() - .expect("one agg"); + assert!(!config.materializations_by_policy_fingerprint.is_empty()); + let agg = config.materializations().values().next().expect("one agg"); assert_eq!(agg.num_aggregates_to_retain, Some(6)); } } diff --git a/data_plane/tests/asapquery_compatibility_process_e2e.rs b/data_plane/tests/asapquery_compatibility_process_e2e.rs index 6d491f7f0..90f2f2cee 100644 --- a/data_plane/tests/asapquery_compatibility_process_e2e.rs +++ b/data_plane/tests/asapquery_compatibility_process_e2e.rs @@ -30,29 +30,32 @@ mod immutable_maintenance_process; // Test-only quotes preserve the fixture's local candidate without a production bypass. fn quote_snapshot_for_test( - snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot, -) -> control_plane::physical::compiler::BackendLocalPlanningSnapshot { + snapshot: control_plane::physical::compiler::BackendLocalPlanningInput, +) -> control_plane::physical::compiler::BackendLocalPlanningInput { quote_snapshot_for_frontend_test(snapshot, false) } fn quote_snapshot_for_frontend_test( - mut snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot, + mut snapshot: control_plane::physical::compiler::BackendLocalPlanningInput, metricsql: bool, -) -> control_plane::physical::compiler::BackendLocalPlanningSnapshot { +) -> control_plane::physical::compiler::BackendLocalPlanningInput { use control_plane::physical::{ - compiler::{PhysicalCompiler, BACKEND_REVISION, PLANNER_REVISION}, + compiler::{PhysicalPlanCompiler, BACKEND_REVISION, PLANNER_REVISION}, workload_cost::{self, WorkloadCostEvidence, WorkloadQuote}, }; - let (request, environment) = snapshot.clone().planning_request().unwrap(); + let (request, environment) = snapshot + .clone() + .into_physical_compilation_request() + .unwrap(); let mut preferred = true; - let quotes = workload_cost::with_exact_alternative(request) + let quotes = workload_cost::enumerate_exact_and_materialized_candidates(request) .unwrap() .into_iter() .filter_map(|candidate| { let plan = if metricsql { - PhysicalCompiler.compile_metricsql(candidate.clone(), environment.clone()) + PhysicalPlanCompiler.compile_metricsql(candidate.clone(), environment.clone()) } else { - PhysicalCompiler.compile(candidate.clone(), environment.clone()) + PhysicalPlanCompiler.compile_promql(candidate.clone(), environment.clone()) } .ok()?; let unit_cost = if preferred { 1.0 } else { 1e12 }; @@ -204,7 +207,7 @@ fn is_warm(response: &Value) -> bool { #[tokio::test] #[ignore = "requires ASAPCollector CollectorPlan schema compatibility; run explicitly after Collector is updated"] async fn erp_measured_kll_collector_to_query_oracle() { - use control_plane::physical::compiler::{BackendLocalPlanningSnapshot, PhysicalCompiler}; + use control_plane::physical::compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}; const QUERY: &str = "quantile_over_time(0.9, erp_latency[5s])"; let artifact: Value = serde_json::from_str(include_str!( "../../control_plane/tests/fixtures/erp-kll-measured.json" @@ -226,10 +229,10 @@ async fn erp_measured_kll_collector_to_query_oracle() { "byte_second_weight": 1e-9, "mode": "hybrid", "runtime": {"allowed_algorithms": ["Kll"], "max_memory_bytes": null} }); - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_value(fixture).unwrap(); - let (mut request, mut environment) = snapshot.planning_request().unwrap(); - request.hybrid_execution = false; - request.queries[0].group_by = vec!["service".into()]; + let snapshot: BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); + let (mut request, mut environment) = snapshot.into_physical_compilation_request().unwrap(); + request.allow_mixed_summary_and_exact_execution = false; + request.queries[0].group_by_labels = vec!["service".into()]; let lifecycle_entry = &mut request .query_workload .as_mut() @@ -240,8 +243,10 @@ async fn erp_measured_kll_collector_to_query_oracle() { lifecycle_entry.time_selection.scope = planner_types::workload::QueryTimeScope::Unknown; environment.target = control_plane::physical::compiler::PhysicalDeploymentTarget::DistributedCollectors; - environment.collector_ids = vec!["erp-collector".into()]; - let plan = PhysicalCompiler.compile(request, environment).unwrap(); + environment.target_collector_ids = vec!["erp-collector".into()]; + let plan = PhysicalPlanCompiler + .compile_promql(request, environment) + .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); assert_eq!(plan.precompute_plan.materializations[0].parameters["k"], 32); let collector = serde_json::to_value(&plan.collector_plans[0]).unwrap(); @@ -486,7 +491,7 @@ async fn registered_temporal_topk_count_sketch_heap() { } async fn registered_temporal_topk(algorithm: planner_types::post_asap::SketchAlgorithm) { - use control_plane::physical::compiler::{BackendLocalPlanningSnapshot, PhysicalCompiler}; + use control_plane::physical::compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}; use planner_types::post_asap::{CompositionOperator, SketchQuery, SummaryFamilyType}; const QUERY: &str = "topk(3, count_over_time(top_endpoint_qps[5s]))"; struct Evidence; @@ -525,17 +530,19 @@ async fn registered_temporal_topk(algorithm: planner_types::post_asap::SketchAlg "source": "deterministic-count-ranking-fixture" } }); - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_value(fixture).unwrap(); - let (mut request, environment) = snapshot.planning_request().unwrap(); + let snapshot: BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); + let (mut request, environment) = snapshot.into_physical_compilation_request().unwrap(); let query = &mut request.queries[0]; - let expr = - control_plane::query_parser::parse_query_expr_canonical(QUERY, query.accuracy.clone()) - .unwrap(); + let expr = control_plane::query_parser::parse_query_expr_canonical( + QUERY, + query.accuracy_target.clone(), + ) + .unwrap(); let model = control_plane::physical::post_asap::cost_model::ForcedFamilyCostModel::new( - query.accuracy.clone(), + query.accuracy_target.clone(), algorithm.clone(), ); - query.post_asap = control_plane::planner_selection::select_summary_with_evidence( + query.selected_plan_root = control_plane::planner_selection::select_summary_with_evidence( &expr, &model, &asap_aware_mapping::DefaultAccuracyModel, @@ -543,7 +550,9 @@ async fn registered_temporal_topk(algorithm: planner_types::post_asap::SketchAlg &Evidence, ) .unwrap(); - let plan = PhysicalCompiler.compile(request, environment).unwrap(); + let plan = PhysicalPlanCompiler + .compile_promql(request, environment) + .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); use data_plane::storage_engines::types::AggregationType; let expected_type = match algorithm { @@ -783,36 +792,39 @@ async fn run_shared_dashboard(multi_pane: bool) { }) .collect(), ); - let mut typed: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let mut typed: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_value(snapshot.clone()).unwrap(); if multi_pane { for entry in typed.query_workload.repeating_queries.as_mut().unwrap() { entry.time_selection.lookback = Some(planner_types::workload::DurationMs(10_000)); } - let (request, _) = typed.clone().planning_request().unwrap(); + let (request, _) = typed.clone().into_physical_compilation_request().unwrap(); for query in request.queries { - let mut candidates = query.window_implementations; + let mut candidates = query.window_realization_candidates; let mut small = candidates[0].clone(); - small.implementation_id = "five-second-pane".into(); + small.realization_id = "five-second-pane".into(); small.layout = asap_types::WindowMaterializationLayout::Pane { pane_secs: 5 }; small.cost.weighted_cost = 0.0; candidates[0].cost.weighted_cost = 10.0; candidates.push(small); typed - .implementation - .window_candidates + .physical_inputs + .window_realization_candidates_by_query .insert(query.query_string, candidates); } } - let (request, environment) = typed.clone().planning_request().unwrap(); + let (request, environment) = typed.clone().into_physical_compilation_request().unwrap(); let candidates = - control_plane::physical::workload_cost::with_exact_alternative(request).unwrap(); + control_plane::physical::workload_cost::enumerate_exact_and_materialized_candidates( + request, + ) + .unwrap(); let quotes = candidates .into_iter() .enumerate() .map(|(index, candidate)| { - let plan = control_plane::physical::compiler::PhysicalCompiler - .compile(candidate.clone(), environment.clone()) + let plan = control_plane::physical::compiler::PhysicalPlanCompiler + .compile_promql(candidate.clone(), environment.clone()) .unwrap(); let manifest = control_plane::physical::workload_cost::manifest(&plan, &candidate.queries) @@ -829,7 +841,7 @@ async fn run_shared_dashboard(multi_pane: bool) { } }) .collect(); - typed.snapshot_version = 2; + typed.schema_version = 2; typed.workload_cost_evidence = Some( control_plane::physical::workload_cost::WorkloadCostEvidence { backend_revision: control_plane::physical::compiler::BACKEND_REVISION.into(), @@ -842,13 +854,13 @@ async fn run_shared_dashboard(multi_pane: bool) { }, ); snapshot = serde_json::to_value(&typed).unwrap(); - let plan = typed.compile().unwrap(); + let plan = typed.compile_promql().unwrap(); assert!(plan.cost_comparison.is_some()); assert_eq!(plan.precompute_plan.materializations.len(), 1); assert_eq!(plan.query_plan.entries.len(), 3); if multi_pane { assert_eq!( - plan.lifecycle_estimates[0].window_implementation_id, + plan.lifecycle_estimates[0].window_realization_id, "five-second-pane" ); for entry in plan.query_plan.entries.values() { @@ -1644,9 +1656,9 @@ async fn collector_free_profile_serves_complete_matrix_and_falls_back_exactly() let materializations = status["materializations"] .as_array() .expect("materialization statuses"); - let planned_snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let planned_snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(&std::fs::read_to_string(&snapshot).unwrap()).unwrap(); - let planned = planned_snapshot.compile().unwrap(); + let planned = planned_snapshot.compile_promql().unwrap(); // Every selected state must be serving; the Planner may share or separate // physical populations, so compare identities rather than a frozen count. let expected = planned @@ -1677,7 +1689,7 @@ async fn collector_free_profile_serves_complete_matrix_and_falls_back_exactly() "collector_plans": invalid.collector_plans, "precompute_plan": invalid.precompute_plan, "transmission_plan": invalid.transmission_plan, "query_plan": invalid.query_plan, "storage_routing": null, "adaptation_evidence": []}); - let built = data_plane::drivers::query::servers::http::build_active_physical_plan( + let built = data_plane::drivers::query::servers::http::validate_and_build_runtime_plan( serde_json::from_value(artifact.clone()).unwrap(), std::sync::Arc::new(data_plane::storage_engines::types::BackendStorageRouting::empty()), ); diff --git a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs index caf5a21f4..1ea601a33 100644 --- a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs +++ b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs @@ -69,7 +69,7 @@ async fn post_full_config(client: &reqwest::Client, stack: &FullStack, json: &Js .unwrap(); let mut artifact = physical_fixture::artifact(&runtime); if runtime - .aggregation_configs + .materializations_by_policy_fingerprint .values() .any(|c| c.metric == "http_requests_total_latency_ms") { @@ -128,7 +128,7 @@ async fn post_full_config(client: &reqwest::Client, stack: &FullStack, json: &Js } use control_plane::types::{AggType, QueryWorkload, WorkloadCharacteristics}; -use data_plane::storage_engines::types::HotReloadStreamingConfig; +use data_plane::storage_engines::types::StreamingConfigHandle; use serde_json::Value as JsonValue; use asap_otel_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; @@ -263,21 +263,21 @@ fn plan_streaming_config_json(workload: &QueryWorkload) -> JsonValue { .expect("emit_backend_streaming_config_json must succeed") } -/// Spin up an in-process backend HTTP server with `HotReloadStreamingConfig` +/// Spin up an in-process backend HTTP server with `StreamingConfigHandle` /// wired through both the query engine and the POST `/api/v1/streaming-config` /// handler. Returns `(port, hot_reload_handle)` — the latter so tests can /// also inspect the current config from the controller's side. -async fn start_backend_http_server() -> (u16, HotReloadStreamingConfig) { +async fn start_backend_http_server() -> (u16, StreamingConfigHandle) { use data_plane::drivers::query::adapters::config::AdapterConfig; use data_plane::drivers::query::servers::{HttpServer, HttpServerConfig}; use data_plane::query_engines::asap_query_engine::engine::ASAPQueryEngine; use data_plane::storage_engines::sketch_db::index::SketchStore; use data_plane::storage_engines::types::StreamingConfig; - let hot_reload = HotReloadStreamingConfig::new(StreamingConfig::default()); - let sketch_index = Arc::new(SketchStore::new()); + let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); + let summary_store = Arc::new(SketchStore::new()); let query_engine = - Arc::new(ASAPQueryEngine::new(15_000).with_sketch_index(sketch_index.clone())); + Arc::new(ASAPQueryEngine::new(15_000).with_sketch_index(summary_store.clone())); let adapter_config = AdapterConfig::prometheus_promql( "http://127.0.0.1:9999".to_string(), // unused — no forwarding in this test @@ -289,7 +289,7 @@ async fn start_backend_http_server() -> (u16, HotReloadStreamingConfig) { adapter_config, }; - let server = HttpServer::new(http_config, query_engine, sketch_index) + let server = HttpServer::new(http_config, query_engine, summary_store) .with_hot_reload_config(hot_reload.clone()); let port = server @@ -345,7 +345,7 @@ fn _wc_anchor() -> WorkloadCharacteristics { /// Full test stack: PrecomputeEngine + SketchStoreSink + OtlpReceiver + /// HttpServer, all sharing the same `SketchStore` and -/// `HotReloadStreamingConfig` so a controller-posted streaming-config +/// `StreamingConfigHandle` so a controller-posted streaming-config /// is visible to the engine's accumulator routing, the engine's window /// outputs land in `SketchStore`, and the query engine reads from the /// same store. @@ -373,17 +373,17 @@ async fn start_full_stack(otlp_http_port: u16, otlp_grpc_port: u16) -> FullStack use data_plane::query_engines::asap_query_engine::engine::ASAPQueryEngine; use data_plane::storage_engines::sketch_db::index::SketchStore; - let sketch_index = Arc::new(SketchStore::new()); - let active = data_plane::storage_engines::types::HotReloadActivePhysicalPlan::new( + let summary_store = Arc::new(SketchStore::new()); + let active = data_plane::storage_engines::types::ActivePhysicalPlanHandle::new( physical_fixture::bootstrap(), ); - let hot_reload = HotReloadStreamingConfig::from_active(active.clone()); + let hot_reload = StreamingConfigHandle::from_active_physical_plan(active.clone()); let series_resolver = Arc::new(SeriesIdResolver::new()); // SketchStoreSink writes precompute output back into SketchStore so // the query engine can find it. let sink = Arc::new(SketchStoreSink::new( - sketch_index.clone(), + summary_store.clone(), hot_reload.clone(), series_resolver.clone(), )); @@ -406,7 +406,7 @@ async fn start_full_stack(otlp_http_port: u16, otlp_grpc_port: u16) -> FullStack hot_reload.clone(), sink, series_resolver.clone(), - sketch_index.clone(), + summary_store.clone(), ); let ingest_state = engine.ingest_state(); tokio::spawn(async move { @@ -441,10 +441,10 @@ async fn start_full_stack(otlp_http_port: u16, otlp_grpc_port: u16) -> FullStack // `sketch_index` via OTLP ingest (the engine's // `precompute_engine` shares the Arc), but the query // path can't see them without this binding. - .with_sketch_index(sketch_index.clone()) + .with_sketch_index(summary_store.clone()) .with_active_physical_plan(active.clone()), ); - let server = HttpServer::new(http_config, query_engine, sketch_index) + let server = HttpServer::new(http_config, query_engine, summary_store) .with_hot_reload_config(hot_reload.clone()) .with_active_physical_plan(active); let backend_port = server @@ -958,7 +958,7 @@ async fn controller_plans_with_grouping_and_backend_parses_grouping_labels() { assert_eq!(active["aggregation_count"], 1); // Walk the streaming_config object to find the registered grouping - // labels. The snapshot path is `streaming_config.aggregation_configs. + // labels. The snapshot path is `streaming_config.materializations_by_policy_fingerprint. // .grouping_labels.`. let cfgs = active["streaming_config"]["aggregation_configs"] .as_object() @@ -1008,7 +1008,7 @@ async fn controller_plans_with_grouping_and_backend_parses_grouping_labels() { // * Modified-OTLP `DdSketchDataPoint` wire encoding + the backend's // OTLP HTTP receiver accept the payload (no 4xx/5xx). // * The full stack (PrecomputeEngine + SketchStoreSink + OtlpReceiver -// + HttpServer all sharing SketchStore + HotReloadStreamingConfig) +// + HttpServer all sharing SketchStore + StreamingConfigHandle) // comes up and stays up under POST + query traffic. // * The OTLP-ingested sketch lands in `SketchStore` keyed by the // right `PolicyFingerprint` (or via the `instances_matching` diff --git a/data_plane/tests/monitor_grpc.rs b/data_plane/tests/monitor_grpc.rs index 31b1d5652..b8901f014 100644 --- a/data_plane/tests/monitor_grpc.rs +++ b/data_plane/tests/monitor_grpc.rs @@ -22,7 +22,7 @@ use asap_otel_proto::monitor::v1::{ edge_to_coord, monitor_service_client::MonitorServiceClient, EdgeToCoord, MonitorRegister, MonitorReport, }; -use data_plane::monitor::{MonitorConfig, MonitorCoordinator, MonitorServiceImpl}; +use data_plane::update_sampling::{MonitorConfig, MonitorCoordinator, MonitorServiceImpl}; async fn start_server(cfgs: Vec) -> String { let coord = MonitorCoordinator::new(cfgs); diff --git a/data_plane/tests/support/distinct_planning_process.rs b/data_plane/tests/support/distinct_planning_process.rs index a67a197ad..058ca9d7b 100644 --- a/data_plane/tests/support/distinct_planning_process.rs +++ b/data_plane/tests/support/distinct_planning_process.rs @@ -1,5 +1,5 @@ use super::*; -use control_plane::physical::compiler::BackendLocalPlanningSnapshot; +use control_plane::physical::compiler::BackendLocalPlanningInput; /// The production compiler, ingest engine and query DAG preserve distinct populations. #[tokio::test] @@ -24,9 +24,9 @@ async fn distinct_range_uses_planner_selected_hll_and_source_labels() { entry["requirements"]["accuracy"] = serde_json::json!({"explicit": {"Epsilon": 0.05}}); fixture["query_workload"]["repeating_queries"] = serde_json::json!([entry]); let plan = quote_snapshot_for_test( - serde_json::from_value::(fixture.clone()).unwrap(), + serde_json::from_value::(fixture.clone()).unwrap(), ) - .compile() + .compile_promql() .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); assert_eq!( @@ -81,7 +81,7 @@ async fn distinct_range_uses_planner_selected_hll_and_source_labels() { wait_until_ready(&client, &format!("{backend}/api/v1/health"), &mut child.0).await; // Source syntax uses the shared parser fork; serving semantics and exact // routing belong to the MetricsQL adapter and its installed query entries. - let snapshot = serde_json::from_value::(fixture).unwrap(); + let snapshot = serde_json::from_value::(fixture).unwrap(); let mut snapshot = snapshot; snapshot.environment.plan_version = 2; let compiled = quote_snapshot_for_frontend_test(snapshot, true) diff --git a/data_plane/tests/support/durable_summary_process.rs b/data_plane/tests/support/durable_summary_process.rs index 626392ae3..263452624 100644 --- a/data_plane/tests/support/durable_summary_process.rs +++ b/data_plane/tests/support/durable_summary_process.rs @@ -13,9 +13,9 @@ async fn persisted_summary_restarts_without_live_reregistration() { .to_owned(); fixture["query_workload"]["repeating_queries"] = serde_json::json!([fixture["query_workload"]["repeating_queries"][2].clone()]); - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); - let plan = quote_snapshot_for_test(snapshot).compile().unwrap(); + let plan = quote_snapshot_for_test(snapshot).compile_promql().unwrap(); let install = data_plane::drivers::query::servers::http::PhysicalPlanInstallRequest { summary_catalog: plan.summary_catalog, collector_plans: plan.collector_plans, diff --git a/data_plane/tests/support/erp_planning_process.rs b/data_plane/tests/support/erp_planning_process.rs index 6e88beac7..59b32c89b 100644 --- a/data_plane/tests/support/erp_planning_process.rs +++ b/data_plane/tests/support/erp_planning_process.rs @@ -1,5 +1,5 @@ use super::*; -use control_plane::physical::{compiler::BackendLocalPlanningSnapshot, erp::ErpShapeObserver}; +use control_plane::physical::{compiler::BackendLocalPlanningInput, erp::ErpShapeObserver}; fn measured_profiles(raw: &[f64]) -> Value { let mut records = Vec::new(); @@ -102,9 +102,8 @@ async fn observed_shape_selects_installed_parameters_and_executes_remote_write() ), control_plane::physical::erp::ErpParameterDecision::Empirical { .. } )); - let snapshot: BackendLocalPlanningSnapshot = - serde_json::from_value(fixture.clone()).unwrap(); - let plan = quote_snapshot_for_test(snapshot).compile().unwrap(); + let snapshot: BackendLocalPlanningInput = serde_json::from_value(fixture.clone()).unwrap(); + let plan = quote_snapshot_for_test(snapshot).compile_promql().unwrap(); assert_eq!( plan.precompute_plan.materializations.len(), 1, diff --git a/data_plane/tests/support/immutable_maintenance_process.rs b/data_plane/tests/support/immutable_maintenance_process.rs index f105cdfa6..a1bbf43fa 100644 --- a/data_plane/tests/support/immutable_maintenance_process.rs +++ b/data_plane/tests/support/immutable_maintenance_process.rs @@ -33,9 +33,9 @@ async fn run_maintenance_process(multi_source: bool, distinct_groups: bool) { entry["demand"]["fixed_interval_at"]["evaluation_phase"] = 0.into(); entry["time_selection"]["lookback"] = 60_000.into(); fixture["query_workload"]["repeating_queries"] = serde_json::json!([entry]); - let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_value(fixture.clone()).unwrap(); - let plan = quote_snapshot_for_test(snapshot).compile().unwrap(); + let plan = quote_snapshot_for_test(snapshot).compile_promql().unwrap(); assert_eq!( plan.precompute_plan.materializations.len(), if multi_source { 3 } else { 2 } @@ -303,15 +303,14 @@ async fn run_maintenance_process(multi_source: bool, distinct_groups: bool) { // singleton-derived output while new source populations can arrive. let mut next_fixture = fixture.clone(); next_fixture["environment"]["plan_version"] = 2.into(); - let next = - quote_snapshot_for_test( - serde_json::from_value::< - control_plane::physical::compiler::BackendLocalPlanningSnapshot, - >(next_fixture) - .unwrap(), + let next = quote_snapshot_for_test( + serde_json::from_value::( + next_fixture, ) - .compile() - .unwrap(); + .unwrap(), + ) + .compile_promql() + .unwrap(); let next_install = data_plane::drivers::query::servers::http::PhysicalPlanInstallRequest { summary_catalog: next.summary_catalog, collector_plans: next.collector_plans, diff --git a/data_plane/tests/support/physical_fixture.rs b/data_plane/tests/support/physical_fixture.rs index e896e9eee..f5fec8feb 100644 --- a/data_plane/tests/support/physical_fixture.rs +++ b/data_plane/tests/support/physical_fixture.rs @@ -4,7 +4,7 @@ use control_plane::{physical::compiler::*, query_plan::*}; use data_plane::{ drivers::query::servers::http::PhysicalPlanInstallRequest, - storage_engines::types::{ActivePhysicalPlan, BackendStorageRouting, StreamingConfig}, + storage_engines::types::{BackendStorageRouting, RuntimePhysicalPlan, StreamingConfig}, }; use std::{collections::BTreeMap, sync::Arc}; @@ -20,7 +20,7 @@ pub fn artifact(config: &StreamingConfig) -> PhysicalPlanInstallRequest { capability_snapshot_id: "transport-fixture".into(), }; let mut configs = config - .aggregation_configs + .materializations_by_policy_fingerprint .values() .cloned() .collect::>(); @@ -37,7 +37,7 @@ pub fn artifact(config: &StreamingConfig) -> PhysicalPlanInstallRequest { let mut precompute = PrecomputePlan::build(envelope.clone(), configs, &["fixture".into()]).unwrap(); precompute.summary_catalog = Some(catalog.reference().unwrap()); - let mut transmission = control_plane::physical::compiler::compile_transmission_plan( + let mut transmission = control_plane::physical::compiler::build_transmission_plan( envelope, &precompute, &BTreeMap::new(), @@ -157,8 +157,8 @@ pub fn artifact(config: &StreamingConfig) -> PhysicalPlanInstallRequest { } #[allow(dead_code)] -pub fn bootstrap() -> ActivePhysicalPlan { - let mut plan = data_plane::drivers::query::servers::http::build_active_physical_plan( +pub fn bootstrap() -> RuntimePhysicalPlan { + let mut plan = data_plane::drivers::query::servers::http::validate_and_build_runtime_plan( artifact(&StreamingConfig::default()), Arc::new(BackendStorageRouting::empty()), ) diff --git a/data_plane/tests/support/univmon_erp_process.rs b/data_plane/tests/support/univmon_erp_process.rs index 7c496d4ab..980166051 100644 --- a/data_plane/tests/support/univmon_erp_process.rs +++ b/data_plane/tests/support/univmon_erp_process.rs @@ -1,5 +1,5 @@ use super::*; -use control_plane::physical::{compiler::BackendLocalPlanningSnapshot, erp::ErpShapeObserver}; +use control_plane::physical::{compiler::BackendLocalPlanningInput, erp::ErpShapeObserver}; use data_plane::precompute_engine::operators::univmon_accumulator::UnivMonAccumulator; use data_plane::storage_engines::types::{AggregateCore, SerializableToSink}; @@ -124,8 +124,8 @@ async fn measured_readout_evidence_selects_and_executes_univmon() { "minimum_confidence": 0.7, "minimum_confidence_margin": 0.05}, "runtime": {"allowed_algorithms": ["Hll", "Kll", "UnivMon"], "max_memory_bytes": null} }); - let snapshot: BackendLocalPlanningSnapshot = serde_json::from_value(fixture.clone()).unwrap(); - let plan = quote_snapshot_for_test(snapshot).compile().unwrap(); + let snapshot: BackendLocalPlanningInput = serde_json::from_value(fixture.clone()).unwrap(); + let plan = quote_snapshot_for_test(snapshot).compile_promql().unwrap(); eprintln!( "UNIVMON_PLANNED {}", serde_json::json!({"query_plan": plan.query_plan, "materializations": plan.precompute_plan.materializations, "lifecycle_estimates": plan.lifecycle_estimates, "executable_dags": plan.precompute_plan.executable_dags, "observation": observation}) @@ -149,9 +149,9 @@ async fn measured_readout_evidence_selects_and_executes_univmon() { .remove("max_frequency_entropy_absolute_bits_error"); } let missing = quote_snapshot_for_test( - serde_json::from_value::(missing_entropy).unwrap(), + serde_json::from_value::(missing_entropy).unwrap(), ) - .compile() + .compile_promql() .unwrap(); use control_plane::query_plan::{QueryPlanNode, QueryReadout}; assert!(missing @@ -300,9 +300,9 @@ async fn measured_readout_evidence_selects_and_executes_univmon() { plan.summary_catalog.reference().unwrap() ); if key.sketch == "univmon" { - let mut live_snapshot: BackendLocalPlanningSnapshot = + let mut live_snapshot: BackendLocalPlanningInput = serde_json::from_value(fixture.clone()).unwrap(); - let policy = live_snapshot.implementation.erp.as_mut().unwrap(); + let policy = live_snapshot.physical_inputs.erp.as_mut().unwrap(); policy.observed_shape_source = Some(control_plane::physical::erp::ErpObservedShapeSource { source: key.source.clone(), @@ -328,7 +328,9 @@ async fn measured_readout_evidence_selects_and_executes_univmon() { .unwrap() .invalid_reason .is_none()); - let replanned = quote_snapshot_for_test(live_snapshot).compile().unwrap(); + let replanned = quote_snapshot_for_test(live_snapshot) + .compile_promql() + .unwrap(); assert!( replanned .precompute_plan From a2744f1b2687197eea5884f67e62520733d8d790 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 13 Sep 2026 11:14:10 -0600 Subject: [PATCH 2/3] docs: explain planning architecture and naming migration --- docs/developer_docs/README.md | 2 + .../architecture-naming-review.zh.md | 192 ++++++++++++++++++ .../control-plane/physical-compiler.md | 48 ++--- .../control-plane/planning-terminology.md | 110 ++++++++++ .../data-structure-ownership-audit.md | 20 +- 5 files changed, 339 insertions(+), 33 deletions(-) create mode 100644 docs/developer_docs/control-plane/architecture-naming-review.zh.md create mode 100644 docs/developer_docs/control-plane/planning-terminology.md diff --git a/docs/developer_docs/README.md b/docs/developer_docs/README.md index 183524ca3..cc35304e9 100644 --- a/docs/developer_docs/README.md +++ b/docs/developer_docs/README.md @@ -27,6 +27,8 @@ cross-repository interfaces live in ## Control plane - [Physical compiler](control-plane/physical-compiler.md) +- [Planning terminology and architecture](control-plane/planning-terminology.md) +- [Architecture naming review (中文)](control-plane/architecture-naming-review.zh.md) - [Plan publication](control-plane/plan-publication.md) - [Workload inputs](control-plane/workload-inputs.md) - [Runtime accuracy feedback](control-plane/runtime-accuracy-feedback.md) diff --git a/docs/developer_docs/control-plane/architecture-naming-review.zh.md b/docs/developer_docs/control-plane/architecture-naming-review.zh.md new file mode 100644 index 000000000..6b0dc2be4 --- /dev/null +++ b/docs/developer_docs/control-plane/architecture-naming-review.zh.md @@ -0,0 +1,192 @@ +# ASAPQuery-backend 架构与命名审查 + +审查基线:GitHub `main`,`b1a58ca810d7b7347f2cc5f30924db56532bbb6b`,2026-09-13。关联 Issue:[#709](https://github.com/ProjectASAP/ASAPQuery-backend/issues/709)。面向维护规划、共享契约、安装及执行代码的开发者。 + +结论:#709 应从“术语替换表”改为“组件边界和数据语义对齐”。优先澄清逻辑候选、物理候选、报价对象、部署状态和存储身份,再决定具体命名。不能因为几个对象都包含计划字段,就把它们合成一个类型;也不能因为都叫 `SummaryNode`,就认为它们只表示近似查询。 + +本文件保留实施前的架构审查与决策依据。当前实现及迁移规则见 [Planning terminology and architecture](planning-terminology.md)。其中明确列出此次命名迁移与保留为后续工作的结构调整。 + +**本 PR 的最终落地与迁移方式** + +下表记录最终采用的名称;后文保留审查时的备选与理由。实施基线已更新为 `cb3153a8`(#710),继续复用 Planner 的精度类型。 + +| 所属边界 / 源码 | 原名称 → 最终名称 | +|---|---| +| [编译输入与编译器](../../../control_plane/src/physical/compiler.rs) | `BackendLocalPlanningSnapshot` → `BackendLocalPlanningInput`;`BackendLocalImplementation` → `BackendLocalPhysicalInputs`;`PlanningQuery` → `QueryCompilationInput`;`PlanningRequest` → `PhysicalCompilationRequest`;`PhysicalCompiler` → `PhysicalPlanCompiler`;`PhysicalPlan` → `CompiledPhysicalPlan` | +| 同上:查询语义 | `post_asap` → `selected_plan_root`;`source` → `legacy_query_source`;`window_secs` → `query_lookback_seconds`;`group_by` → `group_by_labels`;`accuracy` → `accuracy_target`;`lifecycle` → `summary_lifecycle_inputs`;`runtime_policy` → `materialization_runtime_policy` | +| 同上:候选与证据 | `logical_selection` → `planner_selection_trace`;`materialization_policy` → `enabled_materialization_keys`;`evidence` → `topk_membership_evidence_by_query_id`;`hybrid_execution` → `allow_mixed_summary_and_exact_execution`;`synthesized_window_queries` → `compiler_priced_window_query_ids` | +| 同上:窗口与成本 | `WindowImplementationCandidate` → `WindowRealizationCandidate`;`ImplementationCostEvidence` → `WindowRealizationCostQuote`;`LifecycleCostEvidence` → `LifecycleUnitCosts`;`LifecyclePlanningInput` → `SummaryLifecyclePlanningInputs`;`window_implementations` → `window_realization_candidates` | +| 同上:外层配置 | `snapshot_version` → `schema_version`;`implementation` → `physical_inputs`;`window_candidates` → `window_realization_candidates_by_query`;默认 `window_implementation_id` → `default_window_realization_id`;`implementation_cost` → `default_window_cost_quote`;`max_retained_summary_bytes` → `retained_summary_memory_budget_bytes` | +| 同上:部署与保留 | `DeploymentEnvironment` → `PhysicalDeploymentContext`;`collector_ids` → `target_collector_ids`;`query_staleness_margin_ms` → `query_retention_margin_ms` | +| [候选定价与选择](../../../control_plane/src/physical/workload_cost.rs) | `CostDemand` → `CostComponentDemand`;`unit` → `pricing_basis`;`multiplicity` → `occurrences_per_horizon`;`AlternativeCost` → `CandidatePlanEvaluation`;`WorkloadCostComparison` → `CandidatePlanSelectionReport`;`alternative_id` → `candidate_id`;`physical_alternative_id` → `physical_candidate_id`;`alternatives` → `candidate_evaluations` | +| 同上:方法 | `with_exact_alternative` → `enumerate_exact_and_materialized_candidates`;`bind_alternative` → `compile_candidate_for_pricing`;`prepare_manifests` → `compile_candidates_for_pricing`;`select` → `select_lowest_cost_candidate`;`select_metricsql` → `select_lowest_cost_metricsql_candidate` | +| [诊断状态](../../../control_plane/src/physical/workload_cost/status.rs) | `status` 使用 `CandidateEvaluationStatus`;覆盖范围使用 `CandidateSearchScope`;兼容未知字符串与原缺省值。原 `PhysicalQueryFrontend` 移到共享编译入口并命名为 `QueryFrontend`,替代内部布尔参数 | +| [共享运行时算子](../../../crates/asap_types/src/query_plan/residual.rs) | `query_plan::logical::LogicalOperator` → `query_plan::residual::ResidualQueryOperator`;backend 对应模块迁移到 `residual`,保留旧模块导出 | +| [运行时计划与句柄](../../../data_plane/src/storage_engines/types/hot_reload_config.rs) | `ActivePhysicalPlan` → `RuntimePhysicalPlan`;`HotReloadActivePhysicalPlan` → `ActivePhysicalPlanHandle`;`HotReloadStreamingConfig` → `StreamingConfigHandle`;`runtime_config` → `streaming_config`;active handle 的 `snapshot` → `active_snapshot`;`from_active` → `from_active_physical_plan`;`retire_drained` → `mark_drained_plan_retired` | +| [存储元数据](../../../data_plane/src/storage_engines/sketch_db/index/mod.rs) | `SketchInstanceMetadata` → `SummarySeriesMetadata`;`sketch_index` 字段与变量 → `summary_store`;保留 `SketchStore` 类型 | +| 其余调用边界 | `types::QueryWorkload` → `LegacyMetricWorkload`;`ClickHouseSqlWorkload.sds` → `summary_catalog`;`aggregation_configs` → `materializations_by_policy_fingerprint`;`aggregation_id_for_key/value` → `key_policy_fingerprint/value_policy_fingerprint`;`data_plane::monitor` → `update_sampling` | + +迁移规则:**Rust 名称更新,输出 wire 名称保持原样**;反序列化接受新名称作为 alias。旧公共类型导入及主要入口提供 deprecated 转发,但 Rust struct literal 的旧字段拼写无法通过类型别名兼容,源码消费者需要按表迁移。`None` / 空候选集合、浮点频次、报价身份、候选排序、严格小于的选择规则及计划生命周期均保持原义。 + +`erp` 保留并明确为 Error–Resource Profile;`WorkloadQuote.executable` 保留,因为它不代表安装或部署已经通过验证。双模式 streaming handle 的读写行为只补充说明;拆分写 API、删除 legacy 路径、统一 SQL 定价以及 typed ID / 时间单位迁移留作独立工作。英文流程图与边界说明见 [Planning terminology and architecture](planning-terminology.md)。 + +**一、组件职责与当前真实路径** + +| 组件 | 实际职责 / 输入输出 | 命名判断 | +|---|---|---| +| 外部 ASAPPlanner | 解析与语义 IR,合法 summary/exact 候选,精度推理,逻辑选择;backend 提供具体成本和能力约束 | Planner 的语义选择与 backend 的物理候选比较是不同层次,不是两个重复 planner | +| `control_plane::planner_selection` | 适配 Planner 的选择调用、精度及证据;输出语义 DAG 与诊断 trace | `selection` 必须说明是 logical 还是 physical;trace 不是决定执行行为的配置 | +| `physical::compiler` | 输入规范化、窗口候选校验、调用逻辑选择,以及绑定具体物理实现,生成多个一致的计划投影 | `PhysicalPlanCompiler` 比 `PhysicalCompiler` 清楚;整个模块当前职责仍比单纯 lowering 更宽 | +| `physical::workload_cost` | 枚举工作负载级候选、编译、生成报价清单、核验报价、选择最低成本可行候选 | manifest、quote、evaluation、selection report 不应相互替代 | +| `physical::erp` | Error–Resource Profile 的部署适配、分布匹配、经验参数与资源估计 | `empirical_runtime_profile` 是错误展开;输入还包含策略与观测,不只一个 profile | +| `control_plane::clickhouse` | SQL frontend、逻辑选择、物理绑定;支持已有 catalog 输入与自动生成 materialization 两条路径 | `ClickHouseSqlWorkload.sds` 实际是 `SummaryCatalog`;SQL 当前没有走同一套 `workload_cost::select` 整计划报价流程 | +| `asap_types` | 跨组件共享的 catalog、SDS、生产/传输/预计算/查询/发布契约与验证逻辑 | 是共享契约的实际定义方;`control_plane` 中若仅 re-export,不是重复类型 | +| `physical::publication` + backend client + OpAMP | 从编译结果构造发布制品,安装请求、collector 发布与应用确认、backend 激活 | publication 是制品或发布过程;install request 是命令;不是 active runtime 对象 | +| data-plane HTTP 安装 + `PhysicalPlanLifecycle` | 验证跨计划一致性,stage、activate、drain、retire | `ActivePhysicalPlan` 被用于 staged map,名称把类型和生命周期阶段混在一起 | +| OTLP / Remote Write drivers + `SeriesIdResolver` | 协议接入、生产者/世代/帧校验、分配并解析物理 series 身份 | `collector_id` 是特定生产者身份;不能把所有 producer 无条件改叫 collector | +| `PrecomputeEngine` + workers + maintenance runtime | 按安装的 materialization 更新状态;派生维护消费已完成的源状态 | `PrecomputeEngineConfig` 是 worker/队列等引擎设置,`StreamingConfig` 是 materialization 的运行时视图,不是重复配置 | +| `SketchStore` + backfill / persistence | 保存 sketch 和精确聚合状态;SID 索引、覆盖与完整性、恢复和历史填充 | `SketchStore` 已比名称更广;领域表述宜用 summary store。全仓类型迁移可独立进行 | +| query engines / routing / summary execution | 用同一计划快照执行已安装 DAG、读取指定状态、执行精确子树或回退 | 生产路径不能在读请求里重新选择 materialization;legacy helper 应标明边界 | +| `control_plane::monitor` / `replan` | 旧工作负载路径上的指标抓取、违规/过期触发重规划与发布 | 与运行时 ERP 观测不是同一条自动闭环,不应画成一个通用 feedback planner | +| `data_plane::monitor` | 给边缘生产者协调 update-sampling grant | `monitor` 太宽,与 CP 的指标/SLA monitor 不同;建议模块领域名 `update_sampling` | +| `asap_otel_proto` | OTLP 等生成协议类型 | 保持协议边界;不能把协议 DTO 当成语义 catalog | +| `tools` / `scripts` / demos | 校准、离线证据、回放、评测与操作工具;有 JSON 消费者 | 序列化迁移必须审计这些调用方;它们不是运行时执行组件 | + +英文流程架构图见 [Planning terminology and architecture](planning-terminology.md)。下文表格的“当前名称”指审查基线上的名称,右栏是迁移建议。 + +**二、不要把系统画成一条所有入口都相同的流水线** + +PromQL / MetricsQL 的主要计划路径是: + +`工作负载与证据 → Planner 语义选择 → 工作负载级物理候选 → 编译结果与 manifest → provider quote → 最低成本可行候选 → publication → runtime generation` + +这里有三个不同范围的选择:语义 DAG 的选择、单个窗口物理实现的选择、整个工作负载物理候选的成本比较。它们并非重复实现。`with_exact_alternative` 的有界枚举也不保证找到所有未枚举实现的全局最优值。相同成本时,当前严格 `<` 比较保留先出现的候选。[编译器][C1]、[候选与定价][C2] + +`prepare_manifests()` 编译后丢弃 plan,仅返回 manifests 和诊断行;部署选择阶段重新编译候选,再与证据中的 manifest 做完整相等比较。这是“发现/采集成本”和“使用成本部署”两个阶段,不应因为函数都做 compile 就直接删除其中一个。[C2] + +ClickHouse 的自动路径直接从 SQL 选择与绑定构建 `PhysicalPlanPublication`;已有 catalog 路径接收 `sds + precompute_plan + transmission_plan`。它们与 PromQL 共用安装及运行时契约,但不能画成已经共用相同的整计划报价选择过程。建议两个请求对象分别叫 `ClickHouseCompilationInput`、`ClickHouseBindingInput`,或保留现名并明确 automatic / catalog-bound 区别。[C3] + +旧 flat workload → `CollectionPlan` / stage emission 路径仍有实际调用;`PhysicalPlanner`、`PlanNode`、`PostAsapPlan` 等旧模块也留在公开模块树中。不要据名字推断它们都与当前 `PhysicalPlan` 等价,也不要据旧模块存在就断定它们全部处于主路径。[C4] + +**三、#709 中应该修正或收紧的映射** + +| 当前值 / 提议 | 实际语义 | 建议 | +|---|---|---| +| `erp` → `empirical_runtime_profile` | ERP = **Error–Resource Profile**;`ErpPlanningInput` 还含观测、匹配策略、mode、权重和能力 | 保留 `erp` 并写明定义,或字段 `error_resource_planning_input`;不要使用错误全称 | +| `PlanningQuery` → `SelectedQueryInput` | 对象先构造,再被 `select_workload_roots_with_trace(&mut queries, ...)` 更新;并非整个生命周期都已 selected | 倾向 `QueryCompilationInput`,不用类型名承诺当前代码没有保证的阶段 | +| `post_asap` → `selected_summary_plan_root` | Planner 选择的语义 DAG 根,也可装 `KeepPreAsap` exact 根 | `selected_plan_root` / `selected_logical_root`;不要暗示必定 materialized 或 approximate | +| `materialization_candidate_keys()` → `optional_materialization_ids()` | 编译前枚举用的候选 key,与 catalog 中的 `SummaryDefinitionId` 不是一类身份 | `eligible_materialization_keys()`;保留 key 与已绑定 ID 的区别 | +| `materialization_policy` → `enabled_optional_materializations` | `Option>`:`None` 启用全部 eligible,`Some(empty)` 不启用任何可选项 | `enabled_materialization_keys`,保留并明确三态语义;不要默认成普通空集合 | +| `materialization_leaf_contract` → 候选 keys | 函数实际上返回原始 source 的 metric、可选 window、filter contract | `raw_materialization_input_contract`;真正枚举 keys 的函数另行命名 | +| 所有 `leaf` 改为 materialization | DAG 结构叶节点、物化候选、执行中的 exact subtree 是不同对象 | 按消费点改;真正的图论 leaf 可保留,不能机械替换 Planner 术语 | +| `CostDemand.unit` → `cost_unit` | 值为 `horizon` / `query_evaluation`,表达报价的工作量基准,并非 CPU 秒、字节或货币单位 | `pricing_basis` / `demand_basis`;可定义 `PricingBasis` 枚举 | +| `multiplicity` → `occurrences_per_horizon` | 基准工作在 horizon 内的乘数;允许按频率得到浮点估计 | 支持;不可顺手改为整数。总成本是 quote × occurrences | +| `LifecycleCostEvidence` → `LifecycleCostRates` | 同时包含 build/read/retirement 单次成本和 maintenance/retention rate | `LifecycleCostModel` / `LifecycleUnitCosts`,不应全称 rates | +| `WorkloadQuote.executable` → `deployable` | provider 对匹配 manifest 的执行可行性声明;还需 compiler、证据、安装与发布验证 | 保留 `executable` 或用 `provider_feasible`;不要暗示已通过端到端部署检查 | +| `synthesized_window_queries` → `queries_with_derived_window_candidates` | 编译器生成报价的 query ID 集合,决定能否做共享 pane 重定价 | `compiler_priced_window_query_ids`;derived 已用于“由 summary 派生 summary”,易混淆 | +| `collector_ids` → `eligible_collector_ids` | 分布式模式下,每个 ID 都生成 `CollectorPlan` | `target_collector_ids`;代码没有在这些 eligible collector 中再选子集 | +| `query_staleness_margin_ms` → `max_query_staleness_ms` | 为滞后的查询增加保留状态量;自身不实现请求拒绝规则 | `query_retention_margin_ms`,或保留现名并说明用途;避免暗示已有 admission enforcement | +| `BackendLocalImplementation` → `BackendLocalPlanningInputs` | 外层也是 planning input;内层含成本、窗口候选、证据、ERP 与保留约束 | 外层 `BackendLocalPlanningInput`,内层倾向 `BackendLocalPhysicalInputs`;不为这些分类额外创建一组 wrapper | +| `ActivePhysicalPlan` 保持不动,只改 handle | 同一类型也用于 staged 计划和 draining 的旧快照 | 类型倾向 `RuntimePhysicalPlan`;handle 用 `ActivePhysicalPlanHandle`,phase 由 lifecycle 管理 | +| `build_active_physical_plan` → `validate_and_build_active_physical_plan` | 校验与构建,但不执行 activation | `validate_and_build_runtime_plan`;否则名字仍把构建和激活混淆 | +| 泛化的 `snapshot()` → `active_snapshot()` | active-plan handle 与独立 streaming-config handle 的快照含义不同 | 只在能保证 active 语义的 handle 上使用;不可全仓替换 | +| `runtime_policy` → `materialization_runtime_policy` | `RuntimeRulePolicy` 具体控制 sampling、delta、GOS 与 adaptation | 可接受;跨组件说明是 producer/update/transmission policy,不能与候选选择策略混淆 | + +依据:[编译输入与选择 C1][C1]、[候选与定价 C2][C2]、[ERP C5][C5]、[候选 key C6][C6]、[pane 重定价 C7][C7]、[运行时生命周期 C8][C8]。 + +以下映射方向正确,可以作为同一轮小范围迁移: + +| 当前名称 | 建议名称 / 约束 | +|---|---| +| `PlanningRequest` | `PhysicalCompilationRequest`;包含 workload 上下文,不是单 query | +| `PhysicalCompiler` | `PhysicalPlanCompiler` | +| `PhysicalPlan` | `CompiledPhysicalPlan`;候选和获选结果可继续复用此类型,无须新增 `SelectedPhysicalPlan` wrapper | +| `logical_selection` | `planner_selection_trace`;说明只做诊断 | +| `window_implementations` | `window_realization_candidates`;对象 `WindowImplementationCandidate` 也应相应命名 | +| `ImplementationCostEvidence` | `WindowRealizationCostQuote`;保留 model、时效与 workload scope | +| `window_candidates` | `window_realization_candidates_by_query`;注明 key 为注册查询文本,不是 query ID | +| `window_implementation_id` | 默认模板处用 `default_window_realization_id`,已选估计项用 `window_realization_id`,不能全仓统一加 default | +| `PlanningQuery.window_secs` | `query_lookback_seconds`;不可连带重命名其他类型的所有 window 字段 | +| `PlanningQuery.source` | `legacy_query_source` 可接受,但该值也写入 workload cost manifest,不能借改名删除或改变其报价身份 | +| `group_by` / `accuracy` | 在相应输入上用 `group_by_labels` / `accuracy_target`;输出精度结果不能改叫 target | +| `lifecycle` / `LifecyclePlanningInput` | `summary_lifecycle_inputs` / `SummaryLifecyclePlanningInputs` | +| `PlanningRequest.evidence` | `topk_membership_evidence_by_query_id`;输入 map 的 `topk_evidence` 是按查询文本,不应加 by_query_id | +| `hybrid_execution` | `allow_mixed_summary_and_exact_execution`;是允许编译的模式,不是本次请求必然采用混合执行 | +| `bind_alternative` / `prepare_manifests` | `compile_candidate_for_pricing` / `compile_candidates_for_pricing`;后者返回清单及诊断,不返回编译 plans | +| `with_exact_alternative` | `enumerate_exact_and_materialized_candidates`;保留有界搜索与枚举顺序 | +| `AlternativeCost` | `CandidatePlanEvaluation` 可接受,但必须允许尚未定价或编译失败;更中性的 `CandidatePlanDiagnostic` 也符合现有全阶段用途 | +| `WorkloadCostComparison` | `CandidatePlanSelectionReport` | +| `select` / `select_metricsql` | 可用 `select_lowest_cost_candidate` / `select_lowest_cost_metricsql_candidate`;文档写明 feasible 与枚举范围 | +| `metricsql: bool` | PromQL/MetricQL 路径使用 `QueryFrontend`;先审计已有 `PhysicalQueryFrontend`,不要再建第三份枚举;SQL 并未因此自动适配此入口 | +| `CostDemand` | `CostComponentDemand` | +| `publication()` | `to_publication_artifact()`,或沿用 `publication()`;它构造并校验制品,不执行 HTTP 发布 | +| `compile_transmission_plan` | `build_transmission_plan`;从 precompute 和 runtime policies 构造,不一定要把全部参数编码进长函数名 | +| `DeploymentEnvironment` | `PhysicalDeploymentContext`;target、capability、time、generation 都是上下文 | +| `max_retained_summary_bytes` | `retained_summary_memory_budget_bytes`,保留旧缺省值、别名与具体计量定义 | +| `ActivePhysicalPlan.runtime_config` | `streaming_config`,准确表达持有的配置类型 | +| `aggregation_configs` | `materializations_by_policy_fingerprint`;目前 key 是 `u64` 形式 fingerprint,改名不等于已经升级 typed key | +| `get_all_aggregation_configs()` | `materializations()`;明确当前仍返回 map | +| `from_active()` | `from_active_physical_plan()` | +| `retire_drained()` | `mark_drained_plan_retired()`;它标记 lifecycle status,不执行存储 GC | + +**四、全仓中 #709 漏掉的歧义与重复** + +| 对象 | 判断 | 最小处理 | +|---|---|---| +| backend `types::QueryWorkload` 与 Planner `workload::QueryWorkload` | **同名不同义**:前者是单 metric 的 flat legacy 意图;后者是 canonical 工作负载 | legacy 侧叫 `LegacyMetricWorkload` 或导入时显式 alias | +| `PlanNode` / `CollectionPlan` / `PhysicalPlan` / `QueryPlan` | **不同层级**:旧 annotated stage tree、旧采集方案、完整物理 bundle、工作负载的查询执行条目集合 | 优先给旧类型/模块加 legacy 或 stage 语义;保留新契约的区别 | +| `QueryPlan` 与 `QueryPlanEntry` | 前者实际上是按 canonical identity 查找的一组查询执行计划 | 可将前者描述为 query execution catalog;若做 API 迁移再考虑 `QueryExecutionCatalog` / `QueryExecutionPlan`,#709 不必强行扩展 | +| `query_plan::logical::LogicalOperator` | **阶段误导**:内容是已安装的 residual 运行时算子,包括 exact subquery、scan、aggregate | `ResidualQueryOperator`,模块 `residual`;对应 `prepare_logical` 按实际职责命名 | +| `ClickHouseSqlWorkload.sds` | **层级混淆**:保存整个 `SummaryCatalog`,不是单个 Self-Describing Summary | `summary_catalog` | +| `AggregationConfig` 与 `PrecomputeMaterialization` | **同一个类型的兼容别名**,不是两种配置对象 | 新代码用 `PrecomputeMaterialization`;保留旧 alias 过渡,不复制定义 | +| `AggregationIdInfo.aggregation_id_for_key/value` | **历史身份名**:注释明确已是 policy fingerprint 的 u64 | 使用 `key_policy_fingerprint` / `value_policy_fingerprint`;typed ID 迁移单独评估 | +| `SummaryDefinitionId` 与 `PolicyFingerprint` | **合理的语义 wrapper**:前者是 catalog 中的定义引用,后者是兼容内容 fingerprint | 跨 catalog API 优先前者;不能把它与 SID 或 descriptor ID 合并 | +| `SketchInstanceMetadata` | **instance 的粒度错误**:按 SID 管理长期 series 元数据;真正 SDS instance 另外包含具体时间范围 | `SummarySeriesMetadata`,避免与 `SummaryInstance` 混为一个窗口对象 | +| `SketchStore` / `sketch_index` | **名称比职责窄**:对象还保存 exact accumulator、持久化、生命周期及状态读取 | 文档统一称 summary store;`sketch_index` 变量可先改 `summary_store`,全面类型迁移另行处理 | +| `SummaryCatalog` / `sketch_catalog` / SQL catalog / `PolicyRegistry` | **不是重复 catalog**:已安装定义、算法候选/默认参数、源表 schema、运行时 config 派生 lookup | 分别明确 `summary catalog`、`sketch capabilities`、`source schema catalog`、`materialization lookup` | +| `PrecomputePlan.materializations` 与 `StreamingConfig.aggregation_configs` | **合理投影**:安装验证后构建的运行时 map,不是两个应独立编辑的配置真源 | 文档声明 runtime config 来源;保证同一世代,不因字段重复删除验证 | +| `PhysicalPlan` / Publication / InstallRequest / Runtime plan | **合理边界重复**:编译诊断、共享发布制品、带 routing/evidence 的安装命令、持有 Arc 的运行时对象 | 继续使用边界构造和交叉验证;无需一个承载全部阶段的大对象 | +| `HotReloadStreamingConfig.inner` 与 `.active` | **真实双来源**:active 模式读 active plan,`swap` 仍写 inner,写入不成为 snapshot 的结果 | 先将模式写清;若拆为 legacy 可写句柄与 active 只读视图,必须独立定义 API 行为与测试 | +| `summary_exec` / `summary_executor` | 通用执行接口/调度与具体 store-backed 实现,名称近似但不是可直接去重的函数集合 | 更明确的 module docs;若迁移可用 `summary_execution` / `store_summary_reader`,保持执行语义边界 | +| `control_plane::monitor` 与 `data_plane::monitor` | **同名不同功能**:指标/违规监控 vs 更新采样分配 | 用职责名区分,不将它们都当作 ERP 实时反馈 | +| `summary_catalog` / `sds` / `canonical` 中的 re-export | **兼容入口,不是重复实现** | 建立 canonical import 路径后逐步减少旧入口;不要复制 shared DTO | + +依据:[旧 workload C4][C4]、[共享 materialization C9][C9]、[SDS C10][C10]、[catalog C11][C11]、[runtime config C12][C12]、[存储元数据 C13][C13]、[residual operators C14][C14]、[shared publication C15][C15]。 + +**五、必须保留的身份、时间和生命周期区别** + +`plan_id / plan_version` 标识部署决定与世代;`CatalogGeneration` 额外携带 catalog 的摘要。`SummaryDescriptorId` 表达算法/状态语义,`DataDescriptorId` 表达输入 population;`SummaryDefinitionId` 绑定这些定义与物理布局。SID 表达具体物理 series 生命周期,而 `SummaryInstance` 表达具体时间范围和 group 的状态,并携带物理引用、来源与完整性。不能因为底层有些都用 u64,就全部命名为 `id` 或 `materialization_id`。[SDS][C10]、[Catalog][C11]、[存储元数据][C13] + +`window` 至少有 query lookback、evaluation interval、stored pane duration、slide、origin、retention、producer emission cadence 等含义。`PrecomputeMaterialization::stored_window_ms()` 对 FullWindow 与 pane layout 的处理不同,不能全仓把 `window_size` 替换成 `query_lookback_seconds`。改名应保留各字段当前单位;单位转换属于额外行为变更。[C9] + +生命周期至少有三条轴:计划 `Staged / Active / Draining / Retired`,materialization `Materializing / Ready / Serving`,以及具体实例的 completeness。一个计划已 Active,并不表示其每个查询范围都已完整;读路径仍需校验状态。`retire_drained` 只是状态标记,不表示已经回收所有持久化数据。[计划生命周期][C8]、[SDS][C10] + +`snapshot` 对 immutable catalog 和 `Arc` 读取是合适术语,不应一律删除。对于 `BackendLocalPlanningSnapshot`,代码主要将其消费为输入,改成 input 合理;但要同时决定 `snapshot_version` 的 schema 名称和兼容策略。 + +**六、迁移与验收建议** + +1. 先写下字段语义、所属阶段、key 范围和单位;修正 ERP、成本基准、runtime/staged、target collectors 等会误导实现的名称。把 #709 正文与补充评论合并为一份不冲突的映射。 +2. 做内部函数/局部变量改名,优先保持数据结构和算法原样。不要为每个生命周期步骤引入一个新 wrapper。`physical` 的旧路径隔离、hot-reload handle 拆分和 shared schema 版本升级分别处理。 +3. 公共 Rust 字段改名不能靠 type alias 或 forwarding method 保持兼容。先识别实际外部消费者;crate 根文档也声明部分 public module 仅供 workspace 内使用,因此不需要不加区分地给每个 public symbol 建长期兼容层。 +4. 保留 wire 输出时,用 `serde(rename = "old_name")` 固定旧名称,视需要接受新名 alias。仅 `alias = "old_name"` 无法保护旧消费者读取新输出。还需检查手工 JSON/YAML 生成、Python 校准与回放工具、CLI 输出和已有 fixture。 +5. 名称变化可能影响身份:catalog digest 对序列化内容求 SHA-256,编译计划也存在对序列化 materialization 求 hash 的路径。验收应检查 plan/materialization/catalog identity、component keys、manifest 匹配不变,不只是能够 deserialize。 +6. 枚举 `status` 与 `search_scope` 时保留原 wire 表达,或明确版本迁移。现有 status 含 `bind_failed`、`bound`、`evidence_missing`、`rejected`、`evidence_invalid`、`unselected`、`selected`,且 `status` 缺省为空字符串。typed scope 还应保留“bounded inventory 不承诺未枚举最优”的解释信息。 +7. 运行既有 compiler、cost selection、publication、安装 lifecycle、query execution 与 backend process E2E。针对实际 wire/identity 变动补兼容断言:双向读写、缺省行为、None/empty 集合、同成本选择顺序、staged 不生效、旧读者 drain。纯局部改名无需逐字段补镜像测试。 + +额外维护问题:`docs/developer_docs/data-structure-ownership-audit.md` 仍说多个 wire DTO 在 control_plane、尚待搬迁,但当前已经在 `asap_types`;`PrecomputeEngine` 注释还称 Remote Write 已删除,而 driver 与启动代码中已存在。命名 PR 应同步修正相关陈旧说明,否则新的词汇表仍会被旧架构描述覆盖。 + +[C1]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/control_plane/src/physical/compiler.rs +[C2]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/control_plane/src/physical/workload_cost.rs +[C3]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/control_plane/src/clickhouse.rs +[C4]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/control_plane/src/types.rs#L266 +[C5]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/control_plane/src/physical/erp.rs +[C6]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/control_plane/src/query_plan/logical.rs#L1105 +[C7]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/control_plane/src/physical/pane_reuse.rs +[C8]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/data_plane/src/storage_engines/types/hot_reload_config.rs +[C9]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/crates/asap_types/src/aggregation_config.rs +[C10]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/crates/asap_types/src/sds.rs +[C11]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/crates/asap_types/src/summary_catalog.rs +[C12]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/data_plane/src/storage_engines/types/streaming_config.rs +[C13]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/data_plane/src/storage_engines/sketch_db/index/mod.rs#L209 +[C14]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/crates/asap_types/src/query_plan/logical.rs +[C15]: https://github.com/ProjectASAP/ASAPQuery-backend/blob/b1a58ca810d7b7347f2cc5f30924db56532bbb6b/crates/asap_types/src/plan_publication.rs diff --git a/docs/developer_docs/control-plane/physical-compiler.md b/docs/developer_docs/control-plane/physical-compiler.md index 97fdd621d..280f224ec 100644 --- a/docs/developer_docs/control-plane/physical-compiler.md +++ b/docs/developer_docs/control-plane/physical-compiler.md @@ -8,7 +8,7 @@ The compiler consumes ASAPPlanner types pinned to the revision exposed as `physical::compiler::PLANNER_REVISION`, selects from Planner's legal candidate space with backend-owned cost and evidence -inputs, and emits one `PhysicalPlan`. The plan contains one SummaryCatalog plus +inputs, and emits one `CompiledPhysicalPlan`. The plan contains one SummaryCatalog plus CollectorPlan, PrecomputePlan, TransmissionPlan, and QueryPlan projections compiled from the same decision for every target collector. Legacy @@ -28,14 +28,14 @@ compiler never invents a framework or assigns it an optimistic zero cost. The control plane has three public layers: ```text -PlanningRequest + DataWorkload + concrete implementation evidence +PhysicalCompilationRequest + DataWorkload + concrete implementation evidence | ^ | abstract candidates | complete physical costs v | -ASAPPlanner selection <---------- PhysicalCompiler +ASAPPlanner selection <---------- PhysicalPlanCompiler | v -PhysicalCompiler -------> PhysicalPlan +PhysicalPlanCompiler -------> CompiledPhysicalPlan | | | | v v v v Collector Precompute Backend Query @@ -48,7 +48,7 @@ PhysicalCompiler -------> PhysicalPlan - **Physical compiler** enumerates concrete window/pane/state-layout, placement, transport, and runtime implementations without changing the Planner-owned abstract framework. -- **PhysicalPlan** is the only output passed to publication. Its SummaryCatalog, +- **CompiledPhysicalPlan** is the only output passed to publication. Its SummaryCatalog, CollectorPlan, PrecomputePlan, TransmissionPlan, and QueryPlan are created together and share identities. Logical query parsing, summary alternatives, guarantees, and candidate search @@ -59,13 +59,13 @@ remain public ASAPPlanner interfaces. Runtime publication is documented in ### Planning request -`PlanningRequest` is backend-owned request context around Planner's canonical +`PhysicalCompilationRequest` is backend-owned request context around Planner's canonical per-query IR: ```rust -pub struct PlanningRequest { - pub queries: Vec, - pub evidence: HashMap, +pub struct PhysicalCompilationRequest { + pub queries: Vec, + pub topk_membership_evidence_by_query_id: HashMap, pub planner_revision: String, } ``` @@ -74,13 +74,13 @@ Input definitions: | Field | Definition | | --- | --- | -| `queries` | Canonical `QueryExpr`, source, grouping labels, accuracy, stable query ID, `DataWorkload`, and executor-feasible `window_implementations`. | -| `evidence` | Optional typed TopK membership certificates keyed by query ID. | +| `queries` | Selected Planner DAG roots, query identities, source metadata, grouping labels, accuracy targets, lifecycle inputs, and executor-feasible `window_realization_candidates`. | +| `topk_membership_evidence_by_query_id` | Optional typed TopK membership certificates keyed by query ID. | | `planner_revision` | Immutable Planner build/revision used for reproducibility. | TopK evidence is accepted only when its selected lower bound is strictly above the excluded upper bound, its failure probability is valid, its source is -non-empty, and its observation is fresh under `DeploymentEnvironment`. +non-empty, and its observation is fresh under `PhysicalDeploymentContext`. ```rust pub struct TopKMembershipEvidence { @@ -95,7 +95,7 @@ pub struct TopKMembershipEvidence { Why this interface exists: it prevents adapters, protocols, and physical planning from each implementing their own query-to-summary mapping. -Each `WindowImplementationCandidate` carries a backend-owned implementation +Each `WindowRealizationCandidate` carries a backend-owned implementation identity, its Planner `SummaryWindowFramework`, concrete window/pane/state layout, and versioned workload-specific CPU, peak-memory, network, storage, scan, and calibrated weighted-cost evidence. The compiler collapses several @@ -106,18 +106,18 @@ identity. Physical identities never enter post-ASAP IR. ### Physical compiler ```rust -impl PhysicalCompiler { - pub fn compile( +impl PhysicalPlanCompiler { + pub fn compile_promql( &self, - request: PlanningRequest, - environment: DeploymentEnvironment, - ) -> Result; + request: PhysicalCompilationRequest, + environment: PhysicalDeploymentContext, + ) -> Result; } ``` ```rust -pub struct DeploymentEnvironment { - pub collector_ids: Vec, +pub struct PhysicalDeploymentContext { + pub target_collector_ids: Vec, pub capability_snapshot_id: String, pub observed_at_unix_ms: u64, pub max_evidence_age_ms: u64, @@ -127,7 +127,7 @@ pub struct DeploymentEnvironment { pub backend_compat: String, } -pub struct PhysicalPlan { +pub struct CompiledPhysicalPlan { pub envelope: PlanEnvelope, pub summary_catalog: SummaryCatalog, pub collector_plans: Vec, // complete per-target projections @@ -141,7 +141,7 @@ Supporting public types: | Type | Definition | | --- | --- | -| `DeploymentEnvironment` | Target collector IDs, capability snapshot identity, planning time, and evidence freshness policy. | +| `PhysicalDeploymentContext` | Target collector IDs, capability snapshot identity, planning time, and evidence freshness policy. | | `PlanEnvelope` | Shared deterministic `plan_id`, generation time, capability snapshot, and Planner revision. | | `CollectorPlan` | Serializable execution projection consumed by ASAPCollector. | | `PrecomputePlan` | Authoritative materialization, ingest, state-schema, and producer contract consumed directly by the backend runtime. | @@ -331,8 +331,8 @@ identity. ### Add a deployment topology -1. Add a public `DeploymentTopology` variant and its required target fields. -2. Teach `PhysicalCompiler::compile` how selected operators can be placed on it. +1. Add a public `PhysicalDeploymentTarget` variant and its required target fields. +2. Teach `PhysicalPlanCompiler::compile_promql` how selected operators can be placed on it. 3. Reject plans requiring an unavailable stage/capability. 4. Verify the output contains one complete CollectorPlan for every producer and one SummaryCatalog and matching QueryPlan referencing all produced materializations. diff --git a/docs/developer_docs/control-plane/planning-terminology.md b/docs/developer_docs/control-plane/planning-terminology.md new file mode 100644 index 000000000..3e726a934 --- /dev/null +++ b/docs/developer_docs/control-plane/planning-terminology.md @@ -0,0 +1,110 @@ +# Planning terminology and architecture + +Audience: developers changing planning inputs, pricing, publication, or runtime +plan consumers. The [Chinese architecture review](architecture-naming-review.zh.md) +records the reasoning and the original-to-proposed naming checklist for #709. + +```mermaid +flowchart TB + subgraph CP[Control plane] + INPUT["BackendLocalPlanningInput
Workload demand + physical inputs + deployment context"] + LOGICAL["ASAPPlanner + selection adapter
Legal semantic DAG selection"] + REQUEST["PhysicalCompilationRequest
QueryCompilationInput + enabled materialization keys"] + COMPILE["PhysicalPlanCompiler
Compile concrete candidate plans"] + MANIFEST["WorkloadCostManifest
Component implementations and pricing basis"] + QUOTE["WorkloadQuote
Provider feasibility and component prices"] + EVALUATE["CandidatePlanEvaluation
Select the lowest-cost feasible enumerated candidate"] + SQL["ClickHouse SQL selection and binding
Separate compilation path"] + PUBLICATION["PhysicalPlanPublication
PhysicalPlanInstallRequest"] + INPUT --> LOGICAL --> REQUEST --> COMPILE --> MANIFEST + MANIFEST --> QUOTE --> EVALUATE --> PUBLICATION + SQL --> PUBLICATION + end + CONTRACT["Shared asap_types contracts
SummaryCatalog + CollectorPlan + PrecomputePlan
TransmissionPlan + QueryPlan"] + subgraph DP[Data plane] + STAGE["Validate and stage RuntimePhysicalPlan"] + ACTIVE["Activate via ActivePhysicalPlanHandle
One immutable generation for all readers"] + MAINTAIN["Ingest and precompute
StreamingConfigHandle materialization view"] + QUERY["Execute installed query DAG
Residual operators, bound reads, exact subqueries"] + STORE["Summary storage
SummarySeriesMetadata + physical series state"] + STAGE --> ACTIVE + ACTIVE --> MAINTAIN --> STORE + ACTIVE --> QUERY --> STORE + end + PUBLICATION --> CONTRACT --> STAGE + PUBLICATION -->|OpAMP| COLLECTOR[ASAPCollector] + COLLECTOR -->|OTLP frames| MAINTAIN + RAW[Raw time-series samples] -->|Remote Write| MAINTAIN + QUERY -->|Exact execution or fallback| EXACT[Prometheus / VictoriaMetrics / ClickHouse] +``` + +SQL shares publication and runtime contracts; it does not currently use the +time-series workload quote-selection path. Legacy flat metric workloads and +stage emission remain a separate compatibility path. + +## Domain boundaries + +| Value | Meaning | +|---|---| +| `QueryCompilationInput.selected_plan_root` | Selected semantic DAG root, including exact or mixed execution; not necessarily an approximate materialization | +| `WindowRealizationCandidate` | One concrete window framework/layout choice, not a whole-workload candidate | +| `PhysicalCompilationRequest.enabled_materialization_keys` | Optional candidate-key set: `None` enables all eligible keys; an empty set enables none | +| `CompiledPhysicalPlan` | A complete compiled candidate; selection can attach a `CandidatePlanSelectionReport` to the same type | +| `CostComponentDemand.pricing_basis` | Work priced per horizon or per query evaluation, not a currency/resource unit | +| `occurrences_per_horizon` | Floating-point expected occurrences multiplying a component's unit quote | +| `CandidatePlanEvaluation` | Diagnostic state throughout compilation and pricing, including failures and candidates awaiting quotes | +| `LifecycleUnitCosts` | Both one-time costs and per-update/per-second costs; not exclusively rates | +| `erp` | Error–Resource Profile planning inputs, including distribution evidence and matching policy | +| `PhysicalDeploymentContext.target_collector_ids` | Every targeted collector receives a plan; this is not an eligibility pool | +| `query_retention_margin_ms` | Extra retained history for lagging query evaluation; not a separate query-age rejection check | +| `RuntimePhysicalPlan` | Immutable runtime representation, also used while staged and draining | +| `ActivePhysicalPlanHandle` | Shared pointer to the currently active runtime generation | +| `StreamingConfigHandle` | Legacy owned config or a view projected from the active runtime plan | +| `SummarySeriesMetadata` | Metadata for a physical SID; distinct from a time-scoped SDS `SummaryInstance` | + +`SummaryDefinitionId`, `PolicyFingerprint`, descriptor IDs, physical SIDs, +instance IDs, and catalog generations retain their distinct identities. +Likewise, plan activation, materialization readiness, and instance completeness +remain separate conditions. Retiring a drained plan marks lifecycle state; it +does not perform storage garbage collection. + +## Compatibility transition + +New Rust names retain the previous JSON/YAML field names through explicit serde +renames. New spellings are accepted as aliases where fields are deserialized. +This preserves old consumers, exact manifest comparisons, and serialized +identity inputs. Diagnostic enums retain their old string representations, +including unknown strings and the missing-status default. + +Old public type imports and selected method/function names remain deprecated +forwarders. This does **not** preserve old Rust struct-literal field names; +workspace callers migrate with the definitions. External Rust source consumers +must update fields using the review's mapping before these compatibility +imports are removed. No removal date is set until consumer migration is known. + +Wire names containing historical terms, such as `enumerated_local_masks`, are +intentionally retained. New Rust set variables use candidate-key terminology. +Candidate ordering, bounded coverage, strict-less-than tie handling, pricing, +publication validation, and activation behavior are unchanged. + +## Scope of this migration + +The implementation clarifies planning/compiler types, candidate pricing and +diagnostics, publication conversion, runtime generations, streaming views, +series metadata, legacy metric workloads, residual query operators, and the +data-plane update-sampling module. Existing re-exports remain compatibility +entrypoints rather than duplicate implementations. + +The review also identifies follow-up work that is intentionally separate: + +- Splitting the two `StreamingConfigHandle` modes and changing their write API. + Active-plan snapshots read the runtime plan; `swap` still affects only the + legacy backing slot. Plan activation changes authoritative active config. +- Removing legacy stage planning/emission, or unifying SQL and time-series + candidate pricing. +- Renaming the full `SketchStore` public type or redesigning `QueryPlan` as a + differently shaped execution catalog. +- Changing window units, schema versions, identity encodings, or typed ID + ownership. + +These require their own behavior contracts and are not hidden in a naming PR. diff --git a/docs/developer_docs/data-structure-ownership-audit.md b/docs/developer_docs/data-structure-ownership-audit.md index 9174b411d..807ca849c 100644 --- a/docs/developer_docs/data-structure-ownership-audit.md +++ b/docs/developer_docs/data-structure-ownership-audit.md @@ -19,15 +19,17 @@ It classifies operators such as `topk`, `rate`, and `quantile`; the control-plan `QueryShape` describes evaluation lifecycle (`one_shot`, `streaming`, or `periodic`). The distinct names prevent accidental cross-layer use. -## Remaining migration - -`QueryPlan`, `PrecomputePlan`, `TransmissionPlan`, and `CollectorPlan` are wire -contracts currently defined in `control_plane`, so the data-plane crate depends -on the whole control-plane crate to deserialize and execute them. A later change -should move only their serde DTOs and validation-independent identifiers into -`asap_types`, leaving selection, compilation, publication validation, HTTP -handlers, and execution in their current owners. That migration should preserve -the JSON schema and use compile-time conversion at the compiler boundary. +## Shared contracts and compatibility imports + +`QueryPlan`, `PrecomputePlan`, `TransmissionPlan`, `CollectorPlan`, +`SummaryCatalog`, and the publication/install envelopes are now defined in +`asap_types`. Control-plane modules retain compatibility re-exports; these are +not independent DTO implementations. Selection and compilation stay in the +control plane, while installation, ingestion, and query execution stay in the +data plane. Shared contracts retain their validation methods. + +See [planning terminology](control-plane/planning-terminology.md) for the +compiled-plan/runtime-plan distinction and the wire-preserving naming migration. Planner's `planner_types::workload::QueryLanguage` and backend `asap_types::QueryLanguage` also have different scopes. They should remain From 3008da2aacb62f2538916b94d2ae7816ffd2fb02 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 13 Sep 2026 15:23:10 -0600 Subject: [PATCH 3/3] refactor: unify registered workloads with ASAPPlanner --- control_plane/src/emit/mod.rs | 54 +- control_plane/src/lib.rs | 5 +- control_plane/src/main.rs | 157 ++--- .../src/physical/deployment_cost/delta.rs | 19 +- .../src/physical/deployment_cost/mod.rs | 90 +-- .../src/physical/deployment_cost/pareto.rs | 20 +- control_plane/src/physical/plan_cache.rs | 42 +- control_plane/src/physical/stage_split.rs | 9 +- .../src/physical/workload_planner.rs | 178 ++++-- control_plane/src/pipeline.rs | 378 +++++++----- control_plane/src/registered_workload.rs | 569 ++++++++++++++++++ control_plane/src/replan.rs | 90 ++- control_plane/src/store/workload.rs | 151 ++--- control_plane/src/types.rs | 61 +- control_plane/src/workload.rs | 18 +- ...e2e_controller_plans_and_backend_serves.rs | 72 ++- .../architecture-naming-review.zh.md | 27 +- .../control-plane/planning-terminology.md | 64 +- 18 files changed, 1311 insertions(+), 693 deletions(-) create mode 100644 control_plane/src/registered_workload.rs diff --git a/control_plane/src/emit/mod.rs b/control_plane/src/emit/mod.rs index 1cde5aab2..c96caf8a3 100644 --- a/control_plane/src/emit/mod.rs +++ b/control_plane/src/emit/mod.rs @@ -341,19 +341,18 @@ pub fn collect_metric_to_family( // by several capabilities accumulates several families, so its // samples fan into each per-family pipeline at the agent and the // backend serves every (metric, capability) the workload needs. - for (_, workload, _wc) in workload_store.get_all_for_metric(&entry.metric_name) { + for (_, workload) in workload_store.get_all_for_metric(&entry.metric_name) { // If this metric declares an `item_label` (its inner // high-cardinality dimension, e.g. "endpoint") and the // parsed query's own label filters name a value for it (e.g. // `{endpoint="checkout"}`), thread that through as the // `Frequency` intent's actual per-item filter -- see // `bind_workload_typed_with_item_filter`'s doc. - let item_filter = entry.item_label.as_deref().and_then(|label| { - workload - .label_filters - .get(label) - .map(|v| (label, v.as_str())) - }); + let filters = workload.label_filters(); + let item_filter = entry + .item_label + .as_deref() + .and_then(|label| filters.get(label).map(|v| (label, v.as_str()))); let Some(deployment_expr) = crate::physical::workload_planner::bind_workload_typed_with_item_filter( &workload, @@ -404,12 +403,15 @@ pub fn collect_metric_to_grouping_labels( // pre-B2 semantics ("the entry the controller pre-popped first // wins") in the common case AND lets a multi-role metric still // emit a single keep_keys OTTL processor per metric. - if let Some((_, workload, _)) = workload_store + if let Some((_, workload)) = workload_store .get_all_for_metric(&entry.metric_name) .into_iter() .next() { - out.insert(entry.metric_name.clone(), workload.group_by_labels.clone()); + out.insert( + entry.metric_name.clone(), + workload.group_by_labels().clone(), + ); } } out @@ -603,7 +605,7 @@ pub fn collect_cumulative_counter_metrics( let mut seen: std::collections::BTreeSet = std::collections::BTreeSet::new(); for entry in registry.entries() { // `derive_agg_role` reads the WorkloadEntry directly (query - // string + family override), not the lowered LegacyMetricWorkload, so + // string + family override), not the lowered RegisteredWorkload, so // we classify the registry entry. We still consult the // workload_store to confirm the metric was successfully // pre-populated (matching the contract of the sibling @@ -866,24 +868,18 @@ mod runtime_tests { // Reproduces the live demo gap: 3 of 6 (HLL, CountSketch, CMS) silently // drop because the analyzer pre-population path doesn't propagate // `sketch_family_override` from the workload YAML into - // `LegacyMetricWorkload::sketch_type_override`. + // `RegisteredWorkload::sketch_type_override`. /// Mimics the pre-population loop in `main()` — turns each - /// `WorkloadEntry` into a `LegacyMetricWorkload` via the shared `Analyzer`. + /// `WorkloadEntry` into a `RegisteredWorkload` via the shared `Analyzer`. fn populate_store_from_registry(registry: &WorkloadRegistry, store: &WorkloadStore) { use crate::pipeline::Analyzer; - use crate::types; let analyzer = Analyzer::new(); for entry in registry.entries() { let spec = crate::workload::query_spec_for_entry(entry); if let Ok(wl) = analyzer.analyze(spec) { let role = crate::workload::derive_agg_role(entry); - store.set( - &entry.metric_name, - role, - wl, - types::WorkloadCharacteristics::default(), - ); + store.set(&entry.metric_name, role, wl); } } } @@ -1090,12 +1086,12 @@ mod runtime_tests { /// fan into three pipelines. /// /// We populate the store directly with three `(metric, role)` - /// `LegacyMetricWorkload`s — one per capability — so the test pins + /// `RegisteredWorkload`s — one per capability — so the test pins /// `collect_metric_to_family`'s union semantics independently of the /// analyzer's query-string → AggType parsing. #[test] fn collect_metric_to_family_unions_multiple_capabilities_per_metric() { - use crate::types::{AggType, LegacyMetricWorkload, SketchType, WorkloadCharacteristics}; + use crate::types::{AggType, RegisteredWorkload, SketchType}; use crate::workload::AggRole; use planner_types::post_asap::SketchAlgorithm; use std::collections::BTreeSet; @@ -1122,35 +1118,34 @@ mod runtime_tests { let mk = |agg: AggType, override_family: Option, quantiles: Vec| - -> LegacyMetricWorkload { - LegacyMetricWorkload { + -> RegisteredWorkload { + crate::registered_workload::fixtures::WorkloadFixture { metric_name: METRIC.to_string(), label_filters: Default::default(), group_by_labels: Vec::new(), aggregations: vec![agg], time_window: Duration::from_secs(60), repeat_every: None, - accuracy_sla: 0.01, + accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.01), latency_sla: None, sketch_type_override: override_family, exact_required: false, quantiles, } + .build() }; // Quantile → DDSketch (explicit override valid for Quantile). store.set( METRIC, AggRole::Quantile, mk(AggType::Quantile, Some(SketchType::DDSketch), vec![0.99]), - WorkloadCharacteristics::default(), ); // Cardinality → HLL (override valid for the Cardinality class). store.set( METRIC, AggRole::Count, mk(AggType::Cardinality, Some(SketchType::HLL), Vec::new()), - WorkloadCharacteristics::default(), ); // Frequency → CMS. This deployment's capability catalog exposes // CountSketch only for TopK, so the incompatible override is ignored; @@ -1163,7 +1158,6 @@ mod runtime_tests { Some(SketchType::CountSketch), Vec::new(), ), - WorkloadCharacteristics::default(), ); let map = collect_metric_to_family(®istry, &store); @@ -1187,7 +1181,7 @@ mod runtime_tests { // Pre-B3 the WorkloadEntry YAML had no way to declare grouping // labels — the analyzer pulled them only from PromQL `by (...)` // clauses. Bare `quantile_over_time(0.99, metric[30s])` carries no - // `by`, so `LegacyMetricWorkload.group_by_labels` ended up empty, so + // `by`, so `RegisteredWorkload.group_by_labels` ended up empty, so // `collect_metric_to_grouping_labels` returned `{metric: vec![]}`, // so the 5-sketch routing emitter wrote // `keep_keys(datapoint.attributes, [])` — stripping ALL attrs @@ -1196,7 +1190,7 @@ mod runtime_tests { // // Post-B3 a declarative `grouping_labels: [zone]` on WorkloadEntry // is threaded through the pre-pop QuerySpec → analyzer → - // LegacyMetricWorkload.group_by_labels → collect_metric_to_grouping_labels + // RegisteredWorkload.group_by_labels → collect_metric_to_grouping_labels // → the emitter's keep_keys list. Without this round-trip the // end-to-end test's sid catalog stays empty-per-zone. #[test] @@ -1222,7 +1216,7 @@ mod runtime_tests { populate_store_from_registry(®istry, &store); // The analyzer must have threaded grouping_labels into - // LegacyMetricWorkload.group_by_labels. + // RegisteredWorkload.group_by_labels. let map = collect_metric_to_grouping_labels(®istry, &store); assert_eq!( map.get("http_requests_total_latency_ms"), diff --git a/control_plane/src/lib.rs b/control_plane/src/lib.rs index 12e0584da..1c32f784e 100644 --- a/control_plane/src/lib.rs +++ b/control_plane/src/lib.rs @@ -6,8 +6,8 @@ //! Shared execution and installation contracts live in `asap_types`. //! //! `physical::compiler`, `physical::workload_cost`, and `clickhouse` are the -//! current compilation paths. Flat metric workloads and stage emission remain -//! compatibility paths. Their public modules are workspace integration APIs, +//! current compilation paths. Metric stage emission consumes the same canonical +//! workload model through a registration adapter. Public modules are integration APIs, //! not an independent wire schema or a second semantic planner. #![allow( @@ -37,6 +37,7 @@ pub mod planner_selection; pub mod query_parser; pub mod query_plan; pub mod query_planning; +pub mod registered_workload; pub mod replan; pub mod runtime_samples; pub mod sketch_selection; diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index 14db2128b..5cba8760e 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -9,7 +9,6 @@ use control_plane::monitor; use control_plane::opamp; use control_plane::physical; use control_plane::pipeline; -use control_plane::query_parser; use control_plane::replan; use control_plane::runtime_samples; use control_plane::store; @@ -44,7 +43,6 @@ use physical::deployment_cost::tco; use physical::deployment_cost::DeploymentCostPlanner; use physical::plan_cache::CachedDeploymentPlanner; use pipeline::{Analyzer, QuerySpec}; -use query_parser::parse_query_expr_canonical; use replan::Replanner; use store::{PlanStore, WorkloadStore}; use types::AgentCollectorConfig; @@ -255,7 +253,7 @@ async fn main() { // // Critical: thread `sketch_family_override` from each registry entry // into the QuerySpec's `sketch_type` field — that's what populates - // `LegacyMetricWorkload::sketch_type_override`, which the typed planner + // `RegisteredWorkload::sketch_type_override`, which the typed planner // (`bind_workload_typed`) reads to honour MVP §46 entries 5–8 (HLL / // CountSketch / CountMinSketch). Without this stitch the workloads // round-trip through the analyzer with a None override and the @@ -286,11 +284,10 @@ async fn main() { let role = control_plane::workload::derive_agg_role(entry); match analyzer.analyze(spec) { Ok(wl) => { - let wc = types::WorkloadCharacteristics::default(); - let plan = planner.plan(&wl, Some(&wc)); - let metric_name = wl.metric_name.clone(); + let plan = planner.plan(&wl); + let metric_name = wl.metric_name().clone(); plan_store.set(&metric_name, role, plan); - workload_store.set(&metric_name, role, wl, wc); + workload_store.set(&metric_name, role, wl); } Err(e) => { warn!(metric = %entry.metric_name, role = %role, error = %e, @@ -1032,35 +1029,19 @@ fn workload_cost_manifests( // ── Handlers ────────────────────────────────────────────────────────────────── async fn handle_plan(State(st): State, Json(spec): Json) -> impl IntoResponse { - let wc = spec.workload.clone(); - let query_string = spec.query_string.clone(); let workload = match st.analyzer.analyze(spec) { Ok(w) => w, Err(e) => return (StatusCode::UNPROCESSABLE_ENTITY, e.to_string()).into_response(), }; - let plan = st.planner.plan(&workload, Some(&wc)); + let query_string = Some(workload.entry().query.0); + + let plan = st.planner.plan(&workload); // Derive deployment configs from the bound query. Keep its Rc-backed DAG // scoped before any await so the handler future remains Send. let stage_configs = { - let mut bound_physical: Option = None; - if let Some(ref qs) = query_string { - match parse_query_expr_canonical(qs, workload.accuracy.clone()) { - Err(e) => { - warn!(query = %qs, error = %e, "parse_query_expr_canonical failed; skipping algebra pipeline") - } - Ok(qe) => { - // L4 sketch binding: lower the optimised L3 tree to the - // sketch-bound `PhysicalExpr` IR — the typed L5's input. - bound_physical = control_plane::physical::post_asap::bind_query_expr( - &qe, - workload.accuracy.clone(), - ) - .ok(); - } - } - } + let bound_physical = physical::workload_planner::bind_registered_query(&workload).ok(); let stage_configs: Option< std::collections::HashMap< @@ -1085,13 +1066,13 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> // (Quantile vs Sum) would silently overwrite the prior plan. let role = { let entry = control_plane::workload::WorkloadEntry { - metric_name: workload.metric_name.clone(), + metric_name: workload.metric_name().clone(), query_string: query_string.clone(), - accuracy_sla: workload.accuracy_sla, + accuracy_sla: 1.0 - workload.error_bound(), assign_to_role: String::from("agent"), - sketch_family_override: workload.sketch_type_override.clone(), + sketch_family_override: workload.deployment.sketch_type_override.clone(), target_path: None, - grouping_labels: workload.group_by_labels.clone(), + grouping_labels: workload.group_by_labels().clone(), // Role derivation does not depend on sampling; default 1.0. sample_p: 1.0, // Role derivation does not depend on the cardinality hint. @@ -1104,10 +1085,10 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> }; control_plane::workload::derive_agg_role(&entry) }; - st.store.set(&workload.metric_name, role, plan.clone()); + st.store.set(workload.metric_name(), role, plan.clone()); // Persist workload so the replanner can re-run plan() without the original spec. st.workload_store - .set(&workload.metric_name, role, workload.clone(), wc); + .set(workload.metric_name(), role, workload.clone()); // ── Push agent config to agent-role collectors ──────────────────────────── if let Ok(agent_yaml) = generate_agent_collector_config(&plan.agent_config, &st.opamp_endpoint) @@ -1142,8 +1123,8 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> // `agg.grouping = workload.group_by_labels` // patch on the Backend stage below). edge.metric_to_grouping_labels.insert( - workload.metric_name.clone(), - workload.group_by_labels.clone(), + workload.metric_name().clone(), + workload.group_by_labels().clone(), ); // Issue #2: broadcast push — no single agent id // in scope, so emit `$AGENT_ID` placeholder and @@ -1213,7 +1194,7 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> // columns (open-set label naming is // a Step γ TODO in // `intent_algebra::column_resolution`). - // `LegacyMetricWorkload` carries both unambiguously, + // `RegisteredWorkload` carries both unambiguously, // and every aggregation under one workload // shares them — so the patch is uniform. let item_labels = emit::collect_metric_to_item_label( @@ -1222,12 +1203,12 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> ); for agg in &mut be.aggregations { if agg.metric_name.is_empty() { - agg.metric_name = workload.metric_name.clone(); + agg.metric_name = workload.metric_name().clone(); } if agg.window_secs == 0 { - agg.window_secs = workload.time_window.as_secs(); + agg.window_secs = workload.time_window().as_secs(); } - agg.grouping = workload.group_by_labels.clone(); + agg.grouping = workload.group_by_labels().clone(); agg.item_label = item_labels.get(&agg.metric_name).cloned(); } // Option B unification: every typed cumulative @@ -1244,7 +1225,7 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> post_typed_backend_for_role( st.backend_client.as_ref(), &st.backend_routing_cache, - &workload.metric_name, + &workload.metric_name(), role, be, &monitors, @@ -1260,7 +1241,7 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> } } else if physical::stage_split::typed_stage_split_enabled() { warn!( - metric = %workload.metric_name, + metric = %workload.metric_name(), "[USE_TYPED_STAGE_SPLIT] split_typed_three_stage returned None; \ legacy plan output unaffected" ); @@ -1273,7 +1254,7 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> .set_sketch_type(&agent_id, sketch_type.clone()) .await; st.replanner - .register_agent(&agent_id, &workload.metric_name, role) + .register_agent(&agent_id, &workload.metric_name(), role) .await; } @@ -1282,7 +1263,7 @@ async fn handle_plan(State(st): State, Json(spec): Json) -> ( StatusCode::OK, Json(json!({ - "metric": workload.metric_name, + "metric": workload.metric_name(), "sketch_type": plan.agent_config.sketch_type.to_string(), "mode": plan.agent_config.mode.to_string(), "aggregate_by": plan.agent_config.aggregate_by, @@ -1433,7 +1414,7 @@ async fn handle_plan_auto( // the data-plane coordinator derives the live ε-floor p. st.workload_registry.insert_runtime(entry); // (2) Best-effort SKETCH registration: analyze a query for this metric - // and register the LegacyMetricWorkload (with the chosen sketch) so replan + // and register the RegisteredWorkload (with the chosen sketch) so replan // emits a fresh sketch plan. The pipeline analyzer accepts a narrower // grammar than the planner's, so this is non-fatal on rejection. for q in &req.queries { @@ -1443,14 +1424,9 @@ async fn handle_plan_auto( continue; }; if let Ok(mut wl) = st.analyzer.analyze(spec) { - if wl.metric_name == metric { - wl.sketch_type_override = sketch.clone(); - st.workload_store.set( - &metric, - role, - wl, - control_plane::types::WorkloadCharacteristics::default(), - ); + if wl.metric_name() == metric { + wl.deployment.sketch_type_override = sketch.clone(); + st.workload_store.set(&metric, role, wl); break; } } @@ -1484,12 +1460,18 @@ async fn handle_pareto( State(st): State, Json(req): Json, ) -> impl IntoResponse { - let wc = req.spec.workload.clone(); let workload = match st.analyzer.analyze(req.spec) { Ok(w) => w, Err(e) => return (StatusCode::UNPROCESSABLE_ENTITY, e.to_string()).into_response(), }; + let Some(wc) = workload.characteristics_at(chrono::Utc::now().timestamp_millis() as u64) else { + return ( + StatusCode::UNPROCESSABLE_ENTITY, + "fresh data evidence is required for cost comparison", + ) + .into_response(); + }; let frontier = pareto_frontier(&workload, &wc, req.weights, Some(&st.online_store)); if frontier.is_empty() { @@ -1518,7 +1500,7 @@ async fn handle_pareto( ( StatusCode::OK, Json(json!({ - "metric": workload.metric_name, + "metric": workload.metric_name(), "frontier": points, "best": best, })), @@ -1850,7 +1832,7 @@ async fn emit_bootstrap_typed( // stays the source of the edge config. let mut chosen: Option<(String, crate::physical::post_asap::PhysicalExpr)> = None; 'outer: for cand in &candidates { - for (_, wl, _) in st.workload_store.get_all_for_metric(cand) { + for (_, wl) in st.workload_store.get_all_for_metric(cand) { if let Some(expr) = physical::workload_planner::bind_workload_typed(&wl) { chosen = Some((cand.clone(), expr)); break 'outer; @@ -2401,7 +2383,7 @@ mod api_tests { .workload_store .get_all_for_metric("typed_accuracy_metric"); assert_eq!(stored.len(), 1); - assert_eq!(stored[0].1.accuracy, target); + assert_eq!(stored[0].1.accuracy(), target); let plans = state.store.get_all_for_metric("typed_accuracy_metric"); assert_eq!(plans.len(), 1); if matches!(target, AccuracyTarget::Epsilon(_)) { @@ -2491,20 +2473,14 @@ mod api_tests { let (st, app) = test_app(); // Seed one plan directly. use crate::physical::workload_planner::DeploymentPlanCompiler; - let wl = crate::types::LegacyMetricWorkload { - metric_name: "m".into(), - label_filters: std::collections::HashMap::new(), - group_by_labels: vec![], - aggregations: vec![crate::types::AggType::Quantile], - time_window: std::time::Duration::from_secs(300), - repeat_every: None, - accuracy_sla: 0.01, - accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.01), - latency_sla: None, - sketch_type_override: None, - exact_required: false, - quantiles: vec![], - }; + let wl = Analyzer::new() + .analyze( + serde_json::from_value(serde_json::json!({ + "query_string": "quantile_over_time(0.99, m[5m])", "accuracy_sla": 0.99 + })) + .unwrap(), + ) + .unwrap(); st.store.set( "m", control_plane::workload::AggRole::Quantile, @@ -2766,8 +2742,8 @@ mod api_tests { data: types_v2::DataShape::default(), }; let wl = analyzer.analyze(spec).unwrap(); - let wc = types::WorkloadCharacteristics::default(); - let plan = planner.plan(&wl, Some(&wc)); + + let plan = planner.plan(&wl); // B2 (metric, role): pre-populate using the same role the // on_connect callback's `derive_agg_role(entry)` will compute // for this test's workloads.yaml entry (no query_string + no @@ -2779,12 +2755,7 @@ mod api_tests { control_plane::workload::AggRole::Other, plan, ); - workload_store.set( - "http_latency", - control_plane::workload::AggRole::Other, - wl, - wc, - ); + workload_store.set("http_latency", control_plane::workload::AggRole::Other, wl); // Build replanner and late-binding cells. let replanner_cell: Arc>>> = @@ -2911,15 +2882,10 @@ mod api_tests { data: types_v2::DataShape::default(), }; let wl = analyzer.analyze(spec).unwrap(); - let wc = types::WorkloadCharacteristics::default(); - let plan = planner.plan(&wl, Some(&wc)); + + let plan = planner.plan(&wl); plan_store.set("metric_a", control_plane::workload::AggRole::Quantile, plan); - workload_store.set( - "metric_a", - control_plane::workload::AggRole::Quantile, - wl, - wc, - ); + workload_store.set("metric_a", control_plane::workload::AggRole::Quantile, wl); let replanner = Arc::new(Replanner::new( Arc::clone(&planner), @@ -3176,14 +3142,14 @@ mod api_tests { data: types_v2::DataShape::default(), }; let wl = analyzer.analyze(spec).expect("analyze"); - let wc = types::WorkloadCharacteristics::default(); - let plan = state.planner.plan(&wl, Some(&wc)); + + let plan = state.planner.plan(&wl); state .store .set(metric, control_plane::workload::AggRole::Quantile, plan); state .workload_store - .set(metric, control_plane::workload::AggRole::Quantile, wl, wc); + .set(metric, control_plane::workload::AggRole::Quantile, wl); // 4. Swap in the populated registry. state.workload_registry = registry; @@ -3411,14 +3377,14 @@ mod api_tests { data: types_v2::DataShape::default(), }; let wl = analyzer.analyze(spec).expect("analyze"); - let wc = types::WorkloadCharacteristics::default(); - let plan = state.planner.plan(&wl, Some(&wc)); + + let plan = state.planner.plan(&wl); state .store .set(*m, control_plane::workload::AggRole::Quantile, plan); state .workload_store - .set(*m, control_plane::workload::AggRole::Quantile, wl, wc); + .set(*m, control_plane::workload::AggRole::Quantile, wl); } // 4. Swap in the populated registry. @@ -3581,12 +3547,11 @@ mod api_tests { for entry in registry.entries() { let spec = control_plane::workload::query_spec_for_entry(entry); if let Ok(wl) = analyzer.analyze(spec) { - let wc = types::WorkloadCharacteristics::default(); - let plan = state.planner.plan(&wl, Some(&wc)); - let metric_name = wl.metric_name.clone(); + let plan = state.planner.plan(&wl); + let metric_name = wl.metric_name().clone(); let role = control_plane::workload::derive_agg_role(entry); state.store.set(&metric_name, role, plan); - state.workload_store.set(&metric_name, role, wl, wc); + state.workload_store.set(&metric_name, role, wl); } } diff --git a/control_plane/src/physical/deployment_cost/delta.rs b/control_plane/src/physical/deployment_cost/delta.rs index 2f3f70366..de313540f 100644 --- a/control_plane/src/physical/deployment_cost/delta.rs +++ b/control_plane/src/physical/deployment_cost/delta.rs @@ -129,12 +129,12 @@ pub fn delta_benchmark_table() -> HashMap { /// A shorter flush period means: /// • fewer inserts accumulate per period → lower fill rate → better delta /// • more flushes per second → higher total CPU overhead -pub fn flush_period_secs(plan: &CollectionPlan, w: &LegacyMetricWorkload) -> f64 { +pub fn flush_period_secs(plan: &CollectionPlan, w: &RegisteredWorkload) -> f64 { if let Some(wd) = plan.agent_config.window_duration { return wd.as_secs_f64(); } // Batch mode: use repeat_every as a proxy for the batch arrival interval. - w.repeat_every + w.repeat_every() .unwrap_or(Duration::from_secs(1)) .as_secs_f64() .max(0.001) // guard against zero @@ -196,7 +196,7 @@ fn estimate_distinct_keys(inserts_per_flush: f64, wc: &WorkloadCharacteristics) pub fn estimate_fill_rate( wc: &WorkloadCharacteristics, plan: &CollectionPlan, - w: &LegacyMetricWorkload, + w: &RegisteredWorkload, ) -> f64 { let flush_secs = flush_period_secs(plan, w); let inserts_per_flush = wc.samples_per_sec_per_series * wc.series_count as f64 * flush_secs; @@ -313,7 +313,7 @@ pub const RAW_PASSTHROUGH_SAMPLE_RATE_THRESHOLD: f64 = 10.0; /// 7. Otherwise → UseDelta. pub fn decide_delta( plan: &CollectionPlan, - w: &LegacyMetricWorkload, + w: &RegisteredWorkload, wc: &WorkloadCharacteristics, bytes_per_series_per_sec: f64, ) -> (DeltaDecision, TransmissionCostSummary) { @@ -474,21 +474,22 @@ mod tests { use chrono::Utc; use std::collections::HashMap; - fn workload_for(agg: AggType) -> LegacyMetricWorkload { - LegacyMetricWorkload { + fn workload_for(agg: AggType) -> RegisteredWorkload { + crate::registered_workload::fixtures::WorkloadFixture { metric_name: "m".into(), label_filters: HashMap::new(), group_by_labels: vec![], aggregations: vec![agg], time_window: Duration::from_secs(300), repeat_every: Some(Duration::from_secs(10)), - accuracy_sla: 0.01, + accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.01), latency_sla: None, sketch_type_override: None, exact_required: false, quantiles: vec![], } + .build() } fn make_plan(st: SketchType, window: Option) -> CollectionPlan { @@ -549,7 +550,7 @@ mod tests { #[test] fn flush_period_uses_repeat_every_in_batch_mode() { let mut w = workload_for(AggType::Frequency); - w.repeat_every = Some(Duration::from_secs(15)); + w.set_repeat_every(Some(Duration::from_secs(15))); let plan = make_plan(SketchType::CountMinSketch, None); assert_eq!(flush_period_secs(&plan, &w), 15.0); } @@ -557,7 +558,7 @@ mod tests { #[test] fn flush_period_batch_fallback_is_one_second() { let mut w = workload_for(AggType::Frequency); - w.repeat_every = None; + w.set_repeat_every(None); let plan = make_plan(SketchType::CountMinSketch, None); assert_eq!(flush_period_secs(&plan, &w), 1.0); } diff --git a/control_plane/src/physical/deployment_cost/mod.rs b/control_plane/src/physical/deployment_cost/mod.rs index f093e245f..662c9e161 100644 --- a/control_plane/src/physical/deployment_cost/mod.rs +++ b/control_plane/src/physical/deployment_cost/mod.rs @@ -98,7 +98,7 @@ pub struct PlanScore { /// Estimates resource costs for a given plan + workload using the provided cost table. pub fn score_with( plan: &CollectionPlan, - w: &LegacyMetricWorkload, + w: &RegisteredWorkload, table: &HashMap, ) -> PlanScore { let st = &plan.agent_config.sketch_type; @@ -127,13 +127,13 @@ pub fn score_with( cpu_micros_per_sample: costs.cpu_micros_per_sample, memory_bytes: memory, estimated_error: err, - meets_sla: matches!(w.accuracy, crate::types_v2::AccuracyTarget::Epsilon(epsilon) if epsilon > 0.0) + meets_sla: matches!(w.accuracy(), crate::types_v2::AccuracyTarget::Epsilon(epsilon) if epsilon > 0.0) && err <= sla, } } /// Estimates resource costs for a given plan + workload. -pub fn score(plan: &CollectionPlan, w: &LegacyMetricWorkload) -> PlanScore { +pub fn score(plan: &CollectionPlan, w: &RegisteredWorkload) -> PlanScore { let table = benchmark_table(); let st = &plan.agent_config.sketch_type; @@ -165,7 +165,7 @@ pub fn score(plan: &CollectionPlan, w: &LegacyMetricWorkload) -> PlanScore { cpu_micros_per_sample: costs.cpu_micros_per_sample, memory_bytes: memory, estimated_error: err, - meets_sla: matches!(w.accuracy, crate::types_v2::AccuracyTarget::Epsilon(epsilon) if epsilon > 0.0) + meets_sla: matches!(w.accuracy(), crate::types_v2::AccuracyTarget::Epsilon(epsilon) if epsilon > 0.0) && err <= sla, } } @@ -234,30 +234,25 @@ impl DeploymentCostPlanner { /// /// `wc` drives the delta transmission decision: fill rate, flush rate, /// CPU / memory overhead, and raw vs. sketch bandwidth comparison. - /// Pass `None` to use conservative defaults (1 000 series, 100 Hz, - /// 100 B/sample, Zipf distribution, no memory budget). + /// `None` means no usable data evidence: retain the legal rule-based plan + /// without estimating rate-dependent costs from fabricated defaults. pub fn plan( &self, - w: &LegacyMetricWorkload, + w: &RegisteredWorkload, wc: Option<&WorkloadCharacteristics>, ) -> CollectionPlan { - if !matches!(w.accuracy, crate::types_v2::AccuracyTarget::Epsilon(epsilon) if epsilon > 0.0) + if !matches!(w.accuracy(), crate::types_v2::AccuracyTarget::Epsilon(epsilon) if epsilon > 0.0) { return self.inner.plan(w); } - let default_wc; - let wc = match wc { - Some(c) => c, - None => { - default_wc = WorkloadCharacteristics::default(); - &default_wc - } + let Some(wc) = wc else { + return self.inner.plan(w); }; let table = self.cost_table(); // If a specific sketch type is pinned, use it directly. - if let Some(st) = &w.sketch_type_override { + if let Some(st) = &w.deployment.sketch_type_override { let params = default_sketch_params(st, w.error_bound()); let (mode, window_duration) = select_window_strategy(w); let mut plan = self.inner.plan(w); @@ -269,7 +264,8 @@ impl DeploymentCostPlanner { return plan; } - let candidates = crate::physical::sketch_catalog::candidates_for_workload(&w.aggregations); + let candidates = + crate::physical::sketch_catalog::candidates_for_workload(&w.aggregations()); // Start with the rule-based plan as the baseline. let baseline = self.inner.plan(w); @@ -307,7 +303,7 @@ impl DeploymentCostPlanner { /// Runs the delta cost model and writes the decision into the plan using a provided cost table. fn apply_delta_decision_with( plan: &mut CollectionPlan, - w: &LegacyMetricWorkload, + w: &RegisteredWorkload, wc: &WorkloadCharacteristics, table: &HashMap, ) { @@ -351,21 +347,22 @@ mod tests { use std::collections::HashMap; use std::time::Duration; - fn workload(aggs: Vec) -> LegacyMetricWorkload { - LegacyMetricWorkload { + fn workload(aggs: Vec) -> RegisteredWorkload { + crate::registered_workload::fixtures::WorkloadFixture { metric_name: "test".into(), label_filters: HashMap::new(), group_by_labels: vec![], aggregations: aggs, time_window: Duration::from_secs(300), repeat_every: None, - accuracy_sla: 0.01, + accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.01), latency_sla: None, sketch_type_override: None, exact_required: false, quantiles: vec![], } + .build() } fn dummy_plan(st: SketchType) -> CollectionPlan { @@ -409,10 +406,10 @@ mod tests { #[test] fn ddsketch_fails_tight_sla() { - let w = LegacyMetricWorkload { - accuracy_sla: 0.001, - accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.001), - ..workload(vec![AggType::Quantile]) + let w = { + let mut w = workload(vec![AggType::Quantile]); + w.set_accuracy(crate::types_v2::AccuracyTarget::Epsilon(0.001)); + w }; // Force 1% params despite tighter SLA. let mut plan = dummy_plan(SketchType::DDSketch); @@ -434,18 +431,20 @@ mod tests { #[test] fn dim_multiplier_increases_bandwidth() { - let w_few = LegacyMetricWorkload { - group_by_labels: vec!["host".into()], - ..workload(vec![AggType::Quantile]) + let w_few = { + let mut w = workload(vec![AggType::Quantile]); + w.deployment.retained_labels = vec!["host".into()]; + w }; - let w_many = LegacyMetricWorkload { - group_by_labels: vec![ + let w_many = { + let mut w = workload(vec![AggType::Quantile]); + w.deployment.retained_labels = vec![ "host".into(), "service".into(), "zone".into(), "region".into(), - ], - ..workload(vec![AggType::Quantile]) + ]; + w }; let pl = DeploymentPlanCompiler::new(); let s_few = score(&pl.plan(&w_few), &w_few); @@ -455,10 +454,10 @@ mod tests { #[test] fn kll_error_formula() { - let w = LegacyMetricWorkload { - accuracy_sla: 0.02, - accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.02), - ..workload(vec![AggType::Quantile]) + let w = { + let mut w = workload(vec![AggType::Quantile]); + w.set_accuracy(crate::types_v2::AccuracyTarget::Epsilon(0.02)); + w }; let mut plan = dummy_plan(SketchType::KLL); plan.agent_config.sketch_params = SketchParams::KLL { @@ -477,27 +476,28 @@ mod tests { (AggType::Cardinality, 0.01), (AggType::Frequency, 0.02), ] { - let w = LegacyMetricWorkload { - accuracy_sla: sla, - accuracy: crate::types_v2::AccuracyTarget::Epsilon(sla), - ..workload(vec![agg]) + let w = { + let mut w = workload(vec![agg]); + w.set_accuracy(crate::types_v2::AccuracyTarget::Epsilon(sla)); + w }; let plan = pl.plan(&w, None); let s = score(&plan, &w); assert!( s.meets_sla, "agg={} sla={sla}: plan does not meet SLA (error={})", - w.aggregations[0], s.estimated_error + w.aggregations()[0], + s.estimated_error ); } } #[test] fn cost_model_prefers_lower_bandwidth_for_cardinality() { - let w = LegacyMetricWorkload { - accuracy_sla: 0.02, - accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.02), - ..workload(vec![AggType::Cardinality]) + let w = { + let mut w = workload(vec![AggType::Cardinality]); + w.set_accuracy(crate::types_v2::AccuracyTarget::Epsilon(0.02)); + w }; let plan = DeploymentCostPlanner::new().plan(&w, None); assert_eq!( diff --git a/control_plane/src/physical/deployment_cost/pareto.rs b/control_plane/src/physical/deployment_cost/pareto.rs index 379e7debf..cf7f7bfb3 100644 --- a/control_plane/src/physical/deployment_cost/pareto.rs +++ b/control_plane/src/physical/deployment_cost/pareto.rs @@ -95,12 +95,12 @@ pub struct ParetoPoint { /// If `online_store` is `Some` the scoring uses EMA-blended costs; otherwise /// it falls back to the static benchmark table. pub fn pareto_frontier( - workload: &LegacyMetricWorkload, + workload: &RegisteredWorkload, wc: &WorkloadCharacteristics, weights: ObjectiveWeights, online_store: Option<&online::OnlineMetricsStore>, ) -> Vec { - if !matches!(workload.accuracy, crate::types_v2::AccuracyTarget::Epsilon(epsilon) if epsilon > 0.0) + if !matches!(workload.accuracy(), crate::types_v2::AccuracyTarget::Epsilon(epsilon) if epsilon > 0.0) { return Vec::new(); } @@ -182,7 +182,7 @@ fn all_sketch_types() -> Vec { fn apply_delta( st: SketchType, plan: &mut CollectionPlan, - w: &LegacyMetricWorkload, + w: &RegisteredWorkload, wc: &WorkloadCharacteristics, table: &HashMap, ) { @@ -241,21 +241,22 @@ mod tests { use std::collections::HashMap; use std::time::Duration; - fn quantile_workload() -> LegacyMetricWorkload { - LegacyMetricWorkload { + fn quantile_workload() -> RegisteredWorkload { + crate::registered_workload::fixtures::WorkloadFixture { metric_name: "latency".into(), label_filters: HashMap::new(), group_by_labels: vec![], aggregations: vec![AggType::Quantile], time_window: Duration::from_secs(300), repeat_every: None, - accuracy_sla: 0.02, + accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.02), latency_sla: None, sketch_type_override: None, exact_required: false, quantiles: vec![], } + .build() } fn default_wc() -> WorkloadCharacteristics { @@ -369,11 +370,8 @@ mod tests { #[test] fn tight_sla_excludes_inaccurate_sketches() { - let w = LegacyMetricWorkload { - accuracy_sla: 0.001, // very tight — only DDSketch at 0.1% accuracy can meet this - accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.001), - ..quantile_workload() - }; + let mut w = quantile_workload(); + w.set_accuracy(crate::types_v2::AccuracyTarget::Epsilon(0.001)); let f = pareto_frontier(&w, &default_wc(), ObjectiveWeights::default(), None); for p in &f { assert!( diff --git a/control_plane/src/physical/plan_cache.rs b/control_plane/src/physical/plan_cache.rs index c8a01c10f..6c375b8c8 100644 --- a/control_plane/src/physical/plan_cache.rs +++ b/control_plane/src/physical/plan_cache.rs @@ -17,7 +17,7 @@ use std::collections::HashMap; use std::sync::{Arc, RwLock}; use crate::physical::deployment_cost::DeploymentCostPlanner; -use crate::types::{CollectionPlan, LegacyMetricWorkload, WorkloadCharacteristics}; +use crate::types::{CollectionPlan, RegisteredWorkload}; pub struct CachedDeploymentPlanner { inner: DeploymentCostPlanner, @@ -34,12 +34,8 @@ impl CachedDeploymentPlanner { /// Return the baseline plan for this metric, or run the cost model and /// establish a new baseline if this is the first request for the metric. - pub fn plan( - &self, - workload: &LegacyMetricWorkload, - wc: Option<&WorkloadCharacteristics>, - ) -> CollectionPlan { - let key = &workload.metric_name; + pub fn plan(&self, workload: &RegisteredWorkload) -> CollectionPlan { + let key = &workload.metric_name(); // Fast path: return the cached plan if one exists. { @@ -50,7 +46,8 @@ impl CachedDeploymentPlanner { } // Slow path: first request for this metric — run cost optimisation. - let plan = self.inner.plan(workload, wc); + let facts = workload.characteristics_at(chrono::Utc::now().timestamp_millis() as u64); + let plan = self.inner.plan(workload, facts.as_ref()); self.cache .write() .unwrap() @@ -80,21 +77,22 @@ mod tests { use std::collections::HashMap; use std::time::Duration; - fn workload(metric: &str) -> LegacyMetricWorkload { - LegacyMetricWorkload { + fn workload(metric: &str) -> RegisteredWorkload { + crate::registered_workload::fixtures::WorkloadFixture { metric_name: metric.into(), label_filters: HashMap::new(), group_by_labels: vec![], aggregations: vec![AggType::Quantile], time_window: Duration::from_secs(300), repeat_every: None, - accuracy_sla: 0.01, + accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.01), latency_sla: None, sketch_type_override: None, exact_required: false, quantiles: vec![0.99], } + .build() } fn planner() -> CachedDeploymentPlanner { @@ -104,7 +102,7 @@ mod tests { #[test] fn first_call_produces_a_plan() { let p = planner(); - let plan = p.plan(&workload("latency"), None); + let plan = p.plan(&workload("latency")); // Cost model picks the cheapest sketch that meets the SLA; verify // we got a valid plan. transmit_sketch defaults to false (enabled // by DeploymentCostPlanner when appropriate). @@ -114,11 +112,11 @@ mod tests { #[test] fn second_call_returns_same_plan() { let p = planner(); - let first = p.plan(&workload("latency"), None); + let first = p.plan(&workload("latency")); // Change the workload — the baseline planner must ignore it. let mut w2 = workload("latency"); - w2.aggregations = vec![AggType::Cardinality]; - let second = p.plan(&w2, None); + w2.set_accuracy(crate::types_v2::AccuracyTarget::Exact); + let second = p.plan(&w2); assert_eq!( first.agent_config.sketch_type, second.agent_config.sketch_type, "baseline plan must not change even when workload changes" @@ -128,8 +126,8 @@ mod tests { #[test] fn different_metrics_get_independent_plans() { let p = planner(); - let a = p.plan(&workload("metric_a"), None); - let b = p.plan(&workload("metric_b"), None); + let a = p.plan(&workload("metric_a")); + let b = p.plan(&workload("metric_b")); // Both plans are valid (exact sketch type may differ by cost model // internals, but we just check they are independently produced). let _ = (a, b); @@ -139,12 +137,12 @@ mod tests { #[test] fn reset_allows_re_plan() { let p = planner(); - let first = p.plan(&workload("latency"), None); + let first = p.plan(&workload("latency")); p.reset("latency"); assert!(p.baseline_metrics().is_empty()); // After reset the planner will run the cost model again on the same // workload and should produce an equivalent plan. - let second = p.plan(&workload("latency"), None); + let second = p.plan(&workload("latency")); assert_eq!( first.agent_config.sketch_type, second.agent_config.sketch_type, "same workload after reset should produce the same sketch type" @@ -154,9 +152,9 @@ mod tests { #[test] fn baseline_metrics_lists_all_seen_metrics() { let p = planner(); - p.plan(&workload("cpu"), None); - p.plan(&workload("mem"), None); - p.plan(&workload("cpu"), None); // repeat — should not double-count + p.plan(&workload("cpu")); + p.plan(&workload("mem")); + p.plan(&workload("cpu")); // repeat — should not double-count let mut metrics = p.baseline_metrics(); metrics.sort(); assert_eq!(metrics, vec!["cpu", "mem"]); diff --git a/control_plane/src/physical/stage_split.rs b/control_plane/src/physical/stage_split.rs index 9b8383d45..49c456705 100644 --- a/control_plane/src/physical/stage_split.rs +++ b/control_plane/src/physical/stage_split.rs @@ -88,25 +88,26 @@ mod l5_walk_propagation_tests { //! TODO in `intent_algebra::column_resolution`). use crate::physical::colored_dag::StageConfig; - use crate::types::{AggType, LegacyMetricWorkload}; + use crate::types::{AggType, RegisteredWorkload}; use std::collections::HashMap; use std::time::Duration; - fn workload(metric: &str, group_by: Vec, window: Duration) -> LegacyMetricWorkload { - LegacyMetricWorkload { + fn workload(metric: &str, group_by: Vec, window: Duration) -> RegisteredWorkload { + crate::registered_workload::fixtures::WorkloadFixture { metric_name: metric.to_string(), label_filters: HashMap::new(), group_by_labels: group_by, aggregations: vec![AggType::Quantile], time_window: window, repeat_every: None, - accuracy_sla: 0.01, + accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.01), latency_sla: None, sketch_type_override: None, exact_required: false, quantiles: vec![0.99], } + .build() } #[test] diff --git a/control_plane/src/physical/workload_planner.rs b/control_plane/src/physical/workload_planner.rs index ba7337c9a..4e57042ad 100644 --- a/control_plane/src/physical/workload_planner.rs +++ b/control_plane/src/physical/workload_planner.rs @@ -1,4 +1,4 @@ -//! Compatibility compiler from legacy flat workloads to physical deployment +//! Compiler from registered canonical workloads to physical deployment //! plans. Summary selection delegates to ASAPPlanner. use chrono::Utc; @@ -6,6 +6,28 @@ use std::time::Duration; use crate::types::*; +/// Bind the complete registered expression used by the HTTP deployment path. +pub fn bind_registered_query( + w: &RegisteredWorkload, +) -> anyhow::Result { + let query = crate::query_parser::parse_query_expr_canonical(&w.entry().query.0, w.accuracy())?; + if let Some(sketch) = &w.deployment.sketch_type_override { + let cost_model = crate::physical::post_asap::cost_model::ForcedFamilyCostModel::new( + w.accuracy(), + planner_types::post_asap::SketchAlgorithm::from(sketch.clone()), + ); + Ok(crate::physical::post_asap::bind_query_expr_with_cost_model( + &query, + &cost_model, + )?) + } else { + Ok(crate::physical::post_asap::bind_query_expr( + &query, + w.accuracy(), + )?) + } +} + pub const DEFAULT_VALID_FOR: Duration = Duration::from_secs(10 * 60); /// MVP fixture classification used only to translate the demo workload into @@ -36,14 +58,14 @@ fn mvp_deployment_policy( }) } -/// Bind a flat workload to a typed physical expression for stage emission. +/// Derive a typed collector expression from a registered canonical workload. /// /// An explicit sketch override takes precedence when valid for the statistic. /// Otherwise deployment contract rows select the family, with aggregation-type /// defaults for other metrics. Unsupported and raw-passthrough workloads /// return `None`. pub fn bind_workload_typed( - w: &LegacyMetricWorkload, + w: &RegisteredWorkload, ) -> Option { bind_workload_typed_with_evidence(w, None, None) } @@ -53,7 +75,7 @@ pub fn bind_workload_typed( /// evidence is absent; callers that have validated a fresh certificate use /// this entry point instead. pub fn bind_workload_typed_with_topk_evidence( - w: &LegacyMetricWorkload, + w: &RegisteredWorkload, evidence: &crate::physical::compiler::TopKMembershipEvidence, ) -> Option { bind_workload_typed_with_evidence(w, None, Some(evidence)) @@ -64,21 +86,21 @@ pub fn bind_workload_typed_with_topk_evidence( /// query's actual per-item filter value through to the bound /// `SketchQuery::PointCount` -- `None` (what `bind_workload_typed` itself /// passes) gives the bare bucket total, same as before this parameter -/// existed. `LegacyMetricWorkload` itself carries no `item_label` field (adding +/// existed. `RegisteredWorkload` itself carries no `item_label` field (adding /// one would break its 30+ struct-literal construction sites across the /// crate), so callers that know a metric's item_label -- e.g. /// `emit::collect_metric_to_family`'s loop, which already has `entry: /// &WorkloadEntry` and `workload.label_filters` in scope -- pass it in /// directly instead. pub fn bind_workload_typed_with_item_filter( - w: &LegacyMetricWorkload, + w: &RegisteredWorkload, item_filter: Option<(&str, &str)>, ) -> Option { bind_workload_typed_with_evidence(w, item_filter, None) } fn bind_workload_typed_with_evidence( - w: &LegacyMetricWorkload, + w: &RegisteredWorkload, item_filter: Option<(&str, &str)>, topk_evidence: Option<&crate::physical::compiler::TopKMembershipEvidence>, ) -> Option { @@ -98,12 +120,12 @@ fn bind_workload_typed_with_evidence( // `top_endpoint_qps` / `endpoint_request_freq`) parse to // `exact_required: true` and the typed binder declines, so the // 5-sketch routing emitter never sees them. - let metric_is_contract_row = mvp_deployment_policy(&w.metric_name).is_some(); - let operator_pinned_sketch = w.sketch_type_override.is_some(); - if w.exact_required && !metric_is_contract_row && !operator_pinned_sketch { + let metric_is_contract_row = mvp_deployment_policy(&w.metric_name()).is_some(); + let operator_pinned_sketch = w.deployment.sketch_type_override.is_some(); + if w.exact_required() && !metric_is_contract_row && !operator_pinned_sketch { return None; } - if w.aggregations.len() != 1 { + if w.aggregations().len() != 1 && !metric_is_contract_row { return None; } @@ -113,7 +135,7 @@ fn bind_workload_typed_with_evidence( // default. The metric-name match owns the demo contract rows; the // AggType fallback covers everything else. let (statistic, default_kind) = - mvp_deployment_policy(&w.metric_name).unwrap_or_else(|| match w.aggregations[0] { + mvp_deployment_policy(&w.metric_name()).unwrap_or_else(|| match w.aggregations()[0] { AggType::Quantile => (DeploymentIntent::Quantile, SketchAlgorithm::DDSketch), AggType::Cardinality => (DeploymentIntent::Cardinality, SketchAlgorithm::Hll), AggType::Frequency => (DeploymentIntent::Frequency, SketchAlgorithm::Cms), @@ -141,6 +163,7 @@ fn bind_workload_typed_with_evidence( // instead so the binding never produces a nonsense (sketch, stat) // pair. let override_kind: Option = w + .deployment .sketch_type_override .as_ref() .map(|st| SketchAlgorithm::from(st.clone())); @@ -148,7 +171,7 @@ fn bind_workload_typed_with_evidence( // the intent. An invalid override produces no candidate below. let kind = override_kind.unwrap_or(default_kind); - let accuracy = w.accuracy.clone(); + let accuracy = w.accuracy().clone(); let intent_accuracy = accuracy.clone(); // Build the matching L3 `AggIntent` for the picked statistic class. @@ -159,7 +182,7 @@ fn bind_workload_typed_with_evidence( let intent = match statistic { DeploymentIntent::Quantile => L3AggIntent::Quantile { col: None, - q: w.quantiles.first().copied().unwrap_or(0.99), + q: w.quantiles().first().copied().unwrap_or(0.99), accuracy: intent_accuracy, }, DeploymentIntent::Cardinality => L3AggIntent::Cardinality { @@ -180,7 +203,7 @@ fn bind_workload_typed_with_evidence( let scan = QueryExpr::Scan { source: Source::TimeSeries { - metric: w.metric_name.clone(), + metric: w.metric_name().clone(), }, // This synthetic scan only exists to drive `Bind*` rule dispatch // against a representative `Aggregate` shape — the rules key off @@ -215,7 +238,7 @@ fn bind_workload_typed_with_evidence( ), }; let windowed = QueryExpr::TimeRange { - range: w.time_window, + range: w.time_window(), child: Box::new(scan).into(), }; // Planner's weighted Top-K contract deliberately accepts only an @@ -357,30 +380,39 @@ impl DeploymentPlanCompiler { } } - pub fn plan(&self, w: &LegacyMetricWorkload) -> CollectionPlan { + pub fn plan(&self, w: &RegisteredWorkload) -> CollectionPlan { // This legacy scalar cost path cannot certify a failure probability. // Exact/zero-error and EpsilonDelta use raw; the typed binder independently // checks the full requirement against Planner's family guarantees. - if w.exact_required - || !matches!(w.accuracy, crate::types_v2::AccuracyTarget::Epsilon(epsilon) if epsilon > 0.0) + if w.exact_required() + || !matches!(w.accuracy(), crate::types_v2::AccuracyTarget::Epsilon(epsilon) if epsilon > 0.0) { return self.raw_passthrough_plan(w); } - let sketch_type = crate::physical::sketch_catalog::sketch_type_for_agg(&w.aggregations); + if w.deployment.sketch_type_override.is_some() && bind_workload_typed(w).is_none() { + return self.raw_passthrough_plan(w); + } + let sketch_type = w + .deployment + .sketch_type_override + .clone() + .unwrap_or_else(|| { + crate::physical::sketch_catalog::sketch_type_for_agg(&w.aggregations()) + }); let sketch_params = crate::physical::sketch_catalog::build_sketch_params( &self.sketch_defaults, &sketch_type, w.error_bound(), - &w.quantiles, + &w.quantiles(), ); let (mode, window_duration) = select_window_strategy(w); - let mut aggregate_by = w.group_by_labels.clone(); + let mut aggregate_by = w.group_by_labels().clone(); aggregate_by.sort(); let mut label_matchers: Vec = w - .label_filters + .label_filters() .iter() .map(|(k, v)| format!("{k}={v}")) .collect(); @@ -419,11 +451,11 @@ impl DeploymentPlanCompiler { /// Returns a raw-passthrough plan for queries that require exact per-sample /// computation (RSI, MACD, stochastic oscillator, etc.). - fn raw_passthrough_plan(&self, w: &LegacyMetricWorkload) -> CollectionPlan { + fn raw_passthrough_plan(&self, w: &RegisteredWorkload) -> CollectionPlan { let valid_until = Utc::now() + chrono::Duration::seconds(self.valid_for.as_secs() as i64); let mut label_matchers: Vec = w - .label_filters + .label_filters() .iter() .map(|(k, v)| format!("{k}={v}")) .collect(); @@ -467,10 +499,10 @@ pub use crate::physical::sketch_catalog::{build_sketch_params, default_sketch_pa /// /// Rule: if `latency_sla >= time_window` (or unset) → window mode. /// otherwise → batch mode (gateway/backend merges on query). -pub fn select_window_strategy(w: &LegacyMetricWorkload) -> (ProcessorMode, Option) { - match w.latency_sla { - None => (ProcessorMode::Window, Some(w.time_window)), - Some(ls) if ls >= w.time_window => (ProcessorMode::Window, Some(w.time_window)), +pub fn select_window_strategy(w: &RegisteredWorkload) -> (ProcessorMode, Option) { + match w.latency_sla() { + None => (ProcessorMode::Window, Some(w.time_window())), + Some(ls) if ls >= w.time_window() => (ProcessorMode::Window, Some(w.time_window())), _ => (ProcessorMode::Batch, None), } } @@ -482,21 +514,22 @@ mod tests { use super::*; use std::collections::HashMap; - fn workload(aggs: Vec) -> LegacyMetricWorkload { - LegacyMetricWorkload { + fn workload(aggs: Vec) -> RegisteredWorkload { + crate::registered_workload::fixtures::WorkloadFixture { metric_name: "test".into(), label_filters: HashMap::new(), group_by_labels: vec![], aggregations: aggs, time_window: Duration::from_secs(300), repeat_every: None, - accuracy_sla: 0.01, + accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.01), latency_sla: None, sketch_type_override: None, exact_required: false, quantiles: vec![], } + .build() } #[test] @@ -531,7 +564,7 @@ mod tests { (SketchType::CountMinSketch, SketchAlgorithm::DDSketch), ] { let mut w = workload(vec![AggType::Quantile]); - w.sketch_type_override = Some(ov.clone()); + w.deployment.sketch_type_override = Some(ov.clone()); let pe = bind_workload_typed(&w) .unwrap_or_else(|| panic!("bind declined for override {ov:?}")); assert_eq!( @@ -543,20 +576,19 @@ mod tests { } #[test] - fn quantile_priority_wins() { - let plan = DeploymentPlanCompiler::new() - .plan(&workload(vec![AggType::Quantile, AggType::Cardinality])); - assert_eq!( - plan.agent_config.sketch_type, - SketchType::DDSketch, - "quantile should take priority over cardinality" - ); + fn mixed_field_aggregations_require_explicit_queries() { + let spec = serde_json::from_value(serde_json::json!({ + "metric_name": "test", "time_window": "5m", "accuracy_sla": 0.99, + "aggregations": ["quantile", "cardinality"] + })) + .unwrap(); + assert!(crate::pipeline::Analyzer::new().analyze(spec).is_err()); } #[test] fn window_mode_when_latency_geq_time_window() { let mut w = workload(vec![AggType::Quantile]); - w.latency_sla = Some(Duration::from_secs(600)); // 10m >= 5m + w.set_latency_sla(Some(Duration::from_secs(600))); // 10m >= 5m let plan = DeploymentPlanCompiler::new().plan(&w); assert_eq!(plan.agent_config.mode, ProcessorMode::Window); assert_eq!( @@ -568,7 +600,7 @@ mod tests { #[test] fn batch_mode_when_latency_lt_time_window() { let mut w = workload(vec![AggType::Quantile]); - w.latency_sla = Some(Duration::from_secs(60)); // 1m < 5m + w.set_latency_sla(Some(Duration::from_secs(60))); // 1m < 5m let plan = DeploymentPlanCompiler::new().plan(&w); assert_eq!(plan.agent_config.mode, ProcessorMode::Batch); assert_eq!(plan.agent_config.window_duration, None); @@ -577,7 +609,7 @@ mod tests { #[test] fn no_latency_sla_defaults_to_window() { let mut w = workload(vec![AggType::Quantile]); - w.latency_sla = None; + w.set_latency_sla(None); let plan = DeploymentPlanCompiler::new().plan(&w); assert_eq!(plan.agent_config.mode, ProcessorMode::Window); } @@ -585,7 +617,7 @@ mod tests { #[test] fn aggregate_by_sorted() { let mut w = workload(vec![AggType::Quantile]); - w.group_by_labels = vec!["zone".into(), "host.name".into(), "service".into()]; + w.deployment.retained_labels = vec!["zone".into(), "host.name".into(), "service".into()]; let plan = DeploymentPlanCompiler::new().plan(&w); assert_eq!( plan.agent_config.aggregate_by, @@ -596,11 +628,13 @@ mod tests { #[test] fn label_matchers_from_filters() { let mut w = workload(vec![AggType::Quantile]); - w.label_filters = [ - ("env".into(), "prod".into()), - ("service".into(), "web".into()), - ] - .into(); + w.set_label_filters( + [ + ("env".into(), "prod".into()), + ("service".into(), "web".into()), + ] + .into(), + ); let plan = DeploymentPlanCompiler::new().plan(&w); assert_eq!(plan.agent_config.label_matchers.len(), 2); } @@ -608,7 +642,7 @@ mod tests { #[test] fn ddsketch_accuracy_params() { let mut w = workload(vec![AggType::Quantile]); - w.accuracy = crate::types_v2::AccuracyTarget::Epsilon(0.005); + w.set_accuracy(crate::types_v2::AccuracyTarget::Epsilon(0.005)); let plan = DeploymentPlanCompiler::new().plan(&w); match &plan.agent_config.sketch_params { SketchParams::DDSketch { @@ -621,7 +655,7 @@ mod tests { #[test] fn hll_precision_coarse_sla() { let mut w = workload(vec![AggType::Cardinality]); - w.accuracy = crate::types_v2::AccuracyTarget::Epsilon(0.03); + w.set_accuracy(crate::types_v2::AccuracyTarget::Epsilon(0.03)); let plan = DeploymentPlanCompiler::new().plan(&w); match &plan.agent_config.sketch_params { SketchParams::HLL { precision } => { @@ -698,21 +732,22 @@ mod tests { /// contract rows; the AggType still has to be a valid one (the enum /// has no `TopK` variant, so for `top_endpoint_qps` we pass /// `Frequency` and rely on the metric-name reclassification). - fn workload_for(metric: &str, agg: AggType) -> LegacyMetricWorkload { - LegacyMetricWorkload { + fn workload_for(metric: &str, agg: AggType) -> RegisteredWorkload { + crate::registered_workload::fixtures::WorkloadFixture { metric_name: metric.into(), label_filters: HashMap::new(), group_by_labels: vec![], aggregations: vec![agg], time_window: Duration::from_secs(300), repeat_every: None, - accuracy_sla: 0.01, + accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.01), latency_sla: None, sketch_type_override: None, exact_required: false, quantiles: vec![], } + .build() } fn topk_evidence() -> crate::physical::compiler::TopKMembershipEvidence { @@ -806,6 +841,31 @@ mod tests { )); } + /// Canonical binding must preserve pins for both field-only and string requests. + #[test] + fn canonical_binding_preserves_field_only_sketch_pins() { + for sketch in [SketchType::KLL, SketchType::DDSketch] { + for use_query_string in [false, true] { + let mut input = serde_json::json!({ + "metric_name": "latency", "aggregations": ["quantile"], "time_window": "5m", + "accuracy_sla": 0.99, "sketch_type": sketch, + }); + if use_query_string { + input["query_string"] = + serde_json::json!("quantile_over_time(0.99, latency[5m])"); + } + let workload = crate::pipeline::Analyzer::new() + .analyze(serde_json::from_value(input).unwrap()) + .unwrap(); + let bound = bind_registered_query(&workload).unwrap(); + assert_eq!( + extract_family(&bound), + Some(SketchAlgorithm::from(sketch.clone())) + ); + } + } + } + #[test] fn typed_binding_endpoint_request_freq_binds_cms() { // Contract: `endpoint_request_freq` → CMS (Frequency). `Frequency` @@ -869,7 +929,7 @@ mod tests { // for Quantile per the capability matrix, so the override is // honoured. let mut w = workload_for("http_latency_ms", AggType::Quantile); - w.sketch_type_override = Some(SketchType::KLL); + w.deployment.sketch_type_override = Some(SketchType::KLL); let bound = bind_workload_typed(&w).expect("override should still bind"); assert_eq!( extract_family(&bound), @@ -883,7 +943,7 @@ mod tests { // `request_size_bytes`'s contract row is KLL (rank-err); a // workload override of `DDSketch` flips it back to DDSketch. let mut w = workload_for("request_size_bytes", AggType::Quantile); - w.sketch_type_override = Some(SketchType::DDSketch); + w.deployment.sketch_type_override = Some(SketchType::DDSketch); let bound = bind_workload_typed(&w).expect("override should still bind"); assert_eq!( extract_family(&bound), @@ -900,7 +960,7 @@ mod tests { // metric, the planner should accept it instead of falling // back to the canonical CountSketch default. let mut w = workload_for("top_endpoint_qps", AggType::Frequency); - w.sketch_type_override = Some(SketchType::CountMinSketch); + w.deployment.sketch_type_override = Some(SketchType::CountMinSketch); let bound = bind_workload_typed_with_topk_evidence(&w, &topk_evidence()) .expect("CMS Top-K override must bind with evidence"); assert_eq!(extract_family(&bound), Some(SketchAlgorithm::CmsWithHeap)); @@ -926,7 +986,7 @@ mod tests { // matrix rejects the override, and the planner falls back to // the contract-row default (DDSketch for `http_latency_ms`). let mut w = workload_for("http_latency_ms", AggType::Quantile); - w.sketch_type_override = Some(SketchType::HLL); + w.deployment.sketch_type_override = Some(SketchType::HLL); let bound = bind_workload_typed(&w).expect("fallback should bind"); assert_eq!( extract_family(&bound), diff --git a/control_plane/src/pipeline.rs b/control_plane/src/pipeline.rs index d926a8678..b664cdb45 100644 --- a/control_plane/src/pipeline.rs +++ b/control_plane/src/pipeline.rs @@ -4,7 +4,7 @@ use std::collections::{HashMap, HashSet}; use std::time::Duration; use crate::query_parser; -use crate::types::{AggType, LegacyMetricWorkload, SketchType, WorkloadCharacteristics}; +use crate::types::{AggType, RegisteredWorkload, SketchType, WorkloadCharacteristics}; use crate::types_v2::{AccuracyTarget, DataShape, QueryId, QueryLanguage, QueryShape}; // ── Public API ──────────────────────────────────────────────────────────────── @@ -19,8 +19,8 @@ use crate::types_v2::{AccuracyTarget, DataShape, QueryId, QueryLanguage, QuerySh /// 2. **Query string** — supply a raw PromQL string in /// `query_string`. The analyzer parses it and fills in `metric_name`, /// `aggregations`, `group_by_labels`, `label_filters`, and `time_window` -/// automatically. Any explicit fields that are non-empty / non-default -/// **override** the parsed values, so the two approaches compose. +/// automatically. Explicit semantic fields must agree with the expression; +/// conflicting overrides are rejected before registration. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct QuerySpec { /// Raw PromQL query string to parse (SP-1 automatic extraction). @@ -29,49 +29,38 @@ pub struct QuerySpec { #[serde(default)] pub query_string: Option, - /// Metric name override. Required when `query_string` is absent. + /// Metric identity. Required without a query; must match a supplied query. #[serde(default)] pub metric_name: String, #[serde(default)] pub label_filters: HashMap, + /// Additional labels the collector must retain; does not rewrite query GROUP BY. #[serde(default)] pub group_by_labels: Vec, - /// Aggregation type overrides ("quantile", "cardinality", "frequency"). + /// Field-only aggregation ("quantile", "cardinality", "frequency"). /// Required when `query_string` is absent. #[serde(default)] pub aggregations: Vec, - /// Time window override (e.g. "5m"). Required when `query_string` is absent. + /// Time window (e.g. "5m"). Required without a query; otherwise must agree. #[serde(default)] pub time_window: String, #[serde(default)] pub repeat_every: Option, pub accuracy_sla: f64, pub latency_sla: Option, - /// Optional: pin a specific sketch type, bypassing the cost-model planner. + /// Optional implementation constraint, still subject to Planner legality. pub sketch_type: Option, /// Observable data-stream characteristics used for delta / raw-vs-sketch /// bandwidth comparison. Omit to use conservative defaults. #[serde(default)] pub workload: WorkloadCharacteristics, - // ── design.md alignment: new fields, defaulted for back-compat ──────── - // - // These fields converge `QuerySpec` toward the typed schema in - // `control_plane/docs/design.md` §6 `core::workload`. Each is defaulted - // so the existing JSON API surface (POST /api/v1/plan handlers, - // pre-population from `workloads.yaml`, the test fixtures elsewhere - // in the control plane) keeps working without supplying them. The - // accuracy target is consumed by parsing and binding. The other fields - // retain the partial support described in `Analyzer::analyze`. - /// Stable identifier preserved across replan cycles. Optional; - /// auto-derived from `metric_name + accuracy_sla` if omitted - /// (existing API callers don't supply this). + /// Optional stable registration identifier, preserved as metadata. #[serde(default)] pub id: Option, - /// Source language. Inferred from `query_string` syntax / parser - /// dispatch when omitted (existing API callers default to PromQL - /// behavior, which matches today's `query_parser::parse_query`). + /// This metric-registration endpoint accepts PromQL; other languages use + /// their dedicated compilation paths. #[serde(default)] pub language: Option, @@ -82,8 +71,8 @@ pub struct QuerySpec { #[serde(default)] pub accuracy: Option, - /// Per-evaluation $ budget. Optional; the cost model picks freely - /// when unset. + /// Reserved compatibility field. Explicit dollar constraints are rejected + /// because metric registration does not implement them. #[serde(default)] pub dollars: Option, @@ -122,7 +111,7 @@ impl Analyzer { Self } - pub fn analyze(&self, spec: QuerySpec) -> anyhow::Result { + pub fn analyze(&self, spec: QuerySpec) -> anyhow::Result { let accuracy = crate::types_v2::resolve_accuracy_target(spec.accuracy.as_ref(), spec.accuracy_sla) .map_err(|error| anyhow!(error))?; @@ -133,8 +122,8 @@ impl Analyzer { // hard rejections are at L1 because they have no semantically // valid plan: a streaming query over a static dataset, and a // streaming query over a mutable relation (no retraction-aware - // sketches in the catalog yet). Everything else is accepted - // here — downstream rule firing can still narrow further. + // sketches in the catalog yet). Canonical conversion below also checks + // which recurrence and data shapes this deployment path can represent. match (&spec.shape, &spec.data) { (QueryShape::Streaming, DataShape::Batch) => { return Err(anyhow!( @@ -153,18 +142,6 @@ impl Analyzer { _ => {} } - // Compatibility reporting only: all semantic consumers use `accuracy`. - let accuracy_sla = if spec.accuracy.is_none() { - spec.accuracy_sla - } else { - match accuracy { - AccuracyTarget::Exact => 1.0, - AccuracyTarget::Epsilon(epsilon) | AccuracyTarget::EpsilonDelta { epsilon, .. } => { - 1.0 - epsilon - } - } - }; - // ── Step 1: parse query_string if provided ───────────────────────── // Parsing and downstream binding receive this same resolved target. let parsed = spec @@ -187,12 +164,6 @@ impl Analyzer { let aggregations = if !spec.aggregations.is_empty() { parse_agg_types(&spec.aggregations)? } else if let Some(ref p) = parsed { - if p.aggregations.is_empty() && !p.exact_required { - return Err(anyhow!( - "could not infer aggregation type from query_string; \ - provide explicit aggregations" - )); - } p.aggregations.clone() } else { return Err(anyhow!("at least one aggregation is required")); @@ -212,12 +183,8 @@ impl Analyzer { return Err(anyhow!("time_window is required (or provide query_string)")); }; - // ── Step 5: resolve dimensions (group_by + label_filter keys) ────── - // Parsed values are the base; explicit spec fields override / extend. - let parsed_group_by = parsed - .as_ref() - .map(|p| p.group_by_labels.as_slice()) - .unwrap_or(&[]); + // ── Step 5: resolve filters; conflicts are checked against the query ─ + // Collector retention labels remain independent deployment options. let parsed_filters: HashMap = parsed .as_ref() .map(|p| p.label_filters.clone()) @@ -229,12 +196,6 @@ impl Analyzer { m }; - let filter_keys: Vec = merged_filters.keys().cloned().collect(); - let all_group_by: Vec = dedup_dims( - &dedup_dims(parsed_group_by, &spec.group_by_labels), - &filter_keys, - ); - // ── Step 6: scalar fields ────────────────────────────────────────── let repeat_every = spec .repeat_every @@ -250,36 +211,153 @@ impl Analyzer { .transpose() .with_context(|| "invalid latency_sla")?; - let exact_required = parsed.as_ref().map(|p| p.exact_required).unwrap_or(false); - let quantiles = parsed - .as_ref() - .map(|p| p.quantiles.clone()) - .unwrap_or_default(); - - // These remaining compatibility fields do not yet feed the flat planner. - let _ = ( - &spec.shape, - &spec.data, - &spec.id, - &spec.language, - &spec.dollars, - &spec.deployment_model, + use crate::registered_workload::{declared, DeploymentOptions}; + use planner_types::workload::*; + let query = if let Some(query) = &spec.query_string { + let p = parsed.as_ref().expect("parsed above"); + anyhow::ensure!(metric_name == p.metric_name && time_window == p.time_window + && aggregations == p.aggregations && merged_filters == p.label_filters, + "explicit overrides conflict with query_string; update the query expression instead"); + query.clone() + } else { + anyhow::ensure!( + aggregations.len() == 1, + "field-only input requires one aggregation" + ); + let mut filters: Vec<_> = merged_filters + .iter() + .map(|(k, v)| format!("{k}={}", serde_json::to_string(v).expect("string"))) + .collect(); + filters.sort(); + let selector = if filters.is_empty() { + metric_name.clone() + } else { + format!("{}{{{}}}", metric_name, filters.join(",")) + }; + match aggregations[0] { + AggType::Quantile => format!( + "quantile_over_time(0.99, {selector}[{}s])", + time_window.as_secs() + ), + AggType::Cardinality => { + format!("distinct_over_time({selector}[{}s])", time_window.as_secs()) + } + AggType::Frequency => { + format!("count_over_time({selector}[{}s])", time_window.as_secs()) + } + } + }; + anyhow::ensure!( + spec.language.is_none() + || matches!(spec.language, Some(crate::types_v2::QueryLanguage::PromQl)), + "metric registration requires PromQL" ); - - Ok(LegacyMetricWorkload { - metric_name, - label_filters: merged_filters, - group_by_labels: all_group_by, - aggregations, - time_window, - repeat_every, - accuracy, - accuracy_sla, - latency_sla, - sketch_type_override: spec.sketch_type, - exact_required, - quantiles, - }) + anyhow::ensure!( + spec.dollars.is_none(), + "dollars constraints are not supported by metric registration" + ); + let cadence = match spec.shape { + QueryShape::Periodic { every } => { + anyhow::ensure!(repeat_every.is_none_or(|r| r == every), "conflicting repetition intervals"); + Some(every) + }, + QueryShape::Streaming => return Err(anyhow!("streaming demand without a fixed cadence is not supported; specify periodic demand")), + QueryShape::OneShot => repeat_every, + }; + let requirements = QueryRequirements { + accuracy: AccuracyRequirement::Explicit(accuracy), + response_latency: latency_sla + .map(|d| LatencyRequirement::ExplicitMaxMs(d.as_secs_f64() * 1000.0)) + .unwrap_or(LatencyRequirement::Unspecified), + }; + let time_selection = TimeSelection { + scope: QueryTimeScope::RealTime, + lookback: crate::registered_workload::metric_query_range(&query)? + .map(|range| u64::try_from(range.as_millis()).map(DurationMs)) + .transpose()?, + as_of: None, + }; + let (query_batch, repeating_queries) = if let Some(cadence) = cadence { + anyhow::ensure!( + cadence.subsec_nanos().is_multiple_of(1_000_000), + "repetition interval requires whole milliseconds" + ); + let interval = u32::try_from(cadence.as_millis()) + .context("repetition interval exceeds u32 milliseconds")?; + anyhow::ensure!(interval > 0, "repetition interval must be positive"); + ( + None, + Some(vec![RepeatingEntry { + query: Query(query), + demand: RepeatedDemand::FixedInterval(RepetitionInterval(interval)), + requirements, + predictability: Predictability::Predictable { known_at: None }, + time_selection, + }]), + ) + } else { + ( + Some(vec![BatchEntry { + query: Query(query), + requirements, + predictability: Predictability::AdHoc, + invocations: 1, + execute_at: None, + time_selection, + }]), + None, + ) + }; + let wc = &spec.workload; + let rate = wc.series_count as f64 * wc.samples_per_sec_per_series; + anyhow::ensure!( + wc.samples_per_sec_per_series.is_finite() + && wc.samples_per_sec_per_series >= 0.0 + && rate.is_finite(), + "sample rate must be finite and nonnegative" + ); + let arrival = match spec.data { + DataShape::Batch => DataArrival::AtRest, + DataShape::AppendOnlyStream => DataArrival::ContinuouslyIngesting, + DataShape::Mixed => DataArrival::Mixed, + DataShape::Mutable => { + return Err(anyhow!( + "mutable data is not supported by metric registration" + )) + } + }; + let data_workload = Some(DataWorkload { + arrival, + ingestion_rate: declared(Rate(if matches!(spec.data, DataShape::Batch) { + 0.0 + } else { + rate + })), + input_cardinality: declared(wc.series_count), + distribution: declared(match wc.data_distribution { + crate::types::DataDistribution::Zipf => DataDistribution::Zipf, + crate::types::DataDistribution::Uniform => DataDistribution::Uniform, + crate::types::DataDistribution::Bursty => DataDistribution::Bursty, + }), + ingestion_volume: Evidence::default(), + }); + RegisteredWorkload::new( + QueryWorkload { + language: QueryLanguage::PromQL, + query_batch, + repeating_queries, + data_workload, + }, + DeploymentOptions { + sketch_type_override: spec.sketch_type, + query_id: spec.id, + deployment_model: spec.deployment_model, + retained_labels: dedup_dims(&spec.group_by_labels, &[]), + bytes_per_raw_sample: wc.bytes_per_raw_sample, + distinct_keys_per_window: wc.distinct_keys_per_window, + memory_budget_bytes: wc.memory_budget_bytes, + }, + ) } } @@ -301,12 +379,16 @@ pub fn parse_duration(s: &str) -> anyhow::Result { .parse() .map_err(|_| anyhow!("invalid number in duration {:?}", s))?; current_num.clear(); - match ch { - 'h' => total_secs += n * 3600, - 'm' => total_secs += n * 60, - 's' => total_secs += n, + let multiplier = match ch { + 'h' => 3600, + 'm' => 60, + 's' => 1, _ => return Err(anyhow!("unknown unit {:?} in duration {:?}", ch, s)), - } + }; + total_secs = n + .checked_mul(multiplier) + .and_then(|part| total_secs.checked_add(part)) + .ok_or_else(|| anyhow!("duration overflow in {:?}", s))?; } } if !current_num.is_empty() { @@ -394,12 +476,12 @@ mod tests { #[test] fn valid_spec() { let w = Analyzer::new().analyze(basic_spec()).unwrap(); - assert_eq!(w.metric_name, "request_latency"); - assert_eq!(w.accuracy_sla, 0.01); - assert_eq!(w.time_window, Duration::from_secs(300)); - assert_eq!(w.repeat_every, Some(Duration::from_secs(60))); - assert_eq!(w.latency_sla, Some(Duration::from_secs(600))); - assert_eq!(w.aggregations, vec![AggType::Quantile]); + assert_eq!(w.metric_name(), "request_latency"); + assert!(((1.0 - w.error_bound()) - 0.01).abs() < 1e-12); + assert_eq!(w.time_window(), Duration::from_secs(300)); + assert_eq!(w.repeat_every(), Some(Duration::from_secs(60))); + assert_eq!(w.latency_sla(), Some(Duration::from_secs(600))); + assert_eq!(w.aggregations(), vec![AggType::Quantile]); } #[test] @@ -407,22 +489,22 @@ mod tests { let mut spec = basic_spec(); spec.label_filters = [ ("service".into(), "api".into()), - ("host.name".into(), "h1".into()), + ("host_name".into(), "h1".into()), ] .into(); - spec.group_by_labels = vec!["host.name".into(), "region".into()]; + spec.group_by_labels = vec!["host_name".into(), "region".into()]; let w = Analyzer::new().analyze(spec).unwrap(); - for dim in &["host.name", "region", "service"] { + for dim in &["host_name", "region", "service"] { assert!( - w.group_by_labels.contains(&dim.to_string()), + w.group_by_labels().contains(&dim.to_string()), "missing {dim}" ); } // host.name must appear exactly once after dedup assert_eq!( - w.group_by_labels + w.group_by_labels() .iter() - .filter(|d| d.as_str() == "host.name") + .filter(|d| d.as_str() == "host_name") .count(), 1 ); @@ -432,11 +514,7 @@ mod tests { fn multiple_aggregations() { let mut spec = basic_spec(); spec.aggregations = vec!["cardinality".into(), "frequency".into()]; - let w = Analyzer::new().analyze(spec).unwrap(); - assert_eq!( - w.aggregations, - vec![AggType::Cardinality, AggType::Frequency] - ); + assert!(Analyzer::new().analyze(spec).is_err()); } #[test] @@ -550,43 +628,35 @@ mod tests { "sum by (host) (quantile_over_time(0.99, latency[5m]))", )) .unwrap(); - assert_eq!(w.metric_name, "latency"); - assert_eq!(w.aggregations, vec![AggType::Quantile]); - assert_eq!(w.time_window, Duration::from_secs(300)); - assert_eq!(w.quantiles, vec![0.99]); - assert!(w.exact_required); + assert_eq!(w.metric_name(), "latency"); + assert_eq!(w.aggregations(), vec![AggType::Quantile]); + assert_eq!(w.time_window(), Duration::from_secs(300)); + assert_eq!(w.quantiles(), vec![0.99]); + assert!(w.exact_required()); } - /// Explicit metric_name overrides the name derived from query_string. + /// A conflicting metric field cannot change only the stored projection. #[test] fn explicit_metric_name_overrides_parsed() { let mut spec = qs_only("sum by (host) (avg_over_time(cpu[5m]))"); spec.metric_name = "my_custom_metric".into(); - let w = Analyzer::new().analyze(spec).unwrap(); - assert_eq!(w.metric_name, "my_custom_metric"); - // aggregations still come from parse — `avg_over_time` is exact - // (no ASAP-tier sketch substitute for `AggIntent::Avg`), so - // `aggregations` stays empty and `exact_required` flips instead. - assert_eq!(w.aggregations, Vec::::new()); - assert!(w.exact_required); + assert!(Analyzer::new().analyze(spec).is_err()); } - /// Explicit time_window overrides the window derived from query_string. + /// A conflicting window cannot disagree with the canonical expression. #[test] fn explicit_time_window_overrides_parsed() { let mut spec = qs_only("sum by (host) (avg_over_time(cpu[5m]))"); spec.time_window = "1h".into(); - let w = Analyzer::new().analyze(spec).unwrap(); - assert_eq!(w.time_window, Duration::from_secs(3600)); + assert!(Analyzer::new().analyze(spec).is_err()); } - /// Explicit aggregations override those derived from query_string. + /// A conflicting aggregation cannot replace canonical query semantics. #[test] fn explicit_aggregations_override_parsed() { let mut spec = qs_only("sum by (host) (avg_over_time(cpu[5m]))"); // → Quantile spec.aggregations = vec!["cardinality".into()]; - let w = Analyzer::new().analyze(spec).unwrap(); - assert_eq!(w.aggregations, vec![AggType::Cardinality]); + assert!(Analyzer::new().analyze(spec).is_err()); } /// sum_over_time is a stateful exact aggregation; exact_required is set. @@ -597,8 +667,8 @@ mod tests { "sum by (service) (sum_over_time(request_bytes[1h]))", )) .unwrap(); - assert!(w.exact_required, "sum_over_time must set exact_required"); - assert_eq!(w.aggregations, vec![]); + assert!(w.exact_required(), "sum_over_time must set exact_required"); + assert_eq!(w.aggregations(), vec![]); } /// DDSketch quantile φ values are surfaced through the workload. @@ -609,7 +679,7 @@ mod tests { "sum by (host) (quantile_over_time(0.5, latency[5m]))", )) .unwrap(); - assert_eq!(w.quantiles, vec![0.5]); + assert_eq!(w.quantiles(), vec![0.5]); } /// Existing callers that supply all fields explicitly and omit @@ -617,30 +687,30 @@ mod tests { #[test] fn backward_compat_no_query_string() { let w = Analyzer::new().analyze(basic_spec()).unwrap(); - assert_eq!(w.metric_name, "request_latency"); - assert_eq!(w.aggregations, vec![AggType::Quantile]); - assert_eq!(w.time_window, Duration::from_secs(300)); - assert!(!w.exact_required); - assert!(w.quantiles.is_empty()); + assert_eq!(w.metric_name(), "request_latency"); + assert_eq!(w.aggregations(), vec![AggType::Quantile]); + assert_eq!(w.time_window(), Duration::from_secs(300)); + assert!(!w.exact_required()); + assert_eq!(w.quantiles(), vec![0.99]); } // ── design.md alignment tests ───────────────────────────────────────────── /// Typed `accuracy: Some(Epsilon(0.05))` overrides the legacy /// `accuracy_sla: 0.99` (which would translate to `Epsilon(0.01)`), - /// and the resolved value flows through to `LegacyMetricWorkload.accuracy_sla`. + /// and the resolved value flows through to `RegisteredWorkload.accuracy_sla`. #[test] fn typed_accuracy_overrides_legacy_accuracy_sla() { let mut spec = basic_spec(); spec.accuracy_sla = 0.99; // legacy: ε = 0.01 spec.accuracy = Some(AccuracyTarget::Epsilon(0.05)); let w = Analyzer::new().analyze(spec).unwrap(); - // The resolved 1.0 - 0.05 = 0.95 must reach the LegacyMetricWorkload, not + // The resolved 1.0 - 0.05 = 0.95 must reach the RegisteredWorkload, not // the legacy 0.99. assert!( - (w.accuracy_sla - 0.95).abs() < 1e-9, + ((1.0 - w.error_bound()) - 0.95).abs() < 1e-9, "got {}", - w.accuracy_sla + (1.0 - w.error_bound()) ); } @@ -683,10 +753,9 @@ mod tests { let mut spec = basic_spec(); spec.accuracy_sla = 0.2; spec.accuracy = Some(target.clone()); - let mut workload = Analyzer::new().analyze(spec).unwrap(); - assert_eq!(workload.accuracy, target); - // Mutating the deprecated reporting view cannot affect semantic binding. - workload.accuracy_sla = 0.9999; + let workload = Analyzer::new().analyze(spec).unwrap(); + assert_eq!(workload.accuracy(), target); + // Canonical requirements have no independently mutable scalar mirror. let bound = crate::physical::workload_planner::bind_workload_typed(&workload); if target == AccuracyTarget::Exact { assert!( @@ -728,7 +797,7 @@ mod tests { spec.accuracy_sla = 0.5; spec.accuracy = Some(AccuracyTarget::Exact); let w = Analyzer::new().analyze(spec).unwrap(); - assert_eq!(w.accuracy_sla, 1.0); + assert_eq!((1.0 - w.error_bound()), 1.0); } /// L1 rejects `(QueryShape::Streaming, DataShape::Batch)` per the @@ -759,14 +828,13 @@ mod tests { ); } - /// `(QueryShape::Streaming, DataShape::AppendOnlyStream)` — the - /// canonical streaming case — is accepted. + /// Continuous demand needs a supported cadence before metric registration. #[test] - fn l1_accepts_streaming_over_append_only_stream() { + fn rejects_streaming_demand_without_fixed_cadence() { let mut spec = basic_spec(); spec.shape = QueryShape::Streaming; spec.data = DataShape::AppendOnlyStream; - assert!(Analyzer::new().analyze(spec).is_ok()); + assert!(Analyzer::new().analyze(spec).is_err()); } /// JSON without any of the new fields parses correctly via serde — @@ -791,14 +859,14 @@ mod tests { assert_eq!(spec.data, DataShape::AppendOnlyStream); // And the analyzer accepts it. let w = Analyzer::new().analyze(spec).unwrap(); - assert_eq!(w.metric_name, "request_latency"); + assert_eq!(w.metric_name(), "request_latency"); // Legacy accuracy_sla=0.99 round-trips through resolution // (no typed `accuracy` supplied → translate from legacy → // Epsilon(0.01) → back to 1 - 0.01 = 0.99). assert!( - (w.accuracy_sla - 0.99).abs() < 1e-9, + ((1.0 - w.error_bound()) - 0.99).abs() < 1e-9, "got {}", - w.accuracy_sla + (1.0 - w.error_bound()) ); } @@ -816,7 +884,7 @@ mod tests { "id": "q-001", "language": "prom_ql", "accuracy": { "Epsilon": 0.02 }, - "dollars": 0.001, + "dollars": null, "deployment_model": "asaplifecycle", "shape": { "kind": "periodic", "every": { "secs": 60, "nanos": 0 } }, "data": "batch" @@ -825,7 +893,7 @@ mod tests { assert_eq!(spec.id.as_ref().unwrap().as_str(), "q-001"); assert_eq!(spec.language, Some(QueryLanguage::PromQl)); assert_eq!(spec.accuracy, Some(AccuracyTarget::Epsilon(0.02))); - assert_eq!(spec.dollars, Some(0.001)); + assert_eq!(spec.dollars, None); assert_eq!(spec.deployment_model.as_deref(), Some("asaplifecycle")); assert!(matches!(spec.shape, QueryShape::Periodic { .. })); assert_eq!(spec.data, DataShape::Batch); @@ -835,9 +903,9 @@ mod tests { // typed `accuracy: Epsilon(0.02)` overrode the legacy 0.5 → // resolved accuracy_sla in the workload is 1.0 - 0.02 = 0.98. assert!( - (w.accuracy_sla - 0.98).abs() < 1e-9, + ((1.0 - w.error_bound()) - 0.98).abs() < 1e-9, "got {}", - w.accuracy_sla + (1.0 - w.error_bound()) ); } } diff --git a/control_plane/src/registered_workload.rs b/control_plane/src/registered_workload.rs new file mode 100644 index 000000000..c5ce61991 --- /dev/null +++ b/control_plane/src/registered_workload.rs @@ -0,0 +1,569 @@ +//! Registration state: canonical Planner workload plus collector deployment options. +use std::{collections::HashMap, time::Duration}; + +use anyhow::{anyhow, ensure}; +use planner_types::workload::*; + +use crate::{ + query_parser::{self, ParsedQuery}, + types::{AggType, SketchType, WorkloadCharacteristics}, + types_v2::AccuracyTarget, +}; + +/// Facts about the collector deployment, not query semantics or data arrival. +#[derive(Debug, Clone, Default)] +pub struct DeploymentOptions { + pub sketch_type_override: Option, + pub query_id: Option, + pub deployment_model: Option, + pub retained_labels: Vec, + pub bytes_per_raw_sample: u32, + pub distinct_keys_per_window: Option, + pub memory_budget_bytes: Option, +} + +/// The registry owns one canonical query; stage-specific metadata is derived on demand. +#[derive(Debug, Clone)] +pub struct RegisteredWorkload { + workload: QueryWorkload, + pub deployment: DeploymentOptions, +} + +impl RegisteredWorkload { + pub fn new(workload: QueryWorkload, deployment: DeploymentOptions) -> anyhow::Result { + workload + .validate() + .map_err(|e| anyhow!("invalid workload: {e:?}"))?; + ensure!( + workload.language == QueryLanguage::PromQL, + "metric registration requires PromQL" + ); + ensure!( + workload.entries().count() == 1, + "metric registration requires exactly one query entry" + ); + let registered = Self { + workload, + deployment, + }; + let entry = registered.entry(); + query_parser::parse_query_expr_canonical(&entry.query.0, registered.accuracy())?; + let range = metric_query_range(&entry.query.0)?; + ensure!(matches!(entry.recurrence, QueryRecurrence::OneTime { invocations: 1, execute_at: None } | QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(_))), "metric registration supports one invocation or a fixed interval without an evaluation phase"); + ensure!( + entry.time_selection.as_of.is_none(), + "metric registration does not support as_of" + ); + ensure!( + matches!( + entry.time_selection.scope, + QueryTimeScope::RealTime | QueryTimeScope::Unknown + ), + "metric registration requires real-time selection" + ); + ensure!( + entry + .time_selection + .lookback + .is_none_or(|d| range.is_some_and(|r| u128::from(d.0) == r.as_millis())), + "lookback conflicts with the query range" + ); + if let LatencyRequirement::ExplicitMaxMs(ms) = entry.requirements.response_latency { + ensure!( + Duration::try_from_secs_f64(ms / 1000.0).is_ok(), + "latency exceeds supported duration" + ); + } + Ok(registered) + } + + pub fn workload(&self) -> &QueryWorkload { + &self.workload + } + pub fn entry(&self) -> QueryWorkloadEntry { + self.workload + .entries() + .next() + .expect("validated single query") + } + pub fn parsed(&self) -> ParsedQuery { + use planner_types::pre_asap::{AggIntent, QueryExpr}; + let query = self.entry().query.0; + let expr = query_parser::parse_query_expr_canonical(&query, self.accuracy()) + .expect("validated canonical query"); + let mut parsed = query_parser::qe_to_parsed_query(&expr); + // Temporal count is the collector's per-item frequency operation. + // Classify the canonical tree, so formatting cannot change this choice. + if matches!(&expr, QueryExpr::Aggregate { measures, child, .. } + if matches!(measures.as_slice(), [AggIntent::Count { .. }]) + && matches!(child.as_ref(), QueryExpr::TimeRange { .. })) + { + parsed.aggregations = vec![AggType::Frequency]; + parsed.exact_required = false; + } + parsed + } + pub fn metric_name(&self) -> String { + self.parsed().metric_name + } + pub fn label_filters(&self) -> HashMap { + self.parsed().label_filters + } + pub fn group_by_labels(&self) -> Vec { + let parsed = self.parsed(); + let mut labels = parsed.group_by_labels; + for label in &self.deployment.retained_labels { + if !labels.contains(label) { + labels.push(label.clone()); + } + } + let mut filter_labels: Vec<_> = parsed.label_filters.into_keys().collect(); + filter_labels.sort(); + for label in filter_labels { + if !labels.contains(&label) { + labels.push(label); + } + } + labels + } + pub fn aggregations(&self) -> Vec { + self.parsed().aggregations + } + pub fn time_window(&self) -> Duration { + self.parsed().time_window + } + pub fn quantiles(&self) -> Vec { + self.parsed().quantiles + } + pub fn exact_required(&self) -> bool { + self.parsed().exact_required + } + pub fn accuracy(&self) -> AccuracyTarget { + self.entry().requirements.accuracy.target() + } + pub fn error_bound(&self) -> f64 { + match self.accuracy() { + AccuracyTarget::Exact => 0.0, + AccuracyTarget::Epsilon(e) | AccuracyTarget::EpsilonDelta { epsilon: e, .. } => e, + } + } + pub fn repeat_every(&self) -> Option { + match self.entry().recurrence { + QueryRecurrence::Repeated( + RepeatedDemand::FixedInterval(i) + | RepeatedDemand::FixedIntervalAt { interval: i, .. }, + ) => Some(Duration::from_millis(u64::from(i.0))), + _ => None, + } + } + pub fn latency_sla(&self) -> Option { + match self.entry().requirements.response_latency { + LatencyRequirement::ExplicitMaxMs(ms) => Some(Duration::from_secs_f64(ms / 1000.0)), + LatencyRequirement::Unspecified => None, + } + } + /// Cost formulas require fresh evidence. Missing facts remain unavailable. + pub fn characteristics_at(&self, now_ms: u64) -> Option { + if self.deployment.bytes_per_raw_sample == 0 { + return None; + } + let data = self.workload.data_workload.as_ref()?; + let series = *data.input_cardinality.value_at(now_ms)?; + let rate = data.ingestion_rate.value_at(now_ms)?.0; + if series == 0 && rate > 0.0 { + return None; + } + let distribution = data.distribution.value_at(now_ms)?; + Some(WorkloadCharacteristics { + series_count: series, + samples_per_sec_per_series: if series == 0 { + 0.0 + } else { + rate / series as f64 + }, + bytes_per_raw_sample: self.deployment.bytes_per_raw_sample, + distinct_keys_per_window: self.deployment.distinct_keys_per_window, + memory_budget_bytes: self.deployment.memory_budget_bytes, + data_distribution: match distribution { + DataDistribution::Zipf => crate::types::DataDistribution::Zipf, + DataDistribution::Uniform => crate::types::DataDistribution::Uniform, + DataDistribution::Bursty => crate::types::DataDistribution::Bursty, + }, + }) + } +} + +pub(crate) fn metric_query_range(query: &str) -> anyhow::Result> { + use promql_parser::{ + label::MatchOp, + parser::{Expr, LabelModifier}, + util::{walk_expr, ExprVisitor}, + }; + struct Validator { + selectors: usize, + range: Option, + } + impl ExprVisitor for Validator { + type Error = anyhow::Error; + fn pre_visit(&mut self, expr: &Expr) -> anyhow::Result { + let selector = match expr { + Expr::VectorSelector(v) => Some(v), + Expr::MatrixSelector(m) => { + self.range = Some(m.range); + Some(&m.vs) + } + Expr::Subquery(_) => { + anyhow::bail!("metric registration does not support subqueries") + } + Expr::Aggregate(a) if matches!(a.modifier, Some(LabelModifier::Exclude(_))) => { + anyhow::bail!("metric registration does not support without grouping") + } + _ => None, + }; + if let Some(v) = selector { + self.selectors += 1; + ensure!( + v.offset.is_none() && v.at.is_none(), + "metric registration does not support offset or @ modifiers" + ); + ensure!( + v.matchers.or_matchers.is_empty() + && v.matchers + .matchers + .iter() + .all(|m| matches!(m.op, MatchOp::Equal)), + "metric registration supports equality label filters only" + ); + } + Ok(true) + } + } + let expr = promql_parser::parser::parse(query).map_err(|e| anyhow!(e))?; + let mut validator = Validator { + selectors: 0, + range: None, + }; + walk_expr(&mut validator, &expr)?; + ensure!(validator.selectors == 1, "metric registration requires exactly one selector; use the full query compilation API for multi-source expressions"); + Ok(validator.range) +} + +pub(crate) fn declared(value: T) -> Evidence { + Evidence { + value: Some(value), + source: EvidenceSource::Declared, + observed_at_ms: None, + valid_for_ms: None, + } +} + +#[cfg(test)] +pub(crate) mod fixtures { + use super::*; + /// Test inputs are rendered into canonical expressions before entering a planner. + pub struct WorkloadFixture { + pub metric_name: String, + pub label_filters: HashMap, + pub group_by_labels: Vec, + pub aggregations: Vec, + pub time_window: Duration, + pub repeat_every: Option, + pub accuracy: AccuracyTarget, + pub latency_sla: Option, + pub sketch_type_override: Option, + pub exact_required: bool, + pub quantiles: Vec, + } + impl WorkloadFixture { + pub fn build(self) -> RegisteredWorkload { + let filters = self + .label_filters + .iter() + .map(|(k, v)| format!("{k}={}", serde_json::to_string(v).unwrap())) + .collect::>() + .join(","); + let selector = format!( + "{}{{{filters}}}[{}s]", + self.metric_name, + self.time_window.as_secs() + ); + let query = if self.exact_required { + format!("sum_over_time({selector})") + } else { + self.aggregations + .iter() + .map(|agg| match agg { + AggType::Quantile => format!( + "quantile_over_time({}, {selector})", + self.quantiles.first().copied().unwrap_or(0.99) + ), + AggType::Cardinality => format!("distinct_over_time({selector})"), + AggType::Frequency => format!("count_over_time({selector})"), + }) + .collect::>() + .join(" + ") + }; + let requirements = QueryRequirements { + accuracy: AccuracyRequirement::Explicit(self.accuracy), + response_latency: self + .latency_sla + .map(|d| LatencyRequirement::ExplicitMaxMs(d.as_secs_f64() * 1000.0)) + .unwrap_or(LatencyRequirement::Unspecified), + }; + let time_selection = TimeSelection { + scope: QueryTimeScope::RealTime, + lookback: Some(DurationMs(self.time_window.as_millis() as u64)), + as_of: None, + }; + let (query_batch, repeating_queries) = match self.repeat_every { + Some(d) => ( + None, + Some(vec![RepeatingEntry { + query: Query(query), + requirements, + predictability: Predictability::Unknown, + time_selection, + demand: RepeatedDemand::FixedInterval(RepetitionInterval( + d.as_millis().try_into().unwrap(), + )), + }]), + ), + None => ( + Some(vec![BatchEntry { + query: Query(query), + requirements, + predictability: Predictability::Unknown, + time_selection, + invocations: 1, + execute_at: None, + }]), + None, + ), + }; + RegisteredWorkload::new( + QueryWorkload { + language: QueryLanguage::PromQL, + query_batch, + repeating_queries, + data_workload: None, + }, + DeploymentOptions { + sketch_type_override: self.sketch_type_override, + retained_labels: self.group_by_labels, + ..Default::default() + }, + ) + .unwrap() + } + } + impl RegisteredWorkload { + pub fn set_repeat_every(&mut self, cadence: Option) { + let entry = self.entry(); + self.workload.query_batch = None; + self.workload.repeating_queries = None; + match cadence { + Some(d) => { + self.workload.repeating_queries = Some(vec![RepeatingEntry { + query: entry.query, + requirements: entry.requirements, + predictability: entry.predictability, + time_selection: entry.time_selection, + demand: RepeatedDemand::FixedInterval(RepetitionInterval( + d.as_millis().try_into().unwrap(), + )), + }]) + } + None => { + self.workload.query_batch = Some(vec![BatchEntry { + query: entry.query, + requirements: entry.requirements, + predictability: entry.predictability, + time_selection: entry.time_selection, + invocations: 1, + execute_at: None, + }]) + } + } + } + pub fn set_label_filters(&mut self, filters: HashMap) { + let rendered = filters + .iter() + .map(|(k, v)| format!("{k}={}", serde_json::to_string(v).unwrap())) + .collect::>() + .join(","); + if let Some(batch) = &mut self.workload.query_batch { + batch[0].query.0 = batch[0].query.0.replace("{}", &format!("{{{rendered}}}")); + } + if let Some(repeating) = &mut self.workload.repeating_queries { + repeating[0].query.0 = repeating[0] + .query + .0 + .replace("{}", &format!("{{{rendered}}}")); + } + } + pub fn set_accuracy(&mut self, accuracy: AccuracyTarget) { + if let Some(batch) = &mut self.workload.query_batch { + batch[0].requirements.accuracy = AccuracyRequirement::Explicit(accuracy.clone()); + } + if let Some(repeating) = &mut self.workload.repeating_queries { + repeating[0].requirements.accuracy = AccuracyRequirement::Explicit(accuracy); + } + } + pub fn set_latency_sla(&mut self, latency: Option) { + let latency = latency + .map(|d| LatencyRequirement::ExplicitMaxMs(d.as_secs_f64() * 1000.0)) + .unwrap_or(LatencyRequirement::Unspecified); + if let Some(batch) = &mut self.workload.query_batch { + batch[0].requirements.response_latency = latency; + } + if let Some(repeating) = &mut self.workload.repeating_queries { + repeating[0].requirements.response_latency = latency; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + pipeline::{Analyzer, QuerySpec}, + store::workload::WorkloadStore, + workload::AggRole, + }; + + fn spec() -> QuerySpec { + serde_json::from_value(serde_json::json!({ + "query_string": "quantile_over_time(0.9, latency[5m])", + "accuracy_sla": 0.5, + "accuracy": {"EpsilonDelta": {"epsilon": 0.02, "delta": 0.001}}, + "repeat_every": "10s", "latency_sla": "2s" + })) + .unwrap() + } + + /// Registration and retrieval preserve the canonical query, requirements and data evidence. + #[test] + fn canonical_workload_survives_registry_roundtrip() { + let mut input = spec(); + input.workload.series_count = 10; + input.workload.samples_per_sec_per_series = 5.0; + input.workload.distinct_keys_per_window = Some(7); + input.sketch_type = Some(SketchType::KLL); + let registered = Analyzer::new().analyze(input).unwrap(); + let canonical = registered.workload().clone(); + let data = canonical.data_workload.as_ref().unwrap(); + assert_eq!(data.ingestion_rate.value, Some(Rate(50.0))); + assert_eq!(data.input_cardinality.value, Some(10)); + assert_eq!( + canonical.entries().next().unwrap().time_selection.lookback, + Some(DurationMs(300_000)) + ); + let store = WorkloadStore::new(); + store.set("latency", AggRole::Quantile, registered); + let retrieved = store.get("latency", AggRole::Quantile).unwrap(); + assert_eq!(retrieved.workload(), &canonical); + assert_eq!( + retrieved.accuracy(), + AccuracyTarget::EpsilonDelta { + epsilon: 0.02, + delta: 0.001 + } + ); + assert_eq!(retrieved.repeat_every(), Some(Duration::from_secs(10))); + assert_eq!(retrieved.latency_sla(), Some(Duration::from_secs(2))); + assert_eq!( + retrieved.deployment.sketch_type_override, + Some(SketchType::KLL) + ); + let cost = retrieved.characteristics_at(0).unwrap(); + assert_eq!(cost.samples_per_sec_per_series, 5.0); + assert_eq!(cost.distinct_keys_per_window, Some(7)); + } + + /// Missing, stale or inconsistent evidence cannot become a fabricated rate estimate. + #[test] + fn unavailable_data_evidence_stays_unavailable() { + let original = Analyzer::new().analyze(spec()).unwrap(); + let mut canonical = original.workload().clone(); + let data = canonical.data_workload.as_mut().unwrap(); + data.ingestion_rate = Evidence { + value: Some(Rate(50.0)), + source: EvidenceSource::Observed, + observed_at_ms: Some(10), + valid_for_ms: Some(5), + }; + let registered = + RegisteredWorkload::new(canonical.clone(), original.deployment.clone()).unwrap(); + assert!(registered.characteristics_at(15).is_some()); + assert!(registered.characteristics_at(16).is_none()); + canonical.data_workload.as_mut().unwrap().ingestion_rate = Evidence::default(); + let unknown = + RegisteredWorkload::new(canonical.clone(), original.deployment.clone()).unwrap(); + assert!(unknown.characteristics_at(0).is_none()); + let data = canonical.data_workload.as_mut().unwrap(); + data.ingestion_rate = declared(Rate(50.0)); + data.input_cardinality = declared(0); + let inconsistent = RegisteredWorkload::new(canonical, original.deployment).unwrap(); + assert!(inconsistent.characteristics_at(0).is_none()); + } + + /// Unsupported stage projections fail explicitly, before any deployment is emitted. + #[test] + fn rejects_lossy_metric_projections() { + for query in [ + "quantile_over_time(0.9, latency{job!=\"api\"}[5m])", + "quantile_over_time(0.9, latency{job=~\"api.*\"}[5m])", + "sum(a) / sum(b)", + "sum_over_time(latency[5m] offset 1h)", + "sum without(instance)(latency)", + "avg_over_time(latency[10m:1m])", + ] { + let mut input = spec(); + input.query_string = Some(query.into()); + assert!( + Analyzer::new().analyze(input).is_err(), + "must reject {query}" + ); + } + } + + /// Cadence and data arrival are independent, and millisecond cadence conversion is checked. + #[test] + fn recurrence_is_separate_from_arrival() { + let mut input = spec(); + input.repeat_every = None; + input.data = crate::types_v2::DataShape::Batch; + let once = Analyzer::new().analyze(input.clone()).unwrap(); + assert!(matches!( + once.entry().recurrence, + QueryRecurrence::OneTime { .. } + )); + assert_eq!( + once.workload().data_workload.as_ref().unwrap().arrival, + DataArrival::AtRest + ); + input.repeat_every = Some("10s".into()); + let repeating = Analyzer::new().analyze(input.clone()).unwrap(); + assert_eq!(repeating.repeat_every(), Some(Duration::from_secs(10))); + assert_eq!( + repeating.workload().data_workload.as_ref().unwrap().arrival, + DataArrival::AtRest + ); + for cadence in ["0s", "4294968s"] { + input.repeat_every = Some(cadence.into()); + assert!(Analyzer::new().analyze(input.clone()).is_err()); + } + } + + /// Public canonical inputs cannot silently discard known time-selection facts. + #[test] + fn rejects_conflicting_canonical_time_selection() { + let original = Analyzer::new().analyze(spec()).unwrap(); + let mut canonical = original.workload().clone(); + canonical.repeating_queries.as_mut().unwrap()[0] + .time_selection + .lookback = Some(DurationMs(600_000)); + assert!(RegisteredWorkload::new(canonical, original.deployment).is_err()); + } +} diff --git a/control_plane/src/replan.rs b/control_plane/src/replan.rs index 1e309366c..854ff45be 100644 --- a/control_plane/src/replan.rs +++ b/control_plane/src/replan.rs @@ -33,7 +33,7 @@ use crate::physical::plan_cache::CachedDeploymentPlanner; use crate::physical::stage_split; use crate::physical::workload_planner as rules; use crate::store::{PlanStore, WorkloadStore}; -use crate::types::LegacyMetricWorkload; +use crate::types::RegisteredWorkload; use crate::workload::AggRole; fn short_hash(s: &str) -> String { @@ -226,12 +226,12 @@ impl Replanner { role: AggRole, agent_id: &str, ) -> Option { - let (workload, _wc) = self.workload_store.get(metric, role)?; + let workload = self.workload_store.get(metric, role)?; self.try_emit_typed_edge_yaml_for_workload(&workload, agent_id) } /// Same as [`try_emit_typed_edge_yaml`] but takes the - /// `LegacyMetricWorkload` directly. Used by `replan_metric` which already + /// `RegisteredWorkload` directly. Used by `replan_metric` which already /// has the workload in scope. /// /// `agent_id` is threaded into the emitted opamp `X-Agent-ID` header @@ -239,7 +239,7 @@ impl Replanner { /// pass `"$AGENT_ID"` and rely on the agent container's env. fn try_emit_typed_edge_yaml_for_workload( &self, - workload: &LegacyMetricWorkload, + workload: &RegisteredWorkload, agent_id: &str, ) -> Option { let deployment_expr = rules::bind_workload_typed(workload)?; @@ -425,7 +425,7 @@ impl Replanner { return false; } let mut any = false; - for (role, _, _) in pairs { + for (role, _) in pairs { if self.replan_metric_role(metric, role).await { any = true; } @@ -436,7 +436,7 @@ impl Replanner { /// Re-plans a single `(metric, role)` pair and pushes updated configs. /// Returns `true` on success, `false` if the pair is unknown. pub async fn replan_metric_role(&self, metric: &str, role: AggRole) -> bool { - let Some((workload, wc)) = self.workload_store.get(metric, role) else { + let Some(workload) = self.workload_store.get(metric, role) else { warn!(metric, role = %role, "replan requested but workload not found in store"); return false; }; @@ -447,7 +447,8 @@ impl Replanner { // previously established baseline — the whole point of a re-plan is to // re-optimise with current EMA data. self.planner.reset(metric); - let plan = self.planner.plan(&workload, Some(&wc)); + + let plan = self.planner.plan(&workload); self.plan_store.set(metric, role, plan.clone()); // Push agent config only to agents registered for this specific @@ -581,7 +582,7 @@ impl Replanner { /// return `No result for query`. fn build_backend_stage_config( &self, - workload: &LegacyMetricWorkload, + workload: &RegisteredWorkload, role: AggRole, ) -> Option { // ── Typed sketch path (Quantile / Cardinality / TopK / Frequency) ── @@ -596,7 +597,7 @@ impl Replanner { // through `extract_edge_facts` fails, and always leaves // `grouping` empty (`QueryExpr::Aggregate.by` is positional // `ColumnId`s with no label-name resolution today). The - // `LegacyMetricWorkload` carries both unambiguously, and every + // `RegisteredWorkload` carries both unambiguously, and every // aggregation under one workload shares them. // Per-metric item_label (the high-card dimension a CMS/CountSketch // hashes): threaded into the policy params so the data-plane ingest @@ -610,12 +611,12 @@ impl Replanner { .unwrap_or_default(); for agg in &mut be.aggregations { if agg.metric_name.is_empty() { - agg.metric_name = workload.metric_name.clone(); + agg.metric_name = workload.metric_name().clone(); } if agg.window_secs == 0 { - agg.window_secs = workload.time_window.as_secs(); + agg.window_secs = workload.time_window().as_secs(); } - agg.grouping = workload.group_by_labels.clone(); + agg.grouping = workload.group_by_labels().clone(); agg.item_label = item_labels.get(&agg.metric_name).cloned(); } return Some(be); @@ -665,17 +666,17 @@ impl Replanner { AggregationInput, BackendAggregation, BackendStageConfig, }; use planner_types::post_asap::SummaryFamilyType; - let window_secs = workload.time_window.as_secs().max(1); + let window_secs = workload.time_window().as_secs().max(1); Some(BackendStageConfig { aggregations: vec![BackendAggregation { item_label: None, heap_update_mode: None, - aggregation_id: format!("exact-{}-{}", workload.metric_name, role), - metric_name: workload.metric_name.clone(), + aggregation_id: format!("exact-{}-{}", workload.metric_name(), role), + metric_name: workload.metric_name().clone(), family: SummaryFamilyType::ExactAggregate(exact_kind, exact_params), window_secs, spatial_filter: String::new(), - grouping: workload.group_by_labels.clone(), + grouping: workload.group_by_labels().clone(), // ExactAgg consumes raw values at the backend (the agent // ships counter samples; the backend's // SumAccumulator integrates them). @@ -885,21 +886,22 @@ mod tests { )) } - fn test_workload(metric: &str) -> (LegacyMetricWorkload, WorkloadCharacteristics) { - let wl = LegacyMetricWorkload { + fn test_workload(metric: &str) -> (RegisteredWorkload, WorkloadCharacteristics) { + let wl = crate::registered_workload::fixtures::WorkloadFixture { metric_name: metric.into(), label_filters: HashMap::new(), group_by_labels: vec![], aggregations: vec![AggType::Quantile], time_window: Duration::from_secs(300), repeat_every: None, - accuracy_sla: 0.01, + accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.01), latency_sla: None, sketch_type_override: None, exact_required: false, quantiles: vec![], - }; + } + .build(); (wl, WorkloadCharacteristics::default()) } @@ -943,8 +945,8 @@ mod tests { #[tokio::test] async fn replan_known_metric_updates_plan_store() { let r = make_replanner(); - let (wl, wc) = test_workload("latency"); - r.workload_store.set("latency", AggRole::Quantile, wl, wc); + let (wl, _wc) = test_workload("latency"); + r.workload_store.set("latency", AggRole::Quantile, wl); r.plan_store.set("latency", AggRole::Quantile, make_plan()); let ok = r.replan_metric("latency").await; @@ -957,8 +959,8 @@ mod tests { #[tokio::test] async fn replan_expired_replans_only_expired() { let r = make_replanner(); - let (wl, wc) = test_workload("old"); - r.workload_store.set("old", AggRole::Quantile, wl, wc); + let (wl, _wc) = test_workload("old"); + r.workload_store.set("old", AggRole::Quantile, wl); // Insert an already-expired plan. let mut expired_plan = make_plan(); @@ -966,8 +968,8 @@ mod tests { r.plan_store.set("old", AggRole::Quantile, expired_plan); // Insert a still-active plan for "active". - let (awl, awc) = test_workload("active"); - r.workload_store.set("active", AggRole::Quantile, awl, awc); + let (awl, _awc) = test_workload("active"); + r.workload_store.set("active", AggRole::Quantile, awl); r.plan_store.set("active", AggRole::Quantile, make_plan()); r.replan_expired().await; @@ -980,8 +982,8 @@ mod tests { #[tokio::test] async fn register_then_violation_replans_correct_metric() { let r = make_replanner(); - let (wl, wc) = test_workload("req_rate"); - r.workload_store.set("req_rate", AggRole::Quantile, wl, wc); + let (wl, _wc) = test_workload("req_rate"); + r.workload_store.set("req_rate", AggRole::Quantile, wl); r.plan_store.set("req_rate", AggRole::Quantile, make_plan()); r.register_agent("agent-1", "req_rate", AggRole::Quantile) @@ -1010,15 +1012,13 @@ mod tests { #[tokio::test] async fn replan_multi_role_metric_updates_both_plans() { let r = make_replanner(); - let (wl_q, wc_q) = test_workload("http_requests_total"); - let mut wl_s = wl_q.clone(); - wl_s.aggregations = vec![AggType::Quantile]; // analyzer-shaped (test fixture) - let wc_s = wc_q.clone(); + let (wl_q, _wc_q) = test_workload("http_requests_total"); + let wl_s = wl_q.clone(); r.workload_store - .set("http_requests_total", AggRole::Quantile, wl_q, wc_q); + .set("http_requests_total", AggRole::Quantile, wl_q); r.workload_store - .set("http_requests_total", AggRole::Sum, wl_s, wc_s); + .set("http_requests_total", AggRole::Sum, wl_s); r.plan_store .set("http_requests_total", AggRole::Quantile, make_plan()); r.plan_store @@ -1046,10 +1046,9 @@ mod tests { #[tokio::test] async fn replan_metric_role_only_touches_target_role() { let r = make_replanner(); - let (wl, wc) = test_workload("m"); - r.workload_store - .set("m", AggRole::Quantile, wl.clone(), wc.clone()); - r.workload_store.set("m", AggRole::Sum, wl, wc); + let (wl, _wc) = test_workload("m"); + r.workload_store.set("m", AggRole::Quantile, wl.clone()); + r.workload_store.set("m", AggRole::Sum, wl); // Make the Sum-role plan expired and Quantile plan fresh. let mut sum_plan = make_plan(); @@ -1076,10 +1075,9 @@ mod tests { #[tokio::test] async fn agent_serving_multiple_roles_triggers_per_role_replan() { let r = make_replanner(); - let (wl, wc) = test_workload("m"); - r.workload_store - .set("m", AggRole::Quantile, wl.clone(), wc.clone()); - r.workload_store.set("m", AggRole::Sum, wl, wc); + let (wl, _wc) = test_workload("m"); + r.workload_store.set("m", AggRole::Quantile, wl.clone()); + r.workload_store.set("m", AggRole::Sum, wl); r.plan_store.set("m", AggRole::Quantile, make_plan()); r.plan_store.set("m", AggRole::Sum, make_plan()); @@ -1130,8 +1128,8 @@ mod tests { let _env = EnvVarGuard::set("USE_TYPED_STAGE_SPLIT", "1"); let r = make_replanner(); - let (wl, wc) = test_workload("latency"); - r.workload_store.set("latency", AggRole::Quantile, wl, wc); + let (wl, _wc) = test_workload("latency"); + r.workload_store.set("latency", AggRole::Quantile, wl); r.plan_store.set("latency", AggRole::Quantile, make_plan()); let yaml = r @@ -1184,8 +1182,8 @@ mod tests { let _env = EnvVarGuard::unset("USE_TYPED_STAGE_SPLIT"); let r = make_replanner(); - let (wl, wc) = test_workload("latency"); - r.workload_store.set("latency", AggRole::Quantile, wl, wc); + let (wl, _wc) = test_workload("latency"); + r.workload_store.set("latency", AggRole::Quantile, wl); r.plan_store.set("latency", AggRole::Quantile, make_plan()); // Drive the legacy emitter directly — same code diff --git a/control_plane/src/store/workload.rs b/control_plane/src/store/workload.rs index 62a40f800..10d2b2532 100644 --- a/control_plane/src/store/workload.rs +++ b/control_plane/src/store/workload.rs @@ -1,4 +1,4 @@ -//! Persists the `LegacyMetricWorkload` + `WorkloadCharacteristics` associated with +//! Persists the canonical workload and deployment options associated with //! each planned `(metric, AggRole)` pair so that the re-planner can re-run //! `plan()` without needing the original `QuerySpec` HTTP payload. //! @@ -11,14 +11,14 @@ use std::collections::HashMap; use std::sync::RwLock; -use crate::types::{LegacyMetricWorkload, WorkloadCharacteristics}; +use crate::types::RegisteredWorkload; use crate::workload::AggRole; /// Composite key `(metric_name, role)` for the store. pub type WorkloadKey = (String, AggRole); pub struct WorkloadStore { - inner: RwLock>, + inner: RwLock>, } impl Default for WorkloadStore { @@ -41,26 +41,16 @@ impl WorkloadStore { /// only happen when metric AND role coincide — re-registering the /// same `(metric, role)` is the legitimate update path (controller /// HTTP `POST /api/v1/plan` re-issuing the same shape). - pub fn set( - &self, - metric: impl Into, - role: AggRole, - wl: LegacyMetricWorkload, - wc: WorkloadCharacteristics, - ) { + pub fn set(&self, metric: impl Into, role: AggRole, wl: RegisteredWorkload) { self.inner .write() .unwrap() - .insert((metric.into(), role), (wl, wc)); + .insert((metric.into(), role), wl); } - /// Returns a clone of `(workload, characteristics)` for the + /// Returns a clone of the registration for the /// `(metric, role)` pair if known. - pub fn get( - &self, - metric: &str, - role: AggRole, - ) -> Option<(LegacyMetricWorkload, WorkloadCharacteristics)> { + pub fn get(&self, metric: &str, role: AggRole) -> Option { self.inner .read() .unwrap() @@ -68,19 +58,16 @@ impl WorkloadStore { .cloned() } - /// Returns every `(workload, characteristics)` pair registered for + /// Returns every `(role, registration)` pair registered for /// `metric`, across all roles. Empty vec when nothing is registered /// for the metric. Order is unspecified — sort if determinism matters. - pub fn get_all_for_metric( - &self, - metric: &str, - ) -> Vec<(AggRole, LegacyMetricWorkload, WorkloadCharacteristics)> { + pub fn get_all_for_metric(&self, metric: &str) -> Vec<(AggRole, RegisteredWorkload)> { self.inner .read() .unwrap() .iter() .filter(|((m, _), _)| m == metric) - .map(|((_, role), (wl, wc))| (*role, wl.clone(), wc.clone())) + .map(|((_, role), wl)| (*role, wl.clone())) .collect() } @@ -106,34 +93,30 @@ mod tests { use std::collections::HashMap; use std::time::Duration; - fn wl(name: &str) -> LegacyMetricWorkload { - LegacyMetricWorkload { + fn wl(name: &str) -> RegisteredWorkload { + crate::registered_workload::fixtures::WorkloadFixture { metric_name: name.into(), label_filters: HashMap::new(), group_by_labels: vec![], aggregations: vec![AggType::Quantile], time_window: Duration::from_secs(300), repeat_every: None, - accuracy_sla: 0.01, + accuracy: crate::types_v2::AccuracyTarget::Epsilon(0.01), latency_sla: None, sketch_type_override: None, exact_required: false, quantiles: vec![], } + .build() } #[test] fn set_and_get() { let s = WorkloadStore::new(); - s.set( - "latency", - AggRole::Quantile, - wl("latency"), - WorkloadCharacteristics::default(), - ); - let (got, _) = s.get("latency", AggRole::Quantile).unwrap(); - assert_eq!(got.metric_name, "latency"); + s.set("latency", AggRole::Quantile, wl("latency")); + let got = s.get("latency", AggRole::Quantile).unwrap(); + assert_eq!(got.metric_name(), "latency"); } #[test] @@ -145,34 +128,19 @@ mod tests { #[test] fn unknown_role_for_known_metric_returns_none() { let s = WorkloadStore::new(); - s.set( - "m", - AggRole::Quantile, - wl("m"), - WorkloadCharacteristics::default(), - ); + s.set("m", AggRole::Quantile, wl("m")); assert!(s.get("m", AggRole::Sum).is_none()); } #[test] fn overwrite_same_role_replaces() { let s = WorkloadStore::new(); - s.set( - "m", - AggRole::Quantile, - wl("m"), - WorkloadCharacteristics::default(), - ); + s.set("m", AggRole::Quantile, wl("m")); let mut updated = wl("m"); - updated.accuracy_sla = 0.05; - s.set( - "m", - AggRole::Quantile, - updated, - WorkloadCharacteristics::default(), - ); - let (got, _) = s.get("m", AggRole::Quantile).unwrap(); - assert_eq!(got.accuracy_sla, 0.05); + updated.set_accuracy(crate::types_v2::AccuracyTarget::Epsilon(0.05)); + s.set("m", AggRole::Quantile, updated); + let got = s.get("m", AggRole::Quantile).unwrap(); + assert_eq!(got.error_bound(), 0.05); } #[test] @@ -181,51 +149,33 @@ mod tests { // and each entry persists independently of the others. let s = WorkloadStore::new(); let mut wl_q = wl("http_requests_total"); - wl_q.accuracy_sla = 0.01; + wl_q.set_accuracy(crate::types_v2::AccuracyTarget::Epsilon(0.01)); let mut wl_s = wl("http_requests_total"); - wl_s.accuracy_sla = 0.02; + wl_s.set_accuracy(crate::types_v2::AccuracyTarget::Epsilon(0.02)); let mut wl_c = wl("http_requests_total"); - wl_c.accuracy_sla = 0.03; - s.set( - "http_requests_total", - AggRole::Quantile, - wl_q, - WorkloadCharacteristics::default(), - ); - s.set( - "http_requests_total", - AggRole::Sum, - wl_s, - WorkloadCharacteristics::default(), - ); - s.set( - "http_requests_total", - AggRole::Count, - wl_c, - WorkloadCharacteristics::default(), - ); + wl_c.set_accuracy(crate::types_v2::AccuracyTarget::Epsilon(0.03)); + s.set("http_requests_total", AggRole::Quantile, wl_q); + s.set("http_requests_total", AggRole::Sum, wl_s); + s.set("http_requests_total", AggRole::Count, wl_c); // All three persist (the pre-B2 store would have collapsed // them onto one key, only the last survives). assert_eq!( s.get("http_requests_total", AggRole::Quantile) .unwrap() - .0 - .accuracy_sla, + .error_bound(), 0.01 ); assert_eq!( s.get("http_requests_total", AggRole::Sum) .unwrap() - .0 - .accuracy_sla, + .error_bound(), 0.02 ); assert_eq!( s.get("http_requests_total", AggRole::Count) .unwrap() - .0 - .accuracy_sla, + .error_bound(), 0.03 ); // `get_all_for_metric` surfaces all three. @@ -236,18 +186,8 @@ mod tests { #[test] fn remove_clears_only_target_role() { let s = WorkloadStore::new(); - s.set( - "m", - AggRole::Quantile, - wl("m"), - WorkloadCharacteristics::default(), - ); - s.set( - "m", - AggRole::Sum, - wl("m"), - WorkloadCharacteristics::default(), - ); + s.set("m", AggRole::Quantile, wl("m")); + s.set("m", AggRole::Sum, wl("m")); s.remove("m", AggRole::Quantile); assert!(s.get("m", AggRole::Quantile).is_none()); assert!(s.get("m", AggRole::Sum).is_some()); @@ -311,18 +251,13 @@ mod tests { let store = WorkloadStore::new(); for entry in &entries { let role = derive_agg_role(entry); - store.set( - &entry.metric_name, - role, - wl(&entry.metric_name), - WorkloadCharacteristics::default(), - ); + store.set(&entry.metric_name, role, wl(&entry.metric_name)); } // Both distinct roles persist after the loop (vs pre-B2: only // the last `set` survives because the key was metric only). let all = store.get_all_for_metric("http_requests_total"); - let roles: std::collections::HashSet<_> = all.iter().map(|(r, _, _)| *r).collect(); + let roles: std::collections::HashSet<_> = all.iter().map(|(r, _)| *r).collect(); assert!( roles.contains(&crate::workload::AggRole::Sum), "Sum-role plan must survive after the pre-pop loop; got {roles:?}" @@ -336,18 +271,8 @@ mod tests { #[test] fn keys_returns_all_pairs() { let s = WorkloadStore::new(); - s.set( - "a", - AggRole::Quantile, - wl("a"), - WorkloadCharacteristics::default(), - ); - s.set( - "b", - AggRole::Sum, - wl("b"), - WorkloadCharacteristics::default(), - ); + s.set("a", AggRole::Quantile, wl("a")); + s.set("b", AggRole::Sum, wl("b")); let mut keys = s.keys(); keys.sort(); assert_eq!( diff --git a/control_plane/src/types.rs b/control_plane/src/types.rs index f430f760c..b4916d6d1 100644 --- a/control_plane/src/types.rs +++ b/control_plane/src/types.rs @@ -1,31 +1,15 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; use std::time::Duration; // ── Workload characteristics ─────────────────────────────────────────────────── -/// Hint about the statistical distribution of keys in the data stream. -/// Affects fill-rate estimation and therefore delta compression projections. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum DataDistribution { - /// Zipf-distributed keys (s ≈ 1.1). A small number of keys dominate, - /// so only a fraction of sketch cells are touched per window. This is - /// the typical production case. - #[default] - Zipf, - /// All keys are equally probable. Every window fills the sketch more - /// uniformly; delta compression benefit is lower. - Uniform, - /// Traffic arrives in bursts with a concentrated key set. Effective - /// fill rate is lower on average but spikes can reach Uniform levels. - Bursty, -} +pub use planner_types::workload::DataDistribution; /// Observable characteristics of the incoming data stream. /// -/// Callers supply these alongside a [`LegacyMetricWorkload`] so the planner can +/// Compatibility input and transient cost projection of canonical data facts. +/// The registry stores Planner DataWorkload; cost routines use this view to /// compare raw vs. sketch-full vs. sketch-delta transmission costs and /// estimate the CPU / memory overhead at the SDK or agent collector. /// @@ -251,41 +235,7 @@ impl std::fmt::Display for ProcessorMode { // ── Core types ──────────────────────────────────────────────────────────────── -#[derive(Debug, Clone)] -pub struct LegacyMetricWorkload { - pub metric_name: String, - pub label_filters: HashMap, - pub group_by_labels: Vec, - pub aggregations: Vec, - pub time_window: Duration, - pub repeat_every: Option, - /// Authoritative query requirement; never reconstructed from the legacy view. - pub accuracy: crate::types_v2::AccuracyTarget, - /// Deprecated confidence-style view retained for compatibility reporting only. - pub accuracy_sla: f64, - pub latency_sla: Option, - /// When set, the planner must use this sketch type instead of running - /// the cost model. Allows pinning for collectors that support a subset. - pub sketch_type_override: Option, - /// When true, sketches offer no benefit and the plan must use raw - /// pass-through (SP-2–SP-4 collapse to raw-preservation). - /// Set for stateful per-sample queries (RSI, MACD, stochastic, SUM). - pub exact_required: bool, - /// Quantile φ targets implied by the query (e.g. [0.5] for TWAP, - /// [0.0, 1.0] for price range). Empty for non-quantile workloads. - pub quantiles: Vec, -} - -impl LegacyMetricWorkload { - /// Scalar sizing input for legacy cost formulas, not a confidence guarantee. - pub fn error_bound(&self) -> f64 { - match self.accuracy { - crate::types_v2::AccuracyTarget::Exact => 0.0, - crate::types_v2::AccuracyTarget::Epsilon(epsilon) - | crate::types_v2::AccuracyTarget::EpsilonDelta { epsilon, .. } => epsilon, - } - } -} +pub use crate::registered_workload::RegisteredWorkload; // ── Sketch defaults (YAML-configurable) ────────────────────────────────────── @@ -645,6 +595,3 @@ pub struct CollectionPlan { /// Bandwidth and overhead estimates for all three transmission strategies. pub transmission_cost_summary: TransmissionCostSummary, } - -#[deprecated(note = "Use LegacyMetricWorkload; canonical workloads are Planner-owned")] -pub use LegacyMetricWorkload as QueryWorkload; diff --git a/control_plane/src/workload.rs b/control_plane/src/workload.rs index ec64116cc..04f62dfa4 100644 --- a/control_plane/src/workload.rs +++ b/control_plane/src/workload.rs @@ -237,7 +237,7 @@ pub struct WorkloadEntry { /// Optional explicit sketch family override. When set, the planner pins /// this family for the metric (modulo `(sketch, statistic)` validity /// by ASAPPlanner's legal candidate enumeration). Threaded - /// into `LegacyMetricWorkload::sketch_type_override` by the registry pre-pop + /// into `RegisteredWorkload::sketch_type_override` by the registry pre-pop /// path so the typed L4 binding (`bind_workload_typed`) honours it. /// /// MVP-§46 contract entries 5–8 in `deploy/configs/mvp-workload.yaml` @@ -264,12 +264,12 @@ pub struct WorkloadEntry { /// http_requests_total_latency_ms[30s])` carries no `by (...)` /// clause, so the PromQL parser surfaces an EMPTY group_by_labels. /// Without a declarative field the analyzer ends up with an empty - /// `LegacyMetricWorkload.group_by_labels` → an empty `keep_keys` list → + /// `RegisteredWorkload.group_by_labels` → an empty `keep_keys` list → /// the agent strips ALL attrs and mints a single sid per metric /// (instead of one per `(metric, zone)`), defeating the streaming- /// config contract. /// - /// Threaded into `LegacyMetricWorkload::group_by_labels` by the registry + /// Threaded into `RegisteredWorkload::group_by_labels` by the registry /// pre-pop loop in `main`, so it merges with any `by (...)` keys /// the PromQL parser surfaces. Empty / missing ⇒ same behaviour as /// pre-B3 (no allowlist injected). @@ -363,7 +363,7 @@ pub struct WorkloadEntry { /// to reach the planner with *different* flush periods — the startup path /// hardcoded `None`. Threaded into `QuerySpec::repeat_every` by the /// registry pre-pop loop in `main`, which the analyzer parses into - /// [`crate::types::LegacyMetricWorkload::repeat_every`]. + /// [`crate::types::RegisteredWorkload::repeat_every`]. /// /// `None` / missing ⇒ unchanged behaviour (the cost model falls back to /// its window-derived flush rate). @@ -383,7 +383,11 @@ pub fn query_spec_for_entry(entry: &WorkloadEntry) -> crate::pipeline::QuerySpec metric_name: entry.metric_name.clone(), label_filters: Default::default(), group_by_labels: entry.grouping_labels.clone(), - aggregations: vec!["quantile".into()], + aggregations: if entry.query_string.is_some() { + vec![] + } else { + vec!["quantile".into()] + }, time_window: if entry.query_string.is_some() { String::new() } else { @@ -635,13 +639,13 @@ mod tests { let analyzer = crate::pipeline::Analyzer::new(); let declared = analyzer.analyze(query_spec_for_entry(&entries[0])).unwrap(); assert_eq!( - declared.repeat_every, + declared.repeat_every(), Some(std::time::Duration::from_secs(30)) ); // An entry that declares no cadence keeps the historical `None`, so the // cost model falls back to its window-derived flush rate. let undeclared = analyzer.analyze(query_spec_for_entry(&entries[1])).unwrap(); - assert_eq!(undeclared.repeat_every, None); + assert_eq!(undeclared.repeat_every(), None); } /// A cadence the duration parser cannot read is a declaration error, not a diff --git a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs index 1ea601a33..59e9a4046 100644 --- a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs +++ b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs @@ -8,7 +8,7 @@ //! ─PromQL /api/v1/query─► //! query answer //! -//! The control plane drives the streaming-config: a `QueryWorkload` +//! The control plane drives the streaming-config: a `RegisteredWorkload` //! goes through `bind_workload_typed` → `split_typed_three_stage` → //! `emit_backend_streaming_config_json`, the resulting JSON is posted //! to the backend's `/api/v1/streaming-config` endpoint in parser tests. @@ -34,7 +34,6 @@ //! PromQL, asserts the response is well-formed for the planned //! metric. -use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; #[path = "support/physical_fixture.rs"] @@ -127,7 +126,7 @@ async fn post_full_config(client: &reqwest::Client, stack: &FullStack, json: &Js .insert(stack.otlp_http_port, plan); } -use control_plane::types::{AggType, QueryWorkload, WorkloadCharacteristics}; +use control_plane::types::{AggType, RegisteredWorkload, WorkloadCharacteristics}; use data_plane::storage_engines::types::StreamingConfigHandle; use serde_json::Value as JsonValue; @@ -149,7 +148,7 @@ use prost::Message; // ── Helpers ───────────────────────────────────────────────────────────────── -/// Build a `QueryWorkload` with the given parameters. Mirrors the +/// Build a `RegisteredWorkload` with the given parameters. Mirrors the /// `WorkloadAnalyzer` output shape but constructed directly for tests. fn build_workload_with_override( metric_name: &str, @@ -159,21 +158,34 @@ fn build_workload_with_override( group_by_labels: Vec, quantiles: Vec, sketch_type_override: Option, -) -> QueryWorkload { - QueryWorkload { - metric_name: metric_name.to_string(), - label_filters: HashMap::new(), - group_by_labels, - aggregations, - time_window, - repeat_every: None, - accuracy: control_plane::types_v2::AccuracyTarget::Epsilon(accuracy_sla), - accuracy_sla, - latency_sla: None, - sketch_type_override, - exact_required: false, - quantiles, - } +) -> RegisteredWorkload { + let query = match aggregations.as_slice() { + [AggType::Quantile] => format!( + "quantile_over_time({}, {metric_name}[{}s])", + quantiles.first().copied().unwrap_or(0.99), + time_window.as_secs() + ), + [AggType::Cardinality] => format!( + "distinct_over_time({metric_name}[{}s])", + time_window.as_secs() + ), + [AggType::Frequency] => { + format!("count_over_time({metric_name}[{}s])", time_window.as_secs()) + } + _ => panic!("fixture requires one canonical aggregation"), + }; + control_plane::pipeline::Analyzer::new() + .analyze( + serde_json::from_value(serde_json::json!({ + "query_string": query, + "group_by_labels": group_by_labels, + "accuracy_sla": 1.0 - accuracy_sla, + "accuracy": {"Epsilon": accuracy_sla}, + "sketch_type": sketch_type_override, + })) + .unwrap(), + ) + .unwrap() } /// Convenience wrapper — no sketch_type_override. @@ -184,7 +196,7 @@ fn build_workload( time_window: Duration, group_by_labels: Vec, quantiles: Vec, -) -> QueryWorkload { +) -> RegisteredWorkload { build_workload_with_override( metric_name, aggregations, @@ -196,7 +208,7 @@ fn build_workload( ) } -/// Run the controller's planning pipeline end-to-end on a `QueryWorkload` +/// Run the controller's planning pipeline end-to-end on a `RegisteredWorkload` /// and return the `BackendStageConfig` the controller would emit from /// for it — the same object both `emit_backend_streaming_config_json` /// (legacy JSON) and the catalog-backed physical-plan compiler @@ -204,11 +216,11 @@ fn build_workload( /// /// Mirrors the `handle_plan` flow's `StageConfig::Backend(mut be)` /// branch — including the post-emit grouping patch (#245) so the config -/// carries `grouping` from `workload.group_by_labels`. +/// carries `grouping` from `workload.group_by_labels()`. fn plan_backend_stage_config( - workload: &QueryWorkload, + workload: &RegisteredWorkload, ) -> control_plane::physical::colored_dag::BackendStageConfig { - let deployment_expr = if workload.metric_name == "top_endpoint_qps" { + let deployment_expr = if workload.metric_name() == "top_endpoint_qps" { let evidence = control_plane::physical::compiler::TopKMembershipEvidence { selected_lower_bound: 101.0, excluded_upper_bound: 100.0, @@ -242,20 +254,20 @@ fn plan_backend_stage_config( // workload directly, the same way #245 patches grouping. for agg in &mut backend_cfg.aggregations { if agg.metric_name.is_empty() { - agg.metric_name = workload.metric_name.clone(); + agg.metric_name = workload.metric_name().clone(); } if agg.window_secs == 0 { - agg.window_secs = workload.time_window.as_secs(); + agg.window_secs = workload.time_window().as_secs(); } - agg.grouping = workload.group_by_labels.clone(); + agg.grouping = workload.group_by_labels().clone(); } backend_cfg } -/// Run the controller's planning pipeline end-to-end on a `QueryWorkload` +/// Run the controller's planning pipeline end-to-end on a `RegisteredWorkload` /// and return the streaming-config JSON document the controller would /// POST to the backend's `/api/v1/streaming-config` endpoint. -fn plan_streaming_config_json(workload: &QueryWorkload) -> JsonValue { +fn plan_streaming_config_json(workload: &RegisteredWorkload) -> JsonValue { let backend_cfg = plan_backend_stage_config(workload); // No continuous-monitoring (CDM) intents in these tests — pass an empty // slice (the `&[MonitorIntent]` arg added when CDM monitor specs landed). @@ -946,7 +958,7 @@ async fn controller_plans_with_grouping_and_backend_parses_grouping_labels() { assert_eq!( names, vec!["zone"], - "controller must thread workload.group_by_labels → labels.grouping (#245)\n\ + "controller must thread workload.group_by_labels() → labels.grouping (#245)\n\ {streaming_config_json}" ); diff --git a/docs/developer_docs/control-plane/architecture-naming-review.zh.md b/docs/developer_docs/control-plane/architecture-naming-review.zh.md index 8c7e1e3a1..075e6321a 100644 --- a/docs/developer_docs/control-plane/architecture-naming-review.zh.md +++ b/docs/developer_docs/control-plane/architecture-naming-review.zh.md @@ -6,6 +6,31 @@ 本文件保留实施前的架构审查与决策依据。当前实现及迁移规则见 [Planning terminology and architecture](planning-terminology.md)。其中明确列出此次命名迁移与保留为后续工作的结构调整。 +**追加实施:删除扁平 workload,统一 canonical 模型** + +[注册模型](../../../control_plane/src/registered_workload.rs)、[输入转换](../../../control_plane/src/pipeline.rs) 和 [注册存储](../../../control_plane/src/store/workload.rs) 已改为直接使用 ASAPPlanner 的 `QueryWorkload` 及内嵌 `DataWorkload`。`LegacyMetricWorkload` 与旧 `types::QueryWorkload` 别名删除。`RegisteredWorkload` 只组合 canonical workload 与部署选项,不保存第二份扁平查询字段或 `ParsedQuery`。 + +| 原字段 / 对象 | canonical 或部署归属 | 重复判断与迁移含义 | +|---|---|---| +| `metric_name`、过滤条件、聚合、quantiles | `QueryWorkload` 中的完整查询表达式 | 旧字段是表达式的重复投影;现在按需解析,不独立存储 | +| `exact_required` | 从 canonical 表达式推导当前 stage 编译能力 | 不再独立存储,也不替代用户的 accuracy requirement | +| `accuracy` / `accuracy_sla` | entry 的 `requirements.accuracy` | 删除可变标量镜像;入口兼容旧 SLA,显式 `EpsilonDelta` 完整保留 | +| `latency_sla` | `requirements.response_latency` | 查询响应要求,不是数据到达频率 | +| `repeat_every` / `shape` | batch entry 或 repeating entry 的 demand | 明确一次执行与固定周期;毫秒转换检查溢出,不推导数据到达模式 | +| `time_window` | 查询 range 与一致的 `time_selection.lookback` | 不把执行周期、历史时间点和 lookback 混成一个窗口 | +| `series_count`、每系列采样速率 | `DataWorkload.input_cardinality`、`ingestion_rate` | 本单 metric 适配器约定 cardinality 为活跃 series;10 series × 5 Hz = 50 samples/s | +| 分布、来源与证据时效 | `DataWorkload.distribution` / `Evidence` | 存储时保留;未知或过期证据不可退回已知默认值 | +| sketch pin、保留标签、wire bytes、memory cap | `DeploymentOptions` | 是部署约束,不属于查询语义或数据到达事实 | +| `distinct_keys_per_window` | 部署侧每次 flush 的 item 基数估计 | 与 series cardinality 不是同一总体,不合并 | +| `WorkloadCharacteristics` | 兼容输入 DTO / 临时成本投影 | 不再与查询模型一起重复存储;重规划从 canonical evidence 获取有效值 | +| `QueryId` / deployment model | 注册元数据 | 保留标识和路由提示,不伪称已经执行额外约束 | + +迁移包含有意的入口行为变化:字段式 quantile 生成显式 0.99 分位查询,frequency 生成时间范围内的 item count;不再给已有查询硬塞 `quantile`。查询与显式 metric/window/aggregation/filter 冲突时返回错误,要求调用者修改查询表达式。旧 `group_by_labels` 中用于 collector 保留的标签放在部署选项,不改写查询 GROUP BY。 + +单 metric 注册接口拒绝不能无损投影的多 selector、非等值过滤、offset/@、子查询、`without`、历史 time selection、未支持的 cadence 以及未实施的 dollars 约束。更完整的查询继续走已有完整编译接口。数据到达模式与查询周期独立;Batch 表示静态数据,持续 ingestion rate 为零。公开 canonical 构造器也验证这些边界,不能绕过入口校验后静默降级。 + +测试覆盖 canonical 查询与精度/周期/数据证据的存储往返、aggregate sample rate、过期/未知证据、cadence 范围检查及不支持的投影。英文流程架构图见 [Planning terminology and architecture](planning-terminology.md)。下文中提出临时命名 `LegacyMetricWorkload` 的内容保留为审查历史,已被本节的实际删除方案取代。 + **本 PR 的最终落地与迁移方式** 下表记录最终采用的名称;后文保留审查时的备选与理由。实施基线为 `cb3153a8`(#710),继续复用 Planner 的精度类型;随后合并 `47d4332f`(#711),保留最新主分支的接口与历史注释清理。 @@ -24,7 +49,7 @@ | [共享运行时算子](../../../crates/asap_types/src/query_plan/residual.rs) | `query_plan::logical::LogicalOperator` → `query_plan::residual::ResidualQueryOperator`;backend 对应模块迁移到 `residual`,保留旧模块导出 | | [运行时计划与句柄](../../../data_plane/src/storage_engines/types/hot_reload_config.rs) | `ActivePhysicalPlan` → `RuntimePhysicalPlan`;`HotReloadActivePhysicalPlan` → `ActivePhysicalPlanHandle`;`HotReloadStreamingConfig` → `StreamingConfigHandle`;`runtime_config` → `streaming_config`;active handle 的 `snapshot` → `active_snapshot`;`from_active` → `from_active_physical_plan`;`retire_drained` → `mark_drained_plan_retired` | | [存储元数据](../../../data_plane/src/storage_engines/sketch_db/index/mod.rs) | `SketchInstanceMetadata` → `SummarySeriesMetadata`;`sketch_index` 字段与变量 → `summary_store`;保留 `SketchStore` 类型 | -| 其余调用边界 | `types::QueryWorkload` → `LegacyMetricWorkload`;`ClickHouseSqlWorkload.sds` → `summary_catalog`;`aggregation_configs` → `materializations_by_policy_fingerprint`;`aggregation_id_for_key/value` → `key_policy_fingerprint/value_policy_fingerprint`;`data_plane::monitor` → `update_sampling` | +| 其余调用边界 | `types::QueryWorkload` / `LegacyMetricWorkload` 删除,统一到 Planner `QueryWorkload` + `DataWorkload`;`ClickHouseSqlWorkload.sds` → `summary_catalog`;`aggregation_configs` → `materializations_by_policy_fingerprint`;`aggregation_id_for_key/value` → `key_policy_fingerprint/value_policy_fingerprint`;`data_plane::monitor` → `update_sampling` | 迁移规则:**Rust 名称更新,输出 wire 名称保持原样**;反序列化接受新名称作为 alias。旧公共类型导入及主要入口提供 deprecated 转发,但 Rust struct literal 的旧字段拼写无法通过类型别名兼容,源码消费者需要按表迁移。`None` / 空候选集合、浮点频次、报价身份、候选排序、严格小于的选择规则及计划生命周期均保持原义。 diff --git a/docs/developer_docs/control-plane/planning-terminology.md b/docs/developer_docs/control-plane/planning-terminology.md index 3e726a934..9054783b3 100644 --- a/docs/developer_docs/control-plane/planning-terminology.md +++ b/docs/developer_docs/control-plane/planning-terminology.md @@ -7,6 +7,18 @@ records the reasoning and the original-to-proposed naming checklist for #709. ```mermaid flowchart TB subgraph CP[Control plane] + DTO["JSON / YAML compatibility input"] + NORMALIZE["Analyzer
Validate and normalize once"] + WORKLOAD["ASAPPlanner QueryWorkload
Query + requirements + recurrence + time selection"] + DATA["ASAPPlanner DataWorkload
Arrival + rate + cardinality + evidence freshness"] + OPTIONS["DeploymentOptions
Sketch constraint + retained labels + wire size + memory cap"] + REGISTRY["WorkloadStore
Canonical workload + deployment options per metric and role"] + METRIC["Metric deployment planning and replanning
Derive query metadata and fresh cost inputs"] + DTO --> NORMALIZE --> WORKLOAD + DATA --> WORKLOAD + WORKLOAD --> REGISTRY + OPTIONS --> REGISTRY + REGISTRY --> METRIC INPUT["BackendLocalPlanningInput
Workload demand + physical inputs + deployment context"] LOGICAL["ASAPPlanner + selection adapter
Legal semantic DAG selection"] REQUEST["PhysicalCompilationRequest
QueryCompilationInput + enabled materialization keys"] @@ -16,6 +28,7 @@ flowchart TB EVALUATE["CandidatePlanEvaluation
Select the lowest-cost feasible enumerated candidate"] SQL["ClickHouse SQL selection and binding
Separate compilation path"] PUBLICATION["PhysicalPlanPublication
PhysicalPlanInstallRequest"] + WORKLOAD --> INPUT INPUT --> LOGICAL --> REQUEST --> COMPILE --> MANIFEST MANIFEST --> QUOTE --> EVALUATE --> PUBLICATION SQL --> PUBLICATION @@ -33,14 +46,17 @@ flowchart TB end PUBLICATION --> CONTRACT --> STAGE PUBLICATION -->|OpAMP| COLLECTOR[ASAPCollector] + METRIC -->|Stage configuration via OpAMP| COLLECTOR COLLECTOR -->|OTLP frames| MAINTAIN RAW[Raw time-series samples] -->|Remote Write| MAINTAIN QUERY -->|Exact execution or fallback| EXACT[Prometheus / VictoriaMetrics / ClickHouse] ``` SQL shares publication and runtime contracts; it does not currently use the -time-series workload quote-selection path. Legacy flat metric workloads and -stage emission remain a separate compatibility path. +time-series workload quote-selection path. Metric stage emission remains a +separate deployment path, but its stored semantic input now uses the same Planner +`QueryWorkload` and embedded `DataWorkload` types. `LegacyMetricWorkload` and its +old `types::QueryWorkload` alias are deleted. ## Domain boundaries @@ -76,8 +92,8 @@ This preserves old consumers, exact manifest comparisons, and serialized identity inputs. Diagnostic enums retain their old string representations, including unknown strings and the missing-status default. -Old public type imports and selected method/function names remain deprecated -forwarders. This does **not** preserve old Rust struct-literal field names; +Other old public type imports and selected method/function names remain deprecated +forwarders. The flat workload type is removed without a compatibility alias. This does **not** preserve old Rust struct-literal field names; workspace callers migrate with the definitions. External Rust source consumers must update fields using the review's mapping before these compatibility imports are removed. No removal date is set until consumer migration is known. @@ -91,7 +107,7 @@ publication validation, and activation behavior are unchanged. The implementation clarifies planning/compiler types, candidate pricing and diagnostics, publication conversion, runtime generations, streaming views, -series metadata, legacy metric workloads, residual query operators, and the +series metadata, canonical workload registration, residual query operators, and the data-plane update-sampling module. Existing re-exports remain compatibility entrypoints rather than duplicate implementations. @@ -107,4 +123,40 @@ The review also identifies follow-up work that is intentionally separate: - Changing window units, schema versions, identity encodings, or typed ID ownership. -These require their own behavior contracts and are not hidden in a naming PR. +These remain separate from the naming changes and the canonical workload migration. + +## Canonical workload registration + +`RegisteredWorkload` is a backend registration envelope, containing only the +Planner `QueryWorkload` and `DeploymentOptions`. It does not store a parsed query +or a second set of metric, accuracy, window, cadence or data-rate fields. +`ParsedQuery` is an ephemeral adapter view; `WorkloadCharacteristics` is an input +DTO and a transient cost projection. The cached planner obtains that projection +from canonical evidence, and unavailable evidence skips rate-based cost selection. + +For this single-metric adapter, `input_cardinality` denotes active time series and +`ingestion_rate` is the aggregate sample rate: 10 series at 5 Hz produces 50 +samples/s. Distinct item keys per flush remain a separate deployment estimate. +Evidence provenance and freshness survive registry round trips. Unknown or stale +facts do not become declared defaults. Batch data has `AtRest` arrival and no +ongoing ingestion; repetition remains independent of arrival. + +Compatibility field-only inputs generate explicit queries: quantile uses +`quantile_over_time(0.99, ...)`, cardinality uses `distinct_over_time(...)`, and +frequency uses a temporal item count. Filter values are JSON-escaped before query +parsing. Source and label identifiers must be accepted by the PromQL frontend. +The old `group_by_labels` input also supplies collector retention labels; these +are deployment options and do not introduce a query GROUP BY. + +The single-metric registration route accepts one query entry, one selector, +equality filters, one-shot or fixed-interval demand, and supported real-time +selection. It rejects conflicting query/field overrides, multi-selector input, +regex or negative matching, offset/@, subqueries, `without`, unsupported cadence +forms, historical time-selection metadata and unenforced dollar constraints. +Cadences are checked before conversion to Planner milliseconds. The full query +compilation APIs retain their broader query support. + +Both field-only and string requests bind from the registered canonical expression. +Explicit typed accuracy, including epsilon and delta, remains authoritative; +sketch choices remain physical constraints subject to legal binding. IDs and +routing hints are retained as registration metadata.