diff --git a/Cargo.lock b/Cargo.lock index 5b68bc67f..c0dfaf2b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -373,7 +373,7 @@ dependencies = [ [[package]] name = "asap-aware-mapping" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=0402384e589df6e087d6d2d22b463ddc2eea0774#0402384e589df6e087d6d2d22b463ddc2eea0774" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=5e9396033a5a5d5f350bfa675d12770c972ee991#5e9396033a5a5d5f350bfa675d12770c972ee991" dependencies = [ "asap-types", "serde", @@ -384,7 +384,7 @@ dependencies = [ [[package]] name = "asap-frontend-promql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=0402384e589df6e087d6d2d22b463ddc2eea0774#0402384e589df6e087d6d2d22b463ddc2eea0774" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=5e9396033a5a5d5f350bfa675d12770c972ee991#5e9396033a5a5d5f350bfa675d12770c972ee991" dependencies = [ "asap-types", "promql-parser", @@ -393,7 +393,7 @@ dependencies = [ [[package]] name = "asap-frontend-sql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=0402384e589df6e087d6d2d22b463ddc2eea0774#0402384e589df6e087d6d2d22b463ddc2eea0774" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=5e9396033a5a5d5f350bfa675d12770c972ee991#5e9396033a5a5d5f350bfa675d12770c972ee991" dependencies = [ "asap-sql-function-catalog", "asap-types", @@ -415,12 +415,12 @@ dependencies = [ [[package]] name = "asap-sql-function-catalog" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=0402384e589df6e087d6d2d22b463ddc2eea0774#0402384e589df6e087d6d2d22b463ddc2eea0774" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=5e9396033a5a5d5f350bfa675d12770c972ee991#5e9396033a5a5d5f350bfa675d12770c972ee991" [[package]] name = "asap-types" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=0402384e589df6e087d6d2d22b463ddc2eea0774#0402384e589df6e087d6d2d22b463ddc2eea0774" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=5e9396033a5a5d5f350bfa675d12770c972ee991#5e9396033a5a5d5f350bfa675d12770c972ee991" dependencies = [ "serde", "serde_json", @@ -462,6 +462,7 @@ dependencies = [ "asap-types", "base64 0.21.7", "clap 4.6.1", + "promql-parser", "serde", "serde_json", "serde_yaml", diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index 7cb29606b..264534341 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -76,8 +76,8 @@ asap_types.workspace = true # scaffolding, unaware that `data_plane`'s `summary_executor.rs` in *this* # repo is a real one. Vendored locally instead of chased upstream -- see # `data_plane/src/query_engines/asap_query_engine/summary_exec.rs`. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "0402384e589df6e087d6d2d22b463ddc2eea0774" } -asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "0402384e589df6e087d6d2d22b463ddc2eea0774" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "5e9396033a5a5d5f350bfa675d12770c972ee991" } +asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "5e9396033a5a5d5f350bfa675d12770c972ee991" } # L1 adoption (design-target-architecture.md Part B): the PromQL front # end itself, replacing control_plane's own query_parser/promql.rs. @@ -85,8 +85,8 @@ asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = # `planner-types`/`asap-aware-mapping` above -- these three MUST move # together (two revs of the same upstream repo's types in one workspace # resolve to distinct Rust types that won't unify). -asap-frontend-promql = { git = "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/ProjectASAP/ASAPPlanner", rev = "0402384e589df6e087d6d2d22b463ddc2eea0774" } -asap-frontend-sql = { git = "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/ProjectASAP/ASAPPlanner", rev = "0402384e589df6e087d6d2d22b463ddc2eea0774" } +asap-frontend-promql = { git = "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/ProjectASAP/ASAPPlanner", rev = "5e9396033a5a5d5f350bfa675d12770c972ee991" } +asap-frontend-sql = { git = "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/ProjectASAP/ASAPPlanner", rev = "5e9396033a5a5d5f350bfa675d12770c972ee991" } [dev-dependencies] tokio = { version = "1", features = ["full", "test-util"] } diff --git a/control_plane/examples/audit_clickhouse_corpus.rs b/control_plane/examples/audit_clickhouse_corpus.rs index 71fedf79e..e7497b8d1 100644 --- a/control_plane/examples/audit_clickhouse_corpus.rs +++ b/control_plane/examples/audit_clickhouse_corpus.rs @@ -4,7 +4,7 @@ use asap_frontend_sql::SqlCatalog; use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; use control_plane::clickhouse::{ClickHouseSqlWorkload, ClickHouseSqlWorkloadEntry}; use control_plane::physical::compiler::{ - PlanEnvelope, PrecomputePlan, TransmissionPlan, BACKEND_COMPAT, PLANNER_REVISION, + PlanEnvelope, PrecomputePlan, BACKEND_COMPAT, PLANNER_REVISION, }; use planner_types::pre_asap::{Column, DataType, Schema}; use planner_types::types::AccuracyTarget; @@ -56,8 +56,12 @@ fn publication_inputs(schema: &Schema, sql: String) -> ClickHouseSqlWorkload { let mut precompute_plan = PrecomputePlan::build_backend_local(envelope.clone(), vec![materialization.clone()]) .unwrap(); - let mut transmission_plan = - TransmissionPlan::build(envelope, &precompute_plan, &Default::default()).unwrap(); + let mut transmission_plan = control_plane::physical::compiler::compile_transmission_plan( + envelope, + &precompute_plan, + &Default::default(), + ) + .unwrap(); let sds = asap_types::summary_catalog::SummaryCatalog::from_materializations( 27, 1, diff --git a/control_plane/src/clickhouse.rs b/control_plane/src/clickhouse.rs index 19fe715cf..f1334574f 100644 --- a/control_plane/src/clickhouse.rs +++ b/control_plane/src/clickhouse.rs @@ -183,7 +183,7 @@ pub async fn compile_automatic_clickhouse_workload( .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?, ); precompute.executable_dags = installed_dags; - let mut transmission = TransmissionPlan::build( + let mut transmission = crate::physical::compiler::compile_transmission_plan( request.envelope.clone(), &precompute, &std::collections::BTreeMap::new(), @@ -360,7 +360,7 @@ where .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; let mut materialization_nodes = std::collections::BTreeMap::new(); let mut query_nodes = std::collections::BTreeMap::new(); - let executable = QueryPlanEntry::compile_bound_relational_mapped( + let executable = crate::query_plan::compile_bound_relational_mapped( query.sql.clone(), planned.canonical_sql.clone(), &root, @@ -1010,9 +1010,12 @@ mod tests { let mut precompute = PrecomputePlan::build_backend_local(envelope.clone(), vec![config]).unwrap(); precompute.summary_catalog = Some(sds.reference().unwrap()); - let mut transmission = - TransmissionPlan::build(envelope, &precompute, &std::collections::BTreeMap::new()) - .unwrap(); + let mut transmission = crate::physical::compiler::compile_transmission_plan( + envelope, + &precompute, + &std::collections::BTreeMap::new(), + ) + .unwrap(); transmission.summary_catalog = Some(sds.reference().unwrap()); let timestamped = |time_name: &str, value_name: &str| { Schema::with_time_index( @@ -1204,7 +1207,7 @@ mod tests { request.precompute_plan = PrecomputePlan::build_backend_local(envelope.clone(), vec![config]).unwrap(); request.precompute_plan.summary_catalog = Some(request.sds.reference().unwrap()); - request.transmission_plan = TransmissionPlan::build( + request.transmission_plan = crate::physical::compiler::compile_transmission_plan( envelope, &request.precompute_plan, &std::collections::BTreeMap::new(), diff --git a/control_plane/src/emit/backend_push.rs b/control_plane/src/emit/backend_push.rs index d3f4861de..9be93c761 100644 --- a/control_plane/src/emit/backend_push.rs +++ b/control_plane/src/emit/backend_push.rs @@ -260,7 +260,7 @@ async fn push_documents_coupled( warn!(%error, "failed to bind compatibility PrecomputePlan to SummaryCatalog"); return (false, false, 0); } - let transmission_plan = match crate::physical::compiler::TransmissionPlan::build( + let transmission_plan = match crate::physical::compiler::compile_transmission_plan( precompute_plan.envelope.clone(), &precompute_plan, &Default::default(), diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index a79236381..dbf63519d 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -69,6 +69,10 @@ struct AppState { /// agents' `sketch-runtime::PushExporter`. Read by decision /// loops in the replanner. runtime_samples: Arc, + /// The successfully activated typed catalog is authoritative for live ERP + /// input identity; incoming telemetry cannot supply its own descriptors. + active_summary_catalog: + Arc>>>, /// Phase C (MVP v6): shared `BackendClient` for posting /// `StreamingConfig` JSON / YAML to the ASAPQuery-backend's /// `POST /api/v1/streaming-config` endpoint. Phase B had this @@ -487,6 +491,7 @@ async fn main() { opamp_endpoint: opamp_ep, workload_registry: Arc::clone(&workload_registry), runtime_samples: Arc::clone(&runtime_samples_store), + active_summary_catalog: Arc::new(tokio::sync::Mutex::new(None)), backend_client: backend_client_shared, backend_routing_cache: Arc::clone(&backend_routing_cache), }; @@ -611,6 +616,8 @@ struct PhysicalPlanQueryRequest { #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] struct CompileAndPublishPhysicalPlanRequest { + #[serde(default = "default_physical_deployment_target")] + target: physical::compiler::PhysicalDeploymentTarget, #[serde(default)] workload_cost_evidence: Option, queries: Vec, @@ -635,6 +642,10 @@ struct CompileAndPublishPhysicalPlanRequest { apply_timeout_ms: u64, } +fn default_physical_deployment_target() -> physical::compiler::PhysicalDeploymentTarget { + physical::compiler::PhysicalDeploymentTarget::DistributedCollectors +} + fn default_physical_plan_timeout_ms() -> u64 { 10_000 } @@ -704,10 +715,15 @@ async fn compile_and_publish_physical_plan( mut request: CompileAndPublishPhysicalPlanRequest, frontend: PhysicalQueryFrontend, ) -> Response { + // Serialize typed activations so an older response cannot overwrite the + // catalog recorded after a newer backend activation. + let mut active_catalog = st.active_summary_catalog.lock().await; if let Some(erp) = &mut request.erp { if let Err(error) = erp.hydrate_observed_shape(&st.runtime_samples) { return (StatusCode::UNPROCESSABLE_ENTITY, error).into_response(); } + let catalog = active_catalog.clone(); + erp.resolve_population_data_descriptor(catalog.as_deref()); } let (bundle, collector_ids, apply_timeout, adaptation_evidence, _) = match compile_physical_plan_request(request, false, frontend) { @@ -802,6 +818,8 @@ async fn compile_and_publish_physical_plan( .into_response(); } + *active_catalog = Some(Arc::new(bundle.summary_catalog)); + Json(CompileAndPublishPhysicalPlanResponse { cost_comparison: bundle.cost_comparison, plan_id: bundle.envelope.plan_id, @@ -842,6 +860,7 @@ async fn publish_clickhouse_plan( publication: physical::publication::PhysicalPlanPublication, selection_trace: Option, ) -> axum::response::Response { + let mut active_catalog = state.active_summary_catalog.lock().await; let plan_id = publication.summary_catalog.plan_id; let plan_version = publication.summary_catalog.plan_version; let Some(client) = state.backend_client.as_ref() else { @@ -863,6 +882,7 @@ async fn publish_clickhouse_plan( .await; return (StatusCode::BAD_GATEWAY, error.to_string()).into_response(); } + *active_catalog = Some(Arc::new(publication.summary_catalog)); Json(serde_json::json!({ "plan_id": plan_id, "plan_version": plan_version, @@ -888,10 +908,15 @@ fn compile_physical_plan_request( ), (StatusCode, String), > { - if request.queries.is_empty() || request.collector_ids.is_empty() { + if request.queries.is_empty() + || (request.target == physical::compiler::PhysicalDeploymentTarget::DistributedCollectors + && request.collector_ids.is_empty()) + || (request.target == physical::compiler::PhysicalDeploymentTarget::BackendLocalRemoteWrite + && !request.collector_ids.is_empty()) + { return Err(( StatusCode::UNPROCESSABLE_ENTITY, - "queries and collector_ids must both be non-empty".to_string(), + "queries must be non-empty; distributed deployment requires collectors and backend-local deployment requires none".to_string(), )); } if request.max_evidence_age_ms == 0 || request.apply_timeout_ms == 0 { @@ -967,7 +992,8 @@ fn compile_physical_plan_request( let planning_request = physical::compiler::PlanningRequest { query_workload: None, queries, - hybrid_execution: false, + hybrid_execution: request.target + == physical::compiler::PhysicalDeploymentTarget::BackendLocalRemoteWrite, materialization_policy: None, evidence: request.evidence, exact_composition_costs: request.exact_composition_costs, @@ -978,7 +1004,7 @@ fn compile_physical_plan_request( retained_summary_memory_budget_bytes: None, }; let environment = physical::compiler::DeploymentEnvironment { - target: physical::compiler::PhysicalDeploymentTarget::DistributedCollectors, + target: request.target, collector_ids: request.collector_ids.clone(), capability_snapshot_id: request.capability_snapshot_id, observed_at_unix_ms: now, @@ -2242,6 +2268,7 @@ fn test_app_with_backend(backend_url: Option) -> (AppState, axum::Router opamp_endpoint: "ws://ctrl:4320/v1/opamp".into(), workload_registry: Arc::new(WorkloadRegistry::empty()), runtime_samples: runtime_samples::RuntimeSamplesStore::new(64), + active_summary_catalog: Arc::new(tokio::sync::Mutex::new(None)), backend_client, backend_routing_cache: Arc::new(Mutex::new(HashMap::new())), }; @@ -2309,6 +2336,58 @@ mod api_tests { assert_eq!(manifests.as_array().unwrap().len(), 1); } + #[test] + fn backend_local_typed_request_compiles_without_collectors() { + let snapshot: physical::compiler::BackendLocalPlanningSnapshot = serde_json::from_str( + include_str!("../../docs/examples/asapquery-compatibility-demo-snapshot.json"), + ) + .unwrap(); + let (planning, _) = snapshot.planning_request().unwrap(); + let mut query = planning.queries[0].clone(); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as u64; + query.lifecycle.evidence_observed_at_unix_ms = now; + for implementation in &mut query.window_implementations { + implementation.cost.observed_at_unix_ms = now; + } + let planner_types::pre_asap::Source::TimeSeries { metric } = &query.source else { + panic!("expected time series fixture"); + }; + let value = serde_json::json!({ + "target": "backend_local_remote_write", + "queries": [{ + "query_id": query.query_id, "query_string": query.query_string, + "metric": metric, "window_secs": query.window_secs, "accuracy": query.accuracy, + "lifecycle": query.lifecycle, "window_implementations": query.window_implementations + }], + "collector_ids": [], "capability_snapshot_id": "test", + "planner_revision": physical::compiler::PLANNER_REVISION, + "max_evidence_age_ms": 60000, "plan_version": 1, + "activation_unix_ms": now, "backend_compat": physical::compiler::BACKEND_COMPAT + }); + let request = serde_json::from_value(value.clone()).unwrap(); + let (plan, collectors, _, _, _) = + compile_physical_plan_request(request, false, PhysicalQueryFrontend::PromQl).unwrap(); + let plan = plan.unwrap(); + assert!(collectors.is_empty()); + assert!(plan.collector_plans.is_empty()); + assert_eq!( + plan.precompute_plan.ingest.protocol, + physical::compiler::IngestProtocol::PrometheusRemoteWriteV1 + ); + assert!(!plan.precompute_plan.materializations.is_empty()); + let mut distributed = value; + distributed["target"] = serde_json::json!("distributed_collectors"); + assert!(compile_physical_plan_request( + serde_json::from_value(distributed).unwrap(), + false, + PhysicalQueryFrontend::PromQl + ) + .is_err()); + } + async fn body_json(resp: axum::response::Response) -> serde_json::Value { let bytes = resp.into_body().collect().await.unwrap().to_bytes(); serde_json::from_slice(&bytes).unwrap() diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index d357ab4cf..4c491566d 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -258,47 +258,136 @@ pub use asap_types::precompute_plan::{ StateWindowContract, TimestampUnit, }; -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct CollectorMaterialization { - pub query_id: String, - pub materialization: asap_types::sds::SummaryDefinitionId, - pub metric: String, - pub algorithm: String, - pub parameters: Value, - pub group_by: Vec, - pub window_secs: u64, - pub abstract_window_framework: SummaryWindowFramework, - pub window_implementation_id: String, - pub slide_secs: u64, - #[serde( - default, - alias = "paneOriginMs", - skip_serializing_if = "Option::is_none" - )] - pub pane_origin_ms: Option, - pub window_layout: asap_types::WindowMaterializationLayout, - pub evidence_source: Option, - pub lifecycle: CollectorLifecycle, +pub use asap_types::producer_plan::{ + AdaptiveF64Bounds, AdaptiveU64Bounds, CollectorLifecycle, CollectorMaterialization, + CollectorPlan, DeltaPolicy, FrameIdentityContract, GosPolicy, GosThresholdMode, + RuntimeAdaptationEvidence, RuntimeAdaptationPolicy, RuntimeRulePolicy, SamplingEstimator, + SamplingPolicy, SequenceScope, SummaryFrameIdentity, SummaryFrameKind, TransmissionMode, + TransmissionPlan, TransmissionPlanError, TransmissionRule, +}; + +/// Build the fixed physical knob from the controller's canonical +/// epsilon-floor allocator. Degenerate budgets/rates disable sampling. +pub fn sampling_policy_from_accuracy_budget( + epsilon_sampling: f64, + updates_per_window: f64, + estimator: SamplingEstimator, +) -> SamplingPolicy { + if !epsilon_sampling.is_finite() + || epsilon_sampling <= 0.0 + || !updates_per_window.is_finite() + || updates_per_window <= 0.0 + { + return SamplingPolicy::Disabled; + } + let probability = crate::epsilon_alloc::derive_sample_p(epsilon_sampling, updates_per_window); + if probability >= 1.0 { + SamplingPolicy::Disabled + } else { + SamplingPolicy::Fixed { + probability, + estimator, + } + } } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct CollectorLifecycle { - pub kind: String, - pub maintenance_mode: String, - pub evaluation_schedule: String, - pub output_representation: String, +/// Allocate the deterministic staleness share with the same linear-peel +/// composition used by `epsilon_alloc`. `None` means the selected sketch +/// already consumes the budget or communication has no allocated weight. +pub fn gos_policy_from_accuracy_budget( + epsilon_total: f64, + epsilon_sketch: f64, + sites: u32, + edge_cpu_weight: f64, + communication_weight: f64, + threshold_mode: GosThresholdMode, +) -> Option { + if !epsilon_total.is_finite() + || !epsilon_sketch.is_finite() + || !(0.0..=1.0).contains(&epsilon_total) + || epsilon_sketch < 0.0 + || !edge_cpu_weight.is_finite() + || edge_cpu_weight < 0.0 + || !communication_weight.is_finite() + || communication_weight <= 0.0 + { + return None; + } + let (_, epsilon_staleness) = crate::epsilon_alloc::split_budget( + epsilon_total, + epsilon_sketch, + edge_cpu_weight, + communication_weight, + ); + (epsilon_staleness.is_finite() && epsilon_staleness > 0.0).then_some(GosPolicy { + epsilon_staleness, + sites: sites.max(1), + threshold_mode, + }) } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct CollectorPlan { - /// Absent only in legacy artifacts; catalog-aware validation requires it. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub summary_catalog: Option, - pub collector_id: String, - pub envelope: PlanEnvelope, - pub materializations: Vec, - pub transmission_rules: Vec, +pub fn compile_transmission_plan( + envelope: PlanEnvelope, + precompute: &PrecomputePlan, + runtime_policies: &BTreeMap, +) -> Result { + if envelope != precompute.envelope { + return Err(TransmissionPlanError::EnvelopeMismatch); + } + let schemas: BTreeMap<_, _> = precompute + .schemas + .iter() + .map(|schema| (schema.materialization, schema)) + .collect(); + let rules = precompute + .producers + .iter() + .map(|producer| { + let schema = schemas + .get(&producer.materialization) + .expect("validated PrecomputePlan schema binding"); + let materialization = precompute + .materializations + .iter() + .find(|m| m.policy_fingerprint() == producer.materialization.fingerprint()) + .expect("validated PrecomputePlan materialization binding"); + let runtime_policy = runtime_policies + .get(&producer.materialization.fingerprint()) + .cloned() + .unwrap_or_default(); + let mode = if runtime_policy.delta.is_some() { + TransmissionMode::Delta + } else { + TransmissionMode::Full + }; + let emit_every_ms = materialization.window_size.saturating_mul(1_000); + TransmissionRule { + materialization: producer.materialization, + producer_id: producer.producer_id.clone(), + schema_id: producer.schema_id.clone(), + mode, + encoding: schema.encodings[0].clone(), + emit_every_ms, + full_checkpoint_every_ms: (mode == TransmissionMode::Delta) + .then(|| emit_every_ms.saturating_mul(10)), + destination_ref: "asapquery-backend".into(), + runtime_policy, + } + }) + .collect(); + let plan = TransmissionPlan { + summary_catalog: precompute.summary_catalog.clone(), + envelope, + frame_identity: FrameIdentityContract { + identity_version: 1, + sequence_scope: SequenceScope::MaterializationSeriesProducerEpoch, + require_checkpoint_for_full: true, + require_base_checkpoint_for_delta: true, + }, + rules, + }; + plan.validate(precompute)?; + Ok(plan) } /// Complete physical projection of one post-ASAP planning decision. @@ -327,905 +416,6 @@ pub struct MaterializationLifecycleEstimate { pub lifecycle_cost: f64, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -#[serde(rename_all = "snake_case")] -pub enum TransmissionMode { - Full, - Delta, -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum SequenceScope { - MaterializationSeriesProducerEpoch, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct FrameIdentityContract { - pub identity_version: u32, - pub sequence_scope: SequenceScope, - pub require_checkpoint_for_full: bool, - pub require_base_checkpoint_for_delta: bool, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct TransmissionRule { - pub materialization: asap_types::sds::SummaryDefinitionId, - pub producer_id: String, - pub schema_id: String, - pub mode: TransmissionMode, - pub encoding: StateEncoding, - pub emit_every_ms: u64, - pub full_checkpoint_every_ms: Option, - pub destination_ref: String, - /// Plan-owned runtime knobs. These values are part of the immutable plan - /// generation; live feedback may only change them by publishing a - /// successor generation accepted by [`TransmissionPlan::authorize_successor`]. - #[serde(default)] - pub runtime_policy: RuntimeRulePolicy, -} - -/// How the collector admits updates before sketch maintenance. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(tag = "mode", rename_all = "snake_case", deny_unknown_fields)] -pub enum SamplingPolicy { - Disabled, - Fixed { - /// Probability in `(0, 1]`; `1` is valid but should normally be - /// represented by `Disabled`. - probability: f64, - estimator: SamplingEstimator, - }, -} - -impl Default for SamplingPolicy { - fn default() -> Self { - Self::Disabled - } -} - -impl SamplingPolicy { - /// Build the fixed physical knob from the controller's canonical - /// epsilon-floor allocator. Degenerate budgets/rates disable sampling. - pub fn from_accuracy_budget( - epsilon_sampling: f64, - updates_per_window: f64, - estimator: SamplingEstimator, - ) -> Self { - if !epsilon_sampling.is_finite() - || epsilon_sampling <= 0.0 - || !updates_per_window.is_finite() - || updates_per_window <= 0.0 - { - return Self::Disabled; - } - let probability = - crate::epsilon_alloc::derive_sample_p(epsilon_sampling, updates_per_window); - if probability >= 1.0 { - Self::Disabled - } else { - Self::Fixed { - probability, - estimator, - } - } - } -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum SamplingEstimator { - /// Hash-threshold element sampling, used by cardinality summaries. - HashThreshold, - /// Geometric admission/Nitro-style update sampling, used by frequency - /// summaries. The sketch readout carries the corresponding correction. - GeometricAdmission, -} - -/// Norm-adaptive Group-of-Sketches delta gating. GOS is meaningful only for -/// CountSketch families and only when the transmission rule is in delta mode. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct GosPolicy { - pub epsilon_staleness: f64, - pub sites: u32, - pub threshold_mode: GosThresholdMode, -} - -impl GosPolicy { - /// Allocate the deterministic staleness share with the same linear-peel - /// composition used by `epsilon_alloc`. `None` means the selected sketch - /// already consumes the budget or communication has no allocated weight. - pub fn from_accuracy_budget( - epsilon_total: f64, - epsilon_sketch: f64, - sites: u32, - edge_cpu_weight: f64, - communication_weight: f64, - threshold_mode: GosThresholdMode, - ) -> Option { - if !epsilon_total.is_finite() - || !epsilon_sketch.is_finite() - || !(0.0..=1.0).contains(&epsilon_total) - || epsilon_sketch < 0.0 - || !edge_cpu_weight.is_finite() - || edge_cpu_weight < 0.0 - || !communication_weight.is_finite() - || communication_weight <= 0.0 - { - return None; - } - let (_, epsilon_staleness) = crate::epsilon_alloc::split_budget( - epsilon_total, - epsilon_sketch, - edge_cpu_weight, - communication_weight, - ); - (epsilon_staleness.is_finite() && epsilon_staleness > 0.0).then_some(Self { - epsilon_staleness, - sites: sites.max(1), - threshold_mode, - }) - } -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum GosThresholdMode { - Isotropic, - Anisotropic, -} - -/// Sparse-delta semantics within a delta transmission rule. An absolute -/// threshold of zero sends every changed cell. When `gos` is present it -/// replaces the fixed threshold with the GOS norm-adaptive threshold. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct DeltaPolicy { - pub absolute_threshold: f64, - pub gos: Option, -} - -/// Inclusive bounds for one floating-point adaptation knob. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct AdaptiveF64Bounds { - pub min: f64, - pub max: f64, - pub max_step: f64, -} - -/// Inclusive bounds for one integer adaptation knob. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct AdaptiveU64Bounds { - pub min: u64, - pub max: u64, - pub max_step: u64, -} - -/// Guardrails for telemetry-driven runtime adaptation. This is an -/// authorization contract, not an instruction to mutate the active plan. -/// Every accepted change becomes a staged successor PhysicalPlan. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct RuntimeAdaptationPolicy { - pub enabled: bool, - pub not_before_unix_ms: u64, - pub max_evidence_age_ms: u64, - pub min_evidence_samples: u64, - pub sample_probability: Option, - pub emit_every_ms: Option, - pub delta_threshold: Option, - pub gos_epsilon_staleness: Option, -} - -impl Default for RuntimeAdaptationPolicy { - fn default() -> Self { - Self { - enabled: false, - not_before_unix_ms: 0, - max_evidence_age_ms: 0, - min_evidence_samples: 0, - sample_probability: None, - emit_every_ms: None, - delta_threshold: None, - gos_epsilon_staleness: None, - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] -#[serde(deny_unknown_fields)] -pub struct RuntimeRulePolicy { - #[serde(default)] - pub sampling: SamplingPolicy, - pub delta: Option, - #[serde(default)] - pub adaptation: RuntimeAdaptationPolicy, -} - -/// Identity and sufficiency information for evidence authorizing one rule's -/// successor knobs. Raw measurements remain in the runtime-samples store; the -/// authorization boundary needs only their exact provenance and sample count. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct RuntimeAdaptationEvidence { - pub plan_id: u64, - pub plan_version: u64, - pub materialization: asap_types::sds::SummaryDefinitionId, - pub producer_id: String, - pub schema_id: String, - pub producer_version: String, - pub observed_at_unix_ms: u64, - pub sample_count: u64, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct TransmissionPlan { - /// Absent only in legacy artifacts; catalog-aware validation requires it. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub summary_catalog: Option, - pub envelope: PlanEnvelope, - pub frame_identity: FrameIdentityContract, - pub rules: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum SummaryFrameKind { - Full, - Delta, -} - -/// Identity attached to every summary record. Window bounds come from the -/// data point; the remaining fields are carried as reserved `asap.frame.*` -/// attributes until the modified-OTLP schema gains a dedicated message. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct SummaryFrameIdentity { - pub identity_version: u32, - pub plan_id: u64, - pub plan_version: u64, - pub backend_compat: String, - pub materialization: asap_types::sds::SummaryDefinitionId, - /// Canonical producer-side identity for one concrete retained-label group. - pub series_identity: String, - pub schema_id: String, - pub producer_id: String, - pub producer_epoch: String, - pub window_start_unix_nano: u64, - pub window_end_unix_nano: u64, - pub sequence: u64, - pub kind: SummaryFrameKind, - pub encoding: StateEncoding, - pub checkpoint_id: Option, - pub base_checkpoint_id: Option, -} - -#[derive(Debug, Error, PartialEq, Eq)] -pub enum TransmissionPlanError { - #[error("summary catalog mismatch: {0}")] - Catalog(String), - #[error("TransmissionPlan envelope differs from PrecomputePlan")] - EnvelopeMismatch, - #[error("transmission rules do not exactly match precompute producer bindings")] - ProducerSetMismatch, - #[error("invalid transmission rule for producer {0}")] - InvalidRule(String), - #[error("frame identity is invalid: {0}")] - InvalidFrame(String), - #[error("frame has no matching transmission rule")] - UnknownFrame, - #[error("runtime policy for producer {producer_id} is invalid: {reason}")] - InvalidRuntimePolicy { producer_id: String, reason: String }, - #[error("runtime adaptation successor is invalid: {0}")] - InvalidSuccessor(String), - #[error("runtime adaptation evidence for producer {0} is missing or invalid")] - InvalidAdaptationEvidence(String), - #[error("runtime adaptation for producer {producer_id} exceeds guardrails: {knob}")] - AdaptationOutOfBounds { - producer_id: String, - knob: &'static str, - }, -} - -fn validate_catalog_projection( - reference: Option<&asap_types::sds::CatalogGeneration>, - envelope: &PlanEnvelope, - materializations: impl IntoIterator, - catalog: &super::summary_catalog::SummaryCatalog, -) -> Result<(), TransmissionPlanError> { - let expected = catalog - .reference() - .map_err(|error| TransmissionPlanError::Catalog(error.to_string()))?; - if reference != Some(&expected) - || envelope.plan_id != catalog.plan_id - || envelope.plan_version != catalog.plan_version - { - return Err(TransmissionPlanError::Catalog( - "missing or different snapshot reference".into(), - )); - } - for id in materializations { - if !catalog.materializations.contains_key(&id) { - return Err(TransmissionPlanError::Catalog(format!( - "unknown materialization {}", - id.as_u64() - ))); - } - } - Ok(()) -} - -impl CollectorPlan { - pub fn validate_against_catalog( - &self, - catalog: &super::summary_catalog::SummaryCatalog, - ) -> Result<(), TransmissionPlanError> { - validate_catalog_projection( - self.summary_catalog.as_ref(), - &self.envelope, - self.materializations - .iter() - .map(|m| m.materialization) - .chain(self.transmission_rules.iter().map(|r| r.materialization)), - catalog, - ) - } -} - -impl TransmissionPlan { - pub fn validate_against_catalog( - &self, - catalog: &super::summary_catalog::SummaryCatalog, - ) -> Result<(), TransmissionPlanError> { - validate_catalog_projection( - self.summary_catalog.as_ref(), - &self.envelope, - self.rules.iter().map(|r| r.materialization), - catalog, - ) - } - - pub fn build( - envelope: PlanEnvelope, - precompute: &PrecomputePlan, - runtime_policies: &BTreeMap, - ) -> Result { - if envelope != precompute.envelope { - return Err(TransmissionPlanError::EnvelopeMismatch); - } - let schemas: BTreeMap<_, _> = precompute - .schemas - .iter() - .map(|schema| (schema.materialization, schema)) - .collect(); - let rules = precompute - .producers - .iter() - .map(|producer| { - let schema = schemas - .get(&producer.materialization) - .expect("validated PrecomputePlan schema binding"); - let materialization = precompute - .materializations - .iter() - .find(|m| m.policy_fingerprint() == producer.materialization.fingerprint()) - .expect("validated PrecomputePlan materialization binding"); - let runtime_policy = runtime_policies - .get(&producer.materialization.fingerprint()) - .cloned() - .unwrap_or_default(); - let mode = if runtime_policy.delta.is_some() { - TransmissionMode::Delta - } else { - TransmissionMode::Full - }; - let emit_every_ms = materialization.window_size.saturating_mul(1_000); - TransmissionRule { - materialization: producer.materialization, - producer_id: producer.producer_id.clone(), - schema_id: producer.schema_id.clone(), - mode, - encoding: schema.encodings[0].clone(), - emit_every_ms, - full_checkpoint_every_ms: (mode == TransmissionMode::Delta) - .then(|| emit_every_ms.saturating_mul(10)), - destination_ref: "asapquery-backend".into(), - runtime_policy, - } - }) - .collect(); - let plan = Self { - summary_catalog: precompute.summary_catalog.clone(), - envelope, - frame_identity: FrameIdentityContract { - identity_version: 1, - sequence_scope: SequenceScope::MaterializationSeriesProducerEpoch, - require_checkpoint_for_full: true, - require_base_checkpoint_for_delta: true, - }, - rules, - }; - plan.validate(precompute)?; - Ok(plan) - } - - pub fn validate(&self, precompute: &PrecomputePlan) -> Result<(), TransmissionPlanError> { - if self.summary_catalog != precompute.summary_catalog { - return Err(TransmissionPlanError::Catalog( - "transmission and precompute plans reference different catalog snapshots".into(), - )); - } - if self.envelope != precompute.envelope { - return Err(TransmissionPlanError::EnvelopeMismatch); - } - let expected: BTreeSet<_> = precompute - .producers - .iter() - .map(|producer| { - ( - producer.materialization, - producer.producer_id.as_str(), - producer.schema_id.as_str(), - ) - }) - .collect(); - let actual: BTreeSet<_> = self - .rules - .iter() - .map(|rule| { - ( - rule.materialization, - rule.producer_id.as_str(), - rule.schema_id.as_str(), - ) - }) - .collect(); - if expected != actual || actual.len() != self.rules.len() { - return Err(TransmissionPlanError::ProducerSetMismatch); - } - for rule in &self.rules { - let valid_checkpoint_cadence = match (rule.mode, rule.full_checkpoint_every_ms) { - (TransmissionMode::Full, None) => true, - (TransmissionMode::Delta, Some(full_every)) => { - rule.emit_every_ms > 0 - && full_every >= rule.emit_every_ms - && full_every % rule.emit_every_ms == 0 - } - _ => false, - }; - if rule.emit_every_ms == 0 - || rule.destination_ref.is_empty() - || !valid_checkpoint_cadence - { - return Err(TransmissionPlanError::InvalidRule(rule.producer_id.clone())); - } - let schema = precompute - .schemas - .iter() - .find(|schema| schema.materialization == rule.materialization) - .expect("producer set validation guarantees a matching schema"); - if !schema.encodings.contains(&rule.encoding) { - return Err(TransmissionPlanError::InvalidRule(rule.producer_id.clone())); - } - validate_runtime_rule_policy(rule, &schema.family)?; - } - Ok(()) - } - - /// Authorize a telemetry-driven successor without mutating this active - /// plan. Semantic identity, codecs, destination, transmission mode and - /// checkpoint cadence remain fixed. Only explicitly bounded runtime knobs - /// may move, and each changed rule needs fresh evidence attributed to the - /// exact active generation. - pub fn authorize_successor( - &self, - successor: &TransmissionPlan, - evidence: &[RuntimeAdaptationEvidence], - now_unix_ms: u64, - ) -> Result<(), TransmissionPlanError> { - if successor.envelope.plan_id != self.envelope.plan_id - || successor.envelope.plan_version != self.envelope.plan_version.saturating_add(1) - || successor.envelope.backend_compat != self.envelope.backend_compat - || successor.envelope.planner_revision != self.envelope.planner_revision - || successor.envelope.capability_snapshot_id != self.envelope.capability_snapshot_id - || successor.envelope.generated_at_unix_ms < self.envelope.generated_at_unix_ms - || successor.envelope.activation_unix_ms < successor.envelope.generated_at_unix_ms - || successor.rules.len() != self.rules.len() - || successor.frame_identity != self.frame_identity - { - return Err(TransmissionPlanError::InvalidSuccessor( - "successor must be the next version of the same semantic/capability generation" - .into(), - )); - } - - for current in &self.rules { - let Some(next) = successor.rules.iter().find(|candidate| { - candidate.materialization == current.materialization - && candidate.producer_id == current.producer_id - && candidate.schema_id == current.schema_id - }) else { - return Err(TransmissionPlanError::InvalidSuccessor(format!( - "missing rule for producer {}", - current.producer_id - ))); - }; - if current.mode != next.mode - || current.encoding != next.encoding - || current.full_checkpoint_every_ms != next.full_checkpoint_every_ms - || current.destination_ref != next.destination_ref - || current.runtime_policy.adaptation != next.runtime_policy.adaptation - || sampling_estimator(¤t.runtime_policy.sampling) - != sampling_estimator(&next.runtime_policy.sampling) - || delta_shape(¤t.runtime_policy.delta) - != delta_shape(&next.runtime_policy.delta) - { - return Err(TransmissionPlanError::InvalidSuccessor(format!( - "rule identity/codec/mode/guardrails drifted for producer {}", - current.producer_id - ))); - } - if current.emit_every_ms == next.emit_every_ms - && current.runtime_policy.sampling == next.runtime_policy.sampling - && current.runtime_policy.delta == next.runtime_policy.delta - { - continue; - } - let policy = ¤t.runtime_policy.adaptation; - if !policy.enabled || now_unix_ms < policy.not_before_unix_ms { - return Err(TransmissionPlanError::AdaptationOutOfBounds { - producer_id: current.producer_id.clone(), - knob: "adaptation_disabled_or_in_cooldown", - }); - } - let has_evidence = evidence.iter().any(|item| { - item.plan_id == self.envelope.plan_id - && item.plan_version == self.envelope.plan_version - && item.materialization == current.materialization - && item.producer_id == current.producer_id - && item.schema_id == current.schema_id - && !item.producer_version.trim().is_empty() - && item.sample_count >= policy.min_evidence_samples - && item.observed_at_unix_ms <= now_unix_ms - && now_unix_ms.saturating_sub(item.observed_at_unix_ms) - <= policy.max_evidence_age_ms - }); - if !has_evidence { - return Err(TransmissionPlanError::InvalidAdaptationEvidence( - current.producer_id.clone(), - )); - } - authorize_f64_change( - sampling_probability(¤t.runtime_policy.sampling), - sampling_probability(&next.runtime_policy.sampling), - policy.sample_probability.as_ref(), - ¤t.producer_id, - "sample_probability", - )?; - authorize_u64_change( - current.emit_every_ms, - next.emit_every_ms, - policy.emit_every_ms.as_ref(), - ¤t.producer_id, - "emit_every_ms", - )?; - authorize_f64_change( - delta_threshold(¤t.runtime_policy.delta), - delta_threshold(&next.runtime_policy.delta), - policy.delta_threshold.as_ref(), - ¤t.producer_id, - "delta_threshold", - )?; - authorize_f64_change( - gos_epsilon(¤t.runtime_policy.delta), - gos_epsilon(&next.runtime_policy.delta), - policy.gos_epsilon_staleness.as_ref(), - ¤t.producer_id, - "gos_epsilon_staleness", - )?; - } - Ok(()) - } - - pub fn validate_frame( - &self, - frame: &SummaryFrameIdentity, - ) -> Result<(), TransmissionPlanError> { - if frame.identity_version != self.frame_identity.identity_version - || frame.plan_id != self.envelope.plan_id - || frame.plan_version != self.envelope.plan_version - || frame.backend_compat != self.envelope.backend_compat - || frame.series_identity.is_empty() - || frame.producer_epoch.is_empty() - || frame.sequence == 0 - || frame.window_start_unix_nano >= frame.window_end_unix_nano - || (frame.kind == SummaryFrameKind::Full - && self.frame_identity.require_checkpoint_for_full - && frame.checkpoint_id.is_none()) - || (frame.kind == SummaryFrameKind::Delta - && self.frame_identity.require_base_checkpoint_for_delta - && frame.base_checkpoint_id.is_none()) - { - return Err(TransmissionPlanError::InvalidFrame( - "identity/lifecycle/window/checkpoint fields do not satisfy the active contract" - .into(), - )); - } - if self.rules.iter().any(|rule| { - rule.materialization == frame.materialization - && rule.producer_id == frame.producer_id - && rule.schema_id == frame.schema_id - // A delta rule necessarily emits periodic full checkpoints; - // a full-only rule must never emit deltas. - && (frame.kind == SummaryFrameKind::Full - || rule.mode == TransmissionMode::Delta) - && rule.encoding == frame.encoding - }) { - Ok(()) - } else { - Err(TransmissionPlanError::UnknownFrame) - } - } -} - -fn validate_runtime_rule_policy( - rule: &TransmissionRule, - family: &StateFamilyContract, -) -> Result<(), TransmissionPlanError> { - let invalid = |reason: &str| TransmissionPlanError::InvalidRuntimePolicy { - producer_id: rule.producer_id.clone(), - reason: reason.into(), - }; - if let SamplingPolicy::Fixed { - probability, - estimator, - } = &rule.runtime_policy.sampling - { - if !probability.is_finite() || !(0.0..=1.0).contains(probability) || *probability == 0.0 { - return Err(invalid("sample probability must be finite and in (0, 1]")); - } - let supported = matches!( - (family, *estimator), - ( - StateFamilyContract::Sketch { - algorithm: SketchAlgorithm::Hll, - .. - }, - SamplingEstimator::HashThreshold, - ) | ( - StateFamilyContract::Sketch { - algorithm: SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap, - .. - }, - SamplingEstimator::GeometricAdmission, - ) - ); - if !supported { - return Err(invalid( - "sampling estimator is not implemented for the materialization family", - )); - } - } - - if (rule.mode == TransmissionMode::Delta) != rule.runtime_policy.delta.is_some() { - return Err(invalid( - "delta policy must be present exactly when transmission mode is delta", - )); - } - if let Some(delta) = &rule.runtime_policy.delta { - if !delta.absolute_threshold.is_finite() || delta.absolute_threshold < 0.0 { - return Err(invalid("delta threshold must be finite and non-negative")); - } - if !matches!( - family, - StateFamilyContract::Sketch { - algorithm: SketchAlgorithm::DDSketch - | SketchAlgorithm::Hll - | SketchAlgorithm::Cms - | SketchAlgorithm::CmsWithHeap - | SketchAlgorithm::CountSketch - | SketchAlgorithm::CountSketchWithHeap, - .. - } - ) { - return Err(invalid( - "delta transmission is not implemented for the materialization family", - )); - } - if let Some(gos) = &delta.gos { - if !matches!( - family, - StateFamilyContract::Sketch { - algorithm: SketchAlgorithm::CountSketch | SketchAlgorithm::CountSketchWithHeap, - .. - } - ) || !gos.epsilon_staleness.is_finite() - || !(0.0..=1.0).contains(&gos.epsilon_staleness) - || gos.epsilon_staleness == 0.0 - || gos.sites == 0 - { - return Err(invalid( - "GOS requires a CountSketch family, epsilon in (0, 1], and at least one site", - )); - } - } - } - - let adaptation = &rule.runtime_policy.adaptation; - if adaptation.enabled - && (adaptation.max_evidence_age_ms == 0 || adaptation.min_evidence_samples == 0) - { - return Err(invalid( - "enabled adaptation requires non-zero evidence age and sample-count requirements", - )); - } - validate_f64_bounds(adaptation.sample_probability.as_ref(), 0.0, 1.0) - .map_err(|reason| invalid(reason))?; - validate_f64_bounds(adaptation.delta_threshold.as_ref(), 0.0, f64::MAX) - .map_err(|reason| invalid(reason))?; - validate_f64_bounds(adaptation.gos_epsilon_staleness.as_ref(), 0.0, 1.0) - .map_err(|reason| invalid(reason))?; - if let Some(bounds) = &adaptation.emit_every_ms { - if bounds.min == 0 - || bounds.min > bounds.max - || bounds.max_step == 0 - || !(bounds.min..=bounds.max).contains(&rule.emit_every_ms) - { - return Err(invalid("emit interval guardrails are invalid")); - } - } - if let Some(bounds) = &adaptation.sample_probability { - let current = sampling_probability(&rule.runtime_policy.sampling); - if current < bounds.min || current > bounds.max { - return Err(invalid( - "current sampling probability is outside guardrails", - )); - } - } - if let Some(bounds) = &adaptation.delta_threshold { - let Some(current) = rule - .runtime_policy - .delta - .as_ref() - .map(|policy| policy.absolute_threshold) - else { - return Err(invalid("delta guardrails require an active delta policy")); - }; - if current < bounds.min || current > bounds.max { - return Err(invalid("current delta threshold is outside guardrails")); - } - } - if let Some(bounds) = &adaptation.gos_epsilon_staleness { - let Some(current) = rule - .runtime_policy - .delta - .as_ref() - .and_then(|policy| policy.gos.as_ref()) - .map(|gos| gos.epsilon_staleness) - else { - return Err(invalid("GOS guardrails require an active GOS policy")); - }; - if current < bounds.min || current > bounds.max { - return Err(invalid("current GOS epsilon is outside guardrails")); - } - } - Ok(()) -} - -fn validate_f64_bounds( - bounds: Option<&AdaptiveF64Bounds>, - domain_min: f64, - domain_max: f64, -) -> Result<(), &'static str> { - let Some(bounds) = bounds else { - return Ok(()); - }; - if !bounds.min.is_finite() - || !bounds.max.is_finite() - || !bounds.max_step.is_finite() - || bounds.min < domain_min - || bounds.max > domain_max - || bounds.min > bounds.max - || bounds.max_step <= 0.0 - { - Err("floating-point adaptation guardrails are invalid") - } else { - Ok(()) - } -} - -fn sampling_probability(policy: &SamplingPolicy) -> f64 { - match policy { - SamplingPolicy::Disabled => 1.0, - SamplingPolicy::Fixed { probability, .. } => *probability, - } -} - -fn sampling_estimator(policy: &SamplingPolicy) -> Option { - match policy { - SamplingPolicy::Disabled => None, - SamplingPolicy::Fixed { estimator, .. } => Some(*estimator), - } -} - -fn delta_threshold(policy: &Option) -> f64 { - policy - .as_ref() - .map(|policy| policy.absolute_threshold) - .unwrap_or(0.0) -} - -fn gos_epsilon(policy: &Option) -> f64 { - policy - .as_ref() - .and_then(|policy| policy.gos.as_ref()) - .map(|gos| gos.epsilon_staleness) - .unwrap_or(0.0) -} - -fn delta_shape(policy: &Option) -> Option<(Option<(u32, GosThresholdMode)>,)> { - policy.as_ref().map(|policy| { - (policy - .gos - .as_ref() - .map(|gos| (gos.sites, gos.threshold_mode)),) - }) -} - -fn authorize_f64_change( - current: f64, - next: f64, - bounds: Option<&AdaptiveF64Bounds>, - producer_id: &str, - knob: &'static str, -) -> Result<(), TransmissionPlanError> { - if current == next { - return Ok(()); - } - let allowed = bounds.is_some_and(|bounds| { - next.is_finite() - && (bounds.min..=bounds.max).contains(&next) - && (next - current).abs() <= bounds.max_step - }); - if allowed { - Ok(()) - } else { - Err(TransmissionPlanError::AdaptationOutOfBounds { - producer_id: producer_id.into(), - knob, - }) - } -} - -fn authorize_u64_change( - current: u64, - next: u64, - bounds: Option<&AdaptiveU64Bounds>, - producer_id: &str, - knob: &'static str, -) -> Result<(), TransmissionPlanError> { - if current == next { - return Ok(()); - } - let allowed = bounds.is_some_and(|bounds| { - (bounds.min..=bounds.max).contains(&next) && current.abs_diff(next) <= bounds.max_step - }); - if allowed { - Ok(()) - } else { - Err(TransmissionPlanError::AdaptationOutOfBounds { - producer_id: producer_id.into(), - knob, - }) - } -} - #[derive(Debug, Error)] pub enum CompileError { #[error("invalid backend-local workload snapshot: {0}")] @@ -2105,12 +1295,15 @@ impl PhysicalCompiler { query_id: "precompute-plan".into(), reason: error.to_string(), })?; - let mut transmission_plan = - TransmissionPlan::build(envelope.clone(), &precompute_plan, &runtime_policies) - .map_err(|error| CompileError::Query { - query_id: "transmission-plan".into(), - reason: error.to_string(), - })?; + let mut transmission_plan = crate::physical::compiler::compile_transmission_plan( + envelope.clone(), + &precompute_plan, + &runtime_policies, + ) + .map_err(|error| CompileError::Query { + query_id: "transmission-plan".into(), + reason: error.to_string(), + })?; let mut collector_plans = producer_ids .into_iter() .map(|collector_id| CollectorPlan { @@ -2223,7 +1416,7 @@ impl PhysicalCompiler { cumulative_readout: true, }; let mut entry = if request.hybrid_execution { - QueryPlanEntry::compile_bound_composable_mapped( + crate::query_plan::compile_bound_composable_mapped( query.query_id.clone(), canonical.clone(), &query.post_asap, @@ -2240,7 +1433,7 @@ impl PhysicalCompiler { }, ) } else { - QueryPlanEntry::compile_bound_mapped( + crate::query_plan::compile_bound_mapped( query.query_id.clone(), canonical.clone(), &query.post_asap, @@ -2464,6 +1657,53 @@ pub fn select_workload_roots( select_workload_roots_with_erp(queries, roots, evidence, exact_costs, None) } +fn observed_population_matches_root( + policy: &super::erp::ErpPlanningInput, + root: &QueryExpr, +) -> bool { + use asap_types::sds::{PopulationPartitioning, ValueProjectionIdentity}; + use planner_types::pre_asap::{AggIntent, Reduction}; + let (Some(data), Some(observed)) = ( + &policy.resolved_data_descriptor, + &policy.observed_populations, + ) else { + return false; + }; + let QueryExpr::Aggregate { + reduction: Reduction::PerEntity, + measures, + having: None, + child, + .. + } = root + else { + return false; + }; + if measures.is_empty() + || !measures.iter().all(|intent| { + matches!( + intent, + AggIntent::Cardinality { col: None, .. } + | AggIntent::FrequencyL2 { col: None, .. } + | AggIntent::FrequencyEntropy { col: None, .. } + ) + }) + { + return false; + } + let Ok((metric, Some(window), filter)) = raw_time_series_input_contract(child, false) else { + return false; + }; + data.time_series_metric() == Some(metric.as_str()) + && data.population_filter_canonical == filter + && data.value_projection == ValueProjectionIdentity::SampleValue + && data.partitioning == Some(PopulationPartitioning::PerEntity) + && data.group_by_keys.is_empty() + && data.observation_semantics == asap_types::sds::TIMESTAMPED_OBSERVATION_SEMANTICS + && observed.window_end_ms.checked_sub(observed.window_start_ms) + == i64::try_from(window.saturating_mul(1000)).ok() +} + pub fn select_workload_roots_with_erp( queries: &mut [PlanningQuery], roots: Vec>, @@ -2501,6 +1741,17 @@ pub fn select_workload_roots_with_erp( if !matches!(accuracy, AccuracyTarget::Epsilon(_)) { policy.artifact.records.clear(); } + if policy.observed_populations.is_some() + && !roots + .iter() + .all(|(_, root)| observed_population_matches_root(&policy, root)) + { + if let Some(observed) = &mut policy.observed_populations { + observed.invalid_reason = + Some("candidate input differs from observed catalog data semantics".into()); + observed.populations.clear(); + } + } // A benchmark of a different KLL implementation is not evidence // for the collector's sketchlib KLL, even with the same k. policy.artifact.records.retain(|row| { @@ -3102,29 +2353,37 @@ fn select_lifecycle( pub(crate) fn materialization_leaf_contract( node: &SummaryNode, ) -> Result<(String, Option, String), String> { - use planner_types::pre_asap::{CompareOpKind, QueryExpr, ScalarValue}; let SummaryExpr::SummaryAgg { child, .. } = &node.expr else { return Err("materialization requires a SummaryAgg leaf".into()); }; let SummaryExpr::KeepPreAsap(expr) = &child.expr else { return Err("materialization input is not a raw source".into()); }; - let (source, window_secs) = match expr.as_ref() { + raw_time_series_input_contract( + expr, + matches!( + &node.expr, + SummaryExpr::SummaryAgg { + family: SummaryFamilyType::ExactAggregate(..), + .. + } + ), + ) +} + +fn raw_time_series_input_contract( + expr: &QueryExpr, + exact: bool, +) -> Result<(String, Option, String), String> { + use planner_types::pre_asap::{CompareOpKind, ScalarValue}; + let (source, window_secs) = match expr { QueryExpr::TimeRange { child, range } => { if range.as_millis() == 0 || range.as_millis() % 1000 != 0 { return Err("warm producer requires a positive whole-second range".into()); } (child.as_ref(), Some(range.as_secs())) } - QueryExpr::Scan { .. } - if matches!( - &node.expr, - SummaryExpr::SummaryAgg { - family: SummaryFamilyType::ExactAggregate(..), - .. - } - ) => - { + QueryExpr::Scan { .. } if exact => { return Err( "instantaneous sample selection is not a temporal accumulator readout".into(), ); @@ -4350,6 +3609,8 @@ mod tests { byte_second_weight: 1e-9, mode: super::super::erp::ErpAccuracyMode::Hybrid, observed_shape: None, + observed_populations: None, + resolved_data_descriptor: None, observed_shape_source: None, shape_match: None, runtime: super::super::erp::ErpRuntimeCapabilities { @@ -4399,6 +3660,8 @@ mod tests { byte_second_weight: 1e-9, mode: ErpAccuracyMode::Hybrid, observed_shape: None, + observed_populations: None, + resolved_data_descriptor: None, observed_shape_source: None, shape_match: None, runtime: ErpRuntimeCapabilities { @@ -5281,13 +4544,21 @@ mod tests { collector.validate_against_catalog(catalog).unwrap(); collector.envelope.plan_version += 1; assert!(collector.validate_against_catalog(catalog).is_err()); - assert!(validate_catalog_projection( - Some(&catalog.reference().unwrap()), - &bundle.envelope, - [asap_types::PolicyFingerprint(u64::MAX).into()], - catalog - ) - .is_err()); + let mut unknown_materialization = bundle.transmission_plan.clone(); + unknown_materialization.rules.push(TransmissionRule { + materialization: asap_types::PolicyFingerprint(u64::MAX).into(), + producer_id: "foreign".into(), + schema_id: "foreign".into(), + mode: TransmissionMode::Full, + encoding: StateEncoding::ExactAccumulatorV1, + emit_every_ms: 60_000, + full_checkpoint_every_ms: None, + destination_ref: "backend".into(), + runtime_policy: Default::default(), + }); + assert!(unknown_materialization + .validate_against_catalog(catalog) + .is_err()); let encoded = serde_json::to_value(&collector).unwrap(); assert!(encoded["summary_catalog"] .get("summary_descriptors") @@ -5601,10 +4872,14 @@ mod tests { assert_eq!(plan.ingest.timestamp_unit, TimestampUnit::UnixMilliseconds); assert!(plan.producers.is_empty()); plan.validate().expect("valid backend-local projection"); - TransmissionPlan::build(bundle.envelope, &plan, &BTreeMap::new()) - .expect("empty backend-local transmission contract") - .validate(&plan) - .expect("valid empty transmission plan"); + crate::physical::compiler::compile_transmission_plan( + bundle.envelope, + &plan, + &BTreeMap::new(), + ) + .expect("empty backend-local transmission contract") + .validate(&plan) + .expect("valid empty transmission plan"); } #[test] @@ -6064,6 +5339,9 @@ mod tests { ), full_selected ); + // Publication validates stored extent independently of slide cadence. + // In particular the full-window candidate stores 60s at a 10s slide. + bundle.publication().unwrap().validate().unwrap(); assert_eq!(bundle.precompute_plan.materializations[0].window_size, 60); assert_eq!( bundle.precompute_plan.materializations[0].slide_interval, @@ -6272,7 +5550,7 @@ mod tests { #[test] fn runtime_policy_uses_canonical_sampling_and_gos_allocators() { - let sampling = SamplingPolicy::from_accuracy_budget( + let sampling = sampling_policy_from_accuracy_budget( 0.05, 1_000.0, SamplingEstimator::GeometricAdmission, @@ -6283,64 +5561,26 @@ mod tests { assert!((probability - 1.0 / 3.5).abs() < 1e-9); let gos = - GosPolicy::from_accuracy_budget(0.1, 0.03, 4, 1.0, 1.0, GosThresholdMode::Isotropic) + gos_policy_from_accuracy_budget(0.1, 0.03, 4, 1.0, 1.0, GosThresholdMode::Isotropic) .expect("positive communication allocation"); assert!((gos.epsilon_staleness - 0.035).abs() < 1e-12); assert_eq!(gos.sites, 4); } #[test] - fn runtime_policy_is_family_and_mode_checked() { + fn runtime_policy_encoding_is_checked() { let bundle = PhysicalCompiler .compile( request("q", "quantile_over_time(0.99, m[1m])"), environment(10_000), ) .expect("compile"); - let mut rule = bundle.transmission_plan.rules[0].clone(); let mut invalid_encoding = bundle.transmission_plan.clone(); invalid_encoding.rules[0].encoding = StateEncoding::ExactAccumulatorV1; assert!(matches!( invalid_encoding.validate(&bundle.precompute_plan), Err(TransmissionPlanError::InvalidRule(_)) )); - rule.runtime_policy.sampling = SamplingPolicy::Fixed { - probability: 0.5, - estimator: SamplingEstimator::HashThreshold, - }; - let hll = StateFamilyContract::Sketch { - algorithm: SketchAlgorithm::Hll, - parameters: SketchParams::Hll { precision: 14 }, - }; - let count_sketch = StateFamilyContract::Sketch { - algorithm: SketchAlgorithm::CountSketch, - parameters: SketchParams::CountSketch { - width: 128, - depth: 4, - }, - }; - validate_runtime_rule_policy(&rule, &hll).expect("HLL supports hash-threshold sampling"); - assert!(matches!( - validate_runtime_rule_policy(&rule, &count_sketch), - Err(TransmissionPlanError::InvalidRuntimePolicy { .. }) - )); - - rule.runtime_policy.sampling = SamplingPolicy::Disabled; - rule.mode = TransmissionMode::Delta; - rule.full_checkpoint_every_ms = Some(300_000); - rule.runtime_policy.delta = Some(DeltaPolicy { - absolute_threshold: 0.0, - gos: Some(GosPolicy { - epsilon_staleness: 0.02, - sites: 2, - threshold_mode: GosThresholdMode::Isotropic, - }), - }); - validate_runtime_rule_policy(&rule, &count_sketch).expect("CountSketch supports delta GOS"); - assert!(matches!( - validate_runtime_rule_policy(&rule, &hll), - Err(TransmissionPlanError::InvalidRuntimePolicy { .. }) - )); } #[test] diff --git a/control_plane/src/physical/erp.rs b/control_plane/src/physical/erp.rs index a638e38a8..c41fa2fe2 100644 --- a/control_plane/src/physical/erp.rs +++ b/control_plane/src/physical/erp.rs @@ -7,7 +7,7 @@ use asap_aware_mapping::erp::{ use planner_types::post_asap::{SketchAlgorithm, SketchParams}; use serde::{Deserialize, Serialize}; use serde_json::Value; -use std::collections::{BTreeMap, HashMap}; +use std::collections::BTreeMap; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] @@ -22,70 +22,40 @@ pub struct ErpObservedShape { /// instead of silently under-reporting cardinality. #[derive(Debug)] pub struct ErpShapeObserver { - frequencies: HashMap, - interval_updates: HashMap, - max_observed_keys: usize, - max_observed_intervals: usize, - updates: u64, - invalid: bool, + inner: asap_types::erp_observation::BoundedFrequencyObserver, } - impl ErpShapeObserver { - pub fn new(max_observed_keys: usize) -> Result { - Self::with_limits(max_observed_keys, 256) + pub fn observed_key_count(&self) -> usize { + self.inner.observed_key_count() } - - pub fn with_limits( - max_observed_keys: usize, - max_observed_intervals: usize, - ) -> Result { - if max_observed_keys == 0 || max_observed_intervals == 0 { - return Err("observation limits must be positive"); - } + pub fn new(max_keys: usize) -> Result { + Self::with_limits(max_keys, 256) + } + pub fn with_limits(max_keys: usize, max_intervals: usize) -> Result { Ok(Self { - frequencies: HashMap::new(), - interval_updates: HashMap::new(), - max_observed_keys, - max_observed_intervals, - updates: 0, - invalid: false, + inner: asap_types::erp_observation::BoundedFrequencyObserver::new( + max_keys, + max_intervals, + )?, }) } - pub fn observe(&mut self, key: &str, interval: usize) -> Result<(), &'static str> { - if self.invalid { - return Err("ERP observation was invalidated; start a fresh observation window"); - } if key.len() > 4096 { - self.invalid = true; + self.inner.invalidate(); return Err("ERP shape observer key size exceeded"); } - if !self.frequencies.contains_key(key) && self.frequencies.len() == self.max_observed_keys { - self.invalid = true; - return Err("ERP shape observer cardinality cap exceeded"); - } - if !self.interval_updates.contains_key(&interval) - && self.interval_updates.len() == self.max_observed_intervals - { - self.invalid = true; - return Err("ERP shape observer interval cap exceeded"); - } - let Some(updates) = self.updates.checked_add(1) else { - self.invalid = true; - return Err("ERP shape observer count overflow"); - }; - *self.frequencies.entry(key.to_owned()).or_default() += 1; - *self.interval_updates.entry(interval).or_default() += 1; - self.updates = updates; - Ok(()) + self.inner.observe(key.to_owned(), interval) } - pub fn snapshot(&self) -> Option { - if self.frequencies.is_empty() || self.invalid { - return None; - } - let mut counts: Vec<_> = self.frequencies.values().copied().collect(); - counts.sort_unstable_by(|left, right| right.cmp(left)); + ErpObservedShape::from_empirical(&self.inner.snapshot()?) + } +} +impl ErpObservedShape { + pub fn from_empirical( + empirical: &asap_types::erp_observation::EmpiricalFrequencyObservation, + ) -> Option { + let events = empirical.event_count()?; + let counts = &empirical.sorted_counts; let exponent = fit_zipf_exponent(&counts); let fits = [("uniform", 0.0), ("zipf", exponent)] .into_iter() @@ -100,7 +70,7 @@ impl ErpShapeObserver { .iter() .zip(expected) .map(|(count, expected)| { - (*count as f64 / self.updates as f64 - expected / total).abs() + (*count as f64 / events as f64 - expected / total).abs() }) .sum::() / 2.0; @@ -113,13 +83,13 @@ impl ErpShapeObserver { }, goodness_of_fit: distance, // A fit-quality score, not an estimator tail-probability claim. - confidence: (1.0 - distance) * (1.0 - 1.0 / (self.updates as f64).sqrt()), + confidence: (1.0 - distance) * (1.0 - 1.0 / (events as f64).sqrt()), } }) .collect(); - let mut nonzero: Vec<_> = self - .interval_updates - .values() + let mut nonzero: Vec<_> = empirical + .interval_counts + .iter() .copied() .filter(|count| *count > 0) .collect(); @@ -128,8 +98,8 @@ impl ErpShapeObserver { let peak = nonzero.last().copied().unwrap_or(median); Some(ErpObservedShape { observation: ErpShapeObservation { - cardinality: self.frequencies.len() as u64, - observed_events: self.updates, + cardinality: counts.len() as u64, + observed_events: events, fits, empirical_fingerprint: None, }, @@ -362,6 +332,11 @@ pub struct ErpPlanningInput { /// replaced by bounded nearest-profile matching. #[serde(default)] pub observed_shape: Option, + #[serde(default)] + pub observed_populations: + Option>, + #[serde(skip)] + pub resolved_data_descriptor: Option>, /// Runtime-samples ring key from which the backend resolves the freshest /// `erp_observed_shape` payload before compiling a plan. #[serde(default)] @@ -378,25 +353,160 @@ pub struct ErpObservedShapeSource { pub source: String, pub sketch: String, pub implementation: String, + /// Required for catalog-scoped online evidence; absent only for legacy + /// offline runtime records that contain a single shape. + #[serde(default)] + pub population_scope: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct ErpPopulationObservationScope { + pub catalog_generation: asap_types::sds::CatalogGeneration, + pub summary_definition_id: asap_types::sds::SummaryDefinitionId, + pub input_semantics: asap_types::erp_observation::ErpObservationInputSemantics, + pub freshness: asap_types::erp_observation::ErpObservationFreshness, } impl ErpPlanningInput { + /// Resolve evidence against the control plane's accepted catalog, never + /// descriptors supplied by the observation producer. + pub fn resolve_population_data_descriptor( + &mut self, + catalog: Option<&asap_types::summary_catalog::SummaryCatalog>, + ) { + self.resolved_data_descriptor = None; + let Some(populations) = self.observed_populations.as_mut() else { + return; + }; + if populations.invalid_reason.is_some() { + return; + } + let resolved = (|| -> Result<_, String> { + let catalog = catalog.ok_or("no active authoritative catalog for ERP evidence")?; + let generation = catalog.reference().map_err(|error| error.to_string())?; + if generation != populations.catalog_generation { + return Err("ERP evidence catalog differs from the active catalog".into()); + } + let materialization = catalog + .materializations + .get(&populations.summary_definition_id) + .ok_or("ERP evidence summary is absent from the active catalog")?; + let summary = catalog + .summary_descriptors + .get(&materialization.summary_descriptor_id) + .ok_or("ERP summary descriptor is absent from the active catalog")?; + let actual_semantics = match &summary.operator { + asap_types::sds::SummaryOperator::Configured { + aggregation_type: asap_types::AggregationType::HLL, + .. + } => Some( + asap_types::erp_observation::ErpObservationInputSemantics::ScalarSampleValue, + ), + asap_types::sds::SummaryOperator::Configured { + aggregation_type: asap_types::AggregationType::UnivMon, + .. + } => Some( + asap_types::erp_observation::ErpObservationInputSemantics::UnitSampleFrequency, + ), + _ => None, + }; + if actual_semantics != Some(populations.input_semantics) { + return Err( + "ERP observation semantics differ from the installed summary operator".into(), + ); + } + let data = catalog + .data_descriptors + .get(&materialization.data_descriptor_id) + .ok_or("ERP evidence data descriptor is absent from the active catalog")?; + Ok(std::sync::Arc::new(data.clone())) + })(); + match resolved { + Ok(data) => self.resolved_data_descriptor = Some(data), + Err(reason) => { + populations.invalid_reason = Some(reason); + populations.populations.clear(); + } + } + } + pub fn hydrate_observed_shape( &mut self, samples: &crate::runtime_samples::RuntimeSamplesStore, ) -> Result<(), String> { - if self.observed_shape.is_some() { - return Ok(()); - } + let now_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(|error| format!("cannot timestamp ERP observation: {error}"))? + .as_millis() + .try_into() + .map_err(|_| "ERP observation timestamp overflows milliseconds")?; + self.hydrate_observed_shape_at(samples, now_ms) + } + + pub fn hydrate_observed_shape_at( + &mut self, + samples: &crate::runtime_samples::RuntimeSamplesStore, + now_ms: u64, + ) -> Result<(), String> { let Some(source) = &self.observed_shape_source else { return Ok(()); }; + // A reused planning request must resolve new evidence on every compile. + // Never keep an old inline or previously hydrated fit after drift. + self.observed_shape = None; + self.observed_populations = None; let key = crate::runtime_samples::SampleKey { source: source.source.clone(), sketch: source.sketch.clone(), impl_name: source.implementation.clone(), }; - let record = samples.latest(&key).ok_or_else(|| { + let record = samples.latest(&key); + if let Some(scope) = &source.population_scope { + let resolved = (|| -> Result<_, String> { + let record = record + .as_ref() + .ok_or("no live ERP population observation")?; + let value = record + .payload + .get("erp_population_observations") + .ok_or("latest runtime sample has no erp_population_observations")?; + let observed: asap_types::erp_observation::ErpPopulationObservations< + asap_types::erp_observation::EmpiricalFrequencyObservation, + > = serde_json::from_value(value.clone()) + .map_err(|error| format!("invalid ERP population observations: {error}"))?; + observed.validate_identity_and_freshness( + &scope.catalog_generation, + scope.summary_definition_id, + now_ms, + scope.freshness, + )?; + if observed.input_semantics != scope.input_semantics { + return Err("ERP observation input semantics do not match the summary".into()); + } + observed + .try_map_shapes(|empirical| ErpObservedShape::from_empirical(&empirical)) + .ok_or_else(|| "invalid bounded empirical frequency observations".to_owned()) + })(); + // An explicit invalid envelope prevents selection from borrowing an + // older fit or falling back to the artifact's distribution identity. + // The normal ERP -> theoretical -> exact policy handles this miss. + self.observed_populations = Some(resolved.unwrap_or_else(|reason| { + asap_types::erp_observation::ErpPopulationObservations { + schema_version: 1, + catalog_generation: scope.catalog_generation.clone(), + summary_definition_id: scope.summary_definition_id, + observed_at_unix_ms: now_ms, + window_start_ms: 0, + window_end_ms: 0, + input_semantics: scope.input_semantics, + invalid_reason: Some(reason), + populations: Vec::new(), + } + })); + return Ok(()); + } + let record = record.ok_or_else(|| { format!( "no runtime shape sample for {}/{}/{}", source.source, source.sketch, source.implementation @@ -516,6 +626,86 @@ impl ErpPlanningInput { theoretical: SketchParams, required_params: Option<&SketchParams>, ) -> ErpParameterDecision { + if let Some(populations) = &self.observed_populations { + // Every installed partition must support the same configuration. + // Never aggregate their frequency maps into a fictitious population. + let mut best: Option = None; + if populations.invalid_reason.is_none() && !populations.populations.is_empty() { + for row in &self.artifact.records { + if !sketch_name_matches(&row.sketch, &algorithm) { + continue; + } + let Some(params) = parse_params(&algorithm, &row.parameters) else { + continue; + }; + if required_params.is_some_and(|required| required != ¶ms) { + continue; + } + let mut policy = self.clone(); + policy.observed_populations = None; + policy.mode = ErpAccuracyMode::Empirical; + let total_events: f64 = populations + .populations + .iter() + .map(|p| p.shape.observation.observed_events as f64) + .sum(); + let mut errors: f64 = 0.0; + let mut cost = 0.0; + let mut evidence = Vec::new(); + let valid = populations.populations.iter().all(|population| { + policy.observed_shape = Some(population.shape.observation.clone()); + // expected_updates is the definition-wide demand; distribute + // it across partitions instead of charging it once per series. + policy.expected_updates = self.expected_updates + * population.shape.observation.observed_events as f64 + / total_events; + + match policy.select_metric( + algorithm.clone(), + error_metric, + max_error, + theoretical.clone(), + Some(¶ms), + ) { + ErpParameterDecision::Empirical { + record_id, + observed_error, + estimated_cost, + .. + } => { + errors = errors.max(observed_error); + cost += estimated_cost; + evidence.push(record_id); + true + } + _ => false, + } + }); + if valid && cost.is_finite() && best.as_ref().is_none_or(|previous| { + matches!(previous, ErpParameterDecision::Empirical { estimated_cost, .. } if cost < *estimated_cost) + }) { + best = Some(ErpParameterDecision::Empirical { + params, record_id: serde_json::to_string(&evidence).expect("string IDs serialize"), + observed_error: errors, estimated_cost: cost, + }); + } + } + } + return best.unwrap_or_else(|| { + let reason = + "ERP has no configuration valid for every observed population".to_owned(); + if self.mode == ErpAccuracyMode::Hybrid + && self.runtime.supports(&algorithm, &theoretical, None) + { + ErpParameterDecision::TheoreticalFallback { + params: theoretical, + reason, + } + } else { + ErpParameterDecision::ExactFallback { reason } + } + }); + } // Runtime admissibility belongs before ranking: an unusable cheap // profile must not hide a more expensive executable alternative. let mut artifact = self.artifact.clone(); @@ -834,6 +1024,8 @@ mod tests { byte_second_weight: 1e-9, mode, observed_shape: None, + observed_populations: None, + resolved_data_descriptor: None, observed_shape_source: None, shape_match: None, runtime: ErpRuntimeCapabilities::default(), @@ -1135,18 +1327,220 @@ mod tests { source: "edge-a".into(), sketch: "cms".into(), implementation: "oxide".into(), + population_scope: None, }); policy.hydrate_observed_shape(&samples).unwrap(); assert_eq!(policy.observed_shape.unwrap().cardinality, 1000); } + fn online_population_fixture() -> ( + ErpPlanningInput, + asap_types::erp_observation::ErpPopulationObservations, + ) { + use asap_types::erp_observation::*; + let generation = asap_types::sds::CatalogGeneration { + schema_version: 1, + plan_id: 7, + plan_version: 3, + snapshot_sha256: "test-catalog".into(), + }; + let definition = asap_types::PolicyFingerprint(7).into(); + let mut observer = ErpShapeObserver::new(4).unwrap(); + for key in ["a", "a", "b", "b"] { + observer.observe(key, 0).unwrap(); + } + let observed = ErpPopulationObservations { + schema_version: 1, + catalog_generation: generation.clone(), + summary_definition_id: definition, + observed_at_unix_ms: 1_000, + window_start_ms: 0, + window_end_ms: 1_000, + input_semantics: ErpObservationInputSemantics::UnitSampleFrequency, + invalid_reason: None, + populations: vec![ErpPopulationObservation { + population_id: asap_types::sds::SummaryInstanceId::new("partition-a").unwrap(), + shape: observer.snapshot().unwrap(), + }], + }; + let mut policy = input(ErpAccuracyMode::Hybrid); + policy.observed_shape_source = Some(ErpObservedShapeSource { + source: "backend-a".into(), + sketch: "univmon".into(), + implementation: "asap_sketchlib".into(), + population_scope: Some(ErpPopulationObservationScope { + catalog_generation: generation, + summary_definition_id: definition, + input_semantics: ErpObservationInputSemantics::UnitSampleFrequency, + freshness: ErpObservationFreshness { + max_age_ms: 100, + max_future_skew_ms: 5, + }, + }), + }); + (policy, observed) + } + + fn publish_population_fixture( + samples: &crate::runtime_samples::RuntimeSamplesStore, + mut payload: serde_json::Value, + ) { + if let Some(populations) = payload + .pointer_mut("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/erp_population_observations/populations") + .and_then(Value::as_array_mut) + { + for population in populations { + if let Some(cardinality) = population + .pointer("/shape/observation/cardinality") + .and_then(Value::as_u64) + { + population["shape"] = serde_json::json!({"sorted_counts":vec![2;cardinality as usize],"interval_counts":[cardinality*2]}); + } + } + } + samples.append_for_test(crate::runtime_samples::RuntimeRecord { + source: "backend-a".into(), + sketch: "univmon".into(), + impl_name: "asap_sketchlib".into(), + schema_version: 1, + payload, + }); + } + + /// New live evidence replaces hydrated fits; expiry invalidates them even + /// when a caller reuses the same planning request. + #[test] + fn online_hydration_refreshes_and_expires_population_evidence() { + let (mut policy, mut observed) = online_population_fixture(); + let samples = crate::runtime_samples::RuntimeSamplesStore::new(4); + publish_population_fixture( + &samples, + serde_json::json!({"erp_population_observations": observed}), + ); + policy.hydrate_observed_shape_at(&samples, 1_000).unwrap(); + assert_eq!( + policy.observed_populations.as_ref().unwrap().populations[0] + .shape + .observation + .cardinality, + 2 + ); + observed.populations[0].shape.observation.cardinality = 3; + observed.observed_at_unix_ms = 1_010; + publish_population_fixture( + &samples, + serde_json::json!({"erp_population_observations": observed}), + ); + policy.hydrate_observed_shape_at(&samples, 1_010).unwrap(); + assert_eq!( + policy.observed_populations.as_ref().unwrap().populations[0] + .shape + .observation + .cardinality, + 3 + ); + policy.hydrate_observed_shape_at(&samples, 1_111).unwrap(); + let invalid = policy.observed_populations.as_ref().unwrap(); + assert!(invalid.populations.is_empty()); + assert!(invalid.invalid_reason.as_deref().unwrap().contains("stale")); + assert!(policy.observed_shape.is_none()); + } + + /// Only the activated catalog may supply a candidate's data contract. + #[test] + fn online_population_descriptor_requires_authoritative_catalog() { + let mut fixture: Value = serde_json::from_str(include_str!( + "../../../docs/examples/asapquery-compatibility-demo-snapshot.json" + )) + .unwrap(); + let mut query = fixture["query_workload"]["repeating_queries"][3].clone(); + query["query"] = "distinct_over_time(asap_demo_latency_ms[5s])".into(); + query["requirements"]["accuracy"] = serde_json::json!({"explicit":{"Epsilon":0.05}}); + fixture["query_workload"]["repeating_queries"] = serde_json::json!([query]); + let snapshot: crate::physical::compiler::BackendLocalPlanningSnapshot = + serde_json::from_value(fixture).unwrap(); + let plan = snapshot.compile().unwrap(); + let (mut policy, mut observed) = online_population_fixture(); + observed.catalog_generation = plan.summary_catalog.reference().unwrap(); + observed.summary_definition_id = + *plan.summary_catalog.materializations.keys().next().unwrap(); + observed.input_semantics = + asap_types::erp_observation::ErpObservationInputSemantics::ScalarSampleValue; + policy.observed_populations = Some(observed.clone()); + policy.resolve_population_data_descriptor(Some(&plan.summary_catalog)); + let expected = &plan.summary_catalog.materializations[&observed.summary_definition_id] + .data_descriptor_id; + assert_eq!( + &policy.resolved_data_descriptor.as_ref().unwrap().id, + expected + ); + policy.resolve_population_data_descriptor(None); + assert!(policy.resolved_data_descriptor.is_none()); + assert!(policy + .observed_populations + .as_ref() + .unwrap() + .invalid_reason + .is_some()); + assert!(policy + .observed_populations + .as_ref() + .unwrap() + .populations + .is_empty()); + } + + /// Missing, corrupt, foreign and overflow evidence must never recover a + /// previously accepted fit or the artifact's legacy distribution match. + #[test] + fn online_hydration_invalidates_unusable_latest_evidence() { + let (template, observed) = online_population_fixture(); + let mut foreign = observed.clone(); + foreign.catalog_generation.plan_version += 1; + let mut wrong_input = observed.clone(); + wrong_input.input_semantics = + asap_types::erp_observation::ErpObservationInputSemantics::ScalarSampleValue; + let mut overflow = observed.clone(); + overflow.invalid_reason = Some("population limit".into()); + let mut future = observed.clone(); + future.observed_at_unix_ms = 1_006; + for payload in [ + None, + Some(serde_json::json!({})), + Some(serde_json::json!({"erp_population_observations": foreign})), + Some(serde_json::json!({"erp_population_observations": wrong_input})), + Some(serde_json::json!({"erp_population_observations": overflow})), + Some(serde_json::json!({"erp_population_observations": future})), + ] { + let mut policy = template.clone(); + policy.observed_shape = Some(observed.populations[0].shape.observation.clone()); + let samples = crate::runtime_samples::RuntimeSamplesStore::new(4); + if let Some(payload) = payload { + publish_population_fixture(&samples, payload); + } + policy.hydrate_observed_shape_at(&samples, 1_000).unwrap(); + let invalid = policy.observed_populations.as_ref().unwrap(); + assert!(invalid.invalid_reason.is_some()); + assert!(invalid.populations.is_empty()); + assert!(policy.observed_shape.is_none()); + assert!(matches!( + policy.select( + SketchAlgorithm::Hll, + 0.05, + SketchParams::Hll { precision: 12 } + ), + ErpParameterDecision::TheoreticalFallback { .. } + )); + } + } + #[test] fn observer_fails_loud_at_cardinality_cap() { let mut observer = ErpShapeObserver::new(1).unwrap(); observer.observe("a", 0).unwrap(); assert_eq!( observer.observe("b", 0), - Err("ERP shape observer cardinality cap exceeded") + Err("ERP observation key or interval budget exceeded") ); assert!(observer.snapshot().is_none()); assert!(observer.observe("a", 0).is_err()); @@ -1157,14 +1551,16 @@ mod tests { let mut observer = ErpShapeObserver::with_limits(2, 2).unwrap(); observer.observe("a", usize::MAX).unwrap(); observer.observe("a", 0).unwrap(); - assert_eq!(observer.interval_updates.len(), 2); + assert_eq!(observer.inner.snapshot().unwrap().interval_counts.len(), 2); assert!(observer.observe("a", 1).is_err()); assert!(observer.snapshot().is_none()); - let mut observer = ErpShapeObserver::new(2).unwrap(); - observer.observe("a", 0).unwrap(); - observer.updates = u64::MAX; - assert!(observer.observe("a", 0).is_err()); - assert!(observer.snapshot().is_none()); + assert!(ErpObservedShape::from_empirical( + &asap_types::erp_observation::EmpiricalFrequencyObservation { + sorted_counts: vec![u64::MAX, 1], + interval_counts: vec![u64::MAX, 1], + } + ) + .is_none()); } #[test] diff --git a/control_plane/src/physical/publication.rs b/control_plane/src/physical/publication.rs index 42e28d086..63081fa5b 100644 --- a/control_plane/src/physical/publication.rs +++ b/control_plane/src/physical/publication.rs @@ -1,128 +1,7 @@ -//! Canonical publication document for one catalog generation. -use super::compiler::{CollectorPlan, PhysicalPlan, PrecomputePlan, TransmissionPlan}; -use super::summary_catalog::SummaryCatalog; -use crate::query_plan::QueryPlan; -use serde::{Deserialize, Serialize}; +//! Control-plane construction of the shared catalog publication contract. +use super::compiler::PhysicalPlan; +pub use asap_types::plan_publication::{PhysicalPlanInstallRequest, PhysicalPlanPublication}; -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PhysicalPlanPublication { - pub summary_catalog: SummaryCatalog, - pub precompute_plan: PrecomputePlan, - pub collector_plans: Vec, - pub transmission_plan: TransmissionPlan, - pub query_plan: QueryPlan, -} - -/// Complete typed envelope accepted by the data-plane install endpoint. -/// -/// Runtime-only routing and adaptation evidence decorate the compiler-owned -/// publication without changing its catalog generation. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PhysicalPlanInstallRequest { - pub summary_catalog: SummaryCatalog, - #[serde(default)] - pub collector_plans: Vec, - pub precompute_plan: PrecomputePlan, - pub transmission_plan: TransmissionPlan, - pub query_plan: QueryPlan, - pub storage_routing: Option, - #[serde(default)] - pub adaptation_evidence: Vec, -} - -impl PhysicalPlanPublication { - /// Validate every plan against the shared catalog snapshot. - pub fn validate(&self) -> Result<(), String> { - let catalog = &self.summary_catalog; - self.precompute_plan - .validate_against_catalog(catalog) - .map_err(|e| e.to_string())?; - self.transmission_plan - .validate(&self.precompute_plan) - .map_err(|e| e.to_string())?; - self.transmission_plan - .validate_against_catalog(catalog) - .map_err(|e| e.to_string())?; - self.query_plan - .validate_against_catalog(catalog) - .map_err(|e| e.to_string())?; - let materializations = self - .precompute_plan - .materializations - .iter() - .map(|config| (config.policy_fingerprint(), config)) - .collect::>(); - for entry in self.query_plan.entries.values() { - for binding in entry.materialization_bindings() { - let config = materializations - .get(&binding.materialization.fingerprint()) - .copied() - .ok_or("query binding has no precompute materialization")?; - if config.slide_interval.checked_mul(1000) != Some(binding.window_ms) { - return Err("query pane differs from precompute emission interval".into()); - } - if config.pane_origin_ms != binding.pane_origin_ms { - return Err("query pane origin differs from precompute definition".into()); - } - } - } - let mut collectors = std::collections::BTreeSet::new(); - for collector in &self.collector_plans { - if collector.envelope != self.precompute_plan.envelope - || !collectors.insert(&collector.collector_id) - { - return Err("collector envelope mismatch or duplicate collector".into()); - } - collector - .validate_against_catalog(catalog) - .map_err(|e| e.to_string())?; - for rule in &collector.transmission_rules { - if !self.transmission_plan.rules.contains(rule) { - return Err( - "collector transmission rule absent from published transmission plan" - .into(), - ); - } - } - } - for producer in &self.precompute_plan.producers { - let collector = self - .collector_plans - .iter() - .find(|c| c.collector_id == producer.collector_id) - .ok_or("precompute producer has no published CollectorPlan")?; - if !collector - .materializations - .iter() - .any(|m| m.materialization == producer.materialization) - { - return Err( - "collector does not produce referenced precompute materialization".into(), - ); - } - } - Ok(()) - } - - pub fn install_request( - &self, - storage_routing: Option, - adaptation_evidence: Vec, - ) -> Result { - self.validate()?; - Ok(PhysicalPlanInstallRequest { - summary_catalog: self.summary_catalog.clone(), - collector_plans: self.collector_plans.clone(), - precompute_plan: self.precompute_plan.clone(), - transmission_plan: self.transmission_plan.clone(), - query_plan: self.query_plan.clone(), - storage_routing, - adaptation_evidence, - }) - } -} impl PhysicalPlan { pub fn publication(&self) -> Result { let artifact = PhysicalPlanPublication { diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs index 17943b88e..9ffe17971 100644 --- a/control_plane/src/query_plan.rs +++ b/control_plane/src/query_plan.rs @@ -1,854 +1,217 @@ -//! Authoritative backend-executable query DAG. -//! -//! ASAPPlanner owns semantic post-ASAP IR. Physical compilation binds every -//! maintained-summary leaf to one materialization and lowers edges to stable -//! node IDs. Serving executes this graph without reconstructing Planner IR or -//! searching for compatible materializations. +//! Control-plane lowering from Planner IR to the shared installed query DAG. +//! Serving consumes asap_types::query_plan; compilation stays in this component. mod clickhouse_exact; pub mod logical; - -use std::collections::{BTreeMap, BTreeSet}; -use std::rc::Rc; - -use planner_types::post_asap::{SketchQuery, SummaryExpr, SummaryFamilyType, SummaryNode}; +pub use asap_types::query_plan::*; +#[cfg(test)] +use asap_types::PolicyFingerprint; +use planner_types::post_asap::{SummaryExpr, SummaryFamilyType, SummaryNode}; use planner_types::pre_asap::Reduction; -use serde::{Deserialize, Serialize}; -use thiserror::Error; - -pub use asap_types::QueryLanguage; -use asap_types::{sds::SummaryDefinitionId, PolicyFingerprint}; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct QueryPlan { - pub plan_id: u64, - pub plan_version: u64, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub clickhouse_context: Option, - pub entries: BTreeMap, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct ClickHousePlanningContext { - pub tables: std::collections::HashMap, - pub accuracy: planner_types::types::AccuracyTarget, -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct FixedEvaluationRange { - pub start_ms: u64, - pub end_ms: u64, - pub cumulative: bool, -} - -impl QueryPlan { - pub fn empty() -> Self { - Self { - plan_id: 0, - plan_version: 0, - clickhouse_context: None, - entries: BTreeMap::new(), - } - } - - pub fn lookup(&self, promql: &str) -> Result<&QueryPlanEntry, QueryPlanError> { - let identity = canonical_promql(promql)?; - self.lookup_canonical(QueryLanguage::PromQl, &identity) - } - - pub fn lookup_canonical( - &self, - language: QueryLanguage, - identity: &str, - ) -> Result<&QueryPlanEntry, QueryPlanError> { - let key = Self::catalog_key(language, identity); - self.entries - .get(&key) - .filter(|entry| entry.language == language) - .ok_or_else(|| QueryPlanError::QueryNotPlanned(identity.into())) - } - - pub fn catalog_key(language: QueryLanguage, identity: &str) -> String { - match language { - QueryLanguage::PromQl => identity.to_owned(), - QueryLanguage::MetricsQl => format!("metricsql:{identity}"), - QueryLanguage::ClickHouseSql => format!("clickhouse:{identity}"), - } - } - - pub fn lookup_clickhouse( - &self, - canonical_sql: &str, - ) -> Result<&QueryPlanEntry, QueryPlanError> { - self.lookup_canonical(QueryLanguage::ClickHouseSql, canonical_sql) - } - - /// Validate semantic bindings against the authoritative snapshot before use. - pub fn validate_against_catalog( - &self, - catalog: &crate::physical::summary_catalog::SummaryCatalog, - ) -> Result<(), QueryPlanError> { - catalog - .validate() - .map_err(|error| QueryPlanError::Invalid(error.to_string()))?; - if self.plan_id != catalog.plan_id || self.plan_version != catalog.plan_version { - return Err(QueryPlanError::Invalid( - "QueryPlan and SummaryCatalog have different plan identity/version".into(), - )); - } - let available = catalog - .materializations - .keys() - .copied() - .map(Into::into) - .collect(); - self.validate(&available)?; - for entry in self.entries.values() { - for binding in entry.materialization_bindings() { - let identity = catalog - .materializations - .get(&binding.materialization) - .ok_or_else(|| { - QueryPlanError::Invalid( - "query binding references absent catalog materialization".into(), - ) - })?; - let _data = &catalog.data_descriptors[&identity.data_descriptor_id]; - if binding.window_ms == 0 { - return Err(QueryPlanError::Invalid( - "zero physical pane duration".into(), - )); - } - if binding.pane_origin_ms != identity.pane_origin_ms { - return Err(QueryPlanError::Invalid( - "query pane origin differs from catalog definition".into(), - )); - } - } - for node in entry.nodes.values() { - let QueryPlanNode::ExactReadout { input, readout } = node else { - continue; - }; - if !matches!(readout, ExactReadout::Increase | ExactReadout::Rate) { - continue; - } - let Some(QueryPlanNode::ReadMaterialization { binding }) = entry.nodes.get(input) - else { - return Err(QueryPlanError::Invalid( - "counter readout must directly consume one catalog materialization".into(), - )); - }; - let identity = &catalog.materializations[&binding.materialization]; - let descriptor = &catalog.summary_descriptors[&identity.summary_descriptor_id]; - if !matches!( - descriptor.fidelity, - asap_types::sds::FidelityGuarantee::ExactCounter { - full_pane_coverage_required: true, - .. - } - ) { - return Err(QueryPlanError::Invalid( - "rate/increase binding does not reference an exact counter SDS".into(), - )); - } - } - } - Ok(()) - } - - pub fn validate(&self, available: &BTreeSet) -> Result<(), QueryPlanError> { - if self.plan_id != 0 && self.plan_version == 0 { - return Err(QueryPlanError::Invalid( - "non-bootstrap QueryPlan has zero plan_version".into(), - )); - } - for (identity, entry) in &self.entries { - let expected = Self::catalog_key(entry.language, &entry.canonical_query); - if identity != &expected { - return Err(QueryPlanError::Invalid(format!( - "query map key `{identity}` differs from entry identity `{}`", - entry.canonical_query - ))); - } - match entry.language { - QueryLanguage::PromQl | QueryLanguage::MetricsQl - if entry.fixed_evaluation.is_some() => - { - return Err(QueryPlanError::Invalid( - "PromQL query entry carries a ClickHouse fixed evaluation range".into(), - )); - } - QueryLanguage::ClickHouseSql => { - if self.clickhouse_context.is_none() { - return Err(QueryPlanError::Invalid( - "ClickHouse query entry has no planning context".into(), - )); - } - let Some(range) = entry.fixed_evaluation else { - return Err(QueryPlanError::Invalid( - "ClickHouse query entry has no fixed evaluation range".into(), - )); - }; - if range.end_ms <= range.start_ms { - return Err(QueryPlanError::Invalid( - "ClickHouse query entry has an empty evaluation range".into(), - )); - } - } - QueryLanguage::PromQl | QueryLanguage::MetricsQl => {} - } - entry.validate(available)?; - } - Ok(()) - } -} - -pub use asap_types::executable_plan::QueryNodeId; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct QueryPlanEntry { - #[serde(default)] - pub language: QueryLanguage, - pub query_id: String, - #[serde(alias = "canonical_promql")] - pub canonical_query: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub fixed_evaluation: Option, - pub root: QueryNodeId, - pub nodes: BTreeMap, - pub instant: InstantExecution, - pub fallback: FallbackPolicy, -} +use std::collections::BTreeMap; +#[cfg(test)] +use std::collections::BTreeSet; +use std::rc::Rc; -fn topological_order( - root: QueryNodeId, - nodes: &BTreeMap, -) -> Result, QueryPlanError> { - fn visit( - id: QueryNodeId, - nodes: &BTreeMap, - visiting: &mut BTreeSet, - visited: &mut BTreeSet, - out: &mut Vec, - ) -> Result<(), QueryPlanError> { - if visited.contains(&id) { - return Ok(()); - } - if !visiting.insert(id) { - return Err(QueryPlanError::Invalid(format!( - "cycle detected at query node {}", - id.0 - ))); - } - let node = nodes - .get(&id) - .ok_or_else(|| QueryPlanError::Invalid(format!("missing query node {}", id.0)))?; - for input in node.inputs() { - visit(*input, nodes, visiting, visited, out)?; - } - visiting.remove(&id); - visited.insert(id); - out.push(id); - Ok(()) - } - let mut out = Vec::with_capacity(nodes.len()); - visit( +pub fn compile_bound( + query_id: String, + canonical_query: String, + root: &Rc, + instant: InstantExecution, + fallback: FallbackPolicy, + bind: F, +) -> Result +where + F: FnMut( + &Rc, + &SummaryFamilyType, + ) -> Result, +{ + compile_bound_mapped( + query_id, + canonical_query, root, - nodes, - &mut BTreeSet::new(), - &mut BTreeSet::new(), - &mut out, - )?; - Ok(out) -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct InstantExecution { - pub lookback_ms: u64, - pub full_history: bool, - pub cumulative_readout: bool, -} - -impl QueryPlanEntry { - /// Replace an explicit planner fallback cut with a typed external-exact leaf. - /// The control plane chooses the cut; serving only executes the published DAG. - pub fn bind_external_exact_leaf( - &mut self, - node_id: QueryNodeId, - request: ExternalExactRequest, - ) -> Result<(), QueryPlanError> { - if request.language != self.language { - return Err(QueryPlanError::Invalid( - "external exact language differs from its query plan".into(), - )); - } - if !request.input_contracts.is_empty() { - return Err(QueryPlanError::Invalid( - "leaf binding cannot declare DAG input contracts".into(), - )); - } - match self.nodes.get(&node_id) { - Some(QueryPlanNode::ExactFallback { .. }) => {} - Some(_) => { - return Err(QueryPlanError::Invalid( - "external exact binding must replace a planner fallback cut".into(), - )) - } - None => { - return Err(QueryPlanError::Invalid( - "external exact cut node is absent".into(), - )) - } - } - self.nodes.insert( - node_id, - QueryPlanNode::ExternalExact { - request, - inputs: Vec::new(), - }, - ); - Ok(()) - } - - /// Materializations this executable DAG reads, in stable node order. - /// Serving uses this set for readiness accounting; it never performs a - /// catalog candidate search to reconstruct dependencies. - pub fn materialization_bindings(&self) -> Vec<&MaterializationBinding> { - self.nodes - .values() - .filter_map(|node| match node { - QueryPlanNode::ReadMaterialization { binding } => Some(binding), - _ => None, - }) - .collect() - } - - pub fn topological_order(&self) -> Result, QueryPlanError> { - topological_order(self.root, &self.nodes) - } - - pub fn topological_order_from( - &self, - root: QueryNodeId, - ) -> Result, QueryPlanError> { - topological_order(root, &self.nodes) - } - - pub fn compile_bound( - query_id: String, - canonical_query: String, - root: &Rc, - instant: InstantExecution, - fallback: FallbackPolicy, - bind: F, - ) -> Result - where - F: FnMut( - &Rc, - &SummaryFamilyType, - ) -> Result, - { - Self::compile_bound_mapped( - query_id, - canonical_query, - root, - instant, - fallback, - bind, - |_, _| {}, - ) - } - - pub fn compile_bound_mapped( - query_id: String, - canonical_query: String, - root: &Rc, - instant: InstantExecution, - fallback: FallbackPolicy, - mut bind: F, - mut lowered: G, - ) -> Result - where - F: FnMut( - &Rc, - &SummaryFamilyType, - ) -> Result, - G: FnMut(&Rc, QueryNodeId), - { - let mut compiler = DagCompiler { - next_id: 0, - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - bind: &mut bind, - logical_source: None, - preserve_relational: false, - lowered: Some(&mut lowered), - }; - let root = compiler.lower(root)?; - Ok(Self { - language: QueryLanguage::PromQl, - query_id, - canonical_query, - fixed_evaluation: None, - root, - nodes: compiler.nodes, - instant, - fallback, - }) - } - - pub fn compile_bound_relational( - query_id: String, - canonical_query: String, - root: &Rc, - fixed_evaluation: FixedEvaluationRange, - instant: InstantExecution, - fallback: FallbackPolicy, - bind: F, - ) -> Result - where - F: FnMut( - &Rc, - &SummaryFamilyType, - ) -> Result, - { - Self::compile_bound_relational_mapped( - query_id, - canonical_query, - root, - fixed_evaluation, - instant, - fallback, - bind, - |_, _| {}, - ) - } - - /// Preserve Planner-to-runtime node identities for installed SQL DAGs. - pub fn compile_bound_relational_mapped( - query_id: String, - canonical_query: String, - root: &Rc, - fixed_evaluation: FixedEvaluationRange, - instant: InstantExecution, - fallback: FallbackPolicy, - mut bind: F, - mut lowered: G, - ) -> Result - where - F: FnMut( - &Rc, - &SummaryFamilyType, - ) -> Result, - G: FnMut(&Rc, QueryNodeId), - { - let mut compiler = DagCompiler { - next_id: 0, - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - bind: &mut bind, - logical_source: None, - preserve_relational: true, - lowered: Some(&mut lowered), - }; - let root = compiler.lower(root)?; - Ok(Self { - language: QueryLanguage::ClickHouseSql, - query_id, - canonical_query, - fixed_evaluation: Some(fixed_evaluation), - root, - nodes: compiler.nodes, - instant, - fallback, - }) - } - - /// Compile selected summary nodes and verified native residuals into one DAG. - /// This is a distinct physical alternative; native execution remains available. - pub fn compile_bound_composable( - query_id: String, - canonical_query: String, - root: &Rc, - instant: InstantExecution, - fallback: FallbackPolicy, - bind: F, - ) -> Result - where - F: FnMut( - &Rc, - &SummaryFamilyType, - ) -> Result, - { - Self::compile_bound_composable_mapped( - query_id, - canonical_query, - root, - instant, - fallback, - bind, - |_, _| {}, - ) - } - - /// Compile a composable query while exposing the stable mapping from - /// Planner semantic nodes to installed query nodes. The control-plane - /// physical compiler uses this to persist backend placement without - /// relying on pointer values or reconstructing query shape later. - pub fn compile_bound_composable_mapped( - query_id: String, - canonical_query: String, - root: &Rc, - instant: InstantExecution, - fallback: FallbackPolicy, - mut bind: F, - mut lowered: G, - ) -> Result - where - F: FnMut( - &Rc, - &SummaryFamilyType, - ) -> Result, - G: FnMut(&Rc, QueryNodeId), - { - let mut compiler = DagCompiler { - next_id: 0, - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - bind: &mut bind, - logical_source: Some(canonical_query.clone()), - preserve_relational: false, - lowered: Some(&mut lowered), - }; - let root = compiler.lower(root)?; - let mut entry = Self { - language: QueryLanguage::PromQl, - query_id, - canonical_query, - fixed_evaluation: None, - root, - nodes: compiler.nodes, - instant, - fallback, - }; - logical::finalize_residuals(&mut entry)?; - Ok(entry) - } - - /// Validate references, bindings, reachability, and cycles before activation. - pub fn validate(&self, available: &BTreeSet) -> Result<(), QueryPlanError> { - if !self.nodes.contains_key(&self.root) { - return Err(QueryPlanError::Invalid(format!( - "query `{}` has missing root {}", - self.query_id, self.root.0 - ))); - } - for (id, node) in &self.nodes { - if let QueryPlanNode::Logical { operator, inputs } = node { - operator.validate(inputs.len())?; - } - if matches!(node, QueryPlanNode::Scalar { value } if !value.is_finite()) { - return Err(QueryPlanError::Invalid("non-finite scalar constant".into())); - } - if let QueryPlanNode::ExternalExact { request, inputs } = node { - if request.expression.trim().is_empty() { - return Err(QueryPlanError::Invalid( - "external exact expression must not be empty".into(), - )); - } - if request.input_contracts.len() != inputs.len() { - return Err(QueryPlanError::Invalid( - "external exact input contracts must match DAG inputs".into(), - )); - } - if request.input_contracts.iter().any(|contract| { - matches!(contract, ExternalExactInput::CandidateMembership { item_label } if item_label.is_empty()) - }) { - return Err(QueryPlanError::Invalid( - "external exact candidate item label must not be empty".into(), - )); - } - } - if let QueryPlanNode::CandidateTopK { - k, completeness, .. - } = node - { - if *k == 0 { - return Err(QueryPlanError::Invalid( - "CandidateTopK requires k > 0".into(), - )); - } - if matches!( - completeness, - CandidateCompleteness::Certified { guarantee } - if guarantee.metric - != planner_types::post_asap::ErrorMetric::TopKMembership - || guarantee.bound.evaluate().is_none() - || guarantee.failure_probability.evaluate().is_none() - ) { - return Err(QueryPlanError::Invalid( - "invalid CandidateTopK completeness certificate".into(), - )); - } - } - for input in node.inputs() { - if !self.nodes.contains_key(input) { - return Err(QueryPlanError::Invalid(format!( - "query `{}` node {} references missing input {}", - self.query_id, id.0, input.0 - ))); - } - } - if let QueryPlanNode::ReadMaterialization { binding } = node { - if binding.readout_lookback_ms == Some(0) { - return Err(QueryPlanError::Invalid( - "zero semantic readout lookback".into(), - )); - } - if !available.contains(&binding.materialization.fingerprint()) { - return Err(QueryPlanError::Invalid(format!( - "query `{}` node {} references absent materialization {}", - self.query_id, - id.0, - binding.materialization.as_u64() - ))); - } - } - } - let order = self.topological_order()?; - if order.len() != self.nodes.len() { - return Err(QueryPlanError::Invalid(format!( - "query `{}` contains unreachable nodes", - self.query_id - ))); - } - Ok(()) - } -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum FallbackPolicy { - ExactBackend, - Reject, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct MaterializationBinding { - pub materialization: SummaryDefinitionId, - /// Query operator grouping applied while folding those SIDs. - pub output_grouping: PhysicalGrouping, - /// Labels whose values form an item identity inside a keyed sketch. - #[serde(default, alias = "itemLabels", skip_serializing_if = "Vec::is_empty")] - pub item_labels: Vec, - pub window_ms: u64, - /// Unix millisecond timestamp on the materialized pane-boundary grid. - /// Legacy plans deserialize this as unknown and fall back at read time. - #[serde( - default, - alias = "paneOriginMs", - skip_serializing_if = "Option::is_none" - )] - pub pane_origin_ms: Option, - /// Semantic query lookback, independent of the physical pane duration. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub readout_lookback_ms: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(tag = "mode", content = "keys", rename_all = "snake_case")] -pub enum PhysicalGrouping { - PerEntity, - Reduce(Vec), + instant, + fallback, + bind, + |_, _| {}, + ) } -/// Result shape promised by an external exact engine. The backend uses this -/// contract to type-check downstream DAG nodes without depending on an -/// engine-specific response envelope. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum ExternalExactOutput { - Scalar, - InstantVector, - RangeVector, - Relation { schema: serde_json::Value }, -} - -/// How an ordinary DAG input constrains an external exact evaluation. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum ExternalExactInput { - CandidateMembership { item_label: String }, -} - -/// Language-neutral request contract for an exact subtree. Evaluation time is -/// inherited from the containing QueryPlanEntry, avoiding a second time-range -/// envelope that could drift from the installed query plan. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct ExternalExactRequest { - pub language: QueryLanguage, - pub expression: String, - pub output: ExternalExactOutput, - /// Engine parameters forwarded without embedding transport details in the DAG. - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub parameters: BTreeMap, - /// Optional parameter names populated from the query entry's evaluation range. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub start_parameter: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub end_parameter: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub input_contracts: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(tag = "op", rename_all = "snake_case", deny_unknown_fields)] -pub enum QueryPlanNode { - RelationalJoin { - inputs: [QueryNodeId; 2], - join_kind: planner_types::pre_asap::JoinKind, - pred: serde_json::Value, - left_schema: planner_types::post_asap::SummarySchema, - right_schema: planner_types::post_asap::SummarySchema, - output_schema: planner_types::post_asap::SummarySchema, - }, - Relational { - input: QueryNodeId, - /// Serialized planner-owned operation. Keeping the wire form here makes - /// the published catalog Send + Sync even though the planner AST uses Rc. - operation: serde_json::Value, - input_schema: planner_types::post_asap::SummarySchema, - output_schema: planner_types::post_asap::SummarySchema, - }, - Logical { - operator: logical::LogicalOperator, - inputs: Vec, - }, - Scalar { - value: f64, - }, - Binary { - inputs: [QueryNodeId; 2], - operator: planner_types::pre_asap::ArithmeticOpKind, - }, - ReduceSum { - input: QueryNodeId, - grouping: PhysicalGrouping, - }, - ReadMaterialization { - binding: MaterializationBinding, - }, - SummaryEstimate { - input: QueryNodeId, - query: QueryReadout, - }, - ExactReadout { - input: QueryNodeId, - readout: ExactReadout, - }, - SummaryMerge { - inputs: Vec, - }, - /// Use an approximate heap only as a membership sidecar, then rerank the - /// matching exact counter readouts. `inputs[0]` is candidate membership; - /// `inputs[1]` is the authoritative exact value vector. - CandidateTopK { - inputs: [QueryNodeId; 2], - k: u64, - grouping: logical::Grouping, - completeness: CandidateCompleteness, - }, - /// An exact subtree evaluated outside ASAP. Its results enter the query DAG - /// like any other node output and may depend on summary-produced inputs. - ExternalExact { - request: ExternalExactRequest, - inputs: Vec, - }, - ExactFallback { - reason: String, - }, -} - -impl QueryPlanNode { - pub fn inputs(&self) -> &[QueryNodeId] { - match self { - Self::Scalar { .. } | Self::ReadMaterialization { .. } | Self::ExactFallback { .. } => { - &[] - } - Self::Binary { inputs, .. } | Self::RelationalJoin { inputs, .. } => inputs, - Self::ReduceSum { input, .. } - | Self::Relational { input, .. } - | Self::SummaryEstimate { input, .. } - | Self::ExactReadout { input, .. } => std::slice::from_ref(input), - Self::SummaryMerge { inputs } - | Self::Logical { inputs, .. } - | Self::ExternalExact { inputs, .. } => inputs, - Self::CandidateTopK { inputs, .. } => inputs, - } - } +pub fn compile_bound_mapped( + query_id: String, + canonical_query: String, + root: &Rc, + instant: InstantExecution, + fallback: FallbackPolicy, + mut bind: F, + mut lowered: G, +) -> Result +where + F: FnMut( + &Rc, + &SummaryFamilyType, + ) -> Result, + G: FnMut(&Rc, QueryNodeId), +{ + let mut compiler = DagCompiler { + next_id: 0, + nodes: BTreeMap::new(), + seen: BTreeMap::new(), + bind: &mut bind, + logical_source: None, + preserve_relational: false, + lowered: Some(&mut lowered), + }; + let root = compiler.lower(root)?; + Ok(QueryPlanEntry { + language: QueryLanguage::PromQl, + query_id, + canonical_query, + fixed_evaluation: None, + root, + nodes: compiler.nodes, + instant, + fallback, + }) } -pub use planner_types::post_asap::CandidateCompleteness; - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum ExactReadout { - Sum, - Count, - Increase, - Rate, - Max, +pub fn compile_bound_relational( + query_id: String, + canonical_query: String, + root: &Rc, + fixed_evaluation: FixedEvaluationRange, + instant: InstantExecution, + fallback: FallbackPolicy, + bind: F, +) -> Result +where + F: FnMut( + &Rc, + &SummaryFamilyType, + ) -> Result, +{ + compile_bound_relational_mapped( + query_id, + canonical_query, + root, + fixed_evaluation, + instant, + fallback, + bind, + |_, _| {}, + ) } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum QueryReadout { - FrequencyL2, - FrequencyEntropy, - Quantile { - q: f64, - }, - PointCount { - key: planner_types::pre_asap::ColumnRef, - value: Option, - }, - Cardinality, - TopK { - k: usize, - }, +/// Preserve Planner-to-runtime node identities for installed SQL DAGs. +pub fn compile_bound_relational_mapped( + query_id: String, + canonical_query: String, + root: &Rc, + fixed_evaluation: FixedEvaluationRange, + instant: InstantExecution, + fallback: FallbackPolicy, + mut bind: F, + mut lowered: G, +) -> Result +where + F: FnMut( + &Rc, + &SummaryFamilyType, + ) -> Result, + G: FnMut(&Rc, QueryNodeId), +{ + let mut compiler = DagCompiler { + next_id: 0, + nodes: BTreeMap::new(), + seen: BTreeMap::new(), + bind: &mut bind, + logical_source: None, + preserve_relational: true, + lowered: Some(&mut lowered), + }; + let root = compiler.lower(root)?; + Ok(QueryPlanEntry { + language: QueryLanguage::ClickHouseSql, + query_id, + canonical_query, + fixed_evaluation: Some(fixed_evaluation), + root, + nodes: compiler.nodes, + instant, + fallback, + }) } -impl From for QueryReadout { - fn from(query: SketchQuery) -> Self { - match query { - SketchQuery::FrequencyL2 => Self::FrequencyL2, - SketchQuery::FrequencyEntropy => Self::FrequencyEntropy, - SketchQuery::Quantile { q } => Self::Quantile { q }, - SketchQuery::PointCount { key, value } => Self::PointCount { key, value }, - SketchQuery::Cardinality => Self::Cardinality, - SketchQuery::TopK { k } => Self::TopK { k }, - } - } +/// Compile selected summary nodes and verified native residuals into one DAG. +/// This is a distinct physical alternative; native execution remains available. +pub fn compile_bound_composable( + query_id: String, + canonical_query: String, + root: &Rc, + instant: InstantExecution, + fallback: FallbackPolicy, + bind: F, +) -> Result +where + F: FnMut( + &Rc, + &SummaryFamilyType, + ) -> Result, +{ + compile_bound_composable_mapped( + query_id, + canonical_query, + root, + instant, + fallback, + bind, + |_, _| {}, + ) } -impl From for SketchQuery { - fn from(query: QueryReadout) -> Self { - match query { - QueryReadout::FrequencyL2 => Self::FrequencyL2, - QueryReadout::FrequencyEntropy => Self::FrequencyEntropy, - QueryReadout::Quantile { q } => Self::Quantile { q }, - QueryReadout::PointCount { key, value } => Self::PointCount { key, value }, - QueryReadout::Cardinality => Self::Cardinality, - QueryReadout::TopK { k } => Self::TopK { k }, - } - } +/// Compile a composable query while exposing the stable mapping from +/// Planner semantic nodes to installed query nodes. The control-plane +/// physical compiler uses this to persist backend placement without +/// relying on pointer values or reconstructing query shape later. +pub fn compile_bound_composable_mapped( + query_id: String, + canonical_query: String, + root: &Rc, + instant: InstantExecution, + fallback: FallbackPolicy, + mut bind: F, + mut lowered: G, +) -> Result +where + F: FnMut( + &Rc, + &SummaryFamilyType, + ) -> Result, + G: FnMut(&Rc, QueryNodeId), +{ + let mut compiler = DagCompiler { + next_id: 0, + nodes: BTreeMap::new(), + seen: BTreeMap::new(), + bind: &mut bind, + logical_source: Some(canonical_query.clone()), + preserve_relational: false, + lowered: Some(&mut lowered), + }; + let root = compiler.lower(root)?; + let mut entry = QueryPlanEntry { + language: QueryLanguage::PromQl, + query_id, + canonical_query, + fixed_evaluation: None, + root, + nodes: compiler.nodes, + instant, + fallback, + }; + logical::finalize_residuals(&mut entry)?; + Ok(entry) } struct DagCompiler<'a, F> { @@ -1614,24 +977,6 @@ fn physical_grouping( Ok(PhysicalGrouping::Reduce(names)) } -#[derive(Debug, Error)] -pub enum QueryPlanError { - #[error("invalid PromQL query identity: {0}")] - InvalidPromql(String), - #[error("query is absent from the active QueryPlan: {0}")] - QueryNotPlanned(String), - #[error("post-ASAP DAG cannot be represented by the query executor: {0}")] - UnsupportedNode(String), - #[error("invalid QueryPlan: {0}")] - Invalid(String), -} - -pub fn canonical_promql(query: &str) -> Result { - promql_parser::parser::parse(query.trim()) - .map(|expr| expr.to_string()) - .map_err(|error| QueryPlanError::InvalidPromql(error.to_string())) -} - #[cfg(test)] mod tests { use super::*; diff --git a/control_plane/src/query_plan/clickhouse_exact.rs b/control_plane/src/query_plan/clickhouse_exact.rs index 9c8455429..fa227841f 100644 --- a/control_plane/src/query_plan/clickhouse_exact.rs +++ b/control_plane/src/query_plan/clickhouse_exact.rs @@ -77,7 +77,8 @@ fn scalar(expr: &QueryExpr, schema: &Schema) -> Result { let function = match name.as_str() { "map" => "map", "mapconcat" => "mapConcat", - "asap_map_access" => "arrayElement", + "asap_map_access" | "asap_element_access" => "arrayElement", + "asap_struct_field" => "tupleElement", _ => return Err(format!("unsupported exact scalar function {name}")), }; expr.scalar_type(schema).map_err(|e| e.to_string())?; @@ -301,6 +302,53 @@ mod tests { assert!(!sql.contains("{from:")); assert!(!sql.contains("{to:")); } + #[test] + fn typed_list_access_renders_native_element_lookup() { + let schema = Schema::new(vec![Column::new( + "samples", + DataType::List { + element: Box::new(Column::new("item", DataType::Float64, false)), + }, + false, + )]); + let expr = QueryExpr::FunctionCall { + name: "asap_element_access".into(), + args: vec![ + QueryExpr::Column(0), + QueryExpr::Literal(ScalarValue::Int64(-1)), + ], + }; + assert_eq!( + scalar(&expr, &schema).unwrap(), + "arrayElement(`samples`, -1)" + ); + } + + #[test] + fn typed_struct_field_renders_native_lookup() { + let schema = Schema::new(vec![Column::new( + "sample", + DataType::Struct { + fields: vec![ + Column::new("ts", DataType::Int64, false), + Column::new("value", DataType::Float64, true), + ], + }, + false, + )]); + let expr = QueryExpr::FunctionCall { + name: "asap_struct_field".into(), + args: vec![ + QueryExpr::Column(0), + QueryExpr::Literal(ScalarValue::Utf8("value".into())), + ], + }; + assert_eq!( + scalar(&expr, &schema).unwrap(), + "tupleElement(`sample`, 'value')" + ); + } + #[test] fn unsupported_scalar_is_not_forwarded_as_arbitrary_native_code() { let schema = Schema::new(vec![]); @@ -361,8 +409,8 @@ mod original_tests { if sql.contains("sum(value)") && sql.contains("argMax") { use crate::physical::post_asap::{PhysicalExpr, PostAsapPlan}; use crate::query_plan::{ - FallbackPolicy, FixedEvaluationRange, InstantExecution, QueryPlanEntry, - QueryPlanError, QueryPlanNode, + FallbackPolicy, FixedEvaluationRange, InstantExecution, QueryPlanError, + QueryPlanNode, }; let planned = crate::clickhouse::plan_clickhouse_sql(sql, &catalog, AccuracyTarget::Exact) @@ -371,7 +419,7 @@ mod original_tests { let PhysicalExpr::Committed(PostAsapPlan::Summary(root)) = planned.physical else { panic!("missing selected SQL DAG") }; - let entry = QueryPlanEntry::compile_bound_relational( + let entry = crate::query_plan::compile_bound_relational( "test".into(), planned.canonical_sql, &root, diff --git a/control_plane/src/query_plan/logical.rs b/control_plane/src/query_plan/logical.rs index 05d474715..6b20a5a84 100644 --- a/control_plane/src/query_plan/logical.rs +++ b/control_plane/src/query_plan/logical.rs @@ -6,62 +6,13 @@ use promql_parser::{ label::MatchOp, parser::{self, Expr, LabelModifier, Offset, VectorSelector}, }; -use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum LogicalOperator { - /// A maximal exact scalar/vector subtree evaluated by Prometheus. - ExactSubquery { - query: String, - }, - /// Prometheus exact subtree whose selectors are restricted at runtime by - /// the candidate vector produced by its single input. - CandidateExactSubquery { - query: String, - item_label: String, - }, - Scan { - metric: Option, - matchers: Vec, - range_ms: Option, - offset_ms: i64, - }, - UnaryNegate, - VectorToScalar, - Aggregate { - operation: Aggregation, - grouping: Grouping, - }, - /// PromQL `topk(k, vector)` selection over values produced by the child. - /// This is distinct from a frequency-sketch TopK readout: any exact or - /// summary-backed instant-vector child may feed this query-time operator. - TopKSelection { - k: u64, - grouping: Grouping, - }, - Binary { - operation: BinaryOperation, - return_bool: bool, - }, - Temporal { - operation: TemporalOperation, - }, - Sort { - descending: bool, - }, - HistogramQuantile, - Subquery { - range_ms: u64, - step_ms: u64, - offset_ms: i64, - }, -} +pub use asap_types::query_plan::logical::*; /// Stable identity of a Planner-authorized materializable DAG leaf. This is a /// workload-selection key, not another physical materialization definition. -#[derive(Debug, Clone, Serialize, PartialEq)] +#[derive(Debug, Clone, serde::Serialize, PartialEq)] struct MaterializationCandidateIdentity { metric: String, matchers: Vec, @@ -69,63 +20,6 @@ struct MaterializationCandidateIdentity { offset_ms: i64, operation: TemporalOperation, } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct Grouping { - pub labels: Vec, - pub without: bool, -} -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct LabelMatcher { - pub name: String, - pub value: String, - pub operation: LabelMatch, -} -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum LabelMatch { - Equal, - NotEqual, - Regex, - NotRegex, -} -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum Aggregation { - Sum, - Max, - Min, - Avg, - Count, -} -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum BinaryOperation { - Add, - Sub, - Mul, - Div, - Mod, - Pow, - Equal, - NotEqual, - Less, - LessEqual, - Greater, - GreaterEqual, -} -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum TemporalOperation { - Rate, - Increase, - Avg, - Max, - Min, - Sum, - Count, -} fn invalid(message: impl Into) -> QueryPlanError { QueryPlanError::Invalid(message.into()) @@ -143,46 +37,6 @@ fn offset(value: &Option) -> Result { } } -impl LogicalOperator { - pub fn validate(&self, inputs: usize) -> Result<(), QueryPlanError> { - let expected = match self { - Self::Scan { .. } | Self::ExactSubquery { .. } => 0, - Self::CandidateExactSubquery { .. } => 1, - Self::Binary { .. } | Self::HistogramQuantile => 2, - _ => 1, - }; - if inputs != expected { - return Err(invalid("logical operator input arity mismatch")); - } - if matches!( - self, - Self::Scan { - range_ms: Some(0), - .. - } - ) { - return Err(invalid("zero range")); - } - if let Self::ExactSubquery { query } | Self::CandidateExactSubquery { query, .. } = self { - let parsed = parser::parse(query).map_err(|e| invalid(e.to_string()))?; - if matches!(parsed, Expr::MatrixSelector(_) | Expr::Subquery(_)) { - return Err(invalid( - "exact subtree boundary must return scalar or instant vector", - )); - } - } - if let Self::Subquery { - range_ms, step_ms, .. - } = self - { - if *range_ms == 0 || *step_ms == 0 || range_ms / step_ms > 100_000 { - return Err(invalid("invalid or excessive subquery grid")); - } - } - Ok(()) - } -} - struct Lower { nodes: BTreeMap, seen: BTreeMap, @@ -409,53 +263,32 @@ impl Lower { } } -impl QueryPlanEntry { - /// Lower a Planner-authorized native residual into typed backend operations. - /// Callers retain a separate external-native alternative for cost comparison. - pub fn compile_logical( - query_id: String, - canonical_query: String, - instant: InstantExecution, - fallback: FallbackPolicy, - ) -> Result { - let expr = parser::parse(&canonical_query).map_err(|e| invalid(e.to_string()))?; - let mut lower = Lower { - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - }; - let root = lower.lower(&expr)?; - let entry = Self { - language: super::QueryLanguage::PromQl, - query_id, - canonical_query, - fixed_evaluation: None, - root, - nodes: lower.nodes, - instant, - fallback, - }; - entry.validate(&Default::default())?; - Ok(entry) - } - /// Promote only a wholly native entry; never discard selected summary bindings. - pub fn lower_native_residual(&self) -> Result { - if self.nodes.len() != 1 - || !matches!( - self.nodes.get(&self.root), - Some(QueryPlanNode::ExactFallback { .. }) - ) - { - return Err(invalid( - "logical residual promotion requires a whole native root", - )); - } - Self::compile_logical( - self.query_id.clone(), - self.canonical_query.clone(), - self.instant, - self.fallback, - ) - } +/// Lower a Planner-authorized native residual into typed backend operations. +/// Callers retain a separate external-native alternative for cost comparison. +pub fn compile_logical( + query_id: String, + canonical_query: String, + instant: InstantExecution, + fallback: FallbackPolicy, +) -> Result { + let expr = parser::parse(&canonical_query).map_err(|e| invalid(e.to_string()))?; + let mut lower = Lower { + nodes: BTreeMap::new(), + seen: BTreeMap::new(), + }; + let root = lower.lower(&expr)?; + let entry = QueryPlanEntry { + language: super::QueryLanguage::PromQl, + query_id, + canonical_query, + fixed_evaluation: None, + root, + nodes: lower.nodes, + instant, + fallback, + }; + entry.validate(&Default::default())?; + Ok(entry) } /// Match residuals by semantic IR equality, not display text or source names. @@ -555,7 +388,7 @@ mod tests { serde_json::from_str(include_str!("../../tests/fixtures/o11y_queries.json")).unwrap(); for row in corpus["queries"].as_array().unwrap() { let query = row["query"].as_str().unwrap(); - let entry = QueryPlanEntry::compile_logical( + let entry = crate::query_plan::logical::compile_logical( row["id"].as_str().unwrap().into(), query.into(), instant(), @@ -587,7 +420,7 @@ mod tests { #[test] fn repeated_subexpressions_share_node_identity() { // Serialized edges must retain CSE rather than duplicating raw work. - let entry = QueryPlanEntry::compile_logical( + let entry = crate::query_plan::logical::compile_logical( "q".into(), "sum(up) / sum(up)".into(), instant(), @@ -633,7 +466,7 @@ mod tests { 3, ), ] { - let entry = QueryPlanEntry::compile_logical( + let entry = crate::query_plan::logical::compile_logical( "topk".into(), query.into(), instant(), @@ -652,7 +485,7 @@ mod tests { #[test] fn topk_keeps_unsupported_child_as_exact_leaf() { - let entry = QueryPlanEntry::compile_logical( + let entry = crate::query_plan::logical::compile_logical( "topk-subquery".into(), "topk(3, label_replace(memory_bytes, \"dst\", \"$1\", \"src\", \"(.*)\"))".into(), instant(), @@ -681,7 +514,7 @@ mod tests { ("topk by (cluster) (2, m)", vec!["cluster"], false), ("topk without (pod) (2, m)", vec!["pod"], true), ] { - let entry = QueryPlanEntry::compile_logical( + let entry = crate::query_plan::logical::compile_logical( "topk-group".into(), query.into(), instant(), @@ -799,7 +632,7 @@ mod hybrid_tests { .unwrap(); let selected = crate::planner_selection::select_summary_default(&canonical).unwrap(); let entry = - QueryPlanEntry::compile_bound_composable( + crate::query_plan::compile_bound_composable( "hybrid".into(), query.into(), &selected, diff --git a/control_plane/tests/o11y_exact_fallback.rs b/control_plane/tests/o11y_exact_fallback.rs index 6f0b12d80..e43508a03 100644 --- a/control_plane/tests/o11y_exact_fallback.rs +++ b/control_plane/tests/o11y_exact_fallback.rs @@ -26,7 +26,7 @@ fn o11y_non_summary_roots_preserve_exact_query_semantics() { }; assert_eq!(original.as_ref(), &expr, "query semantics changed: {query}"); assert_eq!(node.schema.fields.len(), expr.output_schema().unwrap().columns.len()); - let executable = control_plane::query_plan::QueryPlanEntry::compile_bound( + let executable = control_plane::query_plan::compile_bound( "fixture".into(), query.into(), &node, control_plane::query_plan::InstantExecution { lookback_ms: 300_000, full_history: false, cumulative_readout: false, diff --git a/control_plane/tests/offline_evidence.rs b/control_plane/tests/offline_evidence.rs index 769bdd6bd..b36bd3fdb 100644 --- a/control_plane/tests/offline_evidence.rs +++ b/control_plane/tests/offline_evidence.rs @@ -346,9 +346,7 @@ fn incompatible_evidence_preserves_deployment_behavior() { /// fallback even though the warm tier now supports exact additive binaries. #[test] fn binary_summary_has_explicit_warm_tier_fallback() { - use control_plane::query_plan::{ - FallbackPolicy, InstantExecution, QueryPlanEntry, QueryPlanNode, - }; + use control_plane::query_plan::{FallbackPolicy, InstantExecution, QueryPlanNode}; use planner_types::{post_asap::BinaryOperator, pre_asap::BinaryOpKind}; let child = bound(&model()); let root = std::rc::Rc::new(SummaryNode { @@ -363,7 +361,7 @@ fn binary_summary_has_explicit_warm_tier_fallback() { schema: child.schema.clone(), guarantee: None, }); - let plan = QueryPlanEntry::compile_bound( + let plan = control_plane::query_plan::compile_bound( "test".into(), "left / right".into(), &root, diff --git a/crates/asap_types/Cargo.toml b/crates/asap_types/Cargo.toml index a83ea156c..d6480e602 100644 --- a/crates/asap_types/Cargo.toml +++ b/crates/asap_types/Cargo.toml @@ -4,6 +4,7 @@ version.workspace = true edition.workspace = true [dependencies] +promql-parser.workspace = true base64 = "0.21" tracing.workspace = true serde.workspace = true @@ -33,4 +34,4 @@ sha2 = "0.10" # exactly (`control_plane/Cargo.toml`) -- two different revs of the same # git dependency in one workspace resolve to two distinct Rust types that # won't unify. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "0402384e589df6e087d6d2d22b463ddc2eea0774" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "5e9396033a5a5d5f350bfa675d12770c972ee991" } diff --git a/crates/asap_types/src/aggregation_config.rs b/crates/asap_types/src/aggregation_config.rs index 3e9f5cab3..8254b65b1 100644 --- a/crates/asap_types/src/aggregation_config.rs +++ b/crates/asap_types/src/aggregation_config.rs @@ -103,6 +103,8 @@ pub struct PrecomputeMaterialization { pub grouping_labels: crate::GroupingProjection, #[serde(default, skip_serializing_if = "Option::is_none")] pub partitioning: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub derived_input: Option, pub aggregated_labels: KeyByLabelNames, pub rollup_labels: KeyByLabelNames, pub original_yaml: String, @@ -200,9 +202,39 @@ impl PrecomputeMaterialization { .saturating_mul(1_000) } + pub fn source_identity(&self) -> crate::sds::DataSourceIdentity { + use crate::sds::DataSourceIdentity; + if let Some(input) = &self.derived_input { + DataSourceIdentity::Derived { + input: input.clone(), + } + } else if let Some(table_ref) = &self.table_name { + DataSourceIdentity::Table { + table_ref: table_ref.clone(), + } + } else { + DataSourceIdentity::TimeSeries { + metric: self.metric.clone(), + } + } + } + pub fn population_filter_canonical(&self) -> Result { + if let Some(input) = &self.derived_input { + input.validate()?; + if self.table_name.is_some() + || self.table_population.is_some() + || self.table_timestamp_column.is_some() + || !self.spatial_filter.is_empty() + { + return Err("derived inputs cannot also declare a raw source/filter".into()); + } + } self.effective_value_projection().validate()?; - if self.value_projection.is_some() && self.table_name.is_none() { + if self.value_projection.is_some() + && self.table_name.is_none() + && self.derived_input.is_none() + { return Err("explicit table value projection requires a table source".into()); } if let Some(column) = &self.table_timestamp_column { @@ -255,6 +287,7 @@ impl PrecomputeMaterialization { parameters, grouping_labels: grouping_labels.into(), partitioning: None, + derived_input: None, aggregated_labels, rollup_labels, original_yaml, @@ -400,6 +433,11 @@ impl PrecomputeMaterialization { table_name, value_column, ); + config.derived_input = data + .get("derived_input") + .filter(|v| !v.is_null()) + .map(|v| serde_json::from_value(v.clone())) + .transpose()?; config.partitioning = data .get("partitioning") .filter(|value| !value.is_null()) @@ -595,6 +633,11 @@ impl PrecomputeMaterialization { table_name, value_column, ); + config.derived_input = aggregation_data + .get("derived_input") + .filter(|v| !v.is_null()) + .map(|v| serde_yaml::from_value(v.clone())) + .transpose()?; config.partitioning = aggregation_data .get("partitioning") .filter(|value| !value.is_null()) @@ -640,6 +683,9 @@ impl SerializableToSink for PrecomputeMaterialization { "metric": self.metric, }); + if let Some(input) = &self.derived_input { + json["derived_input"] = serde_json::json!(input); + } // Only include numAggregatesToRetain if it's Some if let Some(num_aggregates) = self.num_aggregates_to_retain { json["numAggregatesToRetain"] = serde_json::json!(num_aggregates); diff --git a/crates/asap_types/src/derived_input.rs b/crates/asap_types/src/derived_input.rs new file mode 100644 index 000000000..a5f6d6dce --- /dev/null +++ b/crates/asap_types/src/derived_input.rs @@ -0,0 +1,383 @@ +//! Content identity for a maintenance sub-DAG cut at existing summary inputs. +//! The executable program remains in OwnedPostAsapDag; this is its catalog key. +use std::collections::{BTreeMap, BTreeSet}; + +use planner_types::post_asap::PostAsapNodeId; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::{executable_plan::OwnedPostAsapDag, sds::SummaryDefinitionId}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DerivedInputIdentity { + pub inputs: BTreeSet, + pub program_sha256: String, +} + +impl DerivedInputIdentity { + pub fn validate(&self) -> Result<(), String> { + if self.inputs.is_empty() + || self.inputs.iter().any(|id| id.fingerprint().is_unset()) + || self.program_sha256.len() != 64 + || !self + .program_sha256 + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + { + return Err("derived input requires summary references and a canonical SHA-256".into()); + } + Ok(()) + } + + /// Hash semantic nodes and edge roles, replacing input frontiers with stable + /// catalog IDs. Query IDs, node numbering, and catalog generations are absent. + pub fn from_dag( + document: &OwnedPostAsapDag, + root: PostAsapNodeId, + frontiers: &BTreeMap, + ) -> Result { + if document.schema_version != crate::executable_plan::OWNED_POST_ASAP_DAG_SCHEMA_VERSION { + return Err("unsupported derived program document version".into()); + } + let decoded = document.decode()?; + let literals: BTreeSet<_> = decoded + .nodes + .iter() + .filter_map(|node| { + matches!( + &node.payload, + planner_types::post_asap::ExecutableOperatorPayload::Fallback { + expression: planner_types::pre_asap::QueryExpr::Literal(_), + } + ) + .then_some(node.id) + }) + .collect(); + let mut incoming: BTreeMap<_, Vec<_>> = BTreeMap::new(); + for edge in &document.edges { + incoming.entry(edge.consumer).or_default().push(edge); + } + let nodes: BTreeMap<_, _> = document.nodes.iter().map(|n| (n.id, n)).collect(); + if frontiers.contains_key(&root) { + return Err("derived program requires a non-frontier root".into()); + } + let mut hashes = BTreeMap::new(); + let mut visiting = BTreeSet::new(); + let mut inputs = BTreeSet::new(); + let mut stack = vec![(root, false)]; + while let Some((id, finish)) = stack.pop() { + if hashes.contains_key(&id) { + continue; + } + let node = nodes + .get(&id) + .ok_or("derived program references missing node")?; + if let Some(summary) = frontiers.get(&id) { + inputs.insert(*summary); + hashes.insert(id, serde_json::json!({"summary": summary})); + continue; + } + if !finish { + if !visiting.insert(id) { + return Err("derived program has a cycle".into()); + } + stack.push((id, true)); + if !incoming.contains_key(&id) && !literals.contains(&id) { + return Err("derived program has an unbound input leaf".into()); + } + for edge in incoming.get(&id).into_iter().flatten() { + stack.push((edge.producer, false)); + } + continue; + } + let mut edges = Vec::new(); + for edge in incoming.get(&id).into_iter().flatten() { + let input = hashes + .get(&edge.producer) + .ok_or("derived input was not evaluated")?; + edges.push( + serde_json::to_vec(&serde_json::json!({ + "input": input, "role": edge.role, "schema": edge.intermediate_schema, + "state": edge.data_state, "grouping": edge.grouping, "window": edge.window, + })) + .map_err(|e| e.to_string())?, + ); + } + edges.sort(); + let bytes = serde_json::to_vec(&serde_json::json!({ + "version": 1, "operator": node.operator, "payload": node.payload, + "state": node.output_state, "schema": node.output_schema, + "guarantee": node.guarantee, "inputs": edges, + })) + .map_err(|e| e.to_string())?; + hashes.insert( + id, + serde_json::json!(format!("{:x}", Sha256::digest(bytes))), + ); + visiting.remove(&id); + } + let identity = Self { + inputs, + program_sha256: hashes[&root] + .as_str() + .ok_or("derived root has no semantic hash")? + .into(), + }; + identity.validate()?; + Ok(identity) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::summary_catalog::SummaryCatalog; + use crate::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; + + fn config() -> PrecomputeMaterialization { + PrecomputeMaterialization::new( + AggregationType::Sum, + String::new(), + Default::default(), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + String::new(), + 10, + 10, + WindowKind::Tumbling, + String::new(), + "m".into(), + None, + None, + None, + ) + } + + #[test] + fn derived_source_is_distinct_and_generation_independent() { + let raw = config(); + let raw_id = SummaryDefinitionId::from(raw.policy_fingerprint()); + let mut derived = raw.clone(); + derived.derived_input = Some(DerivedInputIdentity { + inputs: BTreeSet::from([raw_id]), + program_sha256: "a".repeat(64), + }); + assert_ne!(raw.policy_fingerprint(), derived.policy_fingerprint()); + let a = + SummaryCatalog::from_materializations(1, 1, &[raw.clone(), derived.clone()]).unwrap(); + let b = SummaryCatalog::from_materializations(2, 9, &[raw, derived.clone()]).unwrap(); + assert_eq!(a.materializations, b.materializations); + assert_eq!(a.data_descriptors, b.data_descriptors); + let mut renamed = derived.clone(); + renamed.metric = "output_alias".into(); + assert_eq!(renamed.policy_fingerprint(), derived.policy_fingerprint()); + let json = serde_json::to_value(&derived).unwrap(); + let decoded: PrecomputeMaterialization = serde_json::from_value(json).unwrap(); + assert_eq!(decoded.policy_fingerprint(), derived.policy_fingerprint()); + } + + #[test] + fn raw_utf8_metric_cannot_impersonate_derived_policy_domain() { + let mut derived = config(); + derived.derived_input = Some(DerivedInputIdentity { + inputs: BTreeSet::from([SummaryDefinitionId::from(derived.policy_fingerprint())]), + program_sha256: "d".repeat(64), + }); + let mut raw = derived.clone(); + raw.metric = format!( + "derived-input-v1:{}", + serde_json::to_string(&derived.source_identity()).unwrap() + ); + raw.derived_input = None; + assert_ne!(raw.policy_fingerprint(), derived.policy_fingerprint()); + } + + #[test] + fn catalog_rejects_missing_derived_dependencies_and_raw_source_conflicts() { + let mut derived = config(); + derived.derived_input = Some(DerivedInputIdentity { + inputs: BTreeSet::from([SummaryDefinitionId::from(derived.policy_fingerprint())]), + program_sha256: "b".repeat(64), + }); + assert!(SummaryCatalog::from_materializations(1, 1, &[derived.clone()]).is_err()); + derived.table_name = Some("table".into()); + assert!(derived.population_filter_canonical().is_err()); + } + fn program(source: u32, root: u32) -> OwnedPostAsapDag { + use crate::executable_plan::{OwnedPostAsapEdge, OwnedPostAsapNode}; + use planner_types::post_asap::{ + EdgeRole, ExecutableOperator, ExecutionDataState, GroupingEdgeCompatibility, + WindowEdgeCompatibility, + }; + let state = ExecutionDataState::MAINTENANCE_SUMMARY; + OwnedPostAsapDag { + schema_version: 1, + query_id: "query-a".into(), + root: PostAsapNodeId(root), + nodes: [source, root] + .into_iter() + .map(|id| OwnedPostAsapNode { + id: PostAsapNodeId(id), + operator: ExecutableOperator::SummaryMerge, + payload: serde_json::json!({"kind":"summary_merge"}), + output_state: state, + output_schema: serde_json::json!({"fields":[],"time_index":null}), + guarantee: None, + }) + .collect(), + edges: vec![OwnedPostAsapEdge { + producer: PostAsapNodeId(source), + consumer: PostAsapNodeId(root), + role: EdgeRole::Input, + intermediate_schema: serde_json::json!({"fields":[],"time_index":null}), + data_state: state, + grouping: GroupingEdgeCompatibility::Identical, + window: WindowEdgeCompatibility::NotApplicable, + }], + } + } + + #[test] + fn semantic_signature_ignores_node_and_query_numbering_but_not_inputs() { + let source = SummaryDefinitionId::from(config().policy_fingerprint()); + let a = program(1, 2); + let first = DerivedInputIdentity::from_dag( + &a, + a.root, + &BTreeMap::from([(PostAsapNodeId(1), source)]), + ) + .unwrap(); + let mut b = program(900, 42); + b.query_id = "another-query".into(); + b.nodes.reverse(); + let second = DerivedInputIdentity::from_dag( + &b, + b.root, + &BTreeMap::from([(PostAsapNodeId(900), source)]), + ) + .unwrap(); + assert_eq!(first, second); + b.nodes + .iter_mut() + .find(|n| n.id == b.root) + .unwrap() + .output_schema = serde_json::json!({"fields":[],"time_index":0}); + assert_ne!( + first, + DerivedInputIdentity::from_dag( + &b, + b.root, + &BTreeMap::from([(PostAsapNodeId(900), source)]) + ) + .unwrap() + ); + assert!(DerivedInputIdentity::from_dag(&a, a.root, &BTreeMap::new()).is_err()); + let mut unsupported = a.clone(); + unsupported.schema_version = 999; + assert!(DerivedInputIdentity::from_dag( + &unsupported, + unsupported.root, + &BTreeMap::from([(PostAsapNodeId(1), source)]) + ) + .is_err()); + unsupported = a.clone(); + unsupported.nodes[1].payload = serde_json::json!({"kind":"unknown_operator"}); + assert!(DerivedInputIdentity::from_dag( + &unsupported, + unsupported.root, + &BTreeMap::from([(PostAsapNodeId(1), source)]) + ) + .is_err()); + let mut cycle = a.clone(); + cycle.edges[0].producer = cycle.root; + assert!(DerivedInputIdentity::from_dag(&cycle, cycle.root, &BTreeMap::new()).is_err()); + cycle.edges[0].producer = PostAsapNodeId(999); + assert!(DerivedInputIdentity::from_dag(&cycle, cycle.root, &BTreeMap::new()).is_err()); + } + + #[test] + fn catalog_rejects_self_referential_summary() { + use crate::sds::{ + DataDescriptor, DataSourceIdentity, SummaryDescriptor, ValueProjectionIdentity, + }; + let config = config(); + let input = DerivedInputIdentity { + inputs: BTreeSet::from([SummaryDefinitionId::from(config.policy_fingerprint())]), + program_sha256: "f".repeat(64), + }; + let data = DataDescriptor::new_typed( + DataSourceIdentity::Derived { input }, + ValueProjectionIdentity::SampleValue, + "", + Vec::::new(), + "derived", + ); + assert!(SummaryCatalog::build( + 1, + 1, + [( + config.policy_fingerprint(), + SummaryDescriptor::from_config(&config).unwrap(), + data, + config.window_layout + )] + ) + .is_err()); + } + #[test] + fn installation_rejects_derived_inputs_until_immutable_consumer_is_enabled() { + use crate::precompute_plan::{PlanEnvelope, PrecomputePlan}; + let mut config = config(); + config.derived_input = Some(DerivedInputIdentity { + inputs: BTreeSet::from([SummaryDefinitionId::from(config.policy_fingerprint())]), + program_sha256: "c".repeat(64), + }); + let envelope = PlanEnvelope { + plan_id: 1, + plan_version: 1, + generated_at_unix_ms: 0, + activation_unix_ms: 0, + expiry_unix_ms: None, + backend_compat: "asap-query-backend.v1".into(), + planner_revision: "test".into(), + capability_snapshot_id: "test".into(), + }; + let error = + PrecomputePlan::build(envelope, vec![config], &["producer".into()]).unwrap_err(); + assert!(error.to_string().contains("immutable maintenance consumer")); + } + #[test] + fn literal_leaves_are_hashed_without_inventing_materialization_references() { + use planner_types::{ + post_asap::{ExecutableOperator, ExecutableOperatorPayload}, + pre_asap::{QueryExpr, ScalarValue}, + }; + let mut dag = program(1, 2); + let mut literal = dag.nodes[0].clone(); + literal.id = PostAsapNodeId(3); + literal.operator = ExecutableOperator::Fallback; + literal.payload = serde_json::to_value(ExecutableOperatorPayload::Fallback { + expression: QueryExpr::Literal(ScalarValue::Int64(2)), + }) + .unwrap(); + dag.nodes.push(literal); + let mut edge = dag.edges[0].clone(); + edge.producer = PostAsapNodeId(3); + dag.edges.push(edge); + let source = SummaryDefinitionId::from(config().policy_fingerprint()); + let frontiers = BTreeMap::from([(PostAsapNodeId(1), source)]); + let first = DerivedInputIdentity::from_dag(&dag, dag.root, &frontiers).unwrap(); + assert_eq!(first.inputs, BTreeSet::from([source])); + dag.nodes[2].payload = serde_json::to_value(ExecutableOperatorPayload::Fallback { + expression: QueryExpr::Literal(ScalarValue::Int64(3)), + }) + .unwrap(); + assert_ne!( + first, + DerivedInputIdentity::from_dag(&dag, dag.root, &frontiers).unwrap() + ); + assert!(DerivedInputIdentity::from_dag(&dag, PostAsapNodeId(3), &frontiers).is_err()); + } +} diff --git a/crates/asap_types/src/erp_observation.rs b/crates/asap_types/src/erp_observation.rs new file mode 100644 index 000000000..7693a3c72 --- /dev/null +++ b/crates/asap_types/src/erp_observation.rs @@ -0,0 +1,229 @@ +//! Versioned runtime evidence about the inputs of one installed summary. +//! Shape is generic so the transport contract does not depend on a planner. +use crate::sds::{CatalogGeneration, SummaryDefinitionId, SummaryInstanceId}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ErpObservationInputSemantics { + ScalarSampleValue, + UnitSampleFrequency, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ErpPopulationObservation { + pub population_id: SummaryInstanceId, + pub shape: Shape, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ErpPopulationObservations { + pub schema_version: u32, + pub catalog_generation: CatalogGeneration, + pub summary_definition_id: SummaryDefinitionId, + pub observed_at_unix_ms: u64, + pub window_start_ms: i64, + pub window_end_ms: i64, + pub input_semantics: ErpObservationInputSemantics, + /// Any overflow or unsupported input invalidates the entire envelope. + /// Invalid envelopes contain no population fits. + pub invalid_reason: Option, + pub populations: Vec>, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ErpObservationFreshness { + pub max_age_ms: u64, + pub max_future_skew_ms: u64, +} + +impl ErpPopulationObservations { + pub fn validate_identity_and_freshness( + &self, + expected_generation: &CatalogGeneration, + expected_definition: SummaryDefinitionId, + now_ms: u64, + freshness: ErpObservationFreshness, + ) -> Result<(), &'static str> { + if self.schema_version != 1 + || &self.catalog_generation != expected_generation + || self.summary_definition_id != expected_definition + { + return Err("ERP observation belongs to a different catalog or summary"); + } + if self.invalid_reason.is_some() + || self.populations.is_empty() + || self.window_start_ms >= self.window_end_ms + { + return Err("ERP observation is incomplete or invalid"); + } + if freshness.max_age_ms == 0 + || self.observed_at_unix_ms > now_ms.saturating_add(freshness.max_future_skew_ms) + || now_ms.saturating_sub(self.observed_at_unix_ms) > freshness.max_age_ms + { + return Err("ERP observation is stale or future-dated"); + } + let mut seen = std::collections::BTreeSet::new(); + if self + .populations + .iter() + .any(|p| !seen.insert(&p.population_id)) + { + return Err("ERP observation repeats a population"); + } + Ok(()) + } +} + +/// Empirical sufficient statistics for frequency-shape fitting. No raw keys, +/// raw samples, distribution-family assertion, or planner configuration crosses the wire. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EmpiricalFrequencyObservation { + pub sorted_counts: Vec, + pub interval_counts: Vec, +} +impl EmpiricalFrequencyObservation { + pub fn event_count(&self) -> Option { + if self.sorted_counts.len() > 65_536 + || self.interval_counts.len() > 256 + || self.sorted_counts.is_empty() + || self.sorted_counts.iter().any(|n| *n == 0) + || self.sorted_counts.windows(2).any(|w| w[0] < w[1]) + { + return None; + } + let total = self + .sorted_counts + .iter() + .try_fold(0u64, |sum, n| sum.checked_add(*n))?; + let intervals = self + .interval_counts + .iter() + .try_fold(0u64, |sum, n| sum.checked_add(*n))?; + (total == intervals).then_some(total) + } +} + +/// Bounded exact frequency counting. Fitting and ERP selection remain control-plane work. +#[derive(Debug)] +pub struct BoundedFrequencyObserver { + frequencies: std::collections::HashMap, + intervals: std::collections::HashMap, + max_keys: usize, + max_intervals: usize, + events: u64, + invalid: bool, +} +impl BoundedFrequencyObserver { + pub fn new(max_keys: usize, max_intervals: usize) -> Result { + if max_keys == 0 || max_intervals == 0 { + return Err("observation limits must be positive"); + } + Ok(Self { + frequencies: Default::default(), + intervals: Default::default(), + max_keys, + max_intervals, + events: 0, + invalid: false, + }) + } + pub fn observed_key_count(&self) -> usize { + self.frequencies.len() + } + pub fn invalidate(&mut self) { + self.invalid = true; + } + pub fn observe(&mut self, key: K, interval: usize) -> Result<(), &'static str> { + if self.invalid { + return Err("ERP observation was invalidated"); + } + if (!self.frequencies.contains_key(&key) && self.frequencies.len() >= self.max_keys) + || (!self.intervals.contains_key(&interval) + && self.intervals.len() >= self.max_intervals) + { + self.invalid = true; + return Err("ERP observation key or interval budget exceeded"); + } + let Some(events) = self.events.checked_add(1) else { + self.invalid = true; + return Err("ERP observation count overflow"); + }; + *self.frequencies.entry(key).or_default() += 1; + *self.intervals.entry(interval).or_default() += 1; + self.events = events; + Ok(()) + } + pub fn snapshot(&self) -> Option { + if self.invalid || self.frequencies.is_empty() { + return None; + } + let mut sorted_counts: Vec<_> = self.frequencies.values().copied().collect(); + sorted_counts.sort_unstable_by(|a, b| b.cmp(a)); + Some(EmpiricalFrequencyObservation { + sorted_counts, + interval_counts: self.intervals.values().copied().collect(), + }) + } +} + +impl ErpPopulationObservations { + pub fn try_map_shapes( + self, + mut convert: impl FnMut(S) -> Option, + ) -> Option> { + let populations = self + .populations + .into_iter() + .map(|population| { + Some(ErpPopulationObservation { + population_id: population.population_id, + shape: convert(population.shape)?, + }) + }) + .collect::>>()?; + Some(ErpPopulationObservations { + schema_version: self.schema_version, + catalog_generation: self.catalog_generation, + summary_definition_id: self.summary_definition_id, + observed_at_unix_ms: self.observed_at_unix_ms, + window_start_ms: self.window_start_ms, + window_end_ms: self.window_end_ms, + input_semantics: self.input_semantics, + invalid_reason: self.invalid_reason, + populations, + }) + } +} + +#[cfg(test)] +mod empirical_tests { + use super::*; + #[test] + fn count_overflow_invalidates_instead_of_publishing_prefix() { + let mut observer = BoundedFrequencyObserver::new(2, 2).unwrap(); + observer.observe(1u64, 0).unwrap(); + observer.events = u64::MAX; + assert!(observer.observe(1, 0).is_err()); + assert!(observer.snapshot().is_none()); + } + #[test] + fn malformed_frequency_counts_fail_closed() { + assert!(EmpiricalFrequencyObservation { + sorted_counts: vec![2, 1], + interval_counts: vec![2] + } + .event_count() + .is_none()); + assert!(EmpiricalFrequencyObservation { + sorted_counts: vec![1, 2], + interval_counts: vec![3] + } + .event_count() + .is_none()); + } +} diff --git a/crates/asap_types/src/lib.rs b/crates/asap_types/src/lib.rs index 3c0a23997..e61e299df 100644 --- a/crates/asap_types/src/lib.rs +++ b/crates/asap_types/src/lib.rs @@ -2,13 +2,17 @@ pub mod accumulator_spec; pub mod accuracy; pub mod aggregation_config; pub mod aggregation_type; +pub mod derived_input; pub mod enums; +pub mod erp_observation; pub mod executable_plan; pub mod grouping_projection; pub mod key_by_label_names; pub mod monitor_spec; +pub mod plan_publication; pub mod policy_fingerprint; pub mod policy_registry; +pub mod producer_plan; pub mod query_requirements; pub mod routing_index; pub mod sds; @@ -33,3 +37,5 @@ pub use routing_index::RoutingIndex; pub use storage_backend::*; pub mod precompute_plan; + +pub mod query_plan; diff --git a/crates/asap_types/src/plan_publication.rs b/crates/asap_types/src/plan_publication.rs new file mode 100644 index 000000000..35ce812f5 --- /dev/null +++ b/crates/asap_types/src/plan_publication.rs @@ -0,0 +1,126 @@ +//! Canonical publication document for one catalog generation. +use crate::precompute_plan::PrecomputePlan; +use crate::producer_plan::{CollectorPlan, TransmissionPlan}; +use crate::query_plan::QueryPlan; +use crate::summary_catalog::SummaryCatalog; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PhysicalPlanPublication { + pub summary_catalog: SummaryCatalog, + pub precompute_plan: PrecomputePlan, + pub collector_plans: Vec, + pub transmission_plan: TransmissionPlan, + pub query_plan: QueryPlan, +} + +/// Complete typed envelope accepted by the data-plane install endpoint. +/// +/// Runtime-only routing and adaptation evidence decorate the shared +/// publication without changing its catalog generation. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PhysicalPlanInstallRequest { + pub summary_catalog: SummaryCatalog, + #[serde(default)] + pub collector_plans: Vec, + pub precompute_plan: PrecomputePlan, + pub transmission_plan: TransmissionPlan, + pub query_plan: QueryPlan, + pub storage_routing: Option, + #[serde(default)] + pub adaptation_evidence: Vec, +} + +impl PhysicalPlanPublication { + /// Validate every plan against the shared catalog snapshot. + pub fn validate(&self) -> Result<(), String> { + let catalog = &self.summary_catalog; + self.precompute_plan + .validate_against_catalog(catalog) + .map_err(|e| e.to_string())?; + self.transmission_plan + .validate(&self.precompute_plan) + .map_err(|e| e.to_string())?; + self.transmission_plan + .validate_against_catalog(catalog) + .map_err(|e| e.to_string())?; + self.query_plan + .validate_against_catalog(catalog) + .map_err(|e| e.to_string())?; + let materializations = self + .precompute_plan + .materializations + .iter() + .map(|config| (config.policy_fingerprint(), config)) + .collect::>(); + for entry in self.query_plan.entries.values() { + for binding in entry.materialization_bindings() { + let config = materializations + .get(&binding.materialization.fingerprint()) + .copied() + .ok_or("query binding has no precompute materialization")?; + if config.stored_window_ms() != binding.window_ms { + return Err("query pane differs from precompute stored window".into()); + } + if config.pane_origin_ms != binding.pane_origin_ms { + return Err("query pane origin differs from precompute definition".into()); + } + } + } + let mut collectors = std::collections::BTreeSet::new(); + for collector in &self.collector_plans { + if collector.envelope != self.precompute_plan.envelope + || !collectors.insert(&collector.collector_id) + { + return Err("collector envelope mismatch or duplicate collector".into()); + } + collector + .validate_against_catalog(catalog) + .map_err(|e| e.to_string())?; + for rule in &collector.transmission_rules { + if !self.transmission_plan.rules.contains(rule) { + return Err( + "collector transmission rule absent from published transmission plan" + .into(), + ); + } + } + } + for producer in &self.precompute_plan.producers { + let collector = self + .collector_plans + .iter() + .find(|c| c.collector_id == producer.collector_id) + .ok_or("precompute producer has no published CollectorPlan")?; + if !collector + .materializations + .iter() + .any(|m| m.materialization == producer.materialization) + { + return Err( + "collector does not produce referenced precompute materialization".into(), + ); + } + } + Ok(()) + } + + pub fn install_request( + &self, + storage_routing: Option, + adaptation_evidence: Vec, + ) -> Result { + self.validate()?; + Ok(PhysicalPlanInstallRequest { + summary_catalog: self.summary_catalog.clone(), + collector_plans: self.collector_plans.clone(), + precompute_plan: self.precompute_plan.clone(), + transmission_plan: self.transmission_plan.clone(), + query_plan: self.query_plan.clone(), + storage_routing, + adaptation_evidence, + }) + } +} diff --git a/crates/asap_types/src/policy_fingerprint.rs b/crates/asap_types/src/policy_fingerprint.rs index f9e751651..38afc45ff 100644 --- a/crates/asap_types/src/policy_fingerprint.rs +++ b/crates/asap_types/src/policy_fingerprint.rs @@ -90,7 +90,16 @@ impl PolicyFingerprint { let mut buf: Vec = Vec::with_capacity(512); // 1. metric name - buf.extend_from_slice(cfg.metric.as_bytes()); + if cfg.derived_input.is_some() { + // Raw policies start with UTF-8 metric bytes; 0xff is impossible + // there, so a metric cannot impersonate this source domain. + buf.extend_from_slice(b"\xffderived-input-v1:"); + buf.extend_from_slice( + &serde_json::to_vec(&cfg.source_identity()).expect("typed source identity"), + ); + } else { + buf.extend_from_slice(cfg.metric.as_bytes()); + } buf.push(0); // 2. aggregation_type (Serialize impl is the stable form) diff --git a/crates/asap_types/src/precompute_plan.rs b/crates/asap_types/src/precompute_plan.rs index 257d99dd4..da7b07b0c 100644 --- a/crates/asap_types/src/precompute_plan.rs +++ b/crates/asap_types/src/precompute_plan.rs @@ -334,6 +334,17 @@ impl PrecomputePlan { if !valid_ingest { return Err(PrecomputePlanError::UnsupportedIngestEndpoint); } + // Derived input identity is portable, but raw ingress cannot execute it. + // Installation stays closed until the immutable maintenance consumer is wired. + if self + .materializations + .iter() + .any(|config| config.derived_input.is_some()) + { + return Err(PrecomputePlanError::CatalogContract( + "derived summary input requires an immutable maintenance consumer".into(), + )); + } for (query_id, installed) in &self.executable_dags { if query_id != &installed.document.query_id { return Err(PrecomputePlanError::CatalogContract( diff --git a/crates/asap_types/src/precompute_plan/catalog.rs b/crates/asap_types/src/precompute_plan/catalog.rs index 9ee907798..e9dd2a599 100644 --- a/crates/asap_types/src/precompute_plan/catalog.rs +++ b/crates/asap_types/src/precompute_plan/catalog.rs @@ -1,6 +1,6 @@ //! Catalog consistency checks for the precompute execution plan. use super::*; -use crate::sds::{DataSourceIdentity, SummaryDefinitionId, SummaryDescriptor}; +use crate::sds::{SummaryDefinitionId, SummaryDescriptor}; use crate::summary_catalog::SummaryCatalog; use planner_types::pre_asap::Source; use std::collections::BTreeSet; @@ -70,14 +70,7 @@ impl PrecomputePlan { return Err(invalid("pane origin differs from catalog definition")); } let data = &catalog.data_descriptors[&binding.data_descriptor_id]; - let expected_source = config.table_name.as_ref().map_or_else( - || DataSourceIdentity::TimeSeries { - metric: config.metric.clone(), - }, - |table_ref| DataSourceIdentity::Table { - table_ref: table_ref.clone(), - }, - ); + let expected_source = config.source_identity(); if config.table_name.is_some() && !config.grouping_labels.is_empty() && data.observation_semantics diff --git a/crates/asap_types/src/producer_plan.rs b/crates/asap_types/src/producer_plan.rs new file mode 100644 index 000000000..81d4b36e6 --- /dev/null +++ b/crates/asap_types/src/producer_plan.rs @@ -0,0 +1,879 @@ +//! Installed collector and transmission contracts shared across producers and consumers. +//! Deployment choices and accuracy-budget allocation remain in the control plane. +use crate::precompute_plan::{PlanEnvelope, PrecomputePlan, StateEncoding, StateFamilyContract}; +use planner_types::post_asap::{SketchAlgorithm, SummaryWindowFramework}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::BTreeSet; +use thiserror::Error; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct CollectorMaterialization { + pub query_id: String, + pub materialization: crate::sds::SummaryDefinitionId, + pub metric: String, + pub algorithm: String, + pub parameters: Value, + pub group_by: Vec, + pub window_secs: u64, + pub abstract_window_framework: SummaryWindowFramework, + pub window_implementation_id: String, + pub slide_secs: u64, + #[serde( + default, + alias = "paneOriginMs", + skip_serializing_if = "Option::is_none" + )] + pub pane_origin_ms: Option, + pub window_layout: crate::WindowMaterializationLayout, + pub evidence_source: Option, + pub lifecycle: CollectorLifecycle, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct CollectorLifecycle { + pub kind: String, + pub maintenance_mode: String, + pub evaluation_schedule: String, + pub output_representation: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct CollectorPlan { + /// Absent only in legacy artifacts; catalog-aware validation requires it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub summary_catalog: Option, + pub collector_id: String, + pub envelope: PlanEnvelope, + pub materializations: Vec, + pub transmission_rules: Vec, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum TransmissionMode { + Full, + Delta, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum SequenceScope { + MaterializationSeriesProducerEpoch, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct FrameIdentityContract { + pub identity_version: u32, + pub sequence_scope: SequenceScope, + pub require_checkpoint_for_full: bool, + pub require_base_checkpoint_for_delta: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct TransmissionRule { + pub materialization: crate::sds::SummaryDefinitionId, + pub producer_id: String, + pub schema_id: String, + pub mode: TransmissionMode, + pub encoding: StateEncoding, + pub emit_every_ms: u64, + pub full_checkpoint_every_ms: Option, + pub destination_ref: String, + /// Plan-owned runtime knobs. These values are part of the immutable plan + /// generation; live feedback may only change them by publishing a + /// successor generation accepted by [`TransmissionPlan::authorize_successor`]. + #[serde(default)] + pub runtime_policy: RuntimeRulePolicy, +} + +/// How the collector admits updates before sketch maintenance. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "mode", rename_all = "snake_case", deny_unknown_fields)] +pub enum SamplingPolicy { + Disabled, + Fixed { + /// Probability in `(0, 1]`; `1` is valid but should normally be + /// represented by `Disabled`. + probability: f64, + estimator: SamplingEstimator, + }, +} + +impl Default for SamplingPolicy { + fn default() -> Self { + Self::Disabled + } +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum SamplingEstimator { + /// Hash-threshold element sampling, used by cardinality summaries. + HashThreshold, + /// Geometric admission/Nitro-style update sampling, used by frequency + /// summaries. The sketch readout carries the corresponding correction. + GeometricAdmission, +} + +/// Norm-adaptive Group-of-Sketches delta gating. GOS is meaningful only for +/// CountSketch families and only when the transmission rule is in delta mode. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct GosPolicy { + pub epsilon_staleness: f64, + pub sites: u32, + pub threshold_mode: GosThresholdMode, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum GosThresholdMode { + Isotropic, + Anisotropic, +} + +/// Sparse-delta semantics within a delta transmission rule. An absolute +/// threshold of zero sends every changed cell. When `gos` is present it +/// replaces the fixed threshold with the GOS norm-adaptive threshold. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct DeltaPolicy { + pub absolute_threshold: f64, + pub gos: Option, +} + +/// Inclusive bounds for one floating-point adaptation knob. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct AdaptiveF64Bounds { + pub min: f64, + pub max: f64, + pub max_step: f64, +} + +/// Inclusive bounds for one integer adaptation knob. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdaptiveU64Bounds { + pub min: u64, + pub max: u64, + pub max_step: u64, +} + +/// Guardrails for telemetry-driven runtime adaptation. This is an +/// authorization contract, not an instruction to mutate the active plan. +/// Every accepted change becomes a staged successor PhysicalPlan. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct RuntimeAdaptationPolicy { + pub enabled: bool, + pub not_before_unix_ms: u64, + pub max_evidence_age_ms: u64, + pub min_evidence_samples: u64, + pub sample_probability: Option, + pub emit_every_ms: Option, + pub delta_threshold: Option, + pub gos_epsilon_staleness: Option, +} + +impl Default for RuntimeAdaptationPolicy { + fn default() -> Self { + Self { + enabled: false, + not_before_unix_ms: 0, + max_evidence_age_ms: 0, + min_evidence_samples: 0, + sample_probability: None, + emit_every_ms: None, + delta_threshold: None, + gos_epsilon_staleness: None, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] +#[serde(deny_unknown_fields)] +pub struct RuntimeRulePolicy { + #[serde(default)] + pub sampling: SamplingPolicy, + pub delta: Option, + #[serde(default)] + pub adaptation: RuntimeAdaptationPolicy, +} + +/// Identity and sufficiency information for evidence authorizing one rule's +/// successor knobs. Raw measurements remain in the runtime-samples store; the +/// authorization boundary needs only their exact provenance and sample count. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct RuntimeAdaptationEvidence { + pub plan_id: u64, + pub plan_version: u64, + pub materialization: crate::sds::SummaryDefinitionId, + pub producer_id: String, + pub schema_id: String, + pub producer_version: String, + pub observed_at_unix_ms: u64, + pub sample_count: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct TransmissionPlan { + /// Absent only in legacy artifacts; catalog-aware validation requires it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub summary_catalog: Option, + pub envelope: PlanEnvelope, + pub frame_identity: FrameIdentityContract, + pub rules: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum SummaryFrameKind { + Full, + Delta, +} + +/// Identity attached to every summary record. Window bounds come from the +/// data point; the remaining fields are carried as reserved `asap.frame.*` +/// attributes until the modified-OTLP schema gains a dedicated message. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct SummaryFrameIdentity { + pub identity_version: u32, + pub plan_id: u64, + pub plan_version: u64, + pub backend_compat: String, + pub materialization: crate::sds::SummaryDefinitionId, + /// Canonical producer-side identity for one concrete retained-label group. + pub series_identity: String, + pub schema_id: String, + pub producer_id: String, + pub producer_epoch: String, + pub window_start_unix_nano: u64, + pub window_end_unix_nano: u64, + pub sequence: u64, + pub kind: SummaryFrameKind, + pub encoding: StateEncoding, + pub checkpoint_id: Option, + pub base_checkpoint_id: Option, +} + +#[derive(Debug, Error, PartialEq, Eq)] +pub enum TransmissionPlanError { + #[error("summary catalog mismatch: {0}")] + Catalog(String), + #[error("TransmissionPlan envelope differs from PrecomputePlan")] + EnvelopeMismatch, + #[error("transmission rules do not exactly match precompute producer bindings")] + ProducerSetMismatch, + #[error("invalid transmission rule for producer {0}")] + InvalidRule(String), + #[error("frame identity is invalid: {0}")] + InvalidFrame(String), + #[error("frame has no matching transmission rule")] + UnknownFrame, + #[error("runtime policy for producer {producer_id} is invalid: {reason}")] + InvalidRuntimePolicy { producer_id: String, reason: String }, + #[error("runtime adaptation successor is invalid: {0}")] + InvalidSuccessor(String), + #[error("runtime adaptation evidence for producer {0} is missing or invalid")] + InvalidAdaptationEvidence(String), + #[error("runtime adaptation for producer {producer_id} exceeds guardrails: {knob}")] + AdaptationOutOfBounds { + producer_id: String, + knob: &'static str, + }, +} + +fn validate_catalog_projection( + reference: Option<&crate::sds::CatalogGeneration>, + envelope: &PlanEnvelope, + materializations: impl IntoIterator, + catalog: &crate::summary_catalog::SummaryCatalog, +) -> Result<(), TransmissionPlanError> { + let expected = catalog + .reference() + .map_err(|error| TransmissionPlanError::Catalog(error.to_string()))?; + if reference != Some(&expected) + || envelope.plan_id != catalog.plan_id + || envelope.plan_version != catalog.plan_version + { + return Err(TransmissionPlanError::Catalog( + "missing or different snapshot reference".into(), + )); + } + for id in materializations { + if !catalog.materializations.contains_key(&id) { + return Err(TransmissionPlanError::Catalog(format!( + "unknown materialization {}", + id.as_u64() + ))); + } + } + Ok(()) +} + +impl CollectorPlan { + pub fn validate_against_catalog( + &self, + catalog: &crate::summary_catalog::SummaryCatalog, + ) -> Result<(), TransmissionPlanError> { + validate_catalog_projection( + self.summary_catalog.as_ref(), + &self.envelope, + self.materializations + .iter() + .map(|m| m.materialization) + .chain(self.transmission_rules.iter().map(|r| r.materialization)), + catalog, + ) + } +} + +impl TransmissionPlan { + pub fn validate_against_catalog( + &self, + catalog: &crate::summary_catalog::SummaryCatalog, + ) -> Result<(), TransmissionPlanError> { + validate_catalog_projection( + self.summary_catalog.as_ref(), + &self.envelope, + self.rules.iter().map(|r| r.materialization), + catalog, + ) + } + + pub fn validate(&self, precompute: &PrecomputePlan) -> Result<(), TransmissionPlanError> { + if self.summary_catalog != precompute.summary_catalog { + return Err(TransmissionPlanError::Catalog( + "transmission and precompute plans reference different catalog snapshots".into(), + )); + } + if self.envelope != precompute.envelope { + return Err(TransmissionPlanError::EnvelopeMismatch); + } + let expected: BTreeSet<_> = precompute + .producers + .iter() + .map(|producer| { + ( + producer.materialization, + producer.producer_id.as_str(), + producer.schema_id.as_str(), + ) + }) + .collect(); + let actual: BTreeSet<_> = self + .rules + .iter() + .map(|rule| { + ( + rule.materialization, + rule.producer_id.as_str(), + rule.schema_id.as_str(), + ) + }) + .collect(); + if expected != actual || actual.len() != self.rules.len() { + return Err(TransmissionPlanError::ProducerSetMismatch); + } + for rule in &self.rules { + let valid_checkpoint_cadence = match (rule.mode, rule.full_checkpoint_every_ms) { + (TransmissionMode::Full, None) => true, + (TransmissionMode::Delta, Some(full_every)) => { + rule.emit_every_ms > 0 + && full_every >= rule.emit_every_ms + && full_every % rule.emit_every_ms == 0 + } + _ => false, + }; + if rule.emit_every_ms == 0 + || rule.destination_ref.is_empty() + || !valid_checkpoint_cadence + { + return Err(TransmissionPlanError::InvalidRule(rule.producer_id.clone())); + } + let schema = precompute + .schemas + .iter() + .find(|schema| schema.materialization == rule.materialization) + .expect("producer set validation guarantees a matching schema"); + if !schema.encodings.contains(&rule.encoding) { + return Err(TransmissionPlanError::InvalidRule(rule.producer_id.clone())); + } + validate_runtime_rule_policy(rule, &schema.family)?; + } + Ok(()) + } + + /// Authorize a telemetry-driven successor without mutating this active + /// plan. Semantic identity, codecs, destination, transmission mode and + /// checkpoint cadence remain fixed. Only explicitly bounded runtime knobs + /// may move, and each changed rule needs fresh evidence attributed to the + /// exact active generation. + pub fn authorize_successor( + &self, + successor: &TransmissionPlan, + evidence: &[RuntimeAdaptationEvidence], + now_unix_ms: u64, + ) -> Result<(), TransmissionPlanError> { + if successor.envelope.plan_id != self.envelope.plan_id + || successor.envelope.plan_version != self.envelope.plan_version.saturating_add(1) + || successor.envelope.backend_compat != self.envelope.backend_compat + || successor.envelope.planner_revision != self.envelope.planner_revision + || successor.envelope.capability_snapshot_id != self.envelope.capability_snapshot_id + || successor.envelope.generated_at_unix_ms < self.envelope.generated_at_unix_ms + || successor.envelope.activation_unix_ms < successor.envelope.generated_at_unix_ms + || successor.rules.len() != self.rules.len() + || successor.frame_identity != self.frame_identity + { + return Err(TransmissionPlanError::InvalidSuccessor( + "successor must be the next version of the same semantic/capability generation" + .into(), + )); + } + + for current in &self.rules { + let Some(next) = successor.rules.iter().find(|candidate| { + candidate.materialization == current.materialization + && candidate.producer_id == current.producer_id + && candidate.schema_id == current.schema_id + }) else { + return Err(TransmissionPlanError::InvalidSuccessor(format!( + "missing rule for producer {}", + current.producer_id + ))); + }; + if current.mode != next.mode + || current.encoding != next.encoding + || current.full_checkpoint_every_ms != next.full_checkpoint_every_ms + || current.destination_ref != next.destination_ref + || current.runtime_policy.adaptation != next.runtime_policy.adaptation + || sampling_estimator(¤t.runtime_policy.sampling) + != sampling_estimator(&next.runtime_policy.sampling) + || delta_shape(¤t.runtime_policy.delta) + != delta_shape(&next.runtime_policy.delta) + { + return Err(TransmissionPlanError::InvalidSuccessor(format!( + "rule identity/codec/mode/guardrails drifted for producer {}", + current.producer_id + ))); + } + if current.emit_every_ms == next.emit_every_ms + && current.runtime_policy.sampling == next.runtime_policy.sampling + && current.runtime_policy.delta == next.runtime_policy.delta + { + continue; + } + let policy = ¤t.runtime_policy.adaptation; + if !policy.enabled || now_unix_ms < policy.not_before_unix_ms { + return Err(TransmissionPlanError::AdaptationOutOfBounds { + producer_id: current.producer_id.clone(), + knob: "adaptation_disabled_or_in_cooldown", + }); + } + let has_evidence = evidence.iter().any(|item| { + item.plan_id == self.envelope.plan_id + && item.plan_version == self.envelope.plan_version + && item.materialization == current.materialization + && item.producer_id == current.producer_id + && item.schema_id == current.schema_id + && !item.producer_version.trim().is_empty() + && item.sample_count >= policy.min_evidence_samples + && item.observed_at_unix_ms <= now_unix_ms + && now_unix_ms.saturating_sub(item.observed_at_unix_ms) + <= policy.max_evidence_age_ms + }); + if !has_evidence { + return Err(TransmissionPlanError::InvalidAdaptationEvidence( + current.producer_id.clone(), + )); + } + authorize_f64_change( + sampling_probability(¤t.runtime_policy.sampling), + sampling_probability(&next.runtime_policy.sampling), + policy.sample_probability.as_ref(), + ¤t.producer_id, + "sample_probability", + )?; + authorize_u64_change( + current.emit_every_ms, + next.emit_every_ms, + policy.emit_every_ms.as_ref(), + ¤t.producer_id, + "emit_every_ms", + )?; + authorize_f64_change( + delta_threshold(¤t.runtime_policy.delta), + delta_threshold(&next.runtime_policy.delta), + policy.delta_threshold.as_ref(), + ¤t.producer_id, + "delta_threshold", + )?; + authorize_f64_change( + gos_epsilon(¤t.runtime_policy.delta), + gos_epsilon(&next.runtime_policy.delta), + policy.gos_epsilon_staleness.as_ref(), + ¤t.producer_id, + "gos_epsilon_staleness", + )?; + } + Ok(()) + } + + pub fn validate_frame( + &self, + frame: &SummaryFrameIdentity, + ) -> Result<(), TransmissionPlanError> { + if frame.identity_version != self.frame_identity.identity_version + || frame.plan_id != self.envelope.plan_id + || frame.plan_version != self.envelope.plan_version + || frame.backend_compat != self.envelope.backend_compat + || frame.series_identity.is_empty() + || frame.producer_epoch.is_empty() + || frame.sequence == 0 + || frame.window_start_unix_nano >= frame.window_end_unix_nano + || (frame.kind == SummaryFrameKind::Full + && self.frame_identity.require_checkpoint_for_full + && frame.checkpoint_id.is_none()) + || (frame.kind == SummaryFrameKind::Delta + && self.frame_identity.require_base_checkpoint_for_delta + && frame.base_checkpoint_id.is_none()) + { + return Err(TransmissionPlanError::InvalidFrame( + "identity/lifecycle/window/checkpoint fields do not satisfy the active contract" + .into(), + )); + } + if self.rules.iter().any(|rule| { + rule.materialization == frame.materialization + && rule.producer_id == frame.producer_id + && rule.schema_id == frame.schema_id + // A delta rule necessarily emits periodic full checkpoints; + // a full-only rule must never emit deltas. + && (frame.kind == SummaryFrameKind::Full + || rule.mode == TransmissionMode::Delta) + && rule.encoding == frame.encoding + }) { + Ok(()) + } else { + Err(TransmissionPlanError::UnknownFrame) + } + } +} + +fn validate_runtime_rule_policy( + rule: &TransmissionRule, + family: &StateFamilyContract, +) -> Result<(), TransmissionPlanError> { + let invalid = |reason: &str| TransmissionPlanError::InvalidRuntimePolicy { + producer_id: rule.producer_id.clone(), + reason: reason.into(), + }; + if let SamplingPolicy::Fixed { + probability, + estimator, + } = &rule.runtime_policy.sampling + { + if !probability.is_finite() || !(0.0..=1.0).contains(probability) || *probability == 0.0 { + return Err(invalid("sample probability must be finite and in (0, 1]")); + } + let supported = matches!( + (family, *estimator), + ( + StateFamilyContract::Sketch { + algorithm: SketchAlgorithm::Hll, + .. + }, + SamplingEstimator::HashThreshold, + ) | ( + StateFamilyContract::Sketch { + algorithm: SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap, + .. + }, + SamplingEstimator::GeometricAdmission, + ) + ); + if !supported { + return Err(invalid( + "sampling estimator is not implemented for the materialization family", + )); + } + } + + if (rule.mode == TransmissionMode::Delta) != rule.runtime_policy.delta.is_some() { + return Err(invalid( + "delta policy must be present exactly when transmission mode is delta", + )); + } + if let Some(delta) = &rule.runtime_policy.delta { + if !delta.absolute_threshold.is_finite() || delta.absolute_threshold < 0.0 { + return Err(invalid("delta threshold must be finite and non-negative")); + } + if !matches!( + family, + StateFamilyContract::Sketch { + algorithm: SketchAlgorithm::DDSketch + | SketchAlgorithm::Hll + | SketchAlgorithm::Cms + | SketchAlgorithm::CmsWithHeap + | SketchAlgorithm::CountSketch + | SketchAlgorithm::CountSketchWithHeap, + .. + } + ) { + return Err(invalid( + "delta transmission is not implemented for the materialization family", + )); + } + if let Some(gos) = &delta.gos { + if !matches!( + family, + StateFamilyContract::Sketch { + algorithm: SketchAlgorithm::CountSketch | SketchAlgorithm::CountSketchWithHeap, + .. + } + ) || !gos.epsilon_staleness.is_finite() + || !(0.0..=1.0).contains(&gos.epsilon_staleness) + || gos.epsilon_staleness == 0.0 + || gos.sites == 0 + { + return Err(invalid( + "GOS requires a CountSketch family, epsilon in (0, 1], and at least one site", + )); + } + } + } + + let adaptation = &rule.runtime_policy.adaptation; + if adaptation.enabled + && (adaptation.max_evidence_age_ms == 0 || adaptation.min_evidence_samples == 0) + { + return Err(invalid( + "enabled adaptation requires non-zero evidence age and sample-count requirements", + )); + } + validate_f64_bounds(adaptation.sample_probability.as_ref(), 0.0, 1.0) + .map_err(|reason| invalid(reason))?; + validate_f64_bounds(adaptation.delta_threshold.as_ref(), 0.0, f64::MAX) + .map_err(|reason| invalid(reason))?; + validate_f64_bounds(adaptation.gos_epsilon_staleness.as_ref(), 0.0, 1.0) + .map_err(|reason| invalid(reason))?; + if let Some(bounds) = &adaptation.emit_every_ms { + if bounds.min == 0 + || bounds.min > bounds.max + || bounds.max_step == 0 + || !(bounds.min..=bounds.max).contains(&rule.emit_every_ms) + { + return Err(invalid("emit interval guardrails are invalid")); + } + } + if let Some(bounds) = &adaptation.sample_probability { + let current = sampling_probability(&rule.runtime_policy.sampling); + if current < bounds.min || current > bounds.max { + return Err(invalid( + "current sampling probability is outside guardrails", + )); + } + } + if let Some(bounds) = &adaptation.delta_threshold { + let Some(current) = rule + .runtime_policy + .delta + .as_ref() + .map(|policy| policy.absolute_threshold) + else { + return Err(invalid("delta guardrails require an active delta policy")); + }; + if current < bounds.min || current > bounds.max { + return Err(invalid("current delta threshold is outside guardrails")); + } + } + if let Some(bounds) = &adaptation.gos_epsilon_staleness { + let Some(current) = rule + .runtime_policy + .delta + .as_ref() + .and_then(|policy| policy.gos.as_ref()) + .map(|gos| gos.epsilon_staleness) + else { + return Err(invalid("GOS guardrails require an active GOS policy")); + }; + if current < bounds.min || current > bounds.max { + return Err(invalid("current GOS epsilon is outside guardrails")); + } + } + Ok(()) +} + +fn validate_f64_bounds( + bounds: Option<&AdaptiveF64Bounds>, + domain_min: f64, + domain_max: f64, +) -> Result<(), &'static str> { + let Some(bounds) = bounds else { + return Ok(()); + }; + if !bounds.min.is_finite() + || !bounds.max.is_finite() + || !bounds.max_step.is_finite() + || bounds.min < domain_min + || bounds.max > domain_max + || bounds.min > bounds.max + || bounds.max_step <= 0.0 + { + Err("floating-point adaptation guardrails are invalid") + } else { + Ok(()) + } +} + +fn sampling_probability(policy: &SamplingPolicy) -> f64 { + match policy { + SamplingPolicy::Disabled => 1.0, + SamplingPolicy::Fixed { probability, .. } => *probability, + } +} + +fn sampling_estimator(policy: &SamplingPolicy) -> Option { + match policy { + SamplingPolicy::Disabled => None, + SamplingPolicy::Fixed { estimator, .. } => Some(*estimator), + } +} + +fn delta_threshold(policy: &Option) -> f64 { + policy + .as_ref() + .map(|policy| policy.absolute_threshold) + .unwrap_or(0.0) +} + +fn gos_epsilon(policy: &Option) -> f64 { + policy + .as_ref() + .and_then(|policy| policy.gos.as_ref()) + .map(|gos| gos.epsilon_staleness) + .unwrap_or(0.0) +} + +fn delta_shape(policy: &Option) -> Option<(Option<(u32, GosThresholdMode)>,)> { + policy.as_ref().map(|policy| { + (policy + .gos + .as_ref() + .map(|gos| (gos.sites, gos.threshold_mode)),) + }) +} + +fn authorize_f64_change( + current: f64, + next: f64, + bounds: Option<&AdaptiveF64Bounds>, + producer_id: &str, + knob: &'static str, +) -> Result<(), TransmissionPlanError> { + if current == next { + return Ok(()); + } + let allowed = bounds.is_some_and(|bounds| { + next.is_finite() + && (bounds.min..=bounds.max).contains(&next) + && (next - current).abs() <= bounds.max_step + }); + if allowed { + Ok(()) + } else { + Err(TransmissionPlanError::AdaptationOutOfBounds { + producer_id: producer_id.into(), + knob, + }) + } +} + +fn authorize_u64_change( + current: u64, + next: u64, + bounds: Option<&AdaptiveU64Bounds>, + producer_id: &str, + knob: &'static str, +) -> Result<(), TransmissionPlanError> { + if current == next { + return Ok(()); + } + let allowed = bounds.is_some_and(|bounds| { + (bounds.min..=bounds.max).contains(&next) && current.abs_diff(next) <= bounds.max_step + }); + if allowed { + Ok(()) + } else { + Err(TransmissionPlanError::AdaptationOutOfBounds { + producer_id: producer_id.into(), + knob, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use planner_types::post_asap::SketchParams; + + #[test] + fn runtime_policy_is_family_and_mode_checked() { + let mut rule = TransmissionRule { + materialization: crate::PolicyFingerprint(1).into(), + producer_id: "test".into(), + schema_id: "test".into(), + mode: TransmissionMode::Full, + encoding: StateEncoding::SketchlibProtobufV1, + emit_every_ms: 60_000, + full_checkpoint_every_ms: None, + destination_ref: "backend".into(), + runtime_policy: Default::default(), + }; + rule.runtime_policy.sampling = SamplingPolicy::Fixed { + probability: 0.5, + estimator: SamplingEstimator::HashThreshold, + }; + let hll = StateFamilyContract::Sketch { + algorithm: SketchAlgorithm::Hll, + parameters: SketchParams::Hll { precision: 14 }, + }; + let count_sketch = StateFamilyContract::Sketch { + algorithm: SketchAlgorithm::CountSketch, + parameters: SketchParams::CountSketch { + width: 128, + depth: 4, + }, + }; + validate_runtime_rule_policy(&rule, &hll).expect("HLL supports hash-threshold sampling"); + assert!(matches!( + validate_runtime_rule_policy(&rule, &count_sketch), + Err(TransmissionPlanError::InvalidRuntimePolicy { .. }) + )); + + rule.runtime_policy.sampling = SamplingPolicy::Disabled; + rule.mode = TransmissionMode::Delta; + rule.full_checkpoint_every_ms = Some(300_000); + rule.runtime_policy.delta = Some(DeltaPolicy { + absolute_threshold: 0.0, + gos: Some(GosPolicy { + epsilon_staleness: 0.02, + sites: 2, + threshold_mode: GosThresholdMode::Isotropic, + }), + }); + validate_runtime_rule_policy(&rule, &count_sketch).expect("CountSketch supports delta GOS"); + assert!(matches!( + validate_runtime_rule_policy(&rule, &hll), + Err(TransmissionPlanError::InvalidRuntimePolicy { .. }) + )); + } +} diff --git a/crates/asap_types/src/query_plan.rs b/crates/asap_types/src/query_plan.rs new file mode 100644 index 000000000..945608afe --- /dev/null +++ b/crates/asap_types/src/query_plan.rs @@ -0,0 +1,677 @@ +//! Shared installed QueryPlan contract and activation validation. +//! +//! ASAPPlanner owns semantic post-ASAP IR. Physical compilation binds every +//! maintained-summary leaf to one materialization and lowers edges to stable +//! node IDs. Serving executes this graph without reconstructing Planner IR or +//! searching for compatible materializations. + +pub mod logical; + +use std::collections::{BTreeMap, BTreeSet}; + +use planner_types::post_asap::SketchQuery; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +pub use crate::QueryLanguage; +use crate::{sds::SummaryDefinitionId, PolicyFingerprint}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct QueryPlan { + pub plan_id: u64, + pub plan_version: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub clickhouse_context: Option, + pub entries: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct ClickHousePlanningContext { + pub tables: std::collections::HashMap, + pub accuracy: planner_types::types::AccuracyTarget, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct FixedEvaluationRange { + pub start_ms: u64, + pub end_ms: u64, + pub cumulative: bool, +} + +impl QueryPlan { + pub fn empty() -> Self { + Self { + plan_id: 0, + plan_version: 0, + clickhouse_context: None, + entries: BTreeMap::new(), + } + } + + pub fn lookup(&self, promql: &str) -> Result<&QueryPlanEntry, QueryPlanError> { + let identity = canonical_promql(promql)?; + self.lookup_canonical(QueryLanguage::PromQl, &identity) + } + + pub fn lookup_canonical( + &self, + language: QueryLanguage, + identity: &str, + ) -> Result<&QueryPlanEntry, QueryPlanError> { + let key = Self::catalog_key(language, identity); + self.entries + .get(&key) + .filter(|entry| entry.language == language) + .ok_or_else(|| QueryPlanError::QueryNotPlanned(identity.into())) + } + + pub fn catalog_key(language: QueryLanguage, identity: &str) -> String { + match language { + QueryLanguage::PromQl => identity.to_owned(), + QueryLanguage::MetricsQl => format!("metricsql:{identity}"), + QueryLanguage::ClickHouseSql => format!("clickhouse:{identity}"), + } + } + + pub fn lookup_clickhouse( + &self, + canonical_sql: &str, + ) -> Result<&QueryPlanEntry, QueryPlanError> { + self.lookup_canonical(QueryLanguage::ClickHouseSql, canonical_sql) + } + + /// Validate semantic bindings against the authoritative snapshot before use. + pub fn validate_against_catalog( + &self, + catalog: &crate::summary_catalog::SummaryCatalog, + ) -> Result<(), QueryPlanError> { + catalog + .validate() + .map_err(|error| QueryPlanError::Invalid(error.to_string()))?; + if self.plan_id != catalog.plan_id || self.plan_version != catalog.plan_version { + return Err(QueryPlanError::Invalid( + "QueryPlan and SummaryCatalog have different plan identity/version".into(), + )); + } + let available = catalog + .materializations + .keys() + .copied() + .map(Into::into) + .collect(); + self.validate(&available)?; + for entry in self.entries.values() { + for binding in entry.materialization_bindings() { + let identity = catalog + .materializations + .get(&binding.materialization) + .ok_or_else(|| { + QueryPlanError::Invalid( + "query binding references absent catalog materialization".into(), + ) + })?; + let _data = &catalog.data_descriptors[&identity.data_descriptor_id]; + if binding.window_ms == 0 { + return Err(QueryPlanError::Invalid( + "zero physical pane duration".into(), + )); + } + if binding.pane_origin_ms != identity.pane_origin_ms { + return Err(QueryPlanError::Invalid( + "query pane origin differs from catalog definition".into(), + )); + } + } + for node in entry.nodes.values() { + let QueryPlanNode::ExactReadout { input, readout } = node else { + continue; + }; + if !matches!(readout, ExactReadout::Increase | ExactReadout::Rate) { + continue; + } + let Some(QueryPlanNode::ReadMaterialization { binding }) = entry.nodes.get(input) + else { + return Err(QueryPlanError::Invalid( + "counter readout must directly consume one catalog materialization".into(), + )); + }; + let identity = &catalog.materializations[&binding.materialization]; + let descriptor = &catalog.summary_descriptors[&identity.summary_descriptor_id]; + if !matches!( + descriptor.fidelity, + crate::sds::FidelityGuarantee::ExactCounter { + full_pane_coverage_required: true, + .. + } + ) { + return Err(QueryPlanError::Invalid( + "rate/increase binding does not reference an exact counter SDS".into(), + )); + } + } + } + Ok(()) + } + + pub fn validate(&self, available: &BTreeSet) -> Result<(), QueryPlanError> { + if self.plan_id != 0 && self.plan_version == 0 { + return Err(QueryPlanError::Invalid( + "non-bootstrap QueryPlan has zero plan_version".into(), + )); + } + for (identity, entry) in &self.entries { + let expected = Self::catalog_key(entry.language, &entry.canonical_query); + if identity != &expected { + return Err(QueryPlanError::Invalid(format!( + "query map key `{identity}` differs from entry identity `{}`", + entry.canonical_query + ))); + } + match entry.language { + QueryLanguage::PromQl | QueryLanguage::MetricsQl + if entry.fixed_evaluation.is_some() => + { + return Err(QueryPlanError::Invalid( + "PromQL query entry carries a ClickHouse fixed evaluation range".into(), + )); + } + QueryLanguage::ClickHouseSql => { + if self.clickhouse_context.is_none() { + return Err(QueryPlanError::Invalid( + "ClickHouse query entry has no planning context".into(), + )); + } + let Some(range) = entry.fixed_evaluation else { + return Err(QueryPlanError::Invalid( + "ClickHouse query entry has no fixed evaluation range".into(), + )); + }; + if range.end_ms <= range.start_ms { + return Err(QueryPlanError::Invalid( + "ClickHouse query entry has an empty evaluation range".into(), + )); + } + } + QueryLanguage::PromQl | QueryLanguage::MetricsQl => {} + } + entry.validate(available)?; + } + Ok(()) + } +} + +pub use crate::executable_plan::QueryNodeId; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct QueryPlanEntry { + #[serde(default)] + pub language: QueryLanguage, + pub query_id: String, + #[serde(alias = "canonical_promql")] + pub canonical_query: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fixed_evaluation: Option, + pub root: QueryNodeId, + pub nodes: BTreeMap, + pub instant: InstantExecution, + pub fallback: FallbackPolicy, +} + +fn topological_order( + root: QueryNodeId, + nodes: &BTreeMap, +) -> Result, QueryPlanError> { + fn visit( + id: QueryNodeId, + nodes: &BTreeMap, + visiting: &mut BTreeSet, + visited: &mut BTreeSet, + out: &mut Vec, + ) -> Result<(), QueryPlanError> { + if visited.contains(&id) { + return Ok(()); + } + if !visiting.insert(id) { + return Err(QueryPlanError::Invalid(format!( + "cycle detected at query node {}", + id.0 + ))); + } + let node = nodes + .get(&id) + .ok_or_else(|| QueryPlanError::Invalid(format!("missing query node {}", id.0)))?; + for input in node.inputs() { + visit(*input, nodes, visiting, visited, out)?; + } + visiting.remove(&id); + visited.insert(id); + out.push(id); + Ok(()) + } + let mut out = Vec::with_capacity(nodes.len()); + visit( + root, + nodes, + &mut BTreeSet::new(), + &mut BTreeSet::new(), + &mut out, + )?; + Ok(out) +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct InstantExecution { + pub lookback_ms: u64, + pub full_history: bool, + pub cumulative_readout: bool, +} + +impl QueryPlanEntry { + /// Replace an explicit planner fallback cut with a typed external-exact leaf. + /// The control plane chooses the cut; serving only executes the published DAG. + pub fn bind_external_exact_leaf( + &mut self, + node_id: QueryNodeId, + request: ExternalExactRequest, + ) -> Result<(), QueryPlanError> { + if request.language != self.language { + return Err(QueryPlanError::Invalid( + "external exact language differs from its query plan".into(), + )); + } + if !request.input_contracts.is_empty() { + return Err(QueryPlanError::Invalid( + "leaf binding cannot declare DAG input contracts".into(), + )); + } + match self.nodes.get(&node_id) { + Some(QueryPlanNode::ExactFallback { .. }) => {} + Some(_) => { + return Err(QueryPlanError::Invalid( + "external exact binding must replace a planner fallback cut".into(), + )) + } + None => { + return Err(QueryPlanError::Invalid( + "external exact cut node is absent".into(), + )) + } + } + self.nodes.insert( + node_id, + QueryPlanNode::ExternalExact { + request, + inputs: Vec::new(), + }, + ); + Ok(()) + } + + /// Materializations this executable DAG reads, in stable node order. + /// Serving uses this set for readiness accounting; it never performs a + /// catalog candidate search to reconstruct dependencies. + pub fn materialization_bindings(&self) -> Vec<&MaterializationBinding> { + self.nodes + .values() + .filter_map(|node| match node { + QueryPlanNode::ReadMaterialization { binding } => Some(binding), + _ => None, + }) + .collect() + } + + pub fn topological_order(&self) -> Result, QueryPlanError> { + topological_order(self.root, &self.nodes) + } + + pub fn topological_order_from( + &self, + root: QueryNodeId, + ) -> Result, QueryPlanError> { + topological_order(root, &self.nodes) + } + + /// Validate references, bindings, reachability, and cycles before activation. + pub fn validate(&self, available: &BTreeSet) -> Result<(), QueryPlanError> { + if !self.nodes.contains_key(&self.root) { + return Err(QueryPlanError::Invalid(format!( + "query `{}` has missing root {}", + self.query_id, self.root.0 + ))); + } + for (id, node) in &self.nodes { + if let QueryPlanNode::Logical { operator, inputs } = node { + operator.validate(inputs.len())?; + } + if matches!(node, QueryPlanNode::Scalar { value } if !value.is_finite()) { + return Err(QueryPlanError::Invalid("non-finite scalar constant".into())); + } + if let QueryPlanNode::ExternalExact { request, inputs } = node { + if request.expression.trim().is_empty() { + return Err(QueryPlanError::Invalid( + "external exact expression must not be empty".into(), + )); + } + if request.input_contracts.len() != inputs.len() { + return Err(QueryPlanError::Invalid( + "external exact input contracts must match DAG inputs".into(), + )); + } + if request.input_contracts.iter().any(|contract| { + matches!(contract, ExternalExactInput::CandidateMembership { item_label } if item_label.is_empty()) + }) { + return Err(QueryPlanError::Invalid( + "external exact candidate item label must not be empty".into(), + )); + } + } + if let QueryPlanNode::CandidateTopK { + k, completeness, .. + } = node + { + if *k == 0 { + return Err(QueryPlanError::Invalid( + "CandidateTopK requires k > 0".into(), + )); + } + if matches!( + completeness, + CandidateCompleteness::Certified { guarantee } + if guarantee.metric + != planner_types::post_asap::ErrorMetric::TopKMembership + || guarantee.bound.evaluate().is_none() + || guarantee.failure_probability.evaluate().is_none() + ) { + return Err(QueryPlanError::Invalid( + "invalid CandidateTopK completeness certificate".into(), + )); + } + } + for input in node.inputs() { + if !self.nodes.contains_key(input) { + return Err(QueryPlanError::Invalid(format!( + "query `{}` node {} references missing input {}", + self.query_id, id.0, input.0 + ))); + } + } + if let QueryPlanNode::ReadMaterialization { binding } = node { + if binding.readout_lookback_ms == Some(0) { + return Err(QueryPlanError::Invalid( + "zero semantic readout lookback".into(), + )); + } + if !available.contains(&binding.materialization.fingerprint()) { + return Err(QueryPlanError::Invalid(format!( + "query `{}` node {} references absent materialization {}", + self.query_id, + id.0, + binding.materialization.as_u64() + ))); + } + } + } + let order = self.topological_order()?; + if order.len() != self.nodes.len() { + return Err(QueryPlanError::Invalid(format!( + "query `{}` contains unreachable nodes", + self.query_id + ))); + } + Ok(()) + } +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum FallbackPolicy { + ExactBackend, + Reject, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct MaterializationBinding { + pub materialization: SummaryDefinitionId, + /// Query operator grouping applied while folding those SIDs. + pub output_grouping: PhysicalGrouping, + /// Labels whose values form an item identity inside a keyed sketch. + #[serde(default, alias = "itemLabels", skip_serializing_if = "Vec::is_empty")] + pub item_labels: Vec, + pub window_ms: u64, + /// Unix millisecond timestamp on the materialized pane-boundary grid. + /// Legacy plans deserialize this as unknown and fall back at read time. + #[serde( + default, + alias = "paneOriginMs", + skip_serializing_if = "Option::is_none" + )] + pub pane_origin_ms: Option, + /// Semantic query lookback, independent of the physical pane duration. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub readout_lookback_ms: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "mode", content = "keys", rename_all = "snake_case")] +pub enum PhysicalGrouping { + PerEntity, + Reduce(Vec), +} + +/// Result shape promised by an external exact engine. The backend uses this +/// contract to type-check downstream DAG nodes without depending on an +/// engine-specific response envelope. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum ExternalExactOutput { + Scalar, + InstantVector, + RangeVector, + Relation { schema: serde_json::Value }, +} + +/// How an ordinary DAG input constrains an external exact evaluation. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum ExternalExactInput { + CandidateMembership { item_label: String }, +} + +/// Language-neutral request contract for an exact subtree. Evaluation time is +/// inherited from the containing QueryPlanEntry, avoiding a second time-range +/// envelope that could drift from the installed query plan. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct ExternalExactRequest { + pub language: QueryLanguage, + pub expression: String, + pub output: ExternalExactOutput, + /// Engine parameters forwarded without embedding transport details in the DAG. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub parameters: BTreeMap, + /// Optional parameter names populated from the query entry's evaluation range. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub start_parameter: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub end_parameter: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub input_contracts: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "op", rename_all = "snake_case", deny_unknown_fields)] +pub enum QueryPlanNode { + RelationalJoin { + inputs: [QueryNodeId; 2], + join_kind: planner_types::pre_asap::JoinKind, + pred: serde_json::Value, + left_schema: planner_types::post_asap::SummarySchema, + right_schema: planner_types::post_asap::SummarySchema, + output_schema: planner_types::post_asap::SummarySchema, + }, + Relational { + input: QueryNodeId, + /// Serialized planner-owned operation. Keeping the wire form here makes + /// the published catalog Send + Sync even though the planner AST uses Rc. + operation: serde_json::Value, + input_schema: planner_types::post_asap::SummarySchema, + output_schema: planner_types::post_asap::SummarySchema, + }, + Logical { + operator: logical::LogicalOperator, + inputs: Vec, + }, + Scalar { + value: f64, + }, + Binary { + inputs: [QueryNodeId; 2], + operator: planner_types::pre_asap::ArithmeticOpKind, + }, + ReduceSum { + input: QueryNodeId, + grouping: PhysicalGrouping, + }, + ReadMaterialization { + binding: MaterializationBinding, + }, + SummaryEstimate { + input: QueryNodeId, + query: QueryReadout, + }, + ExactReadout { + input: QueryNodeId, + readout: ExactReadout, + }, + SummaryMerge { + inputs: Vec, + }, + /// Use an approximate heap only as a membership sidecar, then rerank the + /// matching exact counter readouts. `inputs[0]` is candidate membership; + /// `inputs[1]` is the authoritative exact value vector. + CandidateTopK { + inputs: [QueryNodeId; 2], + k: u64, + grouping: logical::Grouping, + completeness: CandidateCompleteness, + }, + /// An exact subtree evaluated outside ASAP. Its results enter the query DAG + /// like any other node output and may depend on summary-produced inputs. + ExternalExact { + request: ExternalExactRequest, + inputs: Vec, + }, + ExactFallback { + reason: String, + }, +} + +impl QueryPlanNode { + pub fn inputs(&self) -> &[QueryNodeId] { + match self { + Self::Scalar { .. } | Self::ReadMaterialization { .. } | Self::ExactFallback { .. } => { + &[] + } + Self::Binary { inputs, .. } | Self::RelationalJoin { inputs, .. } => inputs, + Self::ReduceSum { input, .. } + | Self::Relational { input, .. } + | Self::SummaryEstimate { input, .. } + | Self::ExactReadout { input, .. } => std::slice::from_ref(input), + Self::SummaryMerge { inputs } + | Self::Logical { inputs, .. } + | Self::ExternalExact { inputs, .. } => inputs, + Self::CandidateTopK { inputs, .. } => inputs, + } + } +} + +pub use planner_types::post_asap::CandidateCompleteness; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ExactReadout { + Sum, + Count, + Increase, + Rate, + Max, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum QueryReadout { + FrequencyL2, + FrequencyEntropy, + Quantile { + q: f64, + }, + PointCount { + key: planner_types::pre_asap::ColumnRef, + value: Option, + }, + Cardinality, + TopK { + k: usize, + }, +} + +impl From for QueryReadout { + fn from(query: SketchQuery) -> Self { + match query { + SketchQuery::FrequencyL2 => Self::FrequencyL2, + SketchQuery::FrequencyEntropy => Self::FrequencyEntropy, + SketchQuery::Quantile { q } => Self::Quantile { q }, + SketchQuery::PointCount { key, value } => Self::PointCount { key, value }, + SketchQuery::Cardinality => Self::Cardinality, + SketchQuery::TopK { k } => Self::TopK { k }, + } + } +} + +impl From for SketchQuery { + fn from(query: QueryReadout) -> Self { + match query { + QueryReadout::FrequencyL2 => Self::FrequencyL2, + QueryReadout::FrequencyEntropy => Self::FrequencyEntropy, + QueryReadout::Quantile { q } => Self::Quantile { q }, + QueryReadout::PointCount { key, value } => Self::PointCount { key, value }, + QueryReadout::Cardinality => Self::Cardinality, + QueryReadout::TopK { k } => Self::TopK { k }, + } + } +} + +#[derive(Debug, Error)] +pub enum QueryPlanError { + #[error("invalid PromQL query identity: {0}")] + InvalidPromql(String), + #[error("query is absent from the active QueryPlan: {0}")] + QueryNotPlanned(String), + #[error("post-ASAP DAG cannot be represented by the query executor: {0}")] + UnsupportedNode(String), + #[error("invalid QueryPlan: {0}")] + Invalid(String), +} + +pub fn canonical_promql(query: &str) -> Result { + promql_parser::parser::parse(query.trim()) + .map(|expr| expr.to_string()) + .map_err(|error| QueryPlanError::InvalidPromql(error.to_string())) +} + +#[cfg(test)] +mod contract_tests { + // Installed plans cross producer/query threads without Planner Rc state. + #[test] + fn installed_query_contract_is_send_sync() { + fn assert_send_sync() {} + assert_send_sync::(); + assert_send_sync::(); + } +} diff --git a/crates/asap_types/src/query_plan/logical.rs b/crates/asap_types/src/query_plan/logical.rs new file mode 100644 index 000000000..5c3f6e138 --- /dev/null +++ b/crates/asap_types/src/query_plan/logical.rs @@ -0,0 +1,155 @@ +//! Typed installed residual operators; no Planner selection or AST lowering. +use super::QueryPlanError; +use promql_parser::parser::{self, Expr}; +use serde::{Deserialize, Serialize}; +fn invalid(message: impl Into) -> QueryPlanError { + QueryPlanError::Invalid(message.into()) +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum LogicalOperator { + /// A maximal exact scalar/vector subtree evaluated by Prometheus. + ExactSubquery { + query: String, + }, + /// Prometheus exact subtree whose selectors are restricted at runtime by + /// the candidate vector produced by its single input. + CandidateExactSubquery { + query: String, + item_label: String, + }, + Scan { + metric: Option, + matchers: Vec, + range_ms: Option, + offset_ms: i64, + }, + UnaryNegate, + VectorToScalar, + Aggregate { + operation: Aggregation, + grouping: Grouping, + }, + /// PromQL `topk(k, vector)` selection over values produced by the child. + /// This is distinct from a frequency-sketch TopK readout: any exact or + /// summary-backed instant-vector child may feed this query-time operator. + TopKSelection { + k: u64, + grouping: Grouping, + }, + Binary { + operation: BinaryOperation, + return_bool: bool, + }, + Temporal { + operation: TemporalOperation, + }, + Sort { + descending: bool, + }, + HistogramQuantile, + Subquery { + range_ms: u64, + step_ms: u64, + offset_ms: i64, + }, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct Grouping { + pub labels: Vec, + pub without: bool, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct LabelMatcher { + pub name: String, + pub value: String, + pub operation: LabelMatch, +} +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum LabelMatch { + Equal, + NotEqual, + Regex, + NotRegex, +} +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum Aggregation { + Sum, + Max, + Min, + Avg, + Count, +} +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum BinaryOperation { + Add, + Sub, + Mul, + Div, + Mod, + Pow, + Equal, + NotEqual, + Less, + LessEqual, + Greater, + GreaterEqual, +} +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum TemporalOperation { + Rate, + Increase, + Avg, + Max, + Min, + Sum, + Count, +} + +impl LogicalOperator { + pub fn validate(&self, inputs: usize) -> Result<(), QueryPlanError> { + let expected = match self { + Self::Scan { .. } | Self::ExactSubquery { .. } => 0, + Self::CandidateExactSubquery { .. } => 1, + Self::Binary { .. } | Self::HistogramQuantile => 2, + _ => 1, + }; + if inputs != expected { + return Err(invalid("logical operator input arity mismatch")); + } + if matches!( + self, + Self::Scan { + range_ms: Some(0), + .. + } + ) { + return Err(invalid("zero range")); + } + if let Self::ExactSubquery { query } | Self::CandidateExactSubquery { query, .. } = self { + let parsed = parser::parse(query).map_err(|e| invalid(e.to_string()))?; + if matches!(parsed, Expr::MatrixSelector(_) | Expr::Subquery(_)) { + return Err(invalid( + "exact subtree boundary must return scalar or instant vector", + )); + } + } + if let Self::Subquery { + range_ms, step_ms, .. + } = self + { + if *range_ms == 0 || *step_ms == 0 || range_ms / step_ms > 100_000 { + return Err(invalid("invalid or excessive subquery grid")); + } + } + Ok(()) + } +} diff --git a/crates/asap_types/src/sds.rs b/crates/asap_types/src/sds.rs index 23c3cc48b..6b3133699 100644 --- a/crates/asap_types/src/sds.rs +++ b/crates/asap_types/src/sds.rs @@ -1,5 +1,7 @@ //! Shared SDS metadata contracts. Summary payload bytes remain storage-engine //! owned; catalogs and inventories contain identities and state references only. +pub const TIMESTAMPED_OBSERVATION_SEMANTICS: &str = "asap.timestamped-observations.v2"; + use crate::{AggregationType, PrecomputeMaterialization}; use planner_types::post_asap::{SketchAlgorithm, SketchParams, SummaryFamilyType}; use serde::{Deserialize, Serialize}; @@ -115,6 +117,20 @@ pub struct SummaryInstanceCoordinates { pub group_values: BTreeMap, } +impl SummaryInstanceCoordinates { + pub fn instance_id(&self) -> Result { + let bytes = + serde_json::to_vec(&self.group_values).map_err(|error| SdsError(error.to_string()))?; + SummaryInstanceId::new(format!( + "summary-instance:v1:{}:{}:{}:{}", + self.summary_definition_id.as_u64(), + self.time_range.start_ms, + self.time_range.end_ms, + xxhash_rust::xxh64::xxh64(&bytes, 0) + )) + } +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct SummaryWindowCompletion { @@ -729,8 +745,15 @@ impl FidelityGuarantee { #[serde(deny_unknown_fields)] #[serde(rename_all = "snake_case")] pub enum DataSourceIdentity { - TimeSeries { metric: String }, - Table { table_ref: String }, + TimeSeries { + metric: String, + }, + Table { + table_ref: String, + }, + Derived { + input: crate::derived_input::DerivedInputIdentity, + }, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -829,7 +852,7 @@ impl DataDescriptor { pub fn time_series_metric(&self) -> Option<&str> { match &self.source { DataSourceIdentity::TimeSeries { metric } => Some(metric), - DataSourceIdentity::Table { .. } => None, + DataSourceIdentity::Table { .. } | DataSourceIdentity::Derived { .. } => None, } } @@ -929,6 +952,9 @@ impl DataDescriptor { &self.id } pub fn validate(&self) -> Result<(), SdsError> { + if let DataSourceIdentity::Derived { input } = &self.source { + input.validate().map_err(SdsError)?; + } self.value_projection.validate().map_err(SdsError)?; self.group_by_keys.validate().map_err(SdsError)?; if matches!(self.source, DataSourceIdentity::TimeSeries { .. }) diff --git a/crates/asap_types/src/summary_catalog.rs b/crates/asap_types/src/summary_catalog.rs index 5f31ee43d..d6d404b0e 100644 --- a/crates/asap_types/src/summary_catalog.rs +++ b/crates/asap_types/src/summary_catalog.rs @@ -103,14 +103,7 @@ impl SummaryCatalog { .map(|config| { let summary = SummaryDescriptor::from_config(config) .map_err(|error| SummaryCatalogError::Descriptor(error.to_string()))?; - let source = config.table_name.as_ref().map_or_else( - || DataSourceIdentity::TimeSeries { - metric: config.metric.clone(), - }, - |table_ref| DataSourceIdentity::Table { - table_ref: table_ref.clone(), - }, - ); + let source = config.source_identity(); let value_projection = config.effective_value_projection().clone(); let data = DataDescriptor::new_typed( source, @@ -122,7 +115,7 @@ impl SummaryCatalog { if config.table_name.is_some() && !config.grouping_labels.is_empty() { crate::grouping_projection::TABLE_GROUP_OBSERVATION_SEMANTICS } else { - "asap.timestamped-observations.v2" + crate::sds::TIMESTAMPED_OBSERVATION_SEMANTICS }, ) .with_grouping_projection(config.grouping_labels.clone()) @@ -253,6 +246,54 @@ impl SummaryCatalog { return Err(SummaryCatalogError::MissingDescriptor(id.as_u64())); } } + if !self + .data_descriptors + .values() + .any(|data| matches!(data.source, DataSourceIdentity::Derived { .. })) + { + return Ok(()); + } + // Dependencies must refer to this snapshot and form an acyclic graph. + let mut pending = std::collections::BTreeMap::new(); + let mut consumers: std::collections::BTreeMap<_, Vec<_>> = + std::collections::BTreeMap::new(); + for (id, binding) in &self.materializations { + let dependencies = match &self.data_descriptors[&binding.data_descriptor_id].source { + DataSourceIdentity::Derived { input } => input.inputs.clone(), + _ => Default::default(), + }; + for source in &dependencies { + if !self.materializations.contains_key(source) { + return Err(SummaryCatalogError::Descriptor( + "derived input references missing summary".into(), + )); + } + consumers.entry(*source).or_default().push(*id); + } + pending.insert(*id, dependencies.len()); + } + let mut ready: Vec<_> = pending + .iter() + .filter_map(|(id, count)| (*count == 0).then_some(*id)) + .collect(); + let mut visited = 0; + while let Some(id) = ready.pop() { + visited += 1; + for consumer in consumers.get(&id).into_iter().flatten() { + let count = pending + .get_mut(consumer) + .expect("catalog dependency target"); + *count -= 1; + if *count == 0 { + ready.push(*consumer); + } + } + } + if visited != pending.len() { + return Err(SummaryCatalogError::Descriptor( + "derived summary dependencies have a cycle".into(), + )); + } Ok(()) } } diff --git a/data_plane/Cargo.toml b/data_plane/Cargo.toml index 60ae2ef00..cc5310495 100644 --- a/data_plane/Cargo.toml +++ b/data_plane/Cargo.toml @@ -39,8 +39,8 @@ sha2 = "0.10" # reduction: Reduction, .. }`) are `pre_asap` types, in the same crate now # (not a separate `asap-ir` import). Query serving consumes the compiled # QueryPlan; these types are used at physical-plan compilation boundaries. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "0402384e589df6e087d6d2d22b463ddc2eea0774" } -asap-frontend-promql = { git = "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/ProjectASAP/ASAPPlanner", rev = "0402384e589df6e087d6d2d22b463ddc2eea0774" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "5e9396033a5a5d5f350bfa675d12770c972ee991" } +asap-frontend-promql = { git = "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/ProjectASAP/ASAPPlanner", rev = "5e9396033a5a5d5f350bfa675d12770c972ee991" } # Shared external (workspace) serde.workspace = true @@ -133,7 +133,7 @@ fs2 = "0.4" # none of them. [dev-dependencies] -asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "0402384e589df6e087d6d2d22b463ddc2eea0774" } +asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "5e9396033a5a5d5f350bfa675d12770c972ee991" } tempfile = "3.20.0" criterion = { version = "0.5", features = ["html_reports"] } tokio-tungstenite = "0.21" diff --git a/data_plane/examples/audit_clickhouse_fallback.rs b/data_plane/examples/audit_clickhouse_fallback.rs index 2ada8ebaf..a5bdd1ac0 100644 --- a/data_plane/examples/audit_clickhouse_fallback.rs +++ b/data_plane/examples/audit_clickhouse_fallback.rs @@ -3,9 +3,7 @@ use axum::{ http::{HeaderMap, Method}, }; use control_plane::{ - physical::compiler::{ - PlanEnvelope, PrecomputePlan, TransmissionPlan, BACKEND_COMPAT, PLANNER_REVISION, - }, + physical::compiler::{PlanEnvelope, PrecomputePlan, BACKEND_COMPAT, PLANNER_REVISION}, query_plan::{ClickHousePlanningContext, QueryPlan}, }; use data_plane::{ @@ -76,8 +74,12 @@ async fn main() { let mut precompute_plan = PrecomputePlan::build_backend_local(envelope.clone(), vec![]).unwrap(); precompute_plan.summary_catalog = Some(reference.clone()); - let mut transmission_plan = - TransmissionPlan::build(envelope, &precompute_plan, &BTreeMap::new()).unwrap(); + let mut transmission_plan = control_plane::physical::compiler::compile_transmission_plan( + envelope, + &precompute_plan, + &BTreeMap::new(), + ) + .unwrap(); transmission_plan.summary_catalog = Some(reference); let query_plan = QueryPlan { plan_id: 27, diff --git a/data_plane/src/drivers/ingest/otel.rs b/data_plane/src/drivers/ingest/otel.rs index a8d6e415b..2961bb7aa 100644 --- a/data_plane/src/drivers/ingest/otel.rs +++ b/data_plane/src/drivers/ingest/otel.rs @@ -1714,7 +1714,7 @@ async fn route_modified_otlp_sketches_to_precompute( crate::precompute_engine::frame_lineage::FrameLineageDecision::Apply, ) => { if frame.kind - == control_plane::physical::compiler::SummaryFrameKind::Full + == asap_types::producer_plan::SummaryFrameKind::Full { ingest_state .sketch_index @@ -1754,15 +1754,6 @@ async fn route_modified_otlp_sketches_to_precompute( } } - ingest_state.sketch_snapshots.insert( - series_key.clone(), - crate::precompute_engine::ingest_handler::SnapshotCacheEntry { - core: accumulator.clone_boxed_core(), - window_start: dp.start_time_unix_nano, - }, - ); - ingest_state.note_window_and_sweep(dp.start_time_unix_nano); - use crate::storage_engines::sketch_db::index::{ SketchEncoding, SketchSampleState, }; @@ -1778,7 +1769,7 @@ async fn route_modified_otlp_sketches_to_precompute( ); let encoding = encoding_to_handle(dp.encoding).unwrap_or(SketchEncoding::ProtoFull); - ingest_state.sketch_index.append_sample( + if !ingest_state.sketch_index.append_sample( sid, label_values, window, @@ -1786,7 +1777,18 @@ async fn route_modified_otlp_sketches_to_precompute( bytes: dp.sketch.clone(), encoding, }, + ) { + return Err("summary window is immutable after completion".into()); + } + + ingest_state.sketch_snapshots.insert( + series_key.clone(), + crate::precompute_engine::ingest_handler::SnapshotCacheEntry { + core: accumulator.clone_boxed_core(), + window_start: dp.start_time_unix_nano, + }, ); + ingest_state.note_window_and_sweep(dp.start_time_unix_nano); // Collect the configs whose metric matches this DP. // Detection is independent of the legacy dual-write @@ -2251,7 +2253,7 @@ fn preflight_summary_frames( mut dp: ModifiedOtlpSketchDp, ingest_state: &IngestState, active: &crate::storage_engines::types::ActivePhysicalPlan, - ) -> Result { + ) -> Result { let canonical_name = canonical_sketch_metric_name(metric_name, dp.algorithm.clone()); let frame = take_summary_frame_identity(&mut dp.attrs, dp.start_time_unix_nano, dp.time_unix_nano)?; @@ -2294,7 +2296,7 @@ fn preflight_summary_frames( // A malformed full snapshot must not be discovered after an earlier // frame in the request has already reached SketchStore. - if frame.kind == control_plane::physical::compiler::SummaryFrameKind::Full { + if frame.kind == asap_types::producer_plan::SummaryFrameKind::Full { decode_modified_otlp_sketch_bytes(dp.algorithm.clone(), dp.encoding, &dp.sketch) .map_err(|error| format!("invalid full frame for {metric_name}: {error}"))?; } else { @@ -2452,10 +2454,9 @@ fn take_summary_frame_identity( attrs: &mut HashMap, window_start_unix_nano: u64, window_end_unix_nano: u64, -) -> Result { - use control_plane::physical::compiler::{ - StateEncoding, SummaryFrameIdentity, SummaryFrameKind, - }; +) -> Result { + use asap_types::producer_plan::{SummaryFrameIdentity, SummaryFrameKind}; + use control_plane::physical::compiler::StateEncoding; fn required(attrs: &mut HashMap, key: &str) -> Result { attrs diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index c1ec49919..982100241 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -179,6 +179,15 @@ impl PrometheusRemoteWriteReceiver { self.inner.stats.clone() } + pub(crate) fn install_erp_observation_generation( + &self, + generation: asap_types::sds::CatalogGeneration, + ) { + if let Some(observer) = self.inner.ingest.router.erp_observer() { + observer.install_generation(generation); + } + } + /// Permanently seal this finite source before queuing worker barriers. pub async fn drain(&self) -> Result<(), String> { { @@ -199,10 +208,30 @@ impl PrometheusRemoteWriteReceiver { .and_then(|plan| plan.precompute_plan.summary_catalog.clone()) .ok_or("finite completion requires a catalog generation")?; self.inner.ingest.router.drain().await?; - self.inner - .ingest - .sketch_index - .seal_finite_summary_input(&generation)?; + let flush_deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(30); + loop { + if self + .inner + .ingest + .sketch_index + .seal_finite_summary_input(&generation)? + { + break; + } + if tokio::time::Instant::now() >= flush_deadline { + return Err( + "finite completion is waiting for durable summary payloads; retry drain".into(), + ); + } + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + if let Some(observer) = self.inner.ingest.router.erp_observer() { + let now_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(|error| error.to_string())? + .as_millis() as u64; + observer.publish_finite(&generation, now_ms).await?; + } trim_process_allocator(); Ok(()) } @@ -882,9 +911,10 @@ mod tests { } fn physical_config(streaming: StreamingConfig) -> HotReloadStreamingConfig { + use asap_types::producer_plan::{FrameIdentityContract, SequenceScope, TransmissionPlan}; use control_plane::physical::compiler::{ - FrameIdentityContract, IngestContract, IngestProtocol, PlanEnvelope, PrecomputePlan, - SequenceScope, TimestampUnit, TransmissionPlan, PLANNER_REVISION, + IngestContract, IngestProtocol, PlanEnvelope, PrecomputePlan, TimestampUnit, + PLANNER_REVISION, }; let envelope = PlanEnvelope { plan_id: 7, @@ -943,7 +973,7 @@ mod tests { rules: Vec::new(), }, runtime_config: Arc::new(streaming), - query_plan: Arc::new(control_plane::query_plan::QueryPlan::empty()), + query_plan: Arc::new(asap_types::query_plan::QueryPlan::empty()), storage_routing: Arc::new(BackendStorageRouting::empty()), }; HotReloadStreamingConfig::from_active(HotReloadActivePhysicalPlan::new(active)) @@ -988,6 +1018,7 @@ mod tests { table_name: None, value_projection: None, table_population: None, + derived_input: None, table_timestamp_column: None, partitioning: None, }; @@ -1059,6 +1090,7 @@ mod tests { table_name: None, value_projection: None, table_population: None, + derived_input: None, table_timestamp_column: None, partitioning: None, }; @@ -1483,11 +1515,9 @@ mod tests { .keys() .next() .unwrap(); - let binding = control_plane::query_plan::MaterializationBinding { + let binding = asap_types::query_plan::MaterializationBinding { materialization: asap_types::PolicyFingerprint(policy).into(), - output_grouping: control_plane::query_plan::PhysicalGrouping::Reduce( - vec!["job".into()], - ), + output_grouping: asap_types::query_plan::PhysicalGrouping::Reduce(vec!["job".into()]), item_labels: vec![], window_ms: 60_000, pane_origin_ms: Some(0), diff --git a/data_plane/src/drivers/query/adapters/victoriametrics_http.rs b/data_plane/src/drivers/query/adapters/victoriametrics_http.rs index a75bbd264..e23cf4a9b 100644 --- a/data_plane/src/drivers/query/adapters/victoriametrics_http.rs +++ b/data_plane/src/drivers/query/adapters/victoriametrics_http.rs @@ -104,7 +104,7 @@ impl HttpProtocolAdapter for VictoriaMetricsHttpAdapter { "VictoriaMetrics HTTP / MetricsQL" } fn canonical_plan_identity(&self, query: &str) -> Result, AdapterError> { - control_plane::query_plan::canonical_promql(query) + asap_types::query_plan::canonical_promql(query) .map(Some) .map_err(|error| AdapterError::ParseError(format!("promql-compatible subset: {error}"))) } diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index c456d8e3c..ffc4c6eb1 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -2679,9 +2679,10 @@ mod tests { } async fn setup_remote_write_test_server() -> (u16, PrometheusRemoteWriteReceiver) { + use asap_types::producer_plan::{FrameIdentityContract, SequenceScope, TransmissionPlan}; use control_plane::physical::compiler::{ - FrameIdentityContract, IngestContract, IngestProtocol, PlanEnvelope, PrecomputePlan, - SequenceScope, TimestampUnit, TransmissionPlan, PLANNER_REVISION, + IngestContract, IngestProtocol, PlanEnvelope, PrecomputePlan, TimestampUnit, + PLANNER_REVISION, }; let streaming_config = Arc::new(StreamingConfig::default()); let envelope = PlanEnvelope { @@ -2734,7 +2735,7 @@ mod tests { rules: Vec::new(), }, runtime_config: streaming_config.clone(), - query_plan: Arc::new(control_plane::query_plan::QueryPlan { + query_plan: Arc::new(asap_types::query_plan::QueryPlan { plan_id: 7, plan_version: 1, clickhouse_context: None, @@ -3681,6 +3682,7 @@ aggregations: table_name: None, value_projection: None, table_population: None, + derived_input: None, table_timestamp_column: None, partitioning: None, }; @@ -6045,7 +6047,7 @@ async fn handle_post_streaming_config( (StatusCode::OK, axum::Json(body)).into_response() } -pub use control_plane::physical::publication::PhysicalPlanInstallRequest; +pub use asap_types::plan_publication::PhysicalPlanInstallRequest; /// Decode and cross-validate every backend view before it can become visible. /// Used by both startup artifact loading and the staged HTTP install path. @@ -6216,13 +6218,13 @@ async fn handle_post_physical_plan( .query_plan .entries .values() - .filter(|entry| entry.language == control_plane::query_plan::QueryLanguage::MetricsQl) + .filter(|entry| entry.language == asap_types::query_plan::QueryLanguage::MetricsQl) .count(); let clickhouse_plan_count = active .query_plan .entries .values() - .filter(|entry| entry.language == control_plane::query_plan::QueryLanguage::ClickHouseSql) + .filter(|entry| entry.language == asap_types::query_plan::QueryLanguage::ClickHouseSql) .count(); let plan_version = active.plan_version(); let now = unix_time_ms(); @@ -6271,14 +6273,23 @@ async fn handle_activate_physical_plan( }; let _guard = state.physical_plan_lock.lock().await; let store = Arc::clone(&state.sketch_index); + let remote_write = state.remote_write.clone(); let old = match lifecycle.activate_with_prepare( request.plan_id, request.plan_version, unix_time_ms(), move |plan| match plan.summary_catalog.as_ref() { - Some(catalog) => store - .install_summary_catalog(Arc::clone(catalog)) - .map_err(|error| format!("SummaryCatalog install error: {error}")), + Some(catalog) => { + store + .install_summary_catalog(Arc::clone(catalog)) + .map_err(|error| format!("SummaryCatalog install error: {error}"))?; + if let Some(receiver) = &remote_write { + receiver.install_erp_observation_generation( + catalog.reference().map_err(|error| error.to_string())?, + ); + } + Ok(()) + } None => Err("authoritative SummaryCatalog is unavailable".to_string()), }, ) { @@ -6298,7 +6309,7 @@ async fn handle_activate_physical_plan( .query_plan .entries .values() - .filter(|entry| entry.language == control_plane::query_plan::QueryLanguage::ClickHouseSql) + .filter(|entry| entry.language == asap_types::query_plan::QueryLanguage::ClickHouseSql) .count(); if old.plan_id() != 0 { let draining_id = old.plan_id(); @@ -7179,7 +7190,7 @@ mod catalog_install_tests { let before = handle.snapshot(); let mut candidate = request(); candidate.query_plan.clickhouse_context = - Some(control_plane::query_plan::ClickHousePlanningContext { + Some(asap_types::query_plan::ClickHousePlanningContext { tables: Default::default(), accuracy: planner_types::types::AccuracyTarget::Exact, }); @@ -7188,8 +7199,8 @@ mod catalog_install_tests { .entries .pop_first() .expect("fixture has a query entry"); - entry.language = control_plane::query_plan::QueryLanguage::ClickHouseSql; - entry.fixed_evaluation = Some(control_plane::query_plan::FixedEvaluationRange { + entry.language = asap_types::query_plan::QueryLanguage::ClickHouseSql; + entry.fixed_evaluation = Some(asap_types::query_plan::FixedEvaluationRange { start_ms: 0, end_ms: 1_000, cumulative: true, @@ -7199,7 +7210,7 @@ mod catalog_install_tests { .nodes .values_mut() .find_map(|node| match node { - control_plane::query_plan::QueryPlanNode::ReadMaterialization { binding } => { + asap_types::query_plan::QueryPlanNode::ReadMaterialization { binding } => { Some(binding) } _ => None, @@ -7272,7 +7283,7 @@ mod catalog_install_tests { .values_mut() .flat_map(|entry| entry.nodes.values_mut()) .find_map(|node| match node { - control_plane::query_plan::QueryPlanNode::ReadMaterialization { binding } => { + asap_types::query_plan::QueryPlanNode::ReadMaterialization { binding } => { Some(binding) } _ => None, @@ -7287,22 +7298,20 @@ mod catalog_install_tests { let mut request = request(); let key = request.query_plan.entries.keys().next().unwrap().clone(); let mut entry = request.query_plan.entries.remove(&key).unwrap(); - entry.language = control_plane::query_plan::QueryLanguage::MetricsQl; + entry.language = asap_types::query_plan::QueryLanguage::MetricsQl; let binding = entry .nodes .values_mut() .find_map(|node| match node { - control_plane::query_plan::QueryPlanNode::ReadMaterialization { binding } => { + asap_types::query_plan::QueryPlanNode::ReadMaterialization { binding } => { Some(binding) } _ => None, }) .expect("demo has maintained summaries"); binding.pane_origin_ms = Some(1); - let key = control_plane::query_plan::QueryPlan::catalog_key( - entry.language, - &entry.canonical_query, - ); + let key = + asap_types::query_plan::QueryPlan::catalog_key(entry.language, &entry.canonical_query); request.query_plan.entries.insert(key, entry); let error = install(request).unwrap_err(); assert!(error.contains("pane origin"), "{error}"); diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index 345479b29..eb19b6dd1 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -44,6 +44,9 @@ fn unix_time_ms() -> u64 { #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] struct Args { + /// Publish bounded materialization-input ERP observations after a verified finite-source drain. + #[arg(long)] + erp_runtime_samples_endpoint: Option, /// Runtime component profile. `asapquery` enables backend ingest-time /// materialization and rejects Collector/OTLP-only components. #[arg(long, value_enum, default_value = "distributed")] @@ -768,13 +771,13 @@ async fn main() -> Result<()> { .collect(), executable_dags: Default::default(), }; - let initial_transmission_plan = control_plane::physical::compiler::TransmissionPlan { + let initial_transmission_plan = asap_types::producer_plan::TransmissionPlan { summary_catalog: None, envelope: initial_precompute_plan.envelope.clone(), - frame_identity: control_plane::physical::compiler::FrameIdentityContract { + frame_identity: asap_types::producer_plan::FrameIdentityContract { identity_version: 1, sequence_scope: - control_plane::physical::compiler::SequenceScope::MaterializationSeriesProducerEpoch, + asap_types::producer_plan::SequenceScope::MaterializationSeriesProducerEpoch, require_checkpoint_for_full: true, require_base_checkpoint_for_delta: true, }, @@ -787,7 +790,7 @@ async fn main() -> Result<()> { precompute_plan: initial_precompute_plan, transmission_plan: initial_transmission_plan, runtime_config: streaming_config.clone(), - query_plan: Arc::new(control_plane::query_plan::QueryPlan::empty()), + query_plan: Arc::new(asap_types::query_plan::QueryPlan::empty()), storage_routing: Arc::new( data_plane::storage_engines::types::BackendStorageRouting::empty(), ), @@ -874,6 +877,20 @@ async fn main() -> Result<()> { series_resolver.clone(), sketch_index.clone(), ); + if let Some(endpoint) = args.erp_runtime_samples_endpoint.clone() { + let generation = engine + .ingest_state() + .physical_plan_snapshot() + .and_then(|plan| plan.precompute_plan.summary_catalog.clone()) + .ok_or_else(|| { + std::io::Error::other("ERP observation requires an installed catalog") + })?; + engine + .ingest_state() + .router + .enable_erp_observation(endpoint, generation) + .map_err(std::io::Error::other)?; + } let worker_diagnostics = engine.diagnostics(); let ingest_state = engine.ingest_state(); info!("Starting precompute engine (ingest adapters share its bounded worker queues)"); diff --git a/data_plane/src/precompute_engine/engine.rs b/data_plane/src/precompute_engine/engine.rs index 15f44c40f..8d59efe94 100644 --- a/data_plane/src/precompute_engine/engine.rs +++ b/data_plane/src/precompute_engine/engine.rs @@ -129,7 +129,7 @@ impl PrecomputeEngine { // ConfigReload messages needed. let mut worker_handles = Vec::with_capacity(num_workers); for (id, rx) in receivers.into_iter().enumerate() { - let worker = Worker::new( + let mut worker = Worker::new( id, rx, output_sink.clone(), @@ -148,6 +148,7 @@ impl PrecomputeEngine { self.diagnostics.worker_group_counts[id].clone(), self.diagnostics.worker_watermarks[id].clone(), ); + worker.set_erp_observer(self.ingest_state.router.erp_observer()); let handle = tokio::spawn(async move { worker.run().await; }); diff --git a/data_plane/src/precompute_engine/erp_observer.rs b/data_plane/src/precompute_engine/erp_observer.rs new file mode 100644 index 000000000..47593276b --- /dev/null +++ b/data_plane/src/precompute_engine/erp_observer.rs @@ -0,0 +1,361 @@ +//! Bounded observations of actual materialization inputs, published only after +//! an explicit finite-source completion barrier. No worker timestamp is a seal. +use asap_types::erp_observation::*; +use asap_types::sds::*; +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +const MAX_POPULATIONS: usize = 128; +const MAX_KEYS: usize = 65_536; +const MAX_POPULATION_METADATA_BYTES: usize = 1_048_576; + +struct Population { + coordinates: SummaryInstanceCoordinates, + semantics: ErpObservationInputSemantics, + source: String, + sketch: String, + implementation: String, + observer: BoundedFrequencyObserver, +} +#[derive(Default)] +struct Observations { + generation: Option, + populations: BTreeMap, + extent: BTreeMap, + total_keys: usize, + metadata_bytes: usize, + invalid: Option, +} + +pub struct RuntimeErpObserver { + endpoint: String, + observations: Mutex, +} +impl RuntimeErpObserver { + pub fn new(endpoint: String, generation: CatalogGeneration) -> Arc { + Arc::new(Self { + endpoint, + observations: Mutex::new(Observations { + generation: Some(generation), + ..Default::default() + }), + }) + } + /// Only the accepted catalog activation path may reset an observation epoch. + pub(crate) fn install_generation(&self, generation: CatalogGeneration) { + let mut state = self.observations.lock().unwrap(); + if state.generation.as_ref() != Some(&generation) { + *state = Observations { + generation: Some(generation), + ..Default::default() + }; + } + } + + pub fn observe( + &self, + generation: &CatalogGeneration, + coordinates: SummaryInstanceCoordinates, + config: &asap_types::AggregationConfig, + timestamp_ms: i64, + value: f64, + ) { + use asap_types::{AggregationType, SampleUpdateRule}; + let (semantics, sketch, implementation) = match config.aggregation_type { + AggregationType::HLL => ( + ErpObservationInputSemantics::ScalarSampleValue, + "hll", + "asap-sketchlib-hll-regular-v1", + ), + AggregationType::UnivMon => ( + ErpObservationInputSemantics::UnitSampleFrequency, + "univmon", + "asap-sketchlib-univmon-standard-v1", + ), + _ => return, + }; + let mut state = self.observations.lock().unwrap(); + // Catalog installation establishes identity. Delayed old worker input + // cannot reset the current generation or create a partial replacement fit. + if state.generation.as_ref() != Some(generation) { + return; + } + if state.invalid.is_some() { + return; + } + if !value.is_finite() + || !matches!( + config.sample_update_rule(), + SampleUpdateRule::Value { scale: 1.0 } + ) + { + state.invalid = Some("unsupported non-finite or transformed summary input".into()); + for population in state.populations.values_mut() { + population.observer = BoundedFrequencyObserver::new(1, 64).unwrap(); + } + return; + } + let extent = state + .extent + .entry(coordinates.summary_definition_id) + .or_insert((timestamp_ms, timestamp_ms)); + extent.0 = extent.0.min(timestamp_ms); + extent.1 = extent.1.max(timestamp_ms); + let Ok(id) = coordinates.instance_id() else { + return; + }; + if !state.populations.contains_key(&id) && state.populations.len() >= MAX_POPULATIONS { + state.invalid = Some("ERP population observation budget exceeded".into()); + for population in state.populations.values_mut() { + population.observer = BoundedFrequencyObserver::new(1, 64).unwrap(); + } + return; + } + if !state.populations.contains_key(&id) { + let bytes = coordinates + .group_values + .iter() + .try_fold(0usize, |total, (key, value)| { + total + .checked_add(key.len()) + .and_then(|n| n.checked_add(value.len())) + }); + let total = bytes.and_then(|bytes| state.metadata_bytes.checked_add(bytes)); + if total.is_none_or(|bytes| bytes > MAX_POPULATION_METADATA_BYTES) { + state.invalid = Some("ERP population metadata budget exceeded".into()); + for population in state.populations.values_mut() { + population.observer = BoundedFrequencyObserver::new(1, 64).unwrap(); + } + return; + } + state.metadata_bytes = total.unwrap(); + } + let population = state.populations.entry(id).or_insert_with(|| Population { + source: format!( + "summary-definition:{}", + coordinates.summary_definition_id.as_u64() + ), + coordinates, + semantics, + sketch: sketch.into(), + implementation: implementation.into(), + observer: BoundedFrequencyObserver::new(MAX_KEYS, 64).unwrap(), + }); + let before = population.observer.observed_key_count(); + // Canonical fixed-width numeric identity; no raw sample or arbitrary label copy. + let key = if value == 0.0 { 0 } else { value.to_bits() }; + let range = population.coordinates.time_range; + let duration = range.end_ms.saturating_sub(range.start_ms).max(1); + let offset = timestamp_ms + .saturating_sub(range.start_ms) + .clamp(0, duration); + let interval = ((offset as u128 * 64) / duration as u128).min(63) as usize; + let result = population.observer.observe(key, interval); + let added = population + .observer + .observed_key_count() + .saturating_sub(before); + state.total_keys = state.total_keys.saturating_add(added); + if result.is_err() || state.total_keys > MAX_KEYS { + state.invalid = Some("ERP key observation budget exceeded".into()); + for population in state.populations.values_mut() { + population.observer = BoundedFrequencyObserver::new(1, 64).unwrap(); + } + } + } + + pub async fn publish_finite( + &self, + generation: &CatalogGeneration, + now_ms: u64, + ) -> Result<(), String> { + use control_plane::runtime_samples::feedback::{ + runtime_samples_client::RuntimeSamplesClient, PushBatch, RuntimeRecord, + }; + let records = { + let state = self.observations.lock().unwrap(); + if state.generation.as_ref() != Some(generation) { + return Ok(()); + } + let mut groups: BTreeMap< + (SummaryDefinitionId, i64, i64), + ( + String, + String, + String, + ErpPopulationObservations, + ), + > = BTreeMap::new(); + for (id, population) in &state.populations { + let c = &population.coordinates; + let Some((first, last)) = state.extent.get(&c.summary_definition_id) else { + continue; + }; + if c.time_range.start_ms < *first || c.time_range.end_ms > *last { + continue; + } + let key = ( + c.summary_definition_id, + c.time_range.start_ms, + c.time_range.end_ms, + ); + let entry = groups.entry(key).or_insert_with(|| { + ( + population.source.clone(), + population.sketch.clone(), + population.implementation.clone(), + ErpPopulationObservations { + schema_version: 1, + catalog_generation: generation.clone(), + summary_definition_id: c.summary_definition_id, + observed_at_unix_ms: now_ms, + window_start_ms: c.time_range.start_ms, + window_end_ms: c.time_range.end_ms, + input_semantics: population.semantics, + invalid_reason: state.invalid.clone(), + populations: Vec::new(), + }, + ) + }); + match population.observer.snapshot() { + Some(shape) => entry.3.populations.push(ErpPopulationObservation { + population_id: id.clone(), + shape, + }), + None => entry.3.invalid_reason = Some("ERP population fit unavailable".into()), + } + } + groups + .into_values() + .map(|(source, sketch, impl_name, mut envelope)| { + if envelope.invalid_reason.is_some() { + envelope.populations.clear(); + } + RuntimeRecord { + source, + sketch, + impl_name, + schema_version: 1, + payload_json: serde_json::json!({"erp_population_observations":envelope}) + .to_string(), + } + }) + .collect::>() + }; + if records.is_empty() { + return Ok(()); + } + let expected = records.len() as u64; + let mut client = RuntimeSamplesClient::connect(self.endpoint.clone()) + .await + .map_err(|e| e.to_string())?; + let ack = client + .push(PushBatch { records }) + .await + .map_err(|e| e.to_string())? + .into_inner(); + if ack.accepted != expected { + return Err("control plane rejected ERP observation records".into()); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + fn fixture() -> (CatalogGeneration, asap_types::AggregationConfig) { + let config = asap_types::AggregationConfig::new( + asap_types::AggregationType::HLL, + String::new(), + Default::default(), + asap_types::KeyByLabelNames::new(vec![]), + asap_types::KeyByLabelNames::new(vec![]), + asap_types::KeyByLabelNames::new(vec![]), + String::new(), + 60, + 60, + asap_types::enums::WindowKind::Tumbling, + String::new(), + "m".into(), + None, + None, + None, + ); + ( + CatalogGeneration { + schema_version: 1, + plan_id: 1, + plan_version: 1, + snapshot_sha256: "a".repeat(64), + }, + config, + ) + } + fn coordinate(group: usize) -> SummaryInstanceCoordinates { + SummaryInstanceCoordinates { + summary_definition_id: asap_types::PolicyFingerprint(1).into(), + time_range: HalfOpenTimeRange { + start_ms: 0, + end_ms: 60_000, + }, + group_values: BTreeMap::from([("job".into(), group.to_string())]), + } + } + #[test] + fn observations_keep_partitions_separate_and_invalidate_on_overflow() { + let (generation, config) = fixture(); + let observer = RuntimeErpObserver::new("http://127.0.0.1:1".into(), generation.clone()); + observer.observe(&generation, coordinate(0), &config, 1, 1.0); + observer.observe(&generation, coordinate(0), &config, 2, 1.0); + observer.observe(&generation, coordinate(1), &config, 3, 2.0); + { + let state = observer.observations.lock().unwrap(); + assert_eq!(state.populations.len(), 2); + assert_eq!(state.total_keys, 2); + assert!(state + .populations + .values() + .all(|p| p.observer.observed_key_count() == 1)); + } + for group in 2..=MAX_POPULATIONS { + observer.observe(&generation, coordinate(group), &config, 4, 3.0); + } + let state = observer.observations.lock().unwrap(); + assert!(state.invalid.is_some()); + assert!(state + .populations + .values() + .all(|p| p.observer.snapshot().is_none())); + } + #[test] + fn delayed_generation_cannot_reset_current_population_counts() { + let (generation, config) = fixture(); + let observer = RuntimeErpObserver::new("http://127.0.0.1:1".into(), generation.clone()); + let mut previous = generation.clone(); + previous.plan_version = 0; + observer.install_generation(previous.clone()); + observer.observe(&previous, coordinate(0), &config, 0, 77.0); + observer.install_generation(generation.clone()); + observer.observe(&generation, coordinate(0), &config, 1, 1.0); + let mut stale = generation.clone(); + stale.plan_version = 0; + observer.observe(&stale, coordinate(0), &config, 2, 99.0); + observer.observe(&generation, coordinate(0), &config, 3, 2.0); + let state = observer.observations.lock().unwrap(); + assert_eq!(state.total_keys, 2); + assert_eq!(state.generation.as_ref(), Some(&generation)); + assert_eq!( + state + .populations + .values() + .next() + .unwrap() + .observer + .snapshot() + .unwrap() + .sorted_counts, + vec![1, 1] + ); + } +} diff --git a/data_plane/src/precompute_engine/frame_lineage.rs b/data_plane/src/precompute_engine/frame_lineage.rs index 6598a1384..7f5e0a847 100644 --- a/data_plane/src/precompute_engine/frame_lineage.rs +++ b/data_plane/src/precompute_engine/frame_lineage.rs @@ -5,7 +5,7 @@ //! the wire contract: deltas are only safe to apply after an observed full //! checkpoint, in sequence, within the exact producer/window lineage. -use control_plane::physical::compiler::{SummaryFrameIdentity, SummaryFrameKind}; +use asap_types::producer_plan::{SummaryFrameIdentity, SummaryFrameKind}; use dashmap::mapref::entry::Entry; use thiserror::Error; diff --git a/data_plane/src/precompute_engine/maintenance_runtime.rs b/data_plane/src/precompute_engine/maintenance_runtime.rs index 3d64b8eeb..b49aadff4 100644 --- a/data_plane/src/precompute_engine/maintenance_runtime.rs +++ b/data_plane/src/precompute_engine/maintenance_runtime.rs @@ -654,7 +654,7 @@ mod tests { let binding = BackendExecutableBinding { nodes: BTreeMap::new(), query_sink: PostAsapNodeId(2), - query_plan_sink: control_plane::query_plan::QueryNodeId(2), + query_plan_sink: asap_types::query_plan::QueryNodeId(2), precompute_sinks: vec![PostAsapNodeId(1)], }; let adapter = OperatorAdapter { @@ -841,12 +841,12 @@ mod tests { ( PostAsapNodeId(2), BackendNodeBinding::Query { - query_node: control_plane::query_plan::QueryNodeId(9), + query_node: asap_types::query_plan::QueryNodeId(9), }, ), ]), query_sink: PostAsapNodeId(2), - query_plan_sink: control_plane::query_plan::QueryNodeId(9), + query_plan_sink: asap_types::query_plan::QueryNodeId(9), precompute_sinks: vec![PostAsapNodeId(1)], }; bundle.precompute_plan.executable_dags = BTreeMap::from([( @@ -970,12 +970,12 @@ mod tests { .chain([( PostAsapNodeId(4), BackendNodeBinding::Query { - query_node: control_plane::query_plan::QueryNodeId(9), + query_node: asap_types::query_plan::QueryNodeId(9), }, )]) .collect(), query_sink: PostAsapNodeId(4), - query_plan_sink: control_plane::query_plan::QueryNodeId(9), + query_plan_sink: asap_types::query_plan::QueryNodeId(9), precompute_sinks: vec![PostAsapNodeId(3)], }; let source = sum(2.0); @@ -1043,12 +1043,12 @@ mod tests { ( PostAsapNodeId(2), BackendNodeBinding::Query { - query_node: control_plane::query_plan::QueryNodeId(9), + query_node: asap_types::query_plan::QueryNodeId(9), }, ), ]), query_sink: PostAsapNodeId(2), - query_plan_sink: control_plane::query_plan::QueryNodeId(9), + query_plan_sink: asap_types::query_plan::QueryNodeId(9), precompute_sinks: vec![PostAsapNodeId(1)], }; let adapter = OperatorAdapter { diff --git a/data_plane/src/precompute_engine/mod.rs b/data_plane/src/precompute_engine/mod.rs index c10b4bb1f..068ac13b7 100644 --- a/data_plane/src/precompute_engine/mod.rs +++ b/data_plane/src/precompute_engine/mod.rs @@ -2,6 +2,7 @@ pub mod accumulator_factory; pub mod config; pub mod coordination_checkpoint; mod engine; +pub mod erp_observer; pub mod frame_lineage; pub mod group_key; pub mod ingest_handler; diff --git a/data_plane/src/precompute_engine/output_sink.rs b/data_plane/src/precompute_engine/output_sink.rs index b908f2a8b..46a83a42c 100644 --- a/data_plane/src/precompute_engine/output_sink.rs +++ b/data_plane/src/precompute_engine/output_sink.rs @@ -422,6 +422,7 @@ mod tests { table_name: None, value_projection: None, table_population: None, + derived_input: None, table_timestamp_column: None, partitioning: None, } diff --git a/data_plane/src/precompute_engine/series_router.rs b/data_plane/src/precompute_engine/series_router.rs index bc1c319f1..de8786049 100644 --- a/data_plane/src/precompute_engine/series_router.rs +++ b/data_plane/src/precompute_engine/series_router.rs @@ -155,6 +155,7 @@ impl fmt::Debug for WorkerMessage { /// Routes incoming samples to one of N workers based on a consistent hash. pub struct SeriesRouter { + erp_observer: std::sync::OnceLock>, senders: Vec>, num_workers: usize, } @@ -163,11 +164,27 @@ impl SeriesRouter { pub fn new(senders: Vec>) -> Self { let num_workers = senders.len(); Self { + erp_observer: std::sync::OnceLock::new(), senders, num_workers, } } + pub fn enable_erp_observation( + &self, + endpoint: String, + generation: asap_types::sds::CatalogGeneration, + ) -> Result<(), String> { + self.erp_observer + .set(super::erp_observer::RuntimeErpObserver::new( + endpoint, generation, + )) + .map_err(|_| "ERP observer already configured".into()) + } + pub fn erp_observer(&self) -> Option> { + self.erp_observer.get().cloned() + } + /// Route a pre-grouped batch of group messages to workers concurrently. /// /// Each `GroupSamples` / `AccumulatorInput` message is routed by diff --git a/data_plane/src/precompute_engine/subdag_scheduler.rs b/data_plane/src/precompute_engine/subdag_scheduler.rs index 6f632f44c..6a52b88df 100644 --- a/data_plane/src/precompute_engine/subdag_scheduler.rs +++ b/data_plane/src/precompute_engine/subdag_scheduler.rs @@ -185,12 +185,12 @@ mod tests { .chain([( PostAsapNodeId(4), BackendNodeBinding::Query { - query_node: control_plane::query_plan::QueryNodeId(9), + query_node: asap_types::query_plan::QueryNodeId(9), }, )]) .collect(), query_sink: PostAsapNodeId(4), - query_plan_sink: control_plane::query_plan::QueryNodeId(9), + query_plan_sink: asap_types::query_plan::QueryNodeId(9), precompute_sinks: vec![PostAsapNodeId(3)], } } @@ -376,7 +376,7 @@ mod tests { ( PostAsapNodeId(0), BackendNodeBinding::Query { - query_node: control_plane::query_plan::QueryNodeId(1), + query_node: asap_types::query_plan::QueryNodeId(1), }, ), ( @@ -389,7 +389,7 @@ mod tests { .into_iter() .collect(), query_sink: PostAsapNodeId(0), - query_plan_sink: control_plane::query_plan::QueryNodeId(1), + query_plan_sink: asap_types::query_plan::QueryNodeId(1), precompute_sinks: vec![PostAsapNodeId(1)], }; assert!(matches!( diff --git a/data_plane/src/precompute_engine/worker.rs b/data_plane/src/precompute_engine/worker.rs index 4a83434c3..fcc73edaf 100644 --- a/data_plane/src/precompute_engine/worker.rs +++ b/data_plane/src/precompute_engine/worker.rs @@ -150,6 +150,7 @@ pub struct WorkerRuntimeConfig { /// `(metric, attrs_fingerprint, agg_kind_canonical)` identity contract on /// `SeriesIdResolver`, so one sid uniquely names one bucket. pub struct Worker { + erp_observer: Option>, current_input_revision: Option>, current_catalog_generation: Option>, id: usize, @@ -187,6 +188,12 @@ pub struct Worker { } impl Worker { + pub fn set_erp_observer( + &mut self, + observer: Option>, + ) { + self.erp_observer = observer; + } #[allow(clippy::too_many_arguments)] pub fn new( id: usize, @@ -207,6 +214,7 @@ impl Worker { wall_clock_max_open_grace_period_ms, } = runtime_config; Self { + erp_observer: None, current_input_revision: None, current_catalog_generation: None, id, @@ -594,6 +602,24 @@ impl Worker { record_late_input("append_correction", "raw_sample"); let mut updater = create_accumulator_updater(&state.config); apply_sample(&mut *updater, series_key, *val, *ts, &state.config); + if let (Some(observer), Some(revision)) = + (&self.erp_observer, &input_revision) + { + observer.observe( + &revision.generation, + asap_types::sds::SummaryInstanceCoordinates { + summary_definition_id: policy_fp.into(), + time_range: asap_types::sds::HalfOpenTimeRange { + start_ms: bucket_start, + end_ms: bucket_end, + }, + group_values: group_key.as_population_labels(), + }, + &state.config, + *ts, + *val, + ); + } let key = build_group_key_label_values(group_key); let output = precomputed_output_for_group( window_start as u64, @@ -626,6 +652,23 @@ impl Worker { .or_insert_with(|| create_accumulator_updater(&state.config)); if let Some(value) = value { apply_sample(&mut **updater, series_key, value, *ts, &state.config); + if let (Some(observer), Some(revision)) = (&self.erp_observer, &input_revision) + { + observer.observe( + &revision.generation, + asap_types::sds::SummaryInstanceCoordinates { + summary_definition_id: policy_fp.into(), + time_range: asap_types::sds::HalfOpenTimeRange { + start_ms: bucket_start, + end_ms: bucket_end, + }, + group_values: group_key.as_population_labels(), + }, + &state.config, + *ts, + value, + ); + } } } } diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs index a69da5956..dc7740561 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs @@ -64,14 +64,14 @@ impl CatalogClickHouseAccelerator { async fn prepare_external_exact( &self, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, start_ms: u64, end_ms: u64, request_context: &ClickHouseQueryRequest, ) -> Result { let mut prepared = PreparedExternalLeaves::new(); let leaves = entry.nodes.iter().filter_map(|(id, node)| match node { - control_plane::query_plan::QueryPlanNode::ExternalExact { request, inputs } + asap_types::query_plan::QueryPlanNode::ExternalExact { request, inputs } if request.language == asap_types::QueryLanguage::ClickHouseSql && inputs.is_empty() => { @@ -85,7 +85,7 @@ impl CatalogClickHouseAccelerator { .as_ref() .ok_or_else(|| "ClickHouse exact subtree endpoint unavailable".to_owned())?; let schema = match &bound.output { - control_plane::query_plan::ExternalExactOutput::Relation { schema } => { + asap_types::query_plan::ExternalExactOutput::Relation { schema } => { serde_json::from_value(schema.clone()).map_err(|error| error.to_string())? } _ => return Err("ClickHouse exact subtree must produce a relation".into()), @@ -111,6 +111,9 @@ impl CatalogClickHouseAccelerator { "0".into(), ); parameters.insert("output_format_json_quote_64bit_integers".into(), "0".into()); + // Preserve the distinction between NULL and unsupported NaN/Inf. + // The typed decoder rejects quoted non-finite values and falls back. + parameters.insert("output_format_json_quote_denormals".into(), "1".into()); if let Some(database) = request_context.database() { parameters.insert("database".into(), database.into()); } @@ -144,11 +147,9 @@ impl CatalogClickHouseAccelerator { } fn requested_format(request: &ClickHouseQueryRequest) -> Result { - if let Some(setting) = request - .parameters - .keys() - .find(|key| key.starts_with("output_format_")) - { + if let Some(setting) = request.parameters.keys().find(|key| { + key.starts_with("output_format_") || key.as_str() == "format_tsv_null_representation" + }) { return Err(format!("unsupported output setting {setting}")); } match request @@ -299,6 +300,7 @@ mod tests { for setting in [ "output_format_json_map_as_array_of_tuples", "output_format_json_quote_64bit_integers", + "format_tsv_null_representation", ] { request.parameters.insert(setting.into(), "1".into()); assert!(requested_format(&request).is_err()); @@ -310,14 +312,14 @@ mod tests { precompute_engine::operators::SumAccumulator, storage_engines::sketch_db::index::{AggKind, Capability, SketchInstanceMetadata}, }; - use asap_types::summary_catalog::SummaryCatalog; - use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; - use axum::http::Method; - use control_plane::query_plan::{ + use asap_types::query_plan::{ ClickHousePlanningContext, ExactReadout, ExternalExactOutput, ExternalExactRequest, FallbackPolicy, FixedEvaluationRange, InstantExecution, MaterializationBinding, PhysicalGrouping, QueryLanguage, QueryNodeId, QueryPlan, QueryPlanEntry, QueryPlanNode, }; + use asap_types::summary_catalog::SummaryCatalog; + use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; + use axum::http::Method; struct FixedExactSubtree; @@ -328,6 +330,10 @@ mod tests { request: &ClickHouseQueryRequest, ) -> Result { + assert_eq!( + request.parameters.get("output_format_json_quote_denormals"), + Some(&"1".into()) + ); assert_eq!(request.parameters.get("param_from"), Some(&"0".into())); assert_eq!(request.parameters.get("param_to"), Some(&"2000".into())); Ok(ClickHouseRawResponse { @@ -703,7 +709,7 @@ mod tests { ) .unwrap(); precompute.summary_catalog = Some(sds.reference().unwrap()); - let mut transmission = control_plane::physical::compiler::TransmissionPlan::build( + let mut transmission = control_plane::physical::compiler::compile_transmission_plan( envelope.clone(), &precompute, &BTreeMap::new(), diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/clickhouse_result_adapter.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/clickhouse_result_adapter.rs index a829c548f..cdc46d262 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/clickhouse_result_adapter.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/clickhouse_result_adapter.rs @@ -2,7 +2,7 @@ use super::fallback::ClickHouseRawResponse; use arrow::{ array::{Array, Float64Array, StringArray, TimestampMillisecondArray}, datatypes::{DataType, Field, Schema}, - json::LineDelimitedWriter, + json::{LineDelimitedWriter, WriterBuilder}, record_batch::RecordBatch, util::display::array_value_to_string, }; @@ -118,6 +118,10 @@ impl ClickHouseQueryResult { if column > 0 { output.push(b'\t'); } + if batch.column(column).is_null(row) { + output.extend_from_slice(b"\\N"); + continue; + } if matches!(batch.column(column).data_type(), DataType::Map(..)) { output.extend_from_slice( map_literal(batch.column(column).as_ref(), row)?.as_bytes(), @@ -145,7 +149,9 @@ impl ClickHouseQueryResult { } let row = batch.slice(row, 1); - let mut writer = LineDelimitedWriter::new(&mut output); + let mut writer: LineDelimitedWriter<_> = WriterBuilder::new() + .with_explicit_nulls(true) + .build(&mut output); writer .write_batches(&[&row]) .map_err(|error| ClickHouseResultError::Arrow(error.to_string()))?; @@ -204,7 +210,9 @@ impl ClickHouseQueryResult { let mut rows = Vec::new(); for batch in &self.batches { let mut encoded = Vec::new(); - let mut writer = LineDelimitedWriter::new(&mut encoded); + let mut writer: LineDelimitedWriter<_> = WriterBuilder::new() + .with_explicit_nulls(true) + .build(&mut encoded); writer .write_batches(&[batch]) .map_err(|error| ClickHouseResultError::Arrow(error.to_string()))?; @@ -309,6 +317,38 @@ mod tests { }; use std::sync::Arc; + #[test] + fn nullable_fields_remain_explicit_in_json_and_tsv() { + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![ + Field::new("value", DataType::Float64, true), + Field::new("label", DataType::Utf8, true), + ])), + vec![ + Arc::new(Float64Array::from(vec![Some(1.25), None, Some(2.5)])), + Arc::new(StringArray::from(vec![Some(""), None, Some("\\N")])), + ], + ) + .unwrap(); + let result = ClickHouseQueryResult { + batches: vec![batch], + }; + let json: serde_json::Value = + serde_json::from_slice(&result.encode(ClickHouseFormat::Json).unwrap()).unwrap(); + assert_eq!( + json["data"], + serde_json::json!([{ "value":1.25,"label":"" }, { "value":null,"label":null }, {"value":2.5,"label":"\\N"}]) + ); + let lines = result.encode(ClickHouseFormat::JsonEachRow).unwrap(); + let null_row: serde_json::Value = + serde_json::from_slice(lines.split(|byte| *byte == b'\n').nth(1).unwrap()).unwrap(); + assert_eq!(null_row, serde_json::json!({"value":null,"label":null})); + assert_eq!( + result.encode(ClickHouseFormat::TabSeparated).unwrap(), + b"1.25\t\n\\N\t\\N\n2.5\t\\\\N\n" + ); + } + #[test] fn empty_map_bottom_type_uses_clickhouse_nothing() { let entries = DataType::Struct( diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs index aa9fd90cc..26183c1f6 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs @@ -9,8 +9,8 @@ use crate::{ }, storage_engines::sketch_db::index::SketchStore, }; +use asap_types::query_plan::{QueryNodeId, QueryPlanEntry, QueryPlanNode}; use asap_types::summary_catalog::SummaryCatalog; -use control_plane::query_plan::{QueryNodeId, QueryPlanEntry, QueryPlanNode}; use planner_types::post_asap::ValueOperation; use std::collections::{BTreeMap, BTreeSet}; @@ -65,8 +65,7 @@ fn execute_relation_subtree( if request.language != asap_types::QueryLanguage::ClickHouseSql { return Err("ClickHouse DAG contains an external leaf for another language".into()); } - let control_plane::query_plan::ExternalExactOutput::Relation { schema } = - &request.output + let asap_types::query_plan::ExternalExactOutput::Relation { schema } = &request.output else { return Err("ClickHouse external leaf must declare relation output".into()); }; @@ -276,7 +275,7 @@ fn execute_sql_dag_with_external_unfenced( Some(QueryPlanNode::Relational { output_schema, .. }) | Some(QueryPlanNode::RelationalJoin { output_schema, .. }) => output_schema.clone(), Some(QueryPlanNode::ExternalExact { request, .. }) => { - let control_plane::query_plan::ExternalExactOutput::Relation { schema } = + let asap_types::query_plan::ExternalExactOutput::Relation { schema } = &request.output else { return ClickHouseDagOutcome::Fallback(ClickHouseDagFallback::UnsupportedPlan( diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter.rs index 0bfc9fc91..8ddb70e9c 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter.rs @@ -1,6 +1,7 @@ //! ClickHouse row semantics for planner-owned relational wrappers. mod aggregate; +mod collection; use std::{cmp::Ordering, collections::BTreeMap, sync::Arc}; @@ -14,18 +15,10 @@ use arrow::{ }; use chrono::{DateTime, NaiveDateTime, TimeZone}; use planner_types::{ - post_asap::{SummaryFamilyType, SummaryNode, SummarySchema, ValueOperation}, + post_asap::{SummaryFamilyType, SummarySchema, ValueOperation}, pre_asap::{ArithmeticOpKind, CompareOpKind, DataType, QueryExpr, ScalarValue, SortKey}, }; -use crate::query_engines::{ - asap_query_engine::{ - summary_exec::ExecOutcome, - summary_executor::{GroupState, SummaryValue}, - }, - canonical::relational::RelationalAdapter, -}; - use super::clickhouse_result_adapter::ClickHouseQueryResult; #[derive(Debug, thiserror::Error, PartialEq)] @@ -49,6 +42,8 @@ enum Cell { Bool(bool), Timestamp(i64), Map(Vec<(Cell, Cell)>), + List(Arc<[Cell]>), + Struct(Arc<[Cell]>), } fn json_cell( @@ -68,9 +63,39 @@ fn json_cell( match dtype { DataType::Null if value.is_null() => Ok(Cell::Null), DataType::Null => Err(invalid()), - DataType::List { .. } | DataType::Struct { .. } => Err( - ClickHouseRelationalError::Unsupported("collection value transport".into()), - ), + DataType::List { element } => { + let item_type = clickhouse_type + .strip_prefix("Array(") + .and_then(|inner| inner.strip_suffix(')')) + .ok_or_else(invalid)?; + let items = value.as_array().ok_or_else(invalid)?; + Ok(Cell::List( + items + .iter() + .map(|item| json_cell(item, &element.dtype, element.nullable, item_type)) + .collect::, _>>()? + .into(), + )) + } + DataType::Struct { fields } => { + let types = + collection::tuple_field_types(clickhouse_type, fields).ok_or_else(invalid)?; + let items = value + .as_array() + .filter(|items| items.len() == fields.len()) + .ok_or_else(invalid)?; + Ok(Cell::Struct( + items + .iter() + .zip(fields) + .zip(types) + .map(|((item, field), native)| { + json_cell(item, &field.dtype, field.nullable, native) + }) + .collect::, _>>()? + .into(), + )) + } DataType::Int64 => value.as_i64().map(Cell::Int64).ok_or_else(invalid), DataType::Float64 => value.as_f64().map(Cell::Float64).ok_or_else(invalid), DataType::Utf8 => value @@ -282,20 +307,11 @@ impl ClickHouseRelation { fn map_type_parts(actual: &str) -> Option<(&str, &str)> { let inner = actual.trim().strip_prefix("Map(")?.strip_suffix(')')?; - let mut depth = 0_i32; - let mut quoted = false; - for (index, ch) in inner.char_indices() { - match ch { - '\'' => quoted = !quoted, - '(' if !quoted => depth += 1, - ')' if !quoted => depth -= 1, - ',' if !quoted && depth == 0 => { - return Some((inner[..index].trim(), inner[index + 1..].trim())) - } - _ => {} - } - } - None + let args = collection::arguments(inner)?; + let [key, value] = args.as_slice() else { + return None; + }; + Some((*key, *value)) } fn clickhouse_type_matches(actual: Option<&str>, expected: &DataType, nullable: bool) -> bool { @@ -315,7 +331,18 @@ fn clickhouse_type_matches(actual: Option<&str>, expected: &DataType, nullable: } match expected { DataType::Null => actual == "Nothing", - DataType::List { .. } | DataType::Struct { .. } => false, + DataType::List { element } => { + !nullable + && actual + .strip_prefix("Array(") + .and_then(|inner| inner.strip_suffix(')')) + .is_some_and(|inner| { + clickhouse_type_matches(Some(inner), &element.dtype, element.nullable) + }) + } + DataType::Struct { fields } => { + !nullable && collection::tuple_field_types(actual, fields).is_some() + } DataType::Int64 => actual == "Int64", DataType::Float64 => actual == "Float64", DataType::Utf8 => actual == "String", @@ -436,11 +463,17 @@ impl ClickHouseRelationalAdapter { } for row in &input.rows { for key in keys { - if contains_nan(&eval(&key.expr, row, &schema)?) { + let value = eval(&key.expr, row, &schema)?; + if contains_nan(&value) { return Err(ClickHouseRelationalError::Unsupported( "NaN sort key".into(), )); } + if !matches!(value, Cell::Null) && cell_cmp(&value, &value).is_none() { + return Err(ClickHouseRelationalError::Unsupported( + "unsupported sort key value type".into(), + )); + } } } input @@ -456,81 +489,6 @@ impl ClickHouseRelationalAdapter { } } -impl RelationalAdapter for ClickHouseRelationalAdapter -where - E: crate::query_engines::canonical::executor::SummaryExecutor< - GroupKey = BTreeMap, - State = GroupState, - Value = SummaryValue, - >, -{ - type Relation = ClickHouseRelation; - type Error = ClickHouseRelationalError; - - fn relation_from_outcome( - &self, - node: &SummaryNode, - outcome: ExecOutcome, - ) -> Result { - let fields = fields(node); - let mut rows = Vec::new(); - let mut coverage = None; - let mut coverage_complete = true; - match outcome { - ExecOutcome::Value(groups) => { - for (group, value) in groups { - match value.coverage() { - Some(next) if coverage_complete => { - coverage = intersect_coverage(coverage, Some(next)); - } - None => { - coverage = None; - coverage_complete = false; - } - Some(_) => {} - } - match value { - SummaryValue::Points(points, _) => { - for (timestamp, value) in points { - rows.push(row_from_value(&fields, &group, timestamp, value)?); - } - } - SummaryValue::TopK(_, _) => { - return Err(ClickHouseRelationalError::Unsupported( - "TopK summary rows".into(), - )); - } - } - } - } - ExecOutcome::State(groups) => { - for (group, state, family) in groups { - let value = exact_value(&state, &family)?; - rows.push(row_from_value(&fields, &group, 0, value)?); - } - } - } - Ok(ClickHouseRelation { - rows, - fields, - coverage, - }) - } - - fn apply( - &self, - node: &SummaryNode, - operation: &ValueOperation, - input: Self::Relation, - ) -> Result { - self.apply_operation(operation, &node.schema, input) - } -} - -fn fields(node: &SummaryNode) -> Vec<(String, DataType, bool)> { - fields_from_schema(&node.schema) -} - fn fields_from_schema(schema: &SummarySchema) -> Vec<(String, DataType, bool)> { schema .fields @@ -545,20 +503,6 @@ fn fields_from_schema(schema: &SummarySchema) -> Vec<(String, DataType, bool)> { .collect() } -fn exact_value( - state: &GroupState, - family: &SummaryFamilyType, -) -> Result { - if !matches!(family, SummaryFamilyType::ExactAggregate(..)) { - return Err(ClickHouseRelationalError::Unsupported( - "unfinalized non-exact summary state".into(), - )); - } - state.exact_value(&None).ok_or_else(|| { - ClickHouseRelationalError::Invalid("exact accumulator cannot be finalized".into()) - }) -} - fn row_from_value( fields: &[(String, DataType, bool)], group: &BTreeMap, @@ -645,7 +589,81 @@ fn eval( } QueryExpr::FunctionCall { name, args } => { use planner_types::pre_asap::scalar_signature::MapScalarFunction; - let function = MapScalarFunction::from_name(name).ok_or_else(|| { + if name.eq_ignore_ascii_case("asap_struct_field") { + expr.scalar_type(schema) + .map_err(|error| ClickHouseRelationalError::Invalid(error.to_string()))?; + let DataType::Struct { fields } = args[0] + .scalar_type(schema) + .map_err(|error| ClickHouseRelationalError::Invalid(error.to_string()))? + .0 + else { + unreachable!() + }; + let offset = match &args[1] { + QueryExpr::Literal(ScalarValue::Int64(index)) => { + usize::try_from(index - 1).ok() + } + QueryExpr::Literal(ScalarValue::Utf8(name)) => { + fields.iter().position(|field| &field.name == name) + } + _ => None, + } + .ok_or_else(|| { + ClickHouseRelationalError::Invalid("struct field selector".into()) + })?; + let Cell::Struct(values) = eval(&args[0], row, schema)? else { + return Err(ClickHouseRelationalError::Invalid( + "struct field input".into(), + )); + }; + return values.get(offset).cloned().ok_or_else(|| { + ClickHouseRelationalError::Invalid("struct field value".into()) + }); + } + if name.eq_ignore_ascii_case("asap_element_access") { + let (output_type, _) = expr + .scalar_type(schema) + .map_err(|error| ClickHouseRelationalError::Invalid(error.to_string()))?; + if let DataType::List { element } = args[0] + .scalar_type(schema) + .map_err(|error| ClickHouseRelationalError::Invalid(error.to_string()))? + .0 + { + let Cell::List(values) = eval(&args[0], row, schema)? else { + return Err(ClickHouseRelationalError::Invalid( + "array access input".into(), + )); + }; + let index = match eval(&args[1], row, schema)? { + Cell::Null => return Ok(Cell::Null), + Cell::Int64(index) => index, + _ => { + return Err(ClickHouseRelationalError::Invalid( + "array access index".into(), + )) + } + }; + let offset = if index > 0 { + usize::try_from(index - 1).ok() + } else if index < 0 { + usize::try_from(index.unsigned_abs()) + .ok() + .and_then(|distance| values.len().checked_sub(distance)) + } else { + None + }; + return match offset.and_then(|offset| values.get(offset)) { + Some(value) => Ok(value.clone()), + None => default_collection_element(&output_type, element.nullable), + }; + } + } + let function = (if name.eq_ignore_ascii_case("asap_element_access") { + Some(MapScalarFunction::Access) + } else { + MapScalarFunction::from_name(name) + }) + .ok_or_else(|| { ClickHouseRelationalError::Unsupported(format!("scalar function {name}")) })?; expr.scalar_type(schema) @@ -716,7 +734,7 @@ fn eval( else { unreachable!() }; - default_map_value(&value, value_nullable) + default_collection_element(&value, value_nullable) } } } @@ -726,7 +744,10 @@ fn eval( } } -fn default_map_value(dtype: &DataType, nullable: bool) -> Result { +fn default_collection_element( + dtype: &DataType, + nullable: bool, +) -> Result { if nullable { return Ok(Cell::Null); } @@ -737,9 +758,17 @@ fn default_map_value(dtype: &DataType, nullable: bool) -> Result Cell::Utf8(String::new()), DataType::Bool => Cell::Bool(false), DataType::Map { .. } => Cell::Map(Vec::new()), + DataType::List { .. } => Cell::List(Arc::from([])), + DataType::Struct { fields } => Cell::Struct( + fields + .iter() + .map(|field| default_collection_element(&field.dtype, field.nullable)) + .collect::, _>>()? + .into(), + ), _ => { return Err(ClickHouseRelationalError::Unsupported( - "map missing-key default type".into(), + "collection missing-element default type".into(), )) } }) @@ -865,6 +894,7 @@ fn compare_sort_keys( fn contains_nan(value: &Cell) -> bool { match value { Cell::Float64(value) => value.is_nan(), + Cell::List(values) | Cell::Struct(values) => values.iter().any(contains_nan), Cell::Map(entries) => entries .iter() .any(|(key, value)| contains_nan(key) || contains_nan(value)), @@ -923,14 +953,6 @@ fn cell_cmp(left: &Cell, right: &Cell) -> Option { } } -fn intersect_coverage(current: Option<(u64, u64)>, next: Option<(u64, u64)>) -> Option<(u64, u64)> { - match (current, next) { - (None, next) => next, - (current, None) => current, - (Some(left), Some(right)) => Some((left.0.max(right.0), left.1.min(right.1))), - } -} - fn arrow_type(dtype: &DataType) -> ArrowDataType { match dtype { DataType::Null => ArrowDataType::Null, @@ -1073,52 +1095,202 @@ fn build_array( #[cfg(test)] mod tests { use super::*; - use crate::query_engines::canonical::{ - executor::SummaryExecutor, relational::execute_relational, - }; use planner_types::{ - post_asap::{ExecutionTiming, SketchQuery, SummaryExpr, SummaryField, SummarySchema}, - pre_asap::{ColumnRef, GroupKeys, Predicate, ProjectItem, Reduction}, + post_asap::{SummaryField, SummarySchema}, + pre_asap::{GroupKeys, Predicate, ProjectItem}, }; use std::rc::Rc; - struct MockExecutor; - - impl SummaryExecutor for MockExecutor { - type Handle = (); - type State = GroupState; - type Value = SummaryValue; - type Error = (); - type GroupKey = BTreeMap; - - fn find_candidates( - &self, - _: &SummaryFamilyType, - _: &ColumnRef, - _: &Reduction, - _: &SummaryNode, - ) -> Result, Self::Error> { - unreachable!() - } - - fn fetch_state(&self, _: &Self::Handle) -> Result { + #[test] + fn decodes_declared_array_elements_without_losing_nullability() { + use planner_types::pre_asap::Column; + let dtype = DataType::List { + element: Box::new(Column { + name: "item".into(), + dtype: DataType::Int64, + nullable: true, + table: None, + }), + }; + assert!(clickhouse_type_matches( + Some("Array(Nullable(Int64))"), + &dtype, + false + )); + assert!(!clickhouse_type_matches( + Some("Array(Int64)"), + &dtype, + false + )); + assert!(!clickhouse_type_matches( + Some("Nullable(Array(Nullable(Int64)))"), + &dtype, + true + )); + let value = json_cell( + &serde_json::json!([9007199254740993_i64, null, -7]), + &dtype, + false, + "Array(Nullable(Int64))", + ) + .unwrap(); + let Cell::List(items) = &value else { + panic!("expected list") + }; + assert_eq!( + items.as_ref(), + &[Cell::Int64(9007199254740993), Cell::Null, Cell::Int64(-7)] + ); + let Cell::List(copy) = value.clone() else { unreachable!() - } + }; + assert!(Arc::ptr_eq(items, ©)); + assert!(json_cell( + &serde_json::json!(["wrong"]), + &dtype, + false, + "Array(Nullable(Int64))" + ) + .is_err()); + } - fn merge_states(&self, _: Vec) -> Result { - unreachable!() + #[test] + fn nonfinite_external_array_values_cannot_become_nulls() { + use planner_types::pre_asap::Column; + let dtype = DataType::List { + element: Box::new(Column::new("item", DataType::Float64, true)), + }; + for value in ["inf", "-inf", "nan"] { + assert!(json_cell( + &serde_json::json!(value), + &DataType::Float64, + true, + "Nullable(Float64)" + ) + .is_err()); + assert!(json_cell( + &serde_json::json!([value]), + &dtype, + false, + "Array(Nullable(Float64))" + ) + .is_err()); } + assert_eq!( + json_cell( + &serde_json::json!([null]), + &dtype, + false, + "Array(Nullable(Float64))" + ) + .unwrap(), + Cell::List(vec![Cell::Null].into()) + ); + } - fn readout(&self, _: &Self::State, _: &SketchQuery) -> Result { - unreachable!() + #[test] + fn array_access_uses_signed_indices_and_element_defaults() { + use planner_types::pre_asap::{Column, Schema}; + let function = |name: &str, args| QueryExpr::FunctionCall { + name: name.into(), + args, + }; + let dtype = DataType::List { + element: Box::new(Column::new("item", DataType::Int64, false)), + }; + let schema = Schema::new(vec![ + Column::new("items", dtype, false), + Column::new("index", DataType::Int64, true), + ]); + let access = function( + "asap_element_access", + vec![QueryExpr::Column(0), QueryExpr::Column(1)], + ); + let items = Cell::List(vec![Cell::Int64(10), Cell::Int64(20)].into()); + for (index, expected) in [ + (1, 10), + (2, 20), + (-1, 20), + (-2, 10), + (0, 0), + (3, 0), + (i64::MIN, 0), + (i64::MAX, 0), + ] { + assert_eq!( + eval(&access, &[items.clone(), Cell::Int64(index)], &schema).unwrap(), + Cell::Int64(expected) + ); } + assert_eq!( + eval(&access, &[items, Cell::Null], &schema).unwrap(), + Cell::Null + ); + let zero = function( + "asap_element_access", + vec![ + QueryExpr::Column(0), + QueryExpr::Literal(ScalarValue::Int64(0)), + ], + ); + assert!(eval(&zero, &[Cell::List(Arc::from([])), Cell::Int64(0)], &schema).is_err()); + } - fn logical(&self, _: &QueryExpr) -> Result { - Ok(SummaryValue::Points( - vec![(10, 2.0), (20, 3.0), (30, 1.0)], - Some((0, 40)), - )) - } + #[test] + fn nested_array_tuple_access_preserves_fields_and_defaults() { + use planner_types::pre_asap::{Column, Schema}; + let tuple = DataType::Struct { + fields: vec![ + Column::new("ts", DataType::Int64, false), + Column::new("value", DataType::Float64, true), + ], + }; + let dtype = DataType::List { + element: Box::new(Column::new("item", tuple, false)), + }; + let native = "Array(Tuple(ts Int64, value Nullable(Float64)))"; + assert!(clickhouse_type_matches(Some(native), &dtype, false)); + let samples = json_cell( + &serde_json::json!([[9007199254740993_i64, 2.5], [7, null]]), + &dtype, + false, + native, + ) + .unwrap(); + let schema = Schema::new(vec![Column::new("samples", dtype, false)]); + let field = |index, name: &str| QueryExpr::FunctionCall { + name: "asap_struct_field".into(), + args: vec![ + QueryExpr::FunctionCall { + name: "asap_element_access".into(), + args: vec![ + QueryExpr::Column(0), + QueryExpr::Literal(ScalarValue::Int64(index)), + ], + }, + QueryExpr::Literal(ScalarValue::Utf8(name.into())), + ], + }; + assert_eq!( + eval(&field(1, "ts"), std::slice::from_ref(&samples), &schema).unwrap(), + Cell::Int64(9007199254740993) + ); + assert_eq!( + eval(&field(1, "value"), std::slice::from_ref(&samples), &schema).unwrap(), + Cell::Float64(2.5) + ); + assert_eq!( + eval(&field(-1, "value"), std::slice::from_ref(&samples), &schema).unwrap(), + Cell::Null + ); + assert_eq!( + eval(&field(99, "ts"), std::slice::from_ref(&samples), &schema).unwrap(), + Cell::Int64(0) + ); + assert_eq!( + eval(&field(99, "value"), std::slice::from_ref(&samples), &schema).unwrap(), + Cell::Null + ); } fn schema(fields: &[(&str, DataType)]) -> SummarySchema { @@ -1137,22 +1309,6 @@ mod tests { } } - fn value_node( - child: Rc, - operation: ValueOperation, - schema: SummarySchema, - ) -> Rc { - Rc::new(SummaryNode { - expr: SummaryExpr::ValueOperation { - child, - operation, - timing: ExecutionTiming::ReadTime, - }, - schema, - guarantee: None, - }) - } - /// Map entries remain ordered pairs, including duplicate keys and null values. #[test] fn map_transport_retains_duplicate_keys_and_null_values() { @@ -1199,17 +1355,32 @@ mod tests { #[test] fn executes_filter_project_arithmetic_sort_and_limit_chain() { let input_schema = schema(&[("ts", DataType::Timestamp), ("sum", DataType::Float64)]); - let leaf = Rc::new(SummaryNode { - expr: SummaryExpr::KeepPreAsap(Rc::new(QueryExpr::Literal(ScalarValue::Int64(0)))), - schema: input_schema.clone(), - guarantee: None, - }); + let adapter = ClickHouseRelationalAdapter; + let input = ClickHouseRelation { + rows: vec![ + vec![Cell::Timestamp(10), Cell::Float64(2.0)], + vec![Cell::Timestamp(20), Cell::Float64(3.0)], + vec![Cell::Timestamp(30), Cell::Float64(1.0)], + ], + fields: fields_from_schema(&input_schema), + coverage: Some((0, 40)), + }; + let mut relation = adapter + .apply_filter( + &Predicate(Rc::new(QueryExpr::Compare { + left: Rc::new(QueryExpr::Column(1)), + op: CompareOpKind::Gt, + right: Rc::new(QueryExpr::Literal(ScalarValue::Float64(1.0))), + })), + input, + ) + .unwrap(); + assert_eq!(relation.rows.len(), 2); let projected_schema = schema(&[ ("bucket", DataType::Timestamp), ("score", DataType::Float64), ]); - let projected = value_node( - leaf, + for operation in [ ValueOperation::Project { cols: vec![ ProjectItem { @@ -1227,10 +1398,6 @@ mod tests { ], qualifier: None, }, - projected_schema.clone(), - ); - let sorted = value_node( - projected, ValueOperation::Sort { keys: vec![SortKey { expr: QueryExpr::Column(1), @@ -1239,16 +1406,12 @@ mod tests { }], partition_by: GroupKeys::none(), }, - projected_schema.clone(), - ); - let limited = value_node( - sorted, ValueOperation::Limit { n: 1, offset: 0 }, - projected_schema, - ); - - let relation = execute_relational(&limited, &MockExecutor, &ClickHouseRelationalAdapter) - .expect("supported SQL chain should execute"); + ] { + relation = adapter + .apply_operation(&operation, &projected_schema, relation) + .expect("installed SQL operators should execute"); + } assert_eq!(relation.coverage, Some((0, 40))); let result = relation.into_result().unwrap(); let batch = &result.batches[0]; diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter/collection.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter/collection.rs new file mode 100644 index 000000000..2bb43e7bb --- /dev/null +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/relational_adapter/collection.rs @@ -0,0 +1,120 @@ +//! Native collection metadata is checked against the existing shared schema. + +use planner_types::pre_asap::Column; + +/// Split native type arguments without splitting nested types or quoted names. +pub(super) fn arguments(input: &str) -> Option> { + let mut result = Vec::new(); + let mut start = 0; + let mut depth = 0_usize; + let mut quote = None; + let mut chars = input.char_indices().peekable(); + while let Some((position, ch)) = chars.next() { + if let Some(delimiter) = quote { + if ch == '\\' { + chars.next()?; + } else if ch == delimiter { + if chars.peek().is_some_and(|(_, next)| *next == delimiter) { + chars.next(); + } else { + quote = None; + } + } + continue; + } + match ch { + '\'' | '`' | '"' => quote = Some(ch), + '(' => depth = depth.checked_add(1)?, + ')' => depth = depth.checked_sub(1)?, + ',' if depth == 0 => { + result.push(input[start..position].trim()); + start = position + 1; + } + _ => {} + } + } + if depth != 0 || quote.is_some() { + return None; + } + if !input.is_empty() { + result.push(input[start..].trim()); + } + (!result.iter().any(|argument| argument.is_empty())).then_some(result) +} + +/// Anonymous native Tuple fields have explicit one-based names in the shared +/// Struct schema. Named fields must match their native names exactly. Other +/// Arrow names are never interpreted as an anonymous Tuple. +pub(super) fn tuple_field_types<'a>(native: &'a str, fields: &[Column]) -> Option> { + let inner = native.strip_prefix("Tuple(")?.strip_suffix(')')?; + let args = arguments(inner)?; + if args.len() != fields.len() { + return None; + } + let mut types = Vec::with_capacity(fields.len()); + for (index, (argument, field)) in args.into_iter().zip(fields).enumerate() { + if field.table.is_some() { + return None; + } + if field.name == (index + 1).to_string() + && super::clickhouse_type_matches(Some(argument), &field.dtype, field.nullable) + { + types.push(argument); + continue; + } + // Initial named transport accepts ordinary identifiers. Quoted names + // remain unsupported until a native identifier-decoding contract exists. + let (name, native_type) = argument.split_once(char::is_whitespace)?; + if name != field.name + || name.is_empty() + || !name + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || ch == '_') + || !super::clickhouse_type_matches( + Some(native_type.trim()), + &field.dtype, + field.nullable, + ) + { + return None; + } + types.push(native_type.trim()); + } + Some(types) +} + +#[cfg(test)] +mod tests { + use super::*; + use planner_types::pre_asap::DataType; + + #[test] + fn tuple_metadata_preserves_names_order_and_nested_types() { + let fields = vec![ + Column::new("ts", DataType::Int64, false), + Column::new( + "samples", + DataType::List { + element: Box::new(Column::new("item", DataType::Float64, true)), + }, + false, + ), + ]; + assert_eq!( + tuple_field_types("Tuple(ts Int64, samples Array(Nullable(Float64)))", &fields), + Some(vec!["Int64", "Array(Nullable(Float64))"]) + ); + assert!( + tuple_field_types("Tuple(samples Int64, ts Array(Nullable(Float64)))", &fields) + .is_none() + ); + assert!(tuple_field_types("Tuple(Int64, Array(Nullable(Float64)))", &fields).is_none()); + let anonymous = vec![ + Column::new("1", DataType::Int64, false), + Column::new("2", DataType::Utf8, false), + ]; + assert!(tuple_field_types("Tuple(Int64, String)", &anonymous).is_some()); + assert!(arguments("Map(String, Tuple(Int64, String)), DateTime64(3, 'UTC')").is_some()); + assert!(arguments("Map(String, Tuple(Int64, String)").is_none()); + } +} diff --git a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs index 4ae79947a..6bb5877e2 100644 --- a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs +++ b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs @@ -3,10 +3,10 @@ //! query execution checks only reachable IDs and their requested capabilities. use std::collections::{BTreeMap, BTreeSet}; +use asap_types::query_plan::{ExactReadout, QueryPlanEntry, QueryPlanNode, QueryReadout}; use asap_types::sds::{SummaryDefinitionId, SummaryDescriptor, SummaryOperator}; use asap_types::summary_catalog::SummaryCatalog; use asap_types::AggregationType; -use control_plane::query_plan::{ExactReadout, QueryPlanEntry, QueryPlanNode, QueryReadout}; use crate::query_engines::EngineError; diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index 12d31e207..ccd3a2464 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -10,7 +10,7 @@ struct QueryReadinessRequirement { } fn readiness_requirement( - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, ) -> QueryReadinessRequirement { let bindings = entry.materialization_bindings(); let mut materializations = bindings @@ -128,10 +128,7 @@ impl ASAPQueryEngine { })?; let planned = physical .query_plan - .lookup_canonical( - control_plane::query_plan::QueryLanguage::MetricsQl, - identity, - ) + .lookup_canonical(asap_types::query_plan::QueryLanguage::MetricsQl, identity) .map_err(|error| { crate::query_engines::EngineError::capability_miss("query_plan", error.to_string()) })?; @@ -160,10 +157,7 @@ impl ASAPQueryEngine { })?; let planned = physical .query_plan - .lookup_canonical( - control_plane::query_plan::QueryLanguage::MetricsQl, - identity, - ) + .lookup_canonical(asap_types::query_plan::QueryLanguage::MetricsQl, identity) .map_err(|error| { crate::query_engines::EngineError::capability_miss("query_plan", error.to_string()) })?; @@ -201,7 +195,7 @@ impl ASAPQueryEngine { async fn prepare_logical( &self, physical: &crate::storage_engines::types::ActivePhysicalPlan, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, times: &[u64], ) -> Result { super::catalog_resolver::validate_entry( @@ -266,7 +260,7 @@ impl ASAPQueryEngine { fn execute_logical_entry( &self, physical: &crate::storage_engines::types::ActivePhysicalPlan, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, leaves: &super::logical_dag::PreparedLeaves, at: u64, ) -> Result< @@ -410,7 +404,7 @@ impl ASAPQueryEngine { async fn execute_logical_range( &self, physical: &crate::storage_engines::types::ActivePhysicalPlan, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, start: u64, end: u64, step: u64, @@ -667,10 +661,7 @@ impl ASAPQueryEngine { if let Some(physical) = self.physical_plan_snapshot() { if let Ok(entry) = physical.query_plan.lookup(query) { if entry.nodes.values().any(|node| { - matches!( - node, - control_plane::query_plan::QueryPlanNode::Logical { .. } - ) + matches!(node, asap_types::query_plan::QueryPlanNode::Logical { .. }) }) { return self .execute_logical_range(&physical, entry, start_ms, end_ms, step_ms) @@ -3622,7 +3613,7 @@ mod range_stitch_tests { #[tokio::test] async fn active_metricsql_entry_reaches_the_shared_dag_executor() { - use control_plane::query_plan::{ + use asap_types::query_plan::{ FallbackPolicy, InstantExecution, QueryLanguage, QueryNodeId, QueryPlanEntry, QueryPlanNode, }; @@ -3632,9 +3623,9 @@ mod range_stitch_tests { )) .unwrap(); let mut plan = snapshot.compile().unwrap(); - let identity = control_plane::query_plan::canonical_promql("1 + 2").unwrap(); + let identity = asap_types::query_plan::canonical_promql("1 + 2").unwrap(); plan.query_plan.entries.insert( - control_plane::query_plan::QueryPlan::catalog_key(QueryLanguage::MetricsQl, &identity), + asap_types::query_plan::QueryPlan::catalog_key(QueryLanguage::MetricsQl, &identity), QueryPlanEntry { language: QueryLanguage::MetricsQl, query_id: "vm-scalar".into(), diff --git a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs index 70221144b..aa1537b3e 100644 --- a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs +++ b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs @@ -1,7 +1,7 @@ //! Fetch installed exact cuts from Prometheus before composing them with ASAP state. use super::logical_dag::{PreparedLeaf, PreparedLeaves, Value}; use crate::query_engines::EngineError; -use control_plane::query_plan::{ +use asap_types::query_plan::{ logical::LogicalOperator, ExternalExactInput, ExternalExactRequest, QueryLanguage, QueryNodeId, QueryPlanEntry, QueryPlanNode, }; @@ -441,10 +441,10 @@ pub(super) async fn prepare_external( #[cfg(test)] mod tests { use super::*; - use control_plane::query_plan::{FallbackPolicy, InstantExecution}; + use asap_types::query_plan::{FallbackPolicy, InstantExecution}; fn entry(nodes: BTreeMap) -> QueryPlanEntry { QueryPlanEntry { - language: control_plane::query_plan::QueryLanguage::PromQl, + language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "remote-cut".into(), canonical_query: "a / b".into(), fixed_evaluation: None, @@ -491,7 +491,7 @@ mod tests { request: ExternalExactRequest { language: QueryLanguage::MetricsQl, expression: "sum(rate(m[5m]))".into(), - output: control_plane::query_plan::ExternalExactOutput::InstantVector, + output: asap_types::query_plan::ExternalExactOutput::InstantVector, parameters: BTreeMap::new(), start_parameter: None, end_parameter: None, @@ -526,7 +526,7 @@ mod tests { request: ExternalExactRequest { language: QueryLanguage::PromQl, expression: query.into(), - output: control_plane::query_plan::ExternalExactOutput::InstantVector, + output: asap_types::query_plan::ExternalExactOutput::InstantVector, parameters: BTreeMap::new(), start_parameter: None, end_parameter: None, @@ -630,7 +630,7 @@ mod tests { #[tokio::test] async fn candidate_exact_is_discovered_and_prepared_behind_candidate_topk_root() { - use control_plane::query_plan::{logical::Grouping, CandidateCompleteness}; + use asap_types::query_plan::{logical::Grouping, CandidateCompleteness}; let mut entry = candidate_entry("sum by (job) (rate(m[5m]))"); entry.nodes.insert( QueryNodeId(2), @@ -751,7 +751,7 @@ mod tests { #[tokio::test] async fn exact_leaf_calls_prometheus_and_combines_with_prepared_summary() { // A successful exact branch remains an intermediate, not a whole-root fallback. - use control_plane::query_plan::logical::BinaryOperation; + use asap_types::query_plan::logical::BinaryOperation; use std::sync::{ atomic::{AtomicUsize, Ordering}, Arc, @@ -866,7 +866,7 @@ mod tests { index::{Capability, SketchInstanceMetadata}, }; use crate::storage_engines::types::{KeyByLabelValues, Measurement}; - use control_plane::query_plan::{ + use asap_types::query_plan::{ logical::BinaryOperation, ExactReadout, MaterializationBinding, PhysicalGrouping, }; use std::sync::{ diff --git a/data_plane/src/query_engines/asap_query_engine/live_serve.rs b/data_plane/src/query_engines/asap_query_engine/live_serve.rs index aac4fca19..ccfb7333a 100644 --- a/data_plane/src/query_engines/asap_query_engine/live_serve.rs +++ b/data_plane/src/query_engines/asap_query_engine/live_serve.rs @@ -160,7 +160,7 @@ pub fn serve_instant_from_summary_executor( pub fn serve_from_query_plan( index: &SketchStore, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, t0_ms: u64, t1_ms: u64, is_cumulative: bool, @@ -181,7 +181,7 @@ pub fn serve_from_query_plan( /// timestamps never leak into the public range response. pub fn serve_range_steps_from_query_plan( index: &SketchStore, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, start_ms: u64, end_ms: u64, step_ms: u64, @@ -263,7 +263,7 @@ fn coverage_covers_closed_windows( pub fn serve_instant_from_query_plan( index: &SketchStore, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, now_ms: u64, ) -> Result<(ASAPTierResult, u64), LoweringSkip> { if !summary_executor_live_enabled() { @@ -447,27 +447,27 @@ mod tests { Box::new(crate::precompute_engine::operators::SumAccumulator::with_sum(value)), ); } - let entry = control_plane::query_plan::QueryPlanEntry { - language: control_plane::query_plan::QueryLanguage::PromQl, + let entry = asap_types::query_plan::QueryPlanEntry { + language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "q-sum".into(), canonical_query: "sum_over_time(bytes[1s])".into(), fixed_evaluation: None, - root: control_plane::query_plan::QueryNodeId(0), + root: asap_types::query_plan::QueryNodeId(0), nodes: BTreeMap::from([ ( - control_plane::query_plan::QueryNodeId(0), - control_plane::query_plan::QueryPlanNode::ExactReadout { - input: control_plane::query_plan::QueryNodeId(1), - readout: control_plane::query_plan::ExactReadout::Sum, + asap_types::query_plan::QueryNodeId(0), + asap_types::query_plan::QueryPlanNode::ExactReadout { + input: asap_types::query_plan::QueryNodeId(1), + readout: asap_types::query_plan::ExactReadout::Sum, }, ), ( - control_plane::query_plan::QueryNodeId(1), - control_plane::query_plan::QueryPlanNode::ReadMaterialization { - binding: control_plane::query_plan::MaterializationBinding { + asap_types::query_plan::QueryNodeId(1), + asap_types::query_plan::QueryPlanNode::ReadMaterialization { + binding: asap_types::query_plan::MaterializationBinding { item_labels: Vec::new(), materialization: policy.into(), - output_grouping: control_plane::query_plan::PhysicalGrouping::PerEntity, + output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, window_ms: 1_000, pane_origin_ms: Some(0), readout_lookback_ms: Some(1_000), @@ -475,12 +475,12 @@ mod tests { }, ), ]), - instant: control_plane::query_plan::InstantExecution { + instant: asap_types::query_plan::InstantExecution { lookback_ms: 1_000, full_history: false, cumulative_readout: true, }, - fallback: control_plane::query_plan::FallbackPolicy::ExactBackend, + fallback: asap_types::query_plan::FallbackPolicy::ExactBackend, }; let result = serve_range_steps_from_query_plan(&idx, &entry, 1_000, 3_000, 1_000) diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs index f2c392724..93c124072 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs @@ -4,12 +4,10 @@ use crate::query_engines::{ EngineError, }; use crate::storage_engines::types::KeyByLabelValues; -use control_plane::query_plan::logical::{ +use asap_types::query_plan::logical::{ Aggregation, BinaryOperation, Grouping, LogicalOperator, TemporalOperation, }; -use control_plane::query_plan::{ - CandidateCompleteness, QueryNodeId, QueryPlanEntry, QueryPlanNode, -}; +use asap_types::query_plan::{CandidateCompleteness, QueryNodeId, QueryPlanEntry, QueryPlanNode}; use std::collections::{BTreeMap, BTreeSet}; type Labels = BTreeMap; @@ -730,7 +728,7 @@ fn bucket_quantile(q: f64, mut b: Vec<(f64, f64)>) -> f64 { #[cfg(test)] mod topk_tests { use super::*; - use control_plane::query_plan::{FallbackPolicy, InstantExecution}; + use asap_types::query_plan::{FallbackPolicy, InstantExecution}; fn labels(items: &[(&str, &str)]) -> Labels { items @@ -806,7 +804,7 @@ mod topk_tests { #[test] fn installed_topk_combines_with_prometheus_exact_child() { - let mut entry = QueryPlanEntry::compile_logical( + let mut entry = control_plane::query_plan::logical::compile_logical( "hybrid-topk".into(), "topk(2, m)".into(), InstantExecution { @@ -960,7 +958,7 @@ mod topk_tests { let summary = QueryNodeId(0); let root = QueryNodeId(1); let entry = QueryPlanEntry { - language: control_plane::query_plan::QueryLanguage::PromQl, + language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "summary-rate-topk".into(), canonical_query: "topk(2, rate(requests_total[5m]))".into(), fixed_evaluation: None, @@ -970,7 +968,7 @@ mod topk_tests { summary, QueryPlanNode::ExactReadout { input: QueryNodeId(99), - readout: control_plane::query_plan::ExactReadout::Rate, + readout: asap_types::query_plan::ExactReadout::Rate, }, ), ( @@ -1089,7 +1087,7 @@ mod topk_tests { let value_id = QueryNodeId(1); let root = QueryNodeId(2); let entry = QueryPlanEntry { - language: control_plane::query_plan::QueryLanguage::PromQl, + language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "candidate-topk".into(), canonical_query: "topk(1, rate(requests_total[5m]))".into(), fixed_evaluation: None, diff --git a/data_plane/src/query_engines/asap_query_engine/physical_dag.rs b/data_plane/src/query_engines/asap_query_engine/physical_dag.rs index a917ff23a..ea6d9f63b 100644 --- a/data_plane/src/query_engines/asap_query_engine/physical_dag.rs +++ b/data_plane/src/query_engines/asap_query_engine/physical_dag.rs @@ -6,7 +6,7 @@ use std::collections::BTreeMap; -use control_plane::query_plan::{QueryNodeId, QueryPlanEntry, QueryPlanNode}; +use asap_types::query_plan::{QueryNodeId, QueryPlanEntry, QueryPlanNode}; use thiserror::Error; pub trait QueryNodeRuntime { @@ -146,7 +146,7 @@ mod tests { use std::cell::RefCell; use std::collections::BTreeMap; - use control_plane::query_plan::{FallbackPolicy, InstantExecution, QueryReadout}; + use asap_types::query_plan::{FallbackPolicy, InstantExecution, QueryReadout}; use super::*; @@ -204,7 +204,7 @@ mod tests { .into_iter() .collect(); let entry = QueryPlanEntry { - language: control_plane::query_plan::QueryLanguage::PromQl, + language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "q".into(), canonical_query: "up".into(), fixed_evaluation: None, @@ -277,7 +277,7 @@ mod tests { .into_iter() .collect(); let entry = QueryPlanEntry { - language: control_plane::query_plan::QueryLanguage::PromQl, + language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "q".into(), canonical_query: "up".into(), fixed_evaluation: None, diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index 9c7370487..e3a408caf 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -5,7 +5,7 @@ use std::collections::BTreeMap; use crate::query_engines::asap_query_engine::summary_exec::{execute, ExecOutcome}; -use control_plane::query_plan::{QueryNodeId, QueryPlanNode}; +use asap_types::query_plan::{QueryNodeId, QueryPlanNode}; use control_plane::types_v2::AccuracyTarget; use crate::query_engines::asap_query_engine::physical_dag::{self, QueryNodeRuntime}; @@ -82,7 +82,7 @@ pub fn execute_post_asap_readout( /// Installed QueryPlan materialization resolution occurs before this legacy test helper. pub fn execute_query_plan_readout( index: &SketchStore, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, t0_ms: u64, t1_ms: u64, is_cumulative: bool, @@ -92,8 +92,8 @@ pub fn execute_query_plan_readout( pub fn execute_query_plan_from_readout( index: &SketchStore, - entry: &control_plane::query_plan::QueryPlanEntry, - root: control_plane::query_plan::QueryNodeId, + entry: &asap_types::query_plan::QueryPlanEntry, + root: asap_types::query_plan::QueryNodeId, t0_ms: u64, t1_ms: u64, is_cumulative: bool, @@ -103,7 +103,7 @@ pub fn execute_query_plan_from_readout( pub fn execute_query_plan_instant( index: &SketchStore, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, now_ms: u64, ) -> Result<(PostAsapReadoutOutcome, u64), LoweringSkip> { let t0_ms = if entry.instant.full_history { @@ -525,11 +525,11 @@ fn intersect_coverage(left: Option<(u64, u64)>, right: Option<(u64, u64)>) -> Op } fn reduce_sum_values( - grouping: &control_plane::query_plan::PhysicalGrouping, + grouping: &asap_types::query_plan::PhysicalGrouping, values: &[(BTreeMap, SummaryValue)], coverage: Option<(u64, u64)>, ) -> Result { - let control_plane::query_plan::PhysicalGrouping::Reduce(keys) = grouping else { + let asap_types::query_plan::PhysicalGrouping::Reduce(keys) = grouping else { return Ok(PhysicalQueryOutput::Value(values.to_vec(), coverage)); }; let mut groups = BTreeMap::new(); @@ -568,7 +568,7 @@ fn reduce_sum_values( fn execute_physical_query_plan( index: &SketchStore, - entry: &control_plane::query_plan::QueryPlanEntry, + entry: &asap_types::query_plan::QueryPlanEntry, t0_ms: u64, t1_ms: u64, is_cumulative: bool, @@ -578,8 +578,8 @@ fn execute_physical_query_plan( fn execute_physical_query_payload( index: &SketchStore, - entry: &control_plane::query_plan::QueryPlanEntry, - root: control_plane::query_plan::QueryNodeId, + entry: &asap_types::query_plan::QueryPlanEntry, + root: asap_types::query_plan::QueryNodeId, t0_ms: u64, t1_ms: u64, is_cumulative: bool, @@ -844,7 +844,7 @@ mod tests { }) .collect::>(); let PhysicalQueryOutput::Value(result, _) = reduce_sum_values( - &control_plane::query_plan::PhysicalGrouping::Reduce(vec!["service".into()]), + &asap_types::query_plan::PhysicalGrouping::Reduce(vec!["service".into()]), &values, Some((1000, 2000)), ) @@ -1042,22 +1042,22 @@ mod tests { register_hll(&idx, 2, "worker", &["b", "c"]); let node = plan_promql_to_post_asap(&idx, "count(unique_users)", accuracy()) .expect("compile-stage fixture"); - let canonical = control_plane::query_plan::canonical_promql("count(unique_users)").unwrap(); - let entry = control_plane::query_plan::QueryPlanEntry::compile_bound( + let canonical = asap_types::query_plan::canonical_promql("count(unique_users)").unwrap(); + let entry = control_plane::query_plan::compile_bound( "q-cardinality".into(), canonical, &node, - control_plane::query_plan::InstantExecution { + asap_types::query_plan::InstantExecution { lookback_ms: 60_000, full_history: false, cumulative_readout: true, }, - control_plane::query_plan::FallbackPolicy::ExactBackend, + asap_types::query_plan::FallbackPolicy::ExactBackend, |_node, _family| { - Ok(control_plane::query_plan::MaterializationBinding { + Ok(asap_types::query_plan::MaterializationBinding { item_labels: Vec::new(), materialization: asap_types::PolicyFingerprint(123).into(), - output_grouping: control_plane::query_plan::PhysicalGrouping::PerEntity, + output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, window_ms: 60_000, pane_origin_ms: Some(2_000), readout_lookback_ms: Some(60_000), @@ -1193,27 +1193,27 @@ mod tests { ); } - let entry = control_plane::query_plan::QueryPlanEntry { - language: control_plane::query_plan::QueryLanguage::PromQl, + let entry = asap_types::query_plan::QueryPlanEntry { + language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "q-rate".into(), canonical_query: "rate(requests_total[1m])".into(), fixed_evaluation: None, - root: control_plane::query_plan::QueryNodeId(0), + root: asap_types::query_plan::QueryNodeId(0), nodes: BTreeMap::from([ ( - control_plane::query_plan::QueryNodeId(0), + asap_types::query_plan::QueryNodeId(0), QueryPlanNode::ExactReadout { - input: control_plane::query_plan::QueryNodeId(1), - readout: control_plane::query_plan::ExactReadout::Sum, + input: asap_types::query_plan::QueryNodeId(1), + readout: asap_types::query_plan::ExactReadout::Sum, }, ), ( - control_plane::query_plan::QueryNodeId(1), + asap_types::query_plan::QueryNodeId(1), QueryPlanNode::ReadMaterialization { - binding: control_plane::query_plan::MaterializationBinding { + binding: asap_types::query_plan::MaterializationBinding { item_labels: Vec::new(), materialization: policy.into(), - output_grouping: control_plane::query_plan::PhysicalGrouping::PerEntity, + output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, window_ms: 10_000, pane_origin_ms: Some(0), readout_lookback_ms: Some(60_000), @@ -1221,12 +1221,12 @@ mod tests { }, ), ]), - instant: control_plane::query_plan::InstantExecution { + instant: asap_types::query_plan::InstantExecution { lookback_ms: 60_000, full_history: false, cumulative_readout: true, }, - fallback: control_plane::query_plan::FallbackPolicy::ExactBackend, + fallback: asap_types::query_plan::FallbackPolicy::ExactBackend, }; for (now, expected) in [(60_000, 21.0), (70_000, 27.0), (80_000, 33.0)] { let (outcome, _) = execute_query_plan_instant(&idx, &entry, now).unwrap(); @@ -1289,27 +1289,27 @@ mod tests { accumulator.update(Measurement::new(13.0), 50_000); idx.append_precompute(7, BTreeMap::new(), (0, 60_000), Box::new(accumulator)); - let entry = control_plane::query_plan::QueryPlanEntry { - language: control_plane::query_plan::QueryLanguage::PromQl, + let entry = asap_types::query_plan::QueryPlanEntry { + language: asap_types::query_plan::QueryLanguage::PromQl, query_id: "q-rate".into(), canonical_query: "rate(requests_total[1m])".into(), fixed_evaluation: None, - root: control_plane::query_plan::QueryNodeId(0), + root: asap_types::query_plan::QueryNodeId(0), nodes: BTreeMap::from([ ( - control_plane::query_plan::QueryNodeId(0), + asap_types::query_plan::QueryNodeId(0), QueryPlanNode::ExactReadout { - input: control_plane::query_plan::QueryNodeId(1), - readout: control_plane::query_plan::ExactReadout::Rate, + input: asap_types::query_plan::QueryNodeId(1), + readout: asap_types::query_plan::ExactReadout::Rate, }, ), ( - control_plane::query_plan::QueryNodeId(1), + asap_types::query_plan::QueryNodeId(1), QueryPlanNode::ReadMaterialization { - binding: control_plane::query_plan::MaterializationBinding { + binding: asap_types::query_plan::MaterializationBinding { item_labels: Vec::new(), materialization: policy.into(), - output_grouping: control_plane::query_plan::PhysicalGrouping::PerEntity, + output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, window_ms: 60_000, pane_origin_ms: Some(0), readout_lookback_ms: Some(60_000), @@ -1317,12 +1317,12 @@ mod tests { }, ), ]), - instant: control_plane::query_plan::InstantExecution { + instant: asap_types::query_plan::InstantExecution { lookback_ms: 60_000, full_history: false, cumulative_readout: true, }, - fallback: control_plane::query_plan::FallbackPolicy::ExactBackend, + fallback: asap_types::query_plan::FallbackPolicy::ExactBackend, }; let outcome = execute_query_plan_readout(&idx, &entry, 0, 60_000, true) .expect("execute exact rate DAG"); diff --git a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs index 75ce06ad1..ab3389e05 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs @@ -228,7 +228,7 @@ impl GroupState { /// in QueryPlan so serving never infers semantics from PromQL text. pub fn exact_value_for( &self, - readout: control_plane::query_plan::ExactReadout, + readout: asap_types::query_plan::ExactReadout, key: &Option, range_start_ms: u64, range_end_ms: u64, @@ -237,23 +237,23 @@ impl GroupState { return None; }; let stat = match (readout, agg_type) { - (control_plane::query_plan::ExactReadout::Count, AggregationType::Sum) => { + (asap_types::query_plan::ExactReadout::Count, AggregationType::Sum) => { asap_types::Statistic::Count } ( - control_plane::query_plan::ExactReadout::Sum, + asap_types::query_plan::ExactReadout::Sum, AggregationType::Sum | AggregationType::MultipleSum, ) => asap_types::Statistic::Sum, ( - control_plane::query_plan::ExactReadout::Increase, + asap_types::query_plan::ExactReadout::Increase, AggregationType::Increase | AggregationType::MultipleIncrease, ) => asap_types::Statistic::Increase, ( - control_plane::query_plan::ExactReadout::Rate, + asap_types::query_plan::ExactReadout::Rate, AggregationType::Increase | AggregationType::MultipleIncrease, ) => asap_types::Statistic::Rate, ( - control_plane::query_plan::ExactReadout::Max, + asap_types::query_plan::ExactReadout::Max, AggregationType::MinMax | AggregationType::MultipleMinMax, ) => asap_types::Statistic::Max, _ => return None, @@ -284,7 +284,7 @@ impl GroupState { if matches!( agg_type, AggregationType::MinMax | AggregationType::MultipleMinMax - ) && readout == control_plane::query_plan::ExactReadout::Max + ) && readout == asap_types::query_plan::ExactReadout::Max { return entries .iter() @@ -312,7 +312,7 @@ impl GroupState { ("range_end_ms".to_string(), range_end_ms.to_string()), ]); let merged = merged?; - if readout == control_plane::query_plan::ExactReadout::Count { + if readout == asap_types::query_plan::ExactReadout::Count { return merged.aux_stats().count.map(|count| count as f64); } merged.query_statistic(stat, key, &query_kwargs).ok() @@ -406,7 +406,7 @@ impl SummaryValue { } fn validate_binding_phase( - binding: &control_plane::query_plan::MaterializationBinding, + binding: &asap_types::query_plan::MaterializationBinding, evaluation_ms: u64, ) -> Result<(), SummaryExecutorError> { if i64::try_from(binding.window_ms).is_err() { @@ -435,9 +435,9 @@ impl QueryExecutionContext<'_> { /// are integrity checks and never broaden the candidate set. pub fn read_bound_materialization( &self, - binding: &control_plane::query_plan::MaterializationBinding, + binding: &asap_types::query_plan::MaterializationBinding, ) -> Result, GroupState)>, SummaryExecutorError> { - use control_plane::query_plan::PhysicalGrouping; + use asap_types::query_plan::PhysicalGrouping; let inventory_revision = self.index.summary_update_revision(); let query_range = asap_types::sds::HalfOpenTimeRange { @@ -1410,10 +1410,10 @@ mod tests { #[test] fn pane_only_reads_require_the_planned_evaluation_phase() { - let binding = control_plane::query_plan::MaterializationBinding { + let binding = asap_types::query_plan::MaterializationBinding { item_labels: Vec::new(), materialization: asap_types::PolicyFingerprint(7).into(), - output_grouping: control_plane::query_plan::PhysicalGrouping::PerEntity, + output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, window_ms: 60_000, pane_origin_ms: Some(7_000), readout_lookback_ms: Some(60_000), @@ -1421,7 +1421,7 @@ mod tests { validate_binding_phase(&binding, 67_000).unwrap(); assert!(validate_binding_phase(&binding, 68_000).is_err()); - let legacy = control_plane::query_plan::MaterializationBinding { + let legacy = asap_types::query_plan::MaterializationBinding { item_labels: Vec::new(), pane_origin_ms: None, ..binding @@ -1819,7 +1819,7 @@ mod tests { use crate::precompute_engine::operators::univmon_accumulator::UnivMonAccumulator; use crate::storage_engines::sketch_db::index::SketchEncoding; use crate::storage_engines::types::SerializableToSink; - use control_plane::query_plan::{MaterializationBinding, PhysicalGrouping}; + use asap_types::query_plan::{MaterializationBinding, PhysicalGrouping}; let index = SketchStore::new(); let fp = asap_types::PolicyFingerprint(701); let mut meta = kll_meta(1, "m", &["job"]); diff --git a/data_plane/src/query_engines/canonical/mod.rs b/data_plane/src/query_engines/canonical/mod.rs index 61290336b..c170e5d0e 100644 --- a/data_plane/src/query_engines/canonical/mod.rs +++ b/data_plane/src/query_engines/canonical/mod.rs @@ -1,10 +1,7 @@ -//! Language-independent execution of ASAPPlanner's post-ASAP DAG. +//! Compatibility re-exports for the existing summary executor. //! -//! The implementation remains in its compatibility location while PromQL -//! callers migrate. These re-exports give SQL and PromQL one execution API -//! without changing the existing PromQL types or behavior. - -pub mod relational; +//! Installed SQL serving executes QueryPlan nodes through its typed relational +//! adapter. These aliases remain for callers of the summary execution API. pub mod executor { pub use crate::query_engines::asap_query_engine::summary_exec::{ diff --git a/data_plane/src/query_engines/canonical/relational.rs b/data_plane/src/query_engines/canonical/relational.rs deleted file mode 100644 index 816e97ba6..000000000 --- a/data_plane/src/query_engines/canonical/relational.rs +++ /dev/null @@ -1,62 +0,0 @@ -//! Relational query-time operators over the shared summary executor. -//! -//! The legacy executor intentionally keeps `Value` opaque. This companion -//! adapter lets a language runtime interpret planner-owned `ValueOperation` -//! nodes without widening that trait or changing existing PromQL behavior. - -use planner_types::post_asap::{SummaryExpr, SummaryNode, ValueOperation}; - -use super::executor::{execute, ExecError, ExecOutcome, SummaryExecutor}; - -pub trait RelationalAdapter { - type Relation; - type Error; - - fn relation_from_outcome( - &self, - node: &SummaryNode, - outcome: ExecOutcome, - ) -> Result; - - fn apply( - &self, - node: &SummaryNode, - operation: &ValueOperation, - input: Self::Relation, - ) -> Result; -} - -#[derive(Debug)] -pub enum RelationalExecError { - Executor(ExecError), - Adapter(AdapterError), -} - -/// Recursively evaluates query-time relational nodes and delegates every -/// summary/state node to the unchanged shared summary executor. -pub fn execute_relational( - node: &SummaryNode, - executor: &E, - adapter: &A, -) -> Result> -where - E: SummaryExecutor, - A: RelationalAdapter, -{ - match &node.expr { - SummaryExpr::ValueOperation { - child, operation, .. - } => { - let input = execute_relational(child, executor, adapter)?; - adapter - .apply(node, operation, input) - .map_err(RelationalExecError::Adapter) - } - _ => { - let outcome = execute(node, executor).map_err(RelationalExecError::Executor)?; - adapter - .relation_from_outcome(node, outcome) - .map_err(RelationalExecError::Adapter) - } - } -} diff --git a/data_plane/src/storage_engines/sketch_db/index/admission.rs b/data_plane/src/storage_engines/sketch_db/index/admission.rs index f0d6eceed..19c325108 100644 --- a/data_plane/src/storage_engines/sketch_db/index/admission.rs +++ b/data_plane/src/storage_engines/sketch_db/index/admission.rs @@ -22,7 +22,7 @@ pub(super) struct AdmissionInventory { replay_floors: BTreeMap, observed_extent: Option, finite_complete: bool, - published_series: BTreeSet, + published_series: BTreeMap, pending_revisions: usize, } @@ -182,7 +182,7 @@ impl AdmissionInventory { if self.generation.as_ref() != Some(generation) { return Err("summary series publication catalog generation differs".into()); } - if !self.published_series.contains(&series_id) + if !self.published_series.contains_key(&series_id) && self.published_series.len() >= Self::MAX_WINDOWS { return Err("summary admission series capacity exceeded".into()); @@ -198,11 +198,16 @@ impl AdmissionInventory { return Err("summary coordinate changed series identity".into()); } window.series_id = Some(series_id); - self.published_series.insert(series_id); + let end = u64::try_from(coordinate.time_range.end_ms) + .map_err(|_| "published window end is outside storage timestamp range")?; + self.published_series + .entry(series_id) + .and_modify(|current| *current = (*current).max(end)) + .or_insert(end); Ok(()) } - pub(super) fn seal_finite(&mut self, generation: &CatalogGeneration) -> Result<(), String> { + pub(super) fn validate_finite(&self, generation: &CatalogGeneration) -> Result { if self.generation.as_ref() != Some(generation) { return Err("finite completion catalog generation differs".into()); } @@ -213,11 +218,15 @@ impl AdmissionInventory { { return Err("finite source has unpublished summary windows".into()); } - self.finite_complete = true; - self.revision = self - .revision + self.revision .checked_add(1) - .ok_or("summary admission revision exhausted")?; + .ok_or_else(|| "summary admission revision exhausted".into()) + } + + pub(super) fn seal_finite(&mut self, generation: &CatalogGeneration) -> Result<(), String> { + let revision = self.validate_finite(generation)?; + self.finite_complete = true; + self.revision = revision; Ok(()) } @@ -228,7 +237,7 @@ impl AdmissionInventory { range: HalfOpenTimeRange, ) -> bool { self.finite_complete - && self.published_series.contains(&series_id) + && self.published_series.contains_key(&series_id) && self.observed_extent.is_some_and(|extent| { range.start_ms >= extent.start_ms && range.end_ms <= extent.end_ms }) @@ -244,6 +253,10 @@ impl AdmissionInventory { }) } + pub(super) fn published_frontiers(&self) -> &BTreeMap { + &self.published_series + } + pub(super) fn revision(&self) -> u64 { self.revision } @@ -358,11 +371,18 @@ mod tests { assert!(inventory.has_pending(coordinate.summary_definition_id, coordinate.time_range)); inventory.retire_completed_before(coordinate.summary_definition_id, 1000); assert_eq!(inventory.windows.len(), 1); + inventory + .record_series(&generation, &coordinate, 42) + .unwrap(); inventory .acknowledge(&generation, &coordinate, second) .unwrap(); inventory.retire_completed_before(coordinate.summary_definition_id, 1000); assert!(inventory.windows.is_empty()); + assert_eq!( + inventory.published_frontiers().get(&42), + Some(&(coordinate.time_range.end_ms as u64)) + ); } #[test] diff --git a/data_plane/src/storage_engines/sketch_db/index/epoch_columnar.rs b/data_plane/src/storage_engines/sketch_db/index/epoch_columnar.rs index 4442dfcbc..695c75052 100644 --- a/data_plane/src/storage_engines/sketch_db/index/epoch_columnar.rs +++ b/data_plane/src/storage_engines/sketch_db/index/epoch_columnar.rs @@ -956,6 +956,19 @@ impl SidStoreData { } } + /// Whether any hot or queued-for-flush state belongs to this completion prefix. + pub(crate) fn contains_window_ending_at_or_before(&self, end_ms: u64) -> bool { + self.current_epoch + .iter_entries() + .any(|(window, _, _)| window.1 <= end_ms) + || self.sealed_epochs.values().any(|epoch| { + epoch + .entries + .iter() + .any(|(window, _, _)| window.1 <= end_ms) + }) + } + /// Time-driven seal for the persistence tier: roll every window in /// `current_epoch` whose END is at or before `cutoff_end` into a /// freshly-sealed epoch, leaving the more-recent windows in @@ -1372,6 +1385,21 @@ mod tests { assert_eq!(s.current_epoch.distinct_windows(), 0); } + #[test] + fn completion_prefix_does_not_wait_for_future_windows() { + let mut store = SidStoreData::, i32>::new(); + store.insert((30, 60), vec![], 1); + assert!(!store.contains_window_ending_at_or_before(30)); + store.insert((0, 30), vec![], 2); + assert!(store.contains_window_ending_at_or_before(30)); + store.persistence_enabled = true; + store.seal_aged_windows(31); + assert!(store.contains_window_ending_at_or_before(30)); + store.sealed_epochs.clear(); + assert!(!store.contains_window_ending_at_or_before(30)); + assert!(store.contains_window_ending_at_or_before(60)); + } + #[test] fn seal_aged_windows_noop_when_persistence_disabled() { let mut s = SidStoreData::::new(); diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index ca46a2c69..bc75907fd 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -537,8 +537,8 @@ struct IncompleteSummaryLineage { window_end_unix_ms: u64, } -impl From<&control_plane::physical::compiler::SummaryFrameIdentity> for IncompleteSummaryLineage { - fn from(frame: &control_plane::physical::compiler::SummaryFrameIdentity) -> Self { +impl From<&asap_types::producer_plan::SummaryFrameIdentity> for IncompleteSummaryLineage { + fn from(frame: &asap_types::producer_plan::SummaryFrameIdentity) -> Self { Self { plan_id: frame.plan_id, plan_version: frame.plan_version, @@ -605,6 +605,9 @@ impl Drop for StateMutation<'_> { #[derive(Default)] pub struct SketchStore { + /// Held through each state append; completion takes the exclusive guard. + completed_windows: RwLock>, + completion_flush_before: std::sync::atomic::AtomicU64, admission: RwLock, mutation_revision: std::sync::atomic::AtomicU64, active_mutations: std::sync::atomic::AtomicUsize, @@ -879,17 +882,65 @@ impl SketchStore { pub(crate) fn seal_finite_summary_input( &self, generation: &CatalogGeneration, - ) -> Result<(), String> { + ) -> Result { use std::sync::atomic::Ordering::SeqCst; + // Same order as admitted publication: admission -> metadata -> append fence. + // Closing a receiver alone is insufficient: the fence also rejects writes + // from every other producer once these physical windows are complete. + let mut inventory = self.admission.write().unwrap(); + let frontiers = inventory.published_frontiers().clone(); + let instances = self.instances.read().unwrap(); + let mut records = Vec::new(); + for (sid, end) in &frontiers { + let instance = instances + .get(sid) + .ok_or("completed series has no identity")?; + let mut record = self + .metadata_record(instance) + .ok_or("completed series has no catalog provenance")?; + record.completed_through_ms = record.completed_through_ms.max(Some(*end)); + records.push(record); + } + let mut completed = self.completed_windows.write().unwrap(); let mutation = self.mutation_revision.load(SeqCst); if self.active_mutations.load(SeqCst) != 0 || mutation != self.admitted_mutations.load(SeqCst) { return Err("finite summary completion cannot certify untracked state writes".into()); } - self.admission.write().unwrap().seal_finite(generation)?; + inventory.validate_finite(generation)?; + if self.persistence_read.read().unwrap().is_some() { + if let Some(end) = frontiers.values().max() { + self.completion_flush_before + .fetch_max(end.saturating_add(1), SeqCst); + } + // The flusher evicts an epoch only after its payload and manifest + // are durable. Until then a restart must remain able to replay it. + let pending = frontiers.iter().any(|(sid, end)| { + self.series.get(sid).is_some_and(|data| { + data.read() + .unwrap() + .contains_window_ending_at_or_before(*end) + }) + }); + if pending { + return Ok(false); + } + } + if let Some(writer) = self.persistence_metadata.read().unwrap().as_ref() { + writer + .upsert_all(&records) + .map_err(|error| error.to_string())?; + } + inventory.seal_finite(generation)?; + for (sid, end) in frontiers { + completed + .entry(sid) + .and_modify(|value| *value = (*value).max(end)) + .or_insert(end); + } self.finite_mutation_revision.store(mutation, SeqCst); - Ok(()) + Ok(true) } pub(crate) fn summary_window_known_empty( @@ -1065,6 +1116,12 @@ impl SketchStore { .series .get(series_id) .map(|entry| Arc::clone(entry.value())); + let completed_through = self + .completed_windows + .read() + .unwrap() + .get(series_id) + .copied(); let status = match binding.metadata.status() { AggStatus::Active => SummaryInstanceStatus::Ready, AggStatus::Retired | AggStatus::Expired => SummaryInstanceStatus::Retiring, @@ -1076,16 +1133,16 @@ impl SketchStore { .map_err(|_| "summary instance start exceeds signed timestamp range")?; let end_ms = i64::try_from(window.1) .map_err(|_| "summary instance end exceeds signed timestamp range")?; - let group_bytes = - serde_json::to_vec(&group_values).map_err(|error| error.to_string())?; - let group_fingerprint = xxhash_rust::xxh64::xxh64(&group_bytes, 0); - let instance_id = SummaryInstanceId::new(format!( - "summary-instance:v1:{}:{}:{}:{}", - summary_definition_id.as_u64(), - window.0, - window.1, - group_fingerprint - )) + let group_fingerprint = xxhash_rust::xxh64::xxh64( + &serde_json::to_vec(&group_values).map_err(|error| error.to_string())?, + 0, + ); + let instance_id = asap_types::sds::SummaryInstanceCoordinates { + summary_definition_id, + time_range: asap_types::sds::HalfOpenTimeRange { start_ms, end_ms }, + group_values: group_values.clone(), + } + .instance_id() .map_err(|error| error.to_string())?; let instance = SummaryInstance { instance_id: instance_id.clone(), @@ -1111,10 +1168,11 @@ impl SketchStore { checksum: None, }, status: status.clone(), - // The current payload row does not distinguish a normal - // pane close from a late standalone correction. Report the - // concrete instance without inventing a completeness proof. - completeness: InstanceCompleteness::Unknown, + completeness: if completed_through.is_some_and(|end| window.1 <= end) { + InstanceCompleteness::Complete + } else { + InstanceCompleteness::Unknown + }, lifecycle: InstanceLifecycle::Persistent, observed_at_ms, }; @@ -1213,7 +1271,11 @@ impl SketchStore { series_label_values: BTreeMap, window: TimestampRange, sample: SketchSampleState, - ) { + ) -> bool { + let completed = self.completed_windows.read().unwrap(); + if completed.get(&sid).is_some_and(|end| window.1 <= *end) { + return false; + } let _mutation = self.begin_state_mutation(); let store = self .series @@ -1223,6 +1285,7 @@ impl SketchStore { let mut guard = store.write().unwrap(); guard.insert(window, series_label_values, AggPayload::Sketch(sample)); guard.last_write_unix_ms = now_ms(); + true } /// Build a `SidStoreData` pre-configured for the store's current @@ -1256,7 +1319,11 @@ impl SketchStore { series_label_values: BTreeMap, window: TimestampRange, payload: Box, - ) { + ) -> bool { + let completed = self.completed_windows.read().unwrap(); + if completed.get(&sid).is_some_and(|end| window.1 <= *end) { + return false; + } let _mutation = self.begin_state_mutation(); let max_value = payload .as_any() @@ -1287,6 +1354,7 @@ impl SketchStore { retention_horizon_ms, ); } + true } /// Read a category through the derived in-memory rollup. Returns `None` @@ -2423,6 +2491,7 @@ impl SketchStore { record.summary_definition_id = Some(SummaryDefinitionId::from(m.policy_fp)); record.catalog_generation = Some(Arc::clone(m.catalog_generation.as_ref()?)); } + record.completed_through_ms = self.completed_windows.read().unwrap().get(&m.sid).copied(); record.retired_at_ms = m.retired_at_ms; record.expires_at_ms = m.expires_at_ms; Some(record) @@ -2617,7 +2686,7 @@ impl SketchStore { pub fn mark_summary_lineage_incomplete( &self, sid: u64, - frame: &control_plane::physical::compiler::SummaryFrameIdentity, + frame: &asap_types::producer_plan::SummaryFrameIdentity, ) { self.incomplete_summary_lineages .entry(sid) @@ -2629,7 +2698,7 @@ impl SketchStore { pub fn clear_summary_lineage_incomplete( &self, sid: u64, - frame: &control_plane::physical::compiler::SummaryFrameIdentity, + frame: &asap_types::producer_plan::SummaryFrameIdentity, ) { let key = IncompleteSummaryLineage::from(frame); if let Some(mut lineages) = self.incomplete_summary_lineages.get_mut(&sid) { @@ -2800,7 +2869,7 @@ impl SketchStore { } let window = (output.start_timestamp, output.end_timestamp); - match crate::storage_engines::sketch_db::data::agg_kind_for_config(agg_cfg) { + let accepted = match crate::storage_engines::sketch_db::data::agg_kind_for_config(agg_cfg) { AggKind::Sketch { .. } => self.append_sample( sid, label_values_map, @@ -2816,8 +2885,8 @@ impl SketchStore { window, accumulator.clone_boxed_core(), ), - } - Some(sid) + }; + accepted.then_some(sid) } /// Phase 5 M2.3.6d — eviction-side helper. Removes every sid in the @@ -3012,6 +3081,14 @@ impl SketchStore { let mut registered = 0usize; for rec in records { + if let Some(end) = rec.completed_through_ms { + self.completed_windows + .write() + .unwrap() + .entry(rec.sid) + .and_modify(|current| *current = (*current).max(end)) + .or_insert(end); + } if rec.removed || rec.expires_at_ms.is_some_and(|expiry| expiry <= now_ms()) { continue; } @@ -3109,6 +3186,13 @@ impl SketchStore { // the trait keeps the historical name so the flusher / manifest / // part-writer stay untouched. impl crate::storage_engines::sketch_db::index::persistence::EpochSource for SketchStore { + fn flush_before_ms(&self) -> Option { + let cutoff = self + .completion_flush_before + .load(std::sync::atomic::Ordering::SeqCst); + (cutoff != 0).then_some(cutoff) + } + fn list_sealed_epochs( &self, ) -> Vec { @@ -3536,9 +3620,8 @@ mod tests { #[test] fn incomplete_delta_window_fails_closed_until_matching_full_checkpoint() { - use control_plane::physical::compiler::{ - StateEncoding, SummaryFrameIdentity, SummaryFrameKind, - }; + use asap_types::producer_plan::{SummaryFrameIdentity, SummaryFrameKind}; + use control_plane::physical::compiler::StateEncoding; let idx = SketchStore::new(); idx.register(meta(12)); @@ -4752,6 +4835,168 @@ mod tests { assert_eq!(expired.expires_at_ms, Some(2)); } + #[test] + fn completed_windows_reject_late_updates_after_restart() { + // Completion is a storage admission rule, including legacy producers, + // and survives restart without allowing a correction into consumed state. + let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + serde_json::from_str(include_str!( + "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" + )) + .unwrap(); + let plan = snapshot.compile().unwrap(); + let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); + let directory = tempfile::tempdir().unwrap(); + let store = SketchStore::new(); + store + .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) + .unwrap(); + store.register(meta_with_policy(850, fingerprint)); + let generation = store.active_catalog_generation().unwrap(); + let writer = Arc::new(persistence::metadata::SidMetadataStore::new( + directory.path(), + )); + *store.persistence_metadata.write().unwrap() = Some(writer.clone()); + let coordinate = asap_types::sds::SummaryInstanceCoordinates { + summary_definition_id: fingerprint.into(), + time_range: HalfOpenTimeRange { + start_ms: 0, + end_ms: 30_000, + }, + group_values: BTreeMap::new(), + }; + let revision = store + .admit_summary_updates(&generation, [coordinate.clone()].into()) + .unwrap(); + assert!(store.seal_finite_summary_input(&generation).is_err()); + store + .publish_admitted_summary_update( + &generation, + &coordinate, + revision, + revision, + 120_000, + || { + store + .append_sample(850, BTreeMap::new(), (0, 30_000), sample(1)) + .then_some(850) + }, + ) + .unwrap(); + let stale_record = store + .metadata_record(&store.instances.read().unwrap()[&850]) + .unwrap(); + let before_failed_seal = store.summary_update_revision(); + std::fs::create_dir(writer.path()).unwrap(); + assert!(store.seal_finite_summary_input(&generation).is_err()); + assert!(store.summary_update_revision().matches(before_failed_seal)); + assert!(!store.completed_windows.read().unwrap().contains_key(&850)); + std::fs::remove_dir(writer.path()).unwrap(); + store.seal_finite_summary_input(&generation).unwrap(); + let producers = BTreeMap::from([(fingerprint.into(), "producer".to_string())]); + let inventory = store + .observed_summary_inventory("backend", "store", &producers, 1, 30_000) + .unwrap(); + assert_eq!(inventory.instances.len(), 1); + assert_eq!( + inventory.instances.values().next().unwrap().completeness, + InstanceCompleteness::Complete + ); + assert!(!store.append_sample(850, BTreeMap::new(), (0, 30_000), sample(2))); + assert!(!store.append_precompute( + 850, + BTreeMap::new(), + (0, 30_000), + Box::new(crate::precompute_engine::operators::SumAccumulator::new()) + )); + // A flusher that captured metadata before completion cannot reopen it. + writer.upsert_all(&[stale_record]).unwrap(); + assert_eq!(writer.load().unwrap()[0].completed_through_ms, Some(30_000)); + let restored = SketchStore::new(); + restored + .install_summary_catalog(Arc::new(plan.summary_catalog)) + .unwrap(); + restored.register_recovered_disk_series(directory.path()); + assert!(!restored.append_sample(850, BTreeMap::new(), (0, 30_000), sample(3))); + assert!(restored.append_sample(850, BTreeMap::new(), (30_000, 60_000), sample(4))); + } + + #[test] + fn finite_completion_flushes_payload_before_persisting_immutability() { + // With neither memory pressure nor a hot-tier deadline, completion must + // explicitly flush its payload before persisting a non-replayable window. + let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + serde_json::from_str(include_str!( + "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" + )) + .unwrap(); + let plan = snapshot.compile().unwrap(); + let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); + let directory = tempfile::tempdir().unwrap(); + { + let store = Arc::new(SketchStore::new()); + store + .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) + .unwrap(); + store.register(meta_with_policy(851, fingerprint)); + let mut config = durable_cfg(directory.path().to_path_buf()); + config.hot_window_ms = None; + config.seal_window_count = 100; + let mut persistence = store.start_persistence(config).unwrap(); + let generation = store.active_catalog_generation().unwrap(); + let coordinate = asap_types::sds::SummaryInstanceCoordinates { + summary_definition_id: fingerprint.into(), + time_range: HalfOpenTimeRange { + start_ms: 0, + end_ms: 30_000, + }, + group_values: BTreeMap::new(), + }; + let revision = store + .admit_summary_updates(&generation, [coordinate.clone()].into()) + .unwrap(); + store + .publish_admitted_summary_update( + &generation, + &coordinate, + revision, + revision, + 120_000, + || { + store + .append_sample(851, BTreeMap::new(), (0, 30_000), sample(1)) + .then_some(851) + }, + ) + .unwrap(); + assert!(!store.seal_finite_summary_input(&generation).unwrap()); + assert!(!store.completed_windows.read().unwrap().contains_key(&851)); + assert!(wait_until( + || store.seal_finite_summary_input(&generation).unwrap(), + Duration::from_secs(5) + )); + assert!(!persistence.manifest.live_parts().is_empty()); + persistence.shutdown(); + } + let restored = Arc::new(SketchStore::new()); + restored + .install_summary_catalog(Arc::new(plan.summary_catalog)) + .unwrap(); + let _persistence = restored + .start_persistence(durable_cfg(directory.path().to_path_buf())) + .unwrap(); + assert!(!restored.append_sample(851, BTreeMap::new(), (0, 30_000), sample(2))); + let rows = restored.query_range(851, 0, 30_000); + assert_eq!(rows.len(), 1); + let payloads: Vec<_> = rows + .iter() + .flat_map(|row| row.samples.values()) + .flatten() + .collect(); + assert_eq!(payloads.len(), 1); + assert_eq!(payloads[0].bytes, vec![1]); + } + #[test] fn failed_durable_lifecycle_write_preserves_live_instance() { let store = SketchStore::new(); diff --git a/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs b/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs index 174c4ae5c..30713e75a 100644 --- a/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs +++ b/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs @@ -276,6 +276,7 @@ mod tests { table_name: None, value_projection: None, table_population: None, + derived_input: None, table_timestamp_column: None, partitioning: None, } diff --git a/data_plane/src/storage_engines/sketch_db/persistence/flusher.rs b/data_plane/src/storage_engines/sketch_db/persistence/flusher.rs index c2aad589b..4c8307db2 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/flusher.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/flusher.rs @@ -32,7 +32,7 @@ use super::{PersistError, PersistResult}; /// Handle to a running flusher thread. Dropping the handle signals /// shutdown and joins the thread. pub struct FlusherHandle { - inner: Arc, + pub(super) inner: Arc, thread: Option>, } @@ -58,6 +58,14 @@ pub(crate) struct FlusherShared { pub back_pressure_wait_count: AtomicU64, } +impl FlusherShared { + pub(super) fn allocate_part_id(&self) -> PersistResult { + self.next_part_id + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |id| id.checked_add(1)) + .map_err(|_| PersistError::Internal("part ID exhausted".into())) + } +} + impl FlusherHandle { /// Start a flusher thread. Takes an `EpochSource` (typically the /// store itself, wrapped in `Arc`). @@ -84,12 +92,23 @@ impl FlusherHandle { { // Pick a starting part_id: one past the max currently in the // manifest (so IDs are monotonically increasing across restarts). + let reserved = sid_metadata.load_strict()?.into_iter().flat_map(|r| { + [r.pending_immutable, r.last_immutable] + .into_iter() + .flatten() + .map(|p| p.part_id) + }); let next_id = manifest .live_parts() .iter() .map(|p| p.part_id) + .chain(reserved) .max() - .map(|m| m + 1) + .map(|m| { + m.checked_add(1) + .ok_or_else(|| PersistError::Internal("part ID exhausted".into())) + }) + .transpose()? .unwrap_or(1); let shared = Arc::new(FlusherShared { @@ -296,8 +315,11 @@ fn run_tick(shared: &Arc, source: &S) -> PersistR // it becomes flushable this same tick. (Without `hot_window_ms` the // durable tier is purely memory-pressure driven and phase 1 below // handles eviction.) - if let Some(hot) = cfg.hot_window_ms { - let cutoff = now.saturating_sub(hot); + let flush_cutoff = cfg + .hot_window_ms + .map(|hot| now.saturating_sub(hot)) + .max(source.flush_before_ms()); + if let Some(cutoff) = flush_cutoff { source.seal_aged_epochs(cutoff); } @@ -325,8 +347,7 @@ fn run_tick(shared: &Arc, source: &S) -> PersistR } // Phase 2: time watermark (any epoch older than now - hot_window). - if let Some(hot) = cfg.hot_window_ms { - let cutoff = now.saturating_sub(hot); + if let Some(cutoff) = flush_cutoff { for r in &all { if r.end_ts < cutoff && !selected @@ -367,7 +388,7 @@ fn run_tick(shared: &Arc, source: &S) -> PersistR if !snapshots.is_empty() { // Build one part for the tick. - let part_id = shared.next_part_id.fetch_add(1, Ordering::Relaxed); + let part_id = shared.allocate_part_id()?; let part_dir = part_dir_path(&parts_root(&cfg.disk_path), part_id); let entries_total: usize = snapshots.iter().map(|s| s.len()).sum(); let size_bytes_estimate: u64 = snapshots.iter().map(|s| s.approx_bytes as u64).sum(); @@ -426,6 +447,18 @@ fn run_tick(shared: &Arc, source: &S) -> PersistR .into_iter() .filter(|p| p.max_ts < cutoff) .collect(); + // Read reservations after capturing candidates: a newly published part + // cannot enter the older candidate set after this check. + let reserved: std::collections::HashSet<_> = shared + .sid_metadata + .load_strict()? + .into_iter() + .filter_map(|record| record.pending_immutable.map(|pending| pending.part_id)) + .collect(); + let expired: Vec<_> = expired + .into_iter() + .filter(|part| !reserved.contains(&part.part_id)) + .collect(); for p in &expired { shared.manifest.append_delete(p.part_id)?; let dir = part_dir_path(&parts_root(&cfg.disk_path), p.part_id); diff --git a/data_plane/src/storage_engines/sketch_db/persistence/immutable_output.rs b/data_plane/src/storage_engines/sketch_db/persistence/immutable_output.rs new file mode 100644 index 000000000..0ac85e833 --- /dev/null +++ b/data_plane/src/storage_engines/sketch_db/persistence/immutable_output.rs @@ -0,0 +1,659 @@ +//! Crash-resumable publication of one immutable output window. The SID sidecar +//! reserves the existing part ID before writing; no second payload store is used. +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use super::flusher::{FlusherHandle, FlusherShared}; +use super::manifest::PartEntry; +use super::metadata::SidMetaRecord; +use super::part::{part_dir_path, PartReader, PartWriter}; +use super::source::{EpochSnapshot, EpochSnapshotEntry}; +use super::{PersistError, PersistResult}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ImmutableOutputReservation { + pub part_id: u64, + pub input_digest: [u8; 32], + pub payload_digest: [u8; 32], + pub start_ms: u64, + pub end_ms: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ImmutableWindowPublication { + pub part_id: u64, + pub already_published: bool, +} + +fn invalid(message: &str) -> PersistError { + PersistError::Format(message.into()) +} + +fn validate_identity(current: &SidMetaRecord, record: &SidMetaRecord) -> PersistResult<()> { + if current.sid != record.sid + || current.removed + || current.retired_at_ms.is_some() + || current.expires_at_ms.is_some() + || current.summary_definition_id != record.summary_definition_id + || current.catalog_generation != record.catalog_generation + || current.metric_name != record.metric_name + || current.group_by_keys != record.group_by_keys + || !current.same_kind(record) + { + return Err(invalid("immutable publication metadata identity changed")); + } + Ok(()) +} + +fn fingerprint(snapshot: &EpochSnapshot) -> PersistResult<[u8; 32]> { + if snapshot.entries.is_empty() || snapshot.min_ts >= snapshot.max_ts { + return Err(invalid("immutable publication requires a nonempty window")); + } + let mut rows = Vec::with_capacity(snapshot.entries.len()); + for entry in &snapshot.entries { + if entry.start_ts != snapshot.min_ts || entry.end_ts != snapshot.max_ts { + return Err(invalid("immutable publication spans multiple windows")); + } + let label = entry.label.as_ref().map(|label| label.serialize_to_bytes()); + rows.push(( + label, + &entry.sketch_type_name, + entry.encoding_tag, + &entry.sketch_bytes, + )); + } + rows.sort(); + if rows.windows(2).any(|pair| pair[0].0 == pair[1].0) { + return Err(invalid( + "immutable publication contains duplicate label populations", + )); + } + let mut digest = Sha256::new(); + digest.update(b"asap-immutable-window-v1"); + for value in [ + snapshot.agg_id, + snapshot.min_ts, + snapshot.max_ts, + rows.len() as u64, + ] { + digest.update(value.to_le_bytes()); + } + for (label, kind, encoding, payload) in rows { + digest.update([u8::from(label.is_some())]); + let label = label.as_deref().unwrap_or_default(); + for bytes in [label, kind.as_bytes()] { + digest.update((bytes.len() as u64).to_le_bytes()); + digest.update(bytes); + } + digest.update([encoding]); + digest.update((payload.len() as u64).to_le_bytes()); + digest.update(payload); + } + Ok(digest.finalize().into()) +} + +impl FlusherHandle { + /// Reuse the existing persistence state without transferring ownership of + /// its worker thread. The store can retain a Weak reference to this Arc. + pub(crate) fn publication_handle(&self) -> std::sync::Arc { + std::sync::Arc::clone(&self.inner) + } + pub fn publish_immutable_window( + &self, + record: &SidMetaRecord, + input_digest: [u8; 32], + snapshot: &EpochSnapshot, + ) -> PersistResult { + self.inner + .publish_immutable_window(record, input_digest, snapshot) + } + pub fn resume_pending_immutable_window( + &self, + sid: u64, + ) -> PersistResult> { + self.inner.resume_pending_immutable_window(sid) + } +} + +impl FlusherShared { + /// Find the latest committed result before evaluating a potentially + /// randomized sketch again. This never creates or replaces a payload. + pub fn lookup_immutable_window( + &self, + record: &SidMetaRecord, + input_digest: [u8; 32], + start_ms: u64, + end_ms: u64, + ) -> PersistResult> { + self.sid_metadata.transaction(|records, _| { + let Some(current) = records.get(&record.sid.to_string()) else { + return Ok(None); + }; + validate_identity(current, record)?; + let Some(previous) = ¤t.last_immutable else { + return Ok(None); + }; + if previous.start_ms != start_ms || previous.end_ms != end_ms { + return Ok(None); + } + if previous.input_digest != input_digest { + return Err(invalid("immutable lookup input lineage differs")); + } + self.validate_reserved_part(record.sid, previous)?; + if !self + .manifest + .live_parts() + .iter() + .any(|part| part.part_id == previous.part_id) + { + return Err(invalid("completed immutable part is no longer published")); + } + Ok(Some(ImmutableWindowPublication { + part_id: previous.part_id, + already_published: true, + })) + }) + } + + /// Publish a finalized window. A retry must present exactly the reserved + /// input and payload. This does not guarantee source availability after GC. + pub fn publish_immutable_window( + &self, + record: &SidMetaRecord, + input_digest: [u8; 32], + snapshot: &EpochSnapshot, + ) -> PersistResult { + if snapshot.agg_id != record.sid || record.removed { + return Err(invalid("immutable publication SID is invalid or removed")); + } + let payload_digest = fingerprint(snapshot)?; + self.sid_metadata.transaction(|records, metadata| { + let key = record.sid.to_string(); + let mut current = records.get(&key).cloned().unwrap_or_else(|| record.clone()); + validate_identity(¤t, record)?; + if current + .completed_through_ms + .is_some_and(|end| snapshot.min_ts < end) + { + let Some(previous) = ¤t.last_immutable else { + return Err(invalid( + "completed immutable window has no retained lineage proof", + )); + }; + if previous.input_digest != input_digest + || previous.payload_digest != payload_digest + || previous.start_ms != snapshot.min_ts + || previous.end_ms != snapshot.max_ts + { + return Err(invalid( + "completed immutable retry differs from latest publication", + )); + } + self.validate_reserved_part(record.sid, previous)?; + if !self + .manifest + .live_parts() + .iter() + .any(|part| part.part_id == previous.part_id) + { + return Err(invalid("completed immutable part is no longer published")); + } + return Ok(ImmutableWindowPublication { + part_id: previous.part_id, + already_published: true, + }); + } + let reservation = match current.pending_immutable.clone() { + Some(pending) => { + if pending.input_digest != input_digest + || pending.payload_digest != payload_digest + || pending.start_ms != snapshot.min_ts + || pending.end_ms != snapshot.max_ts + { + return Err(invalid( + "another immutable publication is pending for this SID", + )); + } + pending + } + None => { + let pending = ImmutableOutputReservation { + part_id: self.allocate_part_id()?, + input_digest, + payload_digest, + start_ms: snapshot.min_ts, + end_ms: snapshot.max_ts, + }; + current.pending_immutable = Some(pending.clone()); + records.insert(key.clone(), current.clone()); + metadata.write_records(records)?; + pending + } + }; + let path = part_dir_path(&self.cfg.disk_path.join("parts"), reservation.part_id); + if path.exists() { + // Never overwrite a reserved part silently: callers can recover a + // durable part without sources; damaged/partial parts fail closed. + self.validate_reserved_part(record.sid, &reservation)?; + } else { + PartWriter::write_part(&path, reservation.part_id, std::slice::from_ref(snapshot))?; + std::fs::File::open( + path.parent() + .ok_or_else(|| invalid("missing parts parent"))?, + )? + .sync_all()?; + self.validate_reserved_part(record.sid, &reservation)?; + } + self.finish_reserved_part(&mut current, &reservation)?; + records.insert(key, current); + metadata.write_records(records)?; + Ok(ImmutableWindowPublication { + part_id: reservation.part_id, + already_published: false, + }) + }) + } + + /// Resume only the caller's pending lineage, atomically with metadata + /// validation. A different pending input cannot be acknowledged by this call. + pub fn resume_matching_immutable_window( + &self, + record: &SidMetaRecord, + input_digest: [u8; 32], + start_ms: u64, + end_ms: u64, + ) -> PersistResult> { + self.resume_immutable_window(record.sid, Some((record, input_digest, start_ms, end_ms))) + } + + /// Complete a pending durable part without reconstructing its source input. + /// A reservation whose part was never completed returns an error, preserving + /// the reservation for an explicit recovery policy rather than losing data. + pub fn resume_pending_immutable_window( + &self, + sid: u64, + ) -> PersistResult> { + self.resume_immutable_window(sid, None) + } + + fn resume_immutable_window( + &self, + sid: u64, + expected: Option<(&SidMetaRecord, [u8; 32], u64, u64)>, + ) -> PersistResult> { + self.sid_metadata.transaction(|records, metadata| { + let key = sid.to_string(); + let Some(mut current) = records.get(&key).cloned() else { + return Ok(None); + }; + let Some(pending) = current.pending_immutable.clone() else { + return Ok(None); + }; + if current.removed || current.retired_at_ms.is_some() || current.expires_at_ms.is_some() + { + return Err(invalid("pending immutable SID was removed")); + } + if let Some((record, digest, start, end)) = expected { + validate_identity(¤t, record)?; + if pending.input_digest != digest + || pending.start_ms != start + || pending.end_ms != end + { + return Err(invalid("pending immutable recovery lineage differs")); + } + } + self.validate_reserved_part(sid, &pending)?; + self.finish_reserved_part(&mut current, &pending)?; + records.insert(key, current); + metadata.write_records(records)?; + Ok(Some(ImmutableWindowPublication { + part_id: pending.part_id, + already_published: true, + })) + }) + } + + fn validate_reserved_part( + &self, + sid: u64, + pending: &ImmutableOutputReservation, + ) -> PersistResult<()> { + let reader = PartReader::open(&part_dir_path( + &self.cfg.disk_path.join("parts"), + pending.part_id, + ))?; + if reader.meta.part_id != pending.part_id { + return Err(invalid("reserved part ID mismatch")); + } + let mut entries = Vec::new(); + for index in reader.index_records() { + let row = reader.load_entry(&index)?; + if row.agg_id != sid { + return Err(invalid("reserved part contains another SID")); + } + entries.push(EpochSnapshotEntry { + start_ts: row.start_ts, + end_ts: row.end_ts, + label: row.label, + sketch_type_name: row.sketch_type_name, + encoding_tag: row.encoding_tag, + sketch_bytes: row.sketch_bytes, + }); + } + let snapshot = EpochSnapshot { + agg_id: sid, + epoch_id: 0, + min_ts: pending.start_ms, + max_ts: pending.end_ms, + entries, + approx_bytes: 0, + }; + if fingerprint(&snapshot)? != pending.payload_digest { + return Err(invalid("reserved immutable payload digest mismatch")); + } + Ok(()) + } + + fn finish_reserved_part( + &self, + current: &mut SidMetaRecord, + pending: &ImmutableOutputReservation, + ) -> PersistResult<()> { + let path = part_dir_path(&self.cfg.disk_path.join("parts"), pending.part_id); + // The reservation may have been recovered before its parent-directory + // entry was durable; make that durable before manifest publication too. + std::fs::File::open( + path.parent() + .ok_or_else(|| invalid("missing parts parent"))?, + )? + .sync_all()?; + if !self + .manifest + .live_parts() + .iter() + .any(|part| part.part_id == pending.part_id) + { + let size_bytes = std::fs::read_dir(&path)?.try_fold(0u64, |sum, entry| { + let length = entry?.metadata()?.len(); + sum.checked_add(length) + .ok_or_else(|| std::io::Error::other("part size overflow")) + })?; + self.manifest.append_add(PartEntry { + part_id: pending.part_id, + min_ts: pending.start_ms, + max_ts: pending.end_ms, + size_bytes, + })?; + } + current.completed_through_ms = Some( + current + .completed_through_ms + .unwrap_or(0) + .max(pending.end_ms), + ); + current.last_immutable = Some(pending.clone()); + current.pending_immutable = None; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::super::{ + config::SketchStorePersistenceConfig, + manifest::Manifest, + source::{EpochSource, SealedEpochRef}, + }; + use super::*; + use crate::storage_engines::{sketch_db::data::AggKind, types::AggregationType}; + use std::sync::Arc; + + struct EmptySource; + impl EpochSource for EmptySource { + fn list_sealed_epochs(&self) -> Vec { + vec![] + } + fn snapshot_sealed_epoch(&self, _: u64, _: u64) -> PersistResult> { + Ok(None) + } + fn evict_sealed_epoch(&self, _: u64, _: u64) {} + fn approx_memory_bytes(&self) -> usize { + 0 + } + } + fn handle(path: &std::path::Path) -> FlusherHandle { + let mut config = SketchStorePersistenceConfig::with_memory_limit(1024, path.into()); + config.hot_window_ms = None; + config.delete_older_than_ms = None; + FlusherHandle::start( + config, + Arc::new(Manifest::open_or_init(path).unwrap()), + Arc::new(EmptySource), + ) + .unwrap() + } + fn record() -> SidMetaRecord { + SidMetaRecord::new( + 1, + "derived".into(), + vec![], + &AggKind::ExactAgg { + agg_type: AggregationType::Sum, + parameters_canonical: String::new(), + spatial_filter_canonical: String::new(), + }, + 0, + ) + } + fn snapshot() -> EpochSnapshot { + EpochSnapshot { + agg_id: 1, + epoch_id: 0, + min_ts: 10, + max_ts: 20, + approx_bytes: 8, + entries: vec![EpochSnapshotEntry { + start_ts: 10, + end_ts: 20, + label: None, + sketch_type_name: "SumAccumulator".into(), + encoding_tag: 0, + sketch_bytes: vec![1, 2, 3], + }], + } + } + fn reserve(handle: &FlusherHandle, snapshot: &EpochSnapshot) -> ImmutableOutputReservation { + let pending = ImmutableOutputReservation { + part_id: handle.inner.allocate_part_id().unwrap(), + input_digest: [7; 32], + payload_digest: fingerprint(snapshot).unwrap(), + start_ms: 10, + end_ms: 20, + }; + handle + .inner + .sid_metadata + .transaction(|records, metadata| { + let mut record = record(); + record.pending_immutable = Some(pending.clone()); + records.insert("1".into(), record); + metadata.write_records(records) + }) + .unwrap(); + pending + } + #[test] + fn completed_retry_and_concurrent_publications_do_not_duplicate_parts() { + let temp = tempfile::tempdir().unwrap(); + let handle = handle(temp.path()); + std::thread::scope(|scope| { + let workers: Vec<_> = (0..4) + .map(|_| { + scope.spawn(|| { + handle + .publish_immutable_window(&record(), [7; 32], &snapshot()) + .unwrap() + }) + }) + .collect(); + let ids: Vec<_> = workers + .into_iter() + .map(|worker| worker.join().unwrap().part_id) + .collect(); + assert!(ids.iter().all(|id| *id == ids[0])); + }); + assert_eq!(handle.manifest().live_parts().len(), 1); + assert!(handle + .publish_immutable_window(&record(), [8; 32], &snapshot()) + .is_err()); + let metadata = handle.inner.sid_metadata.load_strict().unwrap(); + assert_eq!(metadata[0].completed_through_ms, Some(20)); + assert!(metadata[0].pending_immutable.is_none()); + let mut different = snapshot(); + different.entries[0].sketch_bytes.push(4); + assert!(handle + .publish_immutable_window(&record(), [7; 32], &different) + .is_err()); + let part_id = handle.manifest().live_parts()[0].part_id; + handle.manifest().append_delete(part_id).unwrap(); + assert!(handle + .publish_immutable_window(&record(), [7; 32], &snapshot()) + .is_err()); + } + #[test] + fn restart_preserves_reserved_part_and_resumes_without_source() { + let temp = tempfile::tempdir().unwrap(); + let mut first = handle(temp.path()); + let state = snapshot(); + let pending = reserve(&first, &state); + let path = part_dir_path(&temp.path().join("parts"), pending.part_id); + PartWriter::write_part(&path, pending.part_id, &[state]).unwrap(); + first.shutdown(); + drop(first); + let (_, recovery) = super::super::recovery::recover(temp.path()).unwrap(); + assert_eq!(recovery.orphan_parts_removed, 0); + let resumed = handle(temp.path()); + assert!(resumed.inner.allocate_part_id().unwrap() > pending.part_id); + let result = resumed.resume_pending_immutable_window(1).unwrap().unwrap(); + assert_eq!(result.part_id, pending.part_id); + assert_eq!(resumed.manifest().live_parts().len(), 1); + assert!(resumed + .resume_pending_immutable_window(1) + .unwrap() + .is_none()); + } + #[test] + fn pending_missing_part_requires_source_and_stale_upsert_preserves_reservation() { + let temp = tempfile::tempdir().unwrap(); + let handle = handle(temp.path()); + let pending = reserve(&handle, &snapshot()); + handle.inner.sid_metadata.upsert_all(&[record()]).unwrap(); + assert_eq!( + handle.inner.sid_metadata.load_strict().unwrap()[0].pending_immutable, + Some(pending.clone()) + ); + assert!(handle.resume_pending_immutable_window(1).is_err()); + assert_eq!( + handle + .publish_immutable_window(&record(), [7; 32], &snapshot()) + .unwrap() + .part_id, + pending.part_id + ); + handle.inner.sid_metadata.upsert_all(&[record()]).unwrap(); + assert!(handle.inner.sid_metadata.load_strict().unwrap()[0] + .pending_immutable + .is_none()); + } + #[test] + fn malformed_metadata_fails_before_part_publication() { + let temp = tempfile::tempdir().unwrap(); + let handle = handle(temp.path()); + std::fs::write(handle.inner.sid_metadata.path(), b"broken").unwrap(); + assert!(handle + .publish_immutable_window(&record(), [7; 32], &snapshot()) + .is_err()); + assert!(handle.manifest().live_parts().is_empty()); + } + #[test] + fn failed_manifest_leaves_recoverable_payload_and_no_completion() { + let temp = tempfile::tempdir().unwrap(); + let handle = handle(temp.path()); + let log = handle.manifest().log_path(); + let backup = log.with_extension("saved"); + std::fs::rename(&log, &backup).unwrap(); + std::fs::create_dir(&log).unwrap(); + assert!(handle + .publish_immutable_window(&record(), [7; 32], &snapshot()) + .is_err()); + let records = handle.inner.sid_metadata.load_strict().unwrap(); + let pending = records[0].pending_immutable.clone().unwrap(); + assert_eq!(records[0].completed_through_ms, None); + assert!(handle.manifest().live_parts().is_empty()); + std::fs::remove_dir(&log).unwrap(); + std::fs::rename(&backup, &log).unwrap(); + let published = handle.resume_pending_immutable_window(1).unwrap().unwrap(); + assert_eq!(published.part_id, pending.part_id); + assert_eq!(handle.manifest().live_parts().len(), 1); + } + #[test] + fn committed_lookup_reuses_payload_without_recomputing_randomized_state() { + let temp = tempfile::tempdir().unwrap(); + let mut first = handle(temp.path()); + let publication = first + .publish_immutable_window(&record(), [7; 32], &snapshot()) + .unwrap(); + first.shutdown(); + drop(first); + let reopened = handle(temp.path()); + let found = reopened + .inner + .lookup_immutable_window(&record(), [7; 32], 10, 20) + .unwrap() + .unwrap(); + assert_eq!(found.part_id, publication.part_id); + assert!(reopened + .inner + .lookup_immutable_window(&record(), [8; 32], 10, 20) + .is_err()); + reopened + .manifest() + .append_delete(publication.part_id) + .unwrap(); + assert!(reopened + .inner + .lookup_immutable_window(&record(), [7; 32], 10, 20) + .is_err()); + } + #[test] + fn matching_resume_rejects_other_lineage_without_mutating_reservation() { + let temp = tempfile::tempdir().unwrap(); + let handle = handle(temp.path()); + let state = snapshot(); + let pending = reserve(&handle, &state); + let path = part_dir_path(&temp.path().join("parts"), pending.part_id); + PartWriter::write_part(&path, pending.part_id, &[state]).unwrap(); + assert!(handle + .inner + .resume_matching_immutable_window(&record(), [8; 32], 10, 20) + .is_err()); + assert!(handle + .inner + .resume_matching_immutable_window(&record(), [7; 32], 20, 30) + .is_err()); + assert!(handle.manifest().live_parts().is_empty()); + assert_eq!( + handle.inner.sid_metadata.load_strict().unwrap()[0].pending_immutable, + Some(pending.clone()) + ); + assert_eq!( + handle + .inner + .resume_matching_immutable_window(&record(), [7; 32], 10, 20) + .unwrap() + .unwrap() + .part_id, + pending.part_id + ); + } +} diff --git a/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs b/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs index 65c0e144c..dfdf7b4bc 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs @@ -319,6 +319,13 @@ pub struct SidMetaRecord { pub expires_at_ms: Option, #[serde(default)] pub removed: bool, + /// No further publication may change a window ending at or before this bound. + #[serde(default)] + pub completed_through_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pending_immutable: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_immutable: Option, } impl SidMetaRecord { @@ -343,9 +350,16 @@ impl SidMetaRecord { retired_at_ms: None, expires_at_ms: None, removed: false, + completed_through_ms: None, + pending_immutable: None, + last_immutable: None, } } + pub(super) fn same_kind(&self, other: &Self) -> bool { + self.agg_kind == other.agg_kind + } + /// Reconstruct the structured [`AggKind`], or `None` for an /// unrecognized sketch kind. pub fn agg_kind(&self) -> Option { @@ -410,6 +424,12 @@ struct SidBindingRec { expires_at_ms: Option, #[serde(default)] removed: bool, + #[serde(default)] + completed_through_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pending_immutable: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + last_immutable: Option, } /// Version-3 normalized sidecar with authoritative catalog provenance. Descriptors appear once and SeriesId bindings hold @@ -480,6 +500,9 @@ impl SdsSidecar { retired_at_ms: record.retired_at_ms, expires_at_ms: record.expires_at_ms, removed: record.removed, + completed_through_ms: record.completed_through_ms, + pending_immutable: record.pending_immutable, + last_immutable: record.last_immutable, }, ); } @@ -533,6 +556,9 @@ impl SdsSidecar { retired_at_ms: binding.retired_at_ms, expires_at_ms: binding.expires_at_ms, removed: binding.removed, + completed_through_ms: binding.completed_through_ms, + pending_immutable: binding.pending_immutable, + last_immutable: binding.last_immutable, }) }) .collect() @@ -638,7 +664,7 @@ impl SidMetadataStore { return Ok(()); } let mut map: HashMap = self - .load()? + .load_strict()? .into_iter() .map(|r| (r.sid.to_string(), r)) .collect(); @@ -649,7 +675,11 @@ impl SidMetadataStore { if let Some(existing) = map.get(&key) { // Lifecycle is monotone for a SeriesId. An older flush snapshot // must not resurrect a retired or removed persisted instance. + next.pending_immutable = existing.pending_immutable.clone(); + next.last_immutable = existing.last_immutable.clone(); next.removed |= existing.removed; + next.completed_through_ms = + existing.completed_through_ms.max(next.completed_through_ms); next.retired_at_ms = existing.retired_at_ms.or(next.retired_at_ms); next.expires_at_ms = match (existing.expires_at_ms, next.expires_at_ms) { (Some(a), Some(b)) => Some(a.min(b)), @@ -670,6 +700,55 @@ impl SidMetadataStore { self.write_atomic(json.as_bytes()) } + pub fn load_strict(&self) -> PersistResult> { + let bytes = match fs::read(&self.path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(error.into()), + }; + let value: serde_json::Value = serde_json::from_slice(&bytes) + .map_err(|error| PersistError::Format(format!("invalid SID metadata: {error}")))?; + if let Some(version) = value.get("schema_version") { + if !matches!(version.as_u64(), Some(2 | 3)) { + return Err(PersistError::Format( + "unsupported SID metadata version".into(), + )); + } + let sidecar: SdsSidecar = serde_json::from_value(value) + .map_err(|error| PersistError::Format(error.to_string()))?; + sidecar.into_records() + } else { + let records: HashMap = serde_json::from_value(value) + .map_err(|error| PersistError::Format(error.to_string()))?; + Ok(records.into_values().collect()) + } + } + + pub(super) fn transaction( + &self, + operation: impl FnOnce(&mut HashMap, &Self) -> PersistResult, + ) -> PersistResult { + let _writer = self + .writer + .lock() + .map_err(|_| PersistError::Internal("SID metadata writer poisoned".into()))?; + let mut records = self + .load_strict()? + .into_iter() + .map(|r| (r.sid.to_string(), r)) + .collect(); + operation(&mut records, self) + } + pub(super) fn write_records( + &self, + records: &HashMap, + ) -> PersistResult<()> { + let sidecar = SdsSidecar::from_records(records.values().cloned()); + let bytes = + serde_json::to_vec(&sidecar).map_err(|e| PersistError::Serialize(e.to_string()))?; + self.write_atomic(&bytes) + } + fn write_atomic(&self, bytes: &[u8]) -> PersistResult<()> { if let Some(parent) = self.path.parent() { fs::create_dir_all(parent)?; @@ -686,9 +765,7 @@ impl SidMetadataStore { } fs::rename(&tmp, &self.path)?; if let Some(parent) = self.path.parent() { - if let Ok(dir) = File::open(parent) { - let _ = dir.sync_all(); - } + File::open(parent)?.sync_all()?; } Ok(()) } diff --git a/data_plane/src/storage_engines/sketch_db/persistence/mod.rs b/data_plane/src/storage_engines/sketch_db/persistence/mod.rs index 349feb92e..552fe4e03 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/mod.rs @@ -27,6 +27,7 @@ pub mod source; pub mod cache; pub mod flusher; +pub mod immutable_output; pub mod recovery; pub use config::SketchStorePersistenceConfig; diff --git a/data_plane/src/storage_engines/sketch_db/persistence/recovery.rs b/data_plane/src/storage_engines/sketch_db/persistence/recovery.rs index 7fb800946..a21402608 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/recovery.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/recovery.rs @@ -34,7 +34,12 @@ pub fn recover(disk_path: &Path) -> PersistResult<(Manifest, RecoveryReport)> { // Validate every live part by reading its meta.bin header. let mut to_drop: Vec = Vec::new(); - let mut referenced: HashSet = HashSet::new(); + let pending: HashSet = super::metadata::SidMetadataStore::new(disk_path) + .load_strict()? + .into_iter() + .filter_map(|record| record.pending_immutable.map(|pending| pending.part_id)) + .collect(); + let mut referenced = pending.clone(); for entry in manifest.live_parts() { referenced.insert(entry.part_id); let part_dir = super::part::part_dir_path(&parts_root, entry.part_id); @@ -61,6 +66,11 @@ pub fn recover(disk_path: &Path) -> PersistResult<(Manifest, RecoveryReport)> { } } + if to_drop.iter().any(|part_id| pending.contains(part_id)) { + return Err(super::PersistError::Format( + "corrupt reserved immutable part".into(), + )); + } for part_id in to_drop { manifest.append_delete(part_id)?; let dir = super::part::part_dir_path(&parts_root, part_id); diff --git a/data_plane/src/storage_engines/sketch_db/persistence/source.rs b/data_plane/src/storage_engines/sketch_db/persistence/source.rs index 117c68544..734cfaf6a 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/source.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/source.rs @@ -97,6 +97,11 @@ pub struct EpochSnapshotEntry { /// * [`approx_memory_bytes`] is cheap (atomic load) and is kept in sync /// with what the flusher has evicted. pub trait EpochSource: Send + Sync { + /// Explicit durability demand from a completed source, independent of the hot tier. + fn flush_before_ms(&self) -> Option { + None + } + fn list_sealed_epochs(&self) -> Vec; /// Time-driven seal: roll every un-sealed `current_epoch` window diff --git a/data_plane/src/storage_engines/types/hot_reload_config.rs b/data_plane/src/storage_engines/types/hot_reload_config.rs index c007f8f5b..88786343c 100644 --- a/data_plane/src/storage_engines/types/hot_reload_config.rs +++ b/data_plane/src/storage_engines/types/hot_reload_config.rs @@ -92,9 +92,9 @@ pub struct ActivePhysicalPlan { /// Present for authoritative installations; legacy bootstrap has no catalog. pub summary_catalog: Option>, pub precompute_plan: asap_types::precompute_plan::PrecomputePlan, - pub transmission_plan: control_plane::physical::compiler::TransmissionPlan, + pub transmission_plan: asap_types::producer_plan::TransmissionPlan, pub runtime_config: Arc, - pub query_plan: Arc, + pub query_plan: Arc, pub storage_routing: Arc, } @@ -672,11 +672,9 @@ mod tests { summary_catalog: None, envelope: envelope.clone(), ingest: asap_types::precompute_plan::IngestContract { - protocol: - asap_types::precompute_plan::IngestProtocol::ModifiedOtlpMetricsV1, + protocol: asap_types::precompute_plan::IngestProtocol::ModifiedOtlpMetricsV1, endpoint_path: "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/v1/metrics".into(), - timestamp_unit: - asap_types::precompute_plan::TimestampUnit::UnixNanoseconds, + timestamp_unit: asap_types::precompute_plan::TimestampUnit::UnixNanoseconds, require_plan_identity: true, require_summary_definition_identity: true, require_registered_producer: true, @@ -684,9 +682,9 @@ mod tests { schemas: Vec::new(), producers: Vec::new(), executable_dags: Default::default(), - materializations: Vec::new(), + materializations: Vec::new(), }, - transmission_plan: control_plane::physical::compiler::TransmissionPlan { + transmission_plan: asap_types::producer_plan::TransmissionPlan { summary_catalog: None, envelope: asap_types::precompute_plan::PlanEnvelope { plan_id, @@ -698,16 +696,17 @@ mod tests { planner_revision: control_plane::physical::compiler::PLANNER_REVISION.into(), capability_snapshot_id: "test".into(), }, - frame_identity: control_plane::physical::compiler::FrameIdentityContract { + frame_identity: asap_types::producer_plan::FrameIdentityContract { identity_version: 1, - sequence_scope: control_plane::physical::compiler::SequenceScope::MaterializationSeriesProducerEpoch, + sequence_scope: + asap_types::producer_plan::SequenceScope::MaterializationSeriesProducerEpoch, require_checkpoint_for_full: true, require_base_checkpoint_for_delta: true, }, rules: Vec::new(), }, runtime_config: Arc::new(StreamingConfig::new(HashMap::new())), - query_plan: Arc::new(control_plane::query_plan::QueryPlan { + query_plan: Arc::new(asap_types::query_plan::QueryPlan { plan_id, plan_version, clickhouse_context: None, diff --git a/data_plane/src/storage_engines/types/streaming_config.rs b/data_plane/src/storage_engines/types/streaming_config.rs index 91a2dc1b3..2298955db 100644 --- a/data_plane/src/storage_engines/types/streaming_config.rs +++ b/data_plane/src/storage_engines/types/streaming_config.rs @@ -133,6 +133,11 @@ impl StreamingConfig { num_aggregates_to_retain, QueryLanguage::PromQl, )?; + if config.derived_input.is_some() { + anyhow::bail!( + "legacy streaming input cannot execute a derived summary program" + ); + } // PR 5: the map key IS the policy-fingerprint u64. // `AggregationConfig::policy_fp_u64()` is the canonical // accessor for this value. @@ -191,6 +196,17 @@ mod tests { assert_eq!(cfg.storage_backend(), StorageBackend::GorillaObjectStore); } + #[test] + fn legacy_yaml_rejects_derived_summary_input() { + let data = serde_yaml::from_str::(&format!( + "aggregations:\n- aggregationType: Sum\n aggregationSubType: ''\n metric: outer\n labels: {{grouping: [], rollup: [], aggregated: []}}\n parameters: {{}}\n windowSize: 10\n windowType: tumbling\n spatialFilter: ''\n derived_input:\n inputs: [1]\n program_sha256: '{}'\n", "a".repeat(64) + )).unwrap(); + let error = StreamingConfig::from_yaml_data(&data).unwrap_err(); + assert!(error + .to_string() + .contains("legacy streaming input cannot execute")); + } + /// PR 5: a streaming-config YAML that omits `aggregationId` /// parses correctly — the backend derives identity from content /// via `PolicyFingerprint::from_config`. The map key is the diff --git a/data_plane/src/tests/test_utilities/engine_factories.rs b/data_plane/src/tests/test_utilities/engine_factories.rs index 3fce2d44b..564d21d7c 100644 --- a/data_plane/src/tests/test_utilities/engine_factories.rs +++ b/data_plane/src/tests/test_utilities/engine_factories.rs @@ -109,6 +109,7 @@ pub fn create_engine_single_pop_with_aggregated( table_name: None, value_projection: None, table_population: None, + derived_input: None, table_timestamp_column: None, partitioning: None, }; @@ -206,6 +207,7 @@ pub fn create_engine_dual_input( table_name: None, value_projection: None, table_population: None, + derived_input: None, table_timestamp_column: None, partitioning: None, }; @@ -233,6 +235,7 @@ pub fn create_engine_dual_input( table_name: None, value_projection: None, table_population: None, + derived_input: None, table_timestamp_column: None, partitioning: None, }; @@ -325,6 +328,7 @@ pub fn create_engine_two_metrics( table_name: None, value_projection: None, table_population: None, + derived_input: None, table_timestamp_column: None, partitioning: None, }; @@ -351,6 +355,7 @@ pub fn create_engine_two_metrics( table_name: None, value_projection: None, table_population: None, + derived_input: None, table_timestamp_column: None, partitioning: None, }; @@ -453,6 +458,7 @@ pub fn create_engine_three_metrics( table_name: None, value_projection: None, table_population: None, + derived_input: None, table_timestamp_column: None, partitioning: None, }; @@ -532,6 +538,7 @@ pub fn create_engine_multi_timestamp( table_name: None, value_projection: None, table_population: None, + derived_input: None, table_timestamp_column: None, partitioning: None, }; @@ -603,6 +610,7 @@ pub fn create_engine_multi_timestamp_with_window( table_name: None, value_projection: None, table_population: None, + derived_input: None, table_timestamp_column: None, partitioning: None, }; diff --git a/data_plane/tests/clickhouse_differential_e2e.rs b/data_plane/tests/clickhouse_differential_e2e.rs index 871bb0b1b..75a07fd46 100644 --- a/data_plane/tests/clickhouse_differential_e2e.rs +++ b/data_plane/tests/clickhouse_differential_e2e.rs @@ -270,13 +270,13 @@ async fn run_mixed_aggregate(aggregate: &str) { assert_eq!( entry.nodes.values().any(|node| matches!( node, - control_plane::query_plan::QueryPlanNode::ExternalExact { .. } + asap_types::query_plan::QueryPlanNode::ExternalExact { .. } )), !grouped ); assert!(entry.nodes.values().any(|node| matches!( node, - control_plane::query_plan::QueryPlanNode::ReadMaterialization { .. } + asap_types::query_plan::QueryPlanNode::ReadMaterialization { .. } ))); let install = publication.install_request(None, Vec::new()).unwrap(); diff --git a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs index a6d9366fd..cc1b733fa 100644 --- a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs +++ b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs @@ -74,9 +74,9 @@ async fn post_full_config(client: &reqwest::Client, stack: &FullStack, json: &Js .any(|c| c.metric == "http_requests_total_latency_ms") { for rule in &mut artifact.transmission_plan.rules { - rule.mode = control_plane::physical::compiler::TransmissionMode::Delta; + rule.mode = asap_types::producer_plan::TransmissionMode::Delta; rule.full_checkpoint_every_ms = Some(rule.emit_every_ms); - rule.runtime_policy.delta = Some(control_plane::physical::compiler::DeltaPolicy { + rule.runtime_policy.delta = Some(asap_types::producer_plan::DeltaPolicy { absolute_threshold: 0.0, gos: None, }); @@ -90,8 +90,7 @@ async fn post_full_config(client: &reqwest::Client, stack: &FullStack, json: &Js { entry.instant.lookback_ms = 1000; for node in entry.nodes.values_mut() { - if let control_plane::query_plan::QueryPlanNode::ReadMaterialization { binding } = node - { + if let asap_types::query_plan::QueryPlanNode::ReadMaterialization { binding } = node { binding.readout_lookback_ms = Some(1000); } } diff --git a/data_plane/tests/support/physical_fixture.rs b/data_plane/tests/support/physical_fixture.rs index 30de3d05d..984fc0d16 100644 --- a/data_plane/tests/support/physical_fixture.rs +++ b/data_plane/tests/support/physical_fixture.rs @@ -37,8 +37,12 @@ pub fn artifact(config: &StreamingConfig) -> PhysicalPlanInstallRequest { let mut precompute = PrecomputePlan::build(envelope.clone(), configs, &["fixture".into()]).unwrap(); precompute.summary_catalog = Some(catalog.reference().unwrap()); - let mut transmission = - TransmissionPlan::build(envelope, &precompute, &BTreeMap::new()).unwrap(); + let mut transmission = control_plane::physical::compiler::compile_transmission_plan( + envelope, + &precompute, + &BTreeMap::new(), + ) + .unwrap(); transmission.summary_catalog = Some(catalog.reference().unwrap()); let mut query_plan = QueryPlan { plan_id: 1, @@ -104,7 +108,7 @@ pub fn artifact(config: &StreamingConfig) -> PhysicalPlanInstallRequest { query_plan.entries.insert( canonical.clone(), QueryPlanEntry { - language: control_plane::query_plan::QueryLanguage::PromQl, + language: asap_types::query_plan::QueryLanguage::PromQl, query_id: canonical.clone(), canonical_query: canonical, fixed_evaluation: None, diff --git a/data_plane/tests/support/univmon_erp_process.rs b/data_plane/tests/support/univmon_erp_process.rs index a37bc601e..9c933b8cb 100644 --- a/data_plane/tests/support/univmon_erp_process.rs +++ b/data_plane/tests/support/univmon_erp_process.rs @@ -195,12 +195,26 @@ async fn measured_readout_evidence_selects_and_executes_univmon() { .await .unwrap(); }); + let runtime_samples = control_plane::runtime_samples::RuntimeSamplesStore::new(8); + let runtime_port = unused_port(); + let runtime_endpoint = format!("http://127.0.0.1:{runtime_port}"); + let runtime_service = + control_plane::runtime_samples::RuntimeSamplesService::new(runtime_samples.clone()) + .into_server(); + let runtime_task = tokio::spawn(async move { + tonic::transport::Server::builder() + .add_service(runtime_service) + .serve(([127, 0, 0, 1], runtime_port).into()) + .await + .unwrap(); + }); let output = tempfile::tempdir().unwrap(); let path = output.path().join("planning.json"); std::fs::write(&path, serde_json::to_vec(&fixture).unwrap()).unwrap(); let port = unused_port(); let mut child = ChildGuard( Command::new(env!("CARGO_BIN_EXE_data_plane")) + .args(["--erp-runtime-samples-endpoint", &runtime_endpoint]) .args(["--profile", "asapquery", "--planning-snapshot"]) .arg(&path) .args([ @@ -231,11 +245,14 @@ async fn measured_readout_evidence_selects_and_executes_univmon() { .unwrap() .as_millis() as i64; let base = now - now.rem_euclid(5000) - 20_000; - let samples: Vec<_> = raw + let mut samples: Vec<_> = raw .iter() .enumerate() .map(|(i, v)| (base + 1 + i as i64, *v)) .collect(); + // Declared finite source includes the preceding boundary; this sample is + // outside the query's left-open range and does not alter its truth. + samples.insert(0, (base, 0.0)); assert_eq!( remote_write( &client, @@ -259,6 +276,69 @@ async fn measured_readout_evidence_selects_and_executes_univmon() { 204 ); drain_precompute(&client, &backend).await; + let keys = runtime_samples.keys(); + assert!( + !keys.is_empty(), + "real worker inputs must reach RuntimeSamples after finite drain" + ); + for key in keys { + let record = runtime_samples.latest(&key).unwrap(); + let observed: asap_types::erp_observation::ErpPopulationObservations< + asap_types::erp_observation::EmpiricalFrequencyObservation, + > = serde_json::from_value(record.payload["erp_population_observations"].clone()).unwrap(); + assert!(observed.invalid_reason.is_none(), "{observed:?}"); + assert!(!observed.populations.is_empty()); + assert_eq!(observed.window_end_ms - observed.window_start_ms, 5000); + assert!(plan + .summary_catalog + .materializations + .contains_key(&observed.summary_definition_id)); + assert_eq!( + observed.catalog_generation, + plan.summary_catalog.reference().unwrap() + ); + if key.sketch == "univmon" { + let mut live_snapshot: BackendLocalPlanningSnapshot = + serde_json::from_value(fixture.clone()).unwrap(); + let policy = live_snapshot.implementation.erp.as_mut().unwrap(); + policy.observed_shape_source = + Some(control_plane::physical::erp::ErpObservedShapeSource { + source: key.source.clone(), + sketch: key.sketch.clone(), + implementation: key.impl_name.clone(), + population_scope: Some( + control_plane::physical::erp::ErpPopulationObservationScope { + catalog_generation: observed.catalog_generation.clone(), + summary_definition_id: observed.summary_definition_id, + input_semantics: observed.input_semantics, + freshness: asap_types::erp_observation::ErpObservationFreshness { + max_age_ms: 60_000, + max_future_skew_ms: 1000, + }, + }, + ), + }); + policy.hydrate_observed_shape(&runtime_samples).unwrap(); + policy.resolve_population_data_descriptor(Some(&plan.summary_catalog)); + assert!(policy + .observed_populations + .as_ref() + .unwrap() + .invalid_reason + .is_none()); + let replanned = live_snapshot.compile().unwrap(); + assert!( + replanned + .precompute_plan + .materializations + .iter() + .any(|m| m.aggregation_type == asap_types::AggregationType::UnivMon), + "actual producer evidence should reach normal Planner selection" + ); + } + } + runtime_task.abort(); + for (i, query) in queries.iter().enumerate() { let result = wait_for_warm_instant( &client, diff --git a/docs/design_docs/asapplanner-integration.md b/docs/design_docs/asapplanner-integration.md index f1f1d082b..c6cc83be6 100644 --- a/docs/design_docs/asapplanner-integration.md +++ b/docs/design_docs/asapplanner-integration.md @@ -106,7 +106,8 @@ Evidence: [selection adapter](../../control_plane/src/planner_selection.rs), [physical compiler](../../control_plane/src/physical/compiler.rs), [legacy workload adapter](../../control_plane/src/physical/workload_planner.rs), -[QueryPlan](../../control_plane/src/query_plan.rs), and +[shared QueryPlan](../../crates/asap_types/src/query_plan.rs), +[query lowering](../../control_plane/src/query_plan.rs), and [bound serving executor](../../data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs). The selection adapter explicitly commits a ranked Planner candidate downstream. Consequently, the figure does not imply that the Planner library deploys or diff --git a/docs/design_docs/summary-catalog-sds-architecture.md b/docs/design_docs/summary-catalog-sds-architecture.md index 048dcbac4..1a364d68e 100644 --- a/docs/design_docs/summary-catalog-sds-architecture.md +++ b/docs/design_docs/summary-catalog-sds-architecture.md @@ -162,8 +162,19 @@ bindings against QueryPlan; precompute execution consumes the shared contract. `PrecomputePlan`, its envelope, ingest, producer, state schema, and catalog consistency checks live in `asap_types::precompute_plan`. The compiler chooses materializations and placement; data-plane installation uses the shared -contract. `QueryPlan` definitions still reside in the control-plane -crate while their remaining compilation methods are separated from wire types. +contract. `asap_types::query_plan` owns QueryPlan, materialization bindings, +logical operator DTOs, and activation validation. The control plane reexports +those types for existing callers and owns the `compile_bound*` and +`logical::compile_logical` functions; Planner traversal and AST lowering do not +move into the shared contract. Data-plane engines import the shared types +directly. No wrapper plan or second wire definition is introduced. + +`asap_types::producer_plan` owns the installed collector and transmission +contracts, frame identities, runtime policy bounds and their validation. The +control plane allocates sampling/GOS budgets and constructs transmission rules +through `sampling_policy_from_accuracy_budget`, `gos_policy_from_accuracy_budget` +and `compile_transmission_plan`. Producers and the data plane import the shared +contracts directly; compilation is not a runtime dependency of those contracts. The implemented ownership split is: @@ -402,6 +413,37 @@ Instance. Merge compatibility additionally requires the operator's merge rules, compatible data scopes and valid instance coverage; sharing descriptors alone does not authorize merging overlapping observations. +### Catalog-scoped runtime ERP evidence + +A runtime observation describes the input of one allocated summary, not an +entire deployment. `ErpPopulationObservations` identifies its catalog generation, +summary definition, observation time, input window and separate summary-instance +populations. The control plane resolves the `DataDescriptor` from its successfully +activated catalog; a telemetry payload cannot provide replacement descriptors. +Alternative sketch parameters may use this evidence only when the compiler +verifies the same data and update semantics. + +The typed physical-plan HTTP endpoints accept `target: backend_local_remote_write` +with an empty `collector_ids` list. Omitting `target` preserves the distributed +collector deployment. Both paths use catalog publication and activation. Typed +activations are serialized, and the accepted catalog is retained only after the +backend acknowledges activation, including ClickHouse publications. + +An ERP `observed_shape_source.population_scope` supplies the expected catalog +and definition, input semantics, and explicit `max_age_ms` / +`max_future_skew_ms` bounds. Each compilation reads the latest runtime record +again. Missing, stale, malformed, foreign or incomplete observations invalidate +all population fits. This is an ERP miss handled by theoretical sizing or exact +execution; it must not restore an older fit or match the artifact's legacy +distribution descriptor. Offline single-shape inputs remain a separate path. + +The initial eligibility is deliberately limited to verified raw per-series +frequency/cardinality readouts over a complete matching window. A 30-second pane +observation does not certify a one-hour input distribution. These checks do not +implement an autonomous drift-triggered replan scheduler, continuous source +completion, or durable restoration of the control plane's active catalog. After +a control-plane restart, live evidence remains ineligible until an authoritative +catalog has been activated again. ### Retired physical series and catalog reactivation @@ -429,3 +471,49 @@ This is an explicit lifetime transition, not cross-generation recovery of arbitr summary state. Legacy records without trustworthy catalog provenance remain unbound. Tombstone reclamation still requires coordinated removal of old physical parts and is not implemented by this transition. + +### Derived summary input identity + +A summary computed from another summary has a different data source from the +original raw table or metric. `PrecomputeMaterialization.derived_input` and +`DataSourceIdentity::Derived` use the same `DerivedInputIdentity`: the referenced +`SummaryDefinitionId`s and a SHA-256 of the maintenance program. The executable +program remains in `OwnedPostAsapDag`; the catalog does not retain another copy. + +The signature replaces materialized input frontiers with stable summary IDs and +hashes the remaining node payloads, schemas, guarantees, and edge semantics. It +excludes query names, plan-local node numbering, and catalog generations. Literal +leaves are hashed directly; raw input leaves still require catalog frontiers. A changed +input definition or transformation creates a new identity. Existing raw-source +identities retain their previous byte representation. Catalog validation rejects +missing input definitions and dependency cycles. + +This contract is a prerequisite, not enabled summary-over-summary execution. +Installation currently rejects derived inputs so they cannot accidentally receive +raw samples through the legacy metric router. Enabling them requires the immutable +maintenance consumer and durable output deduplication protocol; neither raw-table +substitution nor treating late correction fragments as new observations is valid. + +### Immutable completed windows + +Finite Remote Write completion now fences the SummaryStore append boundary, +not just the receiver queue. After all admitted outputs are published, the store +records the greatest published window end for each physical SeriesId. Sketch and +exact-state writes ending at or before that boundary are rejected, including +writes arriving through other producers. A later window remains writable. Observed SDS inventory reports only these frozen +instances as `Complete`; ordinary emitted panes remain `Unknown`. + +The boundary is monotone in the existing SeriesId metadata sidecar and is restored +before recovered identities become writable. A stale background metadata flush +cannot reopen a completed window. The guard belongs to the physical lifetime; +a catalog-authorized replacement SeriesId has its own boundary. + +With persistence enabled, completion explicitly requests the existing flusher to +make the completed prefix durable, even if it is still inside the hot tier. +Completion waits until the corresponding epochs have been evicted after part and +manifest publication; only then does it persist the immutable boundary. An +in-memory deployment provides no restart guarantee. Maintenance consumers still +must atomically publish their output identity before claiming replay-safe consumption. +The existing finite-source completeness proof still rejects untracked writes or +pending admitted work. Continuous producer watermarks and derived-state commit +transactions are separate from this finite-input boundary. diff --git a/docs/developer_docs/univmon-erp-process-validation.md b/docs/developer_docs/univmon-erp-process-validation.md index b3e5849ef..51e65f914 100644 --- a/docs/developer_docs/univmon-erp-process-validation.md +++ b/docs/developer_docs/univmon-erp-process-validation.md @@ -69,3 +69,10 @@ The three queries need different physical configurations in this fixture; they do not demonstrate a single shared state. Planner may share compatible equal- parameter states. The finite replay barrier also does not establish continuous multi-worker population completeness, which is a separate publication contract. + +Finite runtime observations carry bounded empirical frequency counts and temporal +interval counts through the shared metadata contract. The data plane does not +classify distributions or depend on Planner fitting types; the control plane +fits these counts before profile matching. Raw keys are absent from feedback. +An accepted catalog activation establishes the observer generation. Delayed +samples from another generation are ignored and cannot reset current counts.