From f77ad9d42ab66f5c0eee6d63c33be969a8515108 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 3 Sep 2026 19:39:10 -0600 Subject: [PATCH] refactor: align backend summary types with ASAPPlanner --- .../src/backend_plan/from_stage_config.rs | 208 +++--- control_plane/src/backend_plan/mod.rs | 552 ++++++++-------- control_plane/src/emit/backend_push.rs | 14 +- control_plane/src/emit/mod.rs | 8 +- control_plane/src/emit/monitor.rs | 36 +- control_plane/src/emit/otap.rs | 6 +- control_plane/src/emit/stage_config.rs | 405 ++++++------ control_plane/src/emit/telegraf.rs | 8 +- .../src/physical/colored_dag/emitter.rs | 76 +-- .../src/physical/colored_dag/tests.rs | 13 +- control_plane/src/physical/compiler.rs | 26 +- .../deployment_cost/sketch_capability.rs | 44 +- .../src/physical/deployment_cost/wire.rs | 4 +- .../src/physical/post_asap/cost_model.rs | 2 +- .../src/physical/post_asap/matcher.rs | 4 +- control_plane/src/physical/post_asap/tests.rs | 18 +- .../src/physical/runtime_capability.rs | 226 +++---- .../src/physical/workload_planner.rs | 8 +- control_plane/src/query_planning.rs | 46 +- control_plane/src/replan.rs | 43 +- control_plane/src/sketch_selection.rs | 55 +- control_plane/src/types_v2.rs | 56 +- crates/asap_types/src/accumulator_spec.rs | 610 +++++++----------- crates/asap_types/src/lib.rs | 6 +- crates/asap_types/src/monitor_spec.rs | 31 + crates/asap_types/src/storage_backend.rs | 37 ++ data_plane/benches/sketch_db.rs | 40 +- data_plane/examples/sketch_db_diag.rs | 6 +- data_plane/src/drivers/ingest/otel.rs | 372 ++++++----- data_plane/src/drivers/query/servers/http.rs | 6 +- data_plane/src/monitor/coordinator.rs | 20 +- .../precompute_engine/accumulator_factory.rs | 149 +++-- .../src/precompute_engine/ingest_handler.rs | 21 +- .../count_sketch_with_heap_accumulator.rs | 2 +- .../operators/sum_accumulator.rs | 2 +- .../query_engines/asap_query_engine/engine.rs | 79 ++- .../asap_query_engine/live_serve.rs | 120 ++-- .../asap_query_engine/post_asap_planner.rs | 166 +++-- .../asap_query_engine/post_asap_readout.rs | 8 +- .../asap_query_engine/summary_executor.rs | 196 +++--- .../routing/capability_matching.rs | 37 +- .../routing/query_engine_routing.rs | 24 +- data_plane/src/storage_engines/mod.rs | 4 +- .../src/storage_engines/sketch_db/data/mod.rs | 29 +- .../storage_engines/sketch_db/index/mod.rs | 26 +- .../sketch_db/lifecycle/reconcile.rs | 33 +- .../sketch_db/persistence/metadata.rs | 81 +-- .../sketch_db/query/timeline.rs | 27 +- .../storage_engines/types/storage_backend.rs | 142 +--- .../tests/all_sketches_process_oracle_e2e.rs | 136 ++-- ...e2e_controller_plans_and_backend_serves.rs | 12 +- 51 files changed, 1975 insertions(+), 2305 deletions(-) create mode 100644 crates/asap_types/src/storage_backend.rs diff --git a/control_plane/src/backend_plan/from_stage_config.rs b/control_plane/src/backend_plan/from_stage_config.rs index aa154a16..73237768 100644 --- a/control_plane/src/backend_plan/from_stage_config.rs +++ b/control_plane/src/backend_plan/from_stage_config.rs @@ -20,13 +20,12 @@ use std::collections::HashMap; use anyhow::{Context, Result}; -use asap_types::SummaryKind; use asap_types::{MonitorSpec, PolicyFingerprint, PrecomputeMaterialization, QueryLanguage}; use crate::emit::monitor::{agg_id_for_metric, MonitorIntent}; use crate::emit::stage_config::build_backend_aggregation_json; use crate::physical::colored_dag::emitter::{BackendAggregation, BackendStageConfig}; -use crate::physical::runtime_capability::{Capability, SketchKindHandle}; +use crate::physical::runtime_capability::{Capability, SketchAlgorithm}; use asap_types::enums::WindowKind; use planner_types::pre_asap::{ColumnRef, Source}; @@ -53,10 +52,7 @@ pub fn from_stage_config( let fingerprint = fingerprint.policy_fingerprint(); fingerprint_by_agg_id.insert(agg.aggregation_id.as_str(), fingerprint); - let (kind, params) = match &agg.agg_type_override { - Some(exact_type) => exact_kind_params_for_override(exact_type)?, - None => (agg.sketch_kind.clone(), agg.sketch_params.clone()), - }; + let family = agg.family.clone(); materializations.insert( fingerprint, @@ -73,8 +69,7 @@ pub fn from_stage_config( group_by: agg.grouping.clone(), rollup: Vec::new(), spatial_filter: asap_types::utils::normalize_spatial_filter(&agg.spatial_filter), - kind, - params, + family, col: ColumnRef::SampleValue, retention: None, lifecycle: None, @@ -147,25 +142,6 @@ pub fn aggregation_config_for_materialization( .context("build AggregationConfig from synthesized aggregation JSON") } -/// Option B (post-#287) exact-agg override: `s` is already the wire -/// `aggregationType` string (e.g. `"Sum"`) — parse it via -/// `AggregationType::FromStr` (same parser -/// `AggregationConfig::from_yaml_data` uses) and carry it as the -/// matching `SummaryKind`/`SummaryParams` exact-agg pair. -fn exact_kind_params_for_override( - exact_type: &str, -) -> Result<(SummaryKind, asap_types::SummaryParams)> { - use asap_types::SummaryParams; - match exact_type { - "Sum" => Ok((SummaryKind::Sum, SummaryParams::Sum)), - "Count" => Ok((SummaryKind::Count, SummaryParams::Count)), - "MinMax" => Ok((SummaryKind::MinMax, SummaryParams::MinMax)), - "Increase" => Ok((SummaryKind::Increase, SummaryParams::Increase)), - "Rate" => Ok((SummaryKind::Rate, SummaryParams::Rate)), - other => anyhow::bail!("unrecognized agg_type_override {other:?} — no SummaryKind mapping"), - } -} - /// Capability this readout satisfies, given the aggregation it reads /// from. Exact-agg overrides always report `Capability::ExactAgg` /// (mirrors the wire's `aggregationType` bypass — see @@ -180,36 +156,47 @@ fn capability_for_readout( use asap_types::AggregationType; use planner_types::post_asap::SketchQuery; - if let Some(exact_type) = &agg.agg_type_override { - let agg_type: AggregationType = exact_type - .parse() - .map_err(|e: String| anyhow::anyhow!("agg_type_override {exact_type:?}: {e}"))?; - return Ok(Capability::ExactAgg(agg_type)); + match &agg.family { + planner_types::post_asap::SummaryFamilyType::ExactAggregate(kind, _) => { + let agg_type = match kind { + planner_types::post_asap::ExactKind::Sum => AggregationType::Sum, + // The backend implements exact count with its sum-as-count + // accumulator; `ExactKind::Count` remains the canonical + // planner identity at the domain boundary. + planner_types::post_asap::ExactKind::Count => AggregationType::Sum, + planner_types::post_asap::ExactKind::MinMax => AggregationType::MinMax, + planner_types::post_asap::ExactKind::Increase + | planner_types::post_asap::ExactKind::Rate => AggregationType::Increase, + }; + Ok(Capability::ExactAgg(agg_type)) + } + planner_types::post_asap::SummaryFamilyType::Sketch(kind, _) => { + let handle = sketch_algorithm_handle(kind.algorithm())?; + Ok(match op { + SketchQuery::Quantile { .. } => Capability::QuantileApprox(Some(handle)), + SketchQuery::Cardinality => Capability::CardinalityApprox, + SketchQuery::PointCount { .. } => Capability::FrequencyEstimate(Some(handle)), + SketchQuery::TopK { .. } => Capability::FrequencyTopk(Some(handle)), + }) + } + other => anyhow::bail!("unsupported backend summary family {other:?}"), } - - let handle = sketch_kind_handle(&agg.sketch_kind)?; - Ok(match op { - SketchQuery::Quantile { .. } => Capability::QuantileApprox(handle), - SketchQuery::Cardinality => Capability::CardinalityApprox, - SketchQuery::PointCount { .. } => Capability::FrequencyEstimate(handle), - SketchQuery::TopK { .. } => Capability::FrequencyTopk(handle), - }) } -/// Map a `SummaryKind` to the `SketchKindHandle` it identifies as. Only -/// covers the families real `Bind*` rules actually produce for sketch -/// aggregations — mirrors `emit::stage_config::sketch_kind_to_backend_type`'s -/// exhaustive match (and its `unreachable!()` for non-sketch kinds). -fn sketch_kind_handle(kind: &SummaryKind) -> Result { +/// Validate that a Planner `SketchAlgorithm` is implemented by this runtime. +fn sketch_algorithm_handle( + kind: &planner_types::post_asap::SketchAlgorithm, +) -> Result { + use planner_types::post_asap::SketchAlgorithm; Ok(match kind { - SummaryKind::DDSketch => SketchKindHandle::DDSketch, - SummaryKind::Kll => SketchKindHandle::Kll, - SummaryKind::Hll => SketchKindHandle::Hll, - SummaryKind::CountSketch => SketchKindHandle::CountSketch, - SummaryKind::Cms => SketchKindHandle::CountMin, - SummaryKind::CmsWithHeap => SketchKindHandle::CmsWithHeap, - SummaryKind::CountSketchWithHeap => SketchKindHandle::CountSketchWithHeap, - other => anyhow::bail!("no SketchKindHandle mapping for non-sketch SummaryKind {other:?}"), + SketchAlgorithm::DDSketch => SketchAlgorithm::DDSketch, + SketchAlgorithm::Kll => SketchAlgorithm::Kll, + SketchAlgorithm::Hll => SketchAlgorithm::Hll, + SketchAlgorithm::CountSketch => SketchAlgorithm::CountSketch, + SketchAlgorithm::Cms => SketchAlgorithm::Cms, + SketchAlgorithm::CmsWithHeap => SketchAlgorithm::CmsWithHeap, + SketchAlgorithm::CountSketchWithHeap => SketchAlgorithm::CountSketchWithHeap, + other => anyhow::bail!("SketchAlgorithm {other:?} is not implemented by this runtime"), }) } @@ -217,30 +204,31 @@ fn sketch_kind_handle(kind: &SummaryKind) -> Result { mod tests { use super::*; use crate::physical::colored_dag::emitter::{AggregationInput, BackendReadout}; - use asap_types::{ - AggregationType, KeyByLabelNames, SummaryParams, WindowKind as AsapWindowKind, + use asap_types::{AggregationType, KeyByLabelNames, WindowKind as AsapWindowKind}; + use planner_types::post_asap::{ + GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SketchQuery, SummaryFamilyType, }; - use planner_types::post_asap::SketchQuery; use std::collections::HashMap as StdHashMap; fn agg( aggregation_id: &str, metric_name: &str, - sketch_kind: SummaryKind, - sketch_params: SummaryParams, + algorithm: SketchAlgorithm, + sketch_params: SketchParams, grouping: Vec, ) -> BackendAggregation { BackendAggregation { aggregation_id: aggregation_id.to_string(), metric_name: metric_name.to_string(), - sketch_kind, - sketch_params, + family: SummaryFamilyType::Sketch( + SketchKind::new(algorithm, sketch_params), + GroupingStrategy::PerSubpopulationInstance, + ), window_secs: 60, spatial_filter: String::new(), grouping, item_label: None, aggregation_input: AggregationInput::SketchEnvelope, - agg_type_override: None, } } @@ -248,33 +236,37 @@ mod tests { /// (non-JSON-round-trip) reader would build for this fixture, so the /// parity test doesn't just check the implementation against itself. fn hand_built_config(agg: &BackendAggregation) -> PrecomputeMaterialization { - let parameters: StdHashMap = match &agg.sketch_params { - SummaryParams::DDSketch { alpha } => { + let (kind, params) = match &agg.family { + SummaryFamilyType::Sketch(kind, _) => (kind.algorithm(), kind.params()), + other => unreachable!("fixture only uses sketches, got {other:?}"), + }; + let parameters: StdHashMap = match params { + SketchParams::DDSketch { alpha } => { StdHashMap::from([("alpha".to_string(), serde_json::json!(alpha))]) } - SummaryParams::Hll { precision } => { + SketchParams::Hll { precision } => { StdHashMap::from([("precision".to_string(), serde_json::json!(precision))]) } - SummaryParams::CountSketchWithHeap { width, depth, .. } => StdHashMap::from([ + SketchParams::CountSketchWithHeap { width, depth, .. } => StdHashMap::from([ ("w".to_string(), serde_json::json!(width)), ("d".to_string(), serde_json::json!(depth)), ("with_heap".to_string(), serde_json::json!(true)), ]), - SummaryParams::Cms { width, depth } => StdHashMap::from([ + SketchParams::Cms { width, depth } => StdHashMap::from([ ("w".to_string(), serde_json::json!(width)), ("d".to_string(), serde_json::json!(depth)), ]), other => unreachable!("fixture doesn't exercise {other:?}"), }; PrecomputeMaterialization::new( - match agg.sketch_kind { - SummaryKind::DDSketch => AggregationType::DDSketch, - SummaryKind::Kll => AggregationType::DatasketchesKLL, - SummaryKind::Hll => AggregationType::HLL, - SummaryKind::Cms => AggregationType::CountMinSketch, - SummaryKind::CmsWithHeap => AggregationType::CountMinSketchWithHeap, - SummaryKind::CountSketch => AggregationType::CountSketch, - SummaryKind::CountSketchWithHeap => AggregationType::CountSketchWithHeap, + match kind { + SketchAlgorithm::DDSketch => AggregationType::DDSketch, + SketchAlgorithm::Kll => AggregationType::DatasketchesKLL, + SketchAlgorithm::Hll => AggregationType::HLL, + SketchAlgorithm::Cms => AggregationType::CountMinSketch, + SketchAlgorithm::CmsWithHeap => AggregationType::CountMinSketchWithHeap, + SketchAlgorithm::CountSketch => AggregationType::CountSketch, + SketchAlgorithm::CountSketchWithHeap => AggregationType::CountSketchWithHeap, _ => unreachable!("fixture only uses sketch-typed kinds"), }, String::new(), @@ -300,15 +292,15 @@ mod tests { agg( "agg0", "http_latency_ms", - SummaryKind::DDSketch, - SummaryParams::DDSketch { alpha: 0.01 }, + SketchAlgorithm::DDSketch, + SketchParams::DDSketch { alpha: 0.01 }, Vec::new(), ), agg( "agg1", "http_requests_total", - SummaryKind::Hll, - SummaryParams::Hll { precision: 14 }, + SketchAlgorithm::Hll, + SketchParams::Hll { precision: 14 }, vec!["zone".to_string()], ), ], @@ -355,16 +347,24 @@ mod tests { .values() .find(|m| matches!(m.source, Source::TimeSeries { ref metric } if metric == "http_latency_ms")) .expect("ddsketch materialization present"); - assert_eq!(ddsketch.kind, SummaryKind::DDSketch); - assert_eq!(ddsketch.params, SummaryParams::DDSketch { alpha: 0.01 }); + assert!(matches!( + &ddsketch.family, + planner_types::post_asap::SummaryFamilyType::Sketch(kind, _) + if kind.algorithm() == &planner_types::post_asap::SketchAlgorithm::DDSketch + && kind.params() == &planner_types::post_asap::SketchParams::DDSketch { alpha: 0.01 } + )); let hll = plan .materializations .values() .find(|m| matches!(m.source, Source::TimeSeries { ref metric } if metric == "http_requests_total")) .expect("hll materialization present"); - assert_eq!(hll.kind, SummaryKind::Hll); - assert_eq!(hll.params, SummaryParams::Hll { precision: 14 }); + assert!(matches!( + &hll.family, + planner_types::post_asap::SummaryFamilyType::Sketch(kind, _) + if kind.algorithm() == &planner_types::post_asap::SketchAlgorithm::Hll + && kind.params() == &planner_types::post_asap::SketchParams::Hll { precision: 14 } + )); assert_eq!(hll.group_by, vec!["zone".to_string()]); } @@ -387,7 +387,7 @@ mod tests { .expect("routing entry for ddsketch"); assert_eq!( quantile_entry.satisfies, - Capability::QuantileApprox(SketchKindHandle::DDSketch) + Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch)) ); let hll_fp = plan @@ -411,8 +411,8 @@ mod tests { agg( "agg0", "endpoint_count", - SummaryKind::CountSketchWithHeap, - SummaryParams::CountSketchWithHeap { + SketchAlgorithm::CountSketchWithHeap, + SketchParams::CountSketchWithHeap { width: 2048, depth: 5, heap_size: 10, @@ -422,8 +422,8 @@ mod tests { agg( "agg1", "endpoint_hits", - SummaryKind::Cms, - SummaryParams::Cms { + SketchAlgorithm::Cms, + SketchParams::Cms { width: 4096, depth: 4, }, @@ -446,7 +446,7 @@ mod tests { }; let plan = from_stage_config(&cfg, &[], 1, 0).expect("build plan"); let topk_entry = plan.routing.iter().find(|r| { - r.satisfies == Capability::FrequencyTopk(SketchKindHandle::CountSketchWithHeap) + r.satisfies == Capability::FrequencyTopk(Some(SketchAlgorithm::CountSketchWithHeap)) }); assert!( topk_entry.is_some(), @@ -457,7 +457,7 @@ mod tests { let freq_entry = plan .routing .iter() - .find(|r| r.satisfies == Capability::FrequencyEstimate(SketchKindHandle::CountMin)); + .find(|r| r.satisfies == Capability::FrequencyEstimate(Some(SketchAlgorithm::Cms))); assert!( freq_entry.is_some(), "expected a FrequencyEstimate routing entry: {:?}", @@ -467,14 +467,19 @@ mod tests { #[test] fn exact_agg_override_reports_exact_agg_capability() { - let mut a = agg( - "agg0", - "http_requests_total", - SummaryKind::Sum, // sentinel value, suppressed by the override - SummaryParams::Sum, - vec!["zone".to_string()], - ); - a.agg_type_override = Some("Sum".to_string()); + let a = BackendAggregation { + aggregation_id: "agg0".into(), + metric_name: "http_requests_total".into(), + family: SummaryFamilyType::ExactAggregate( + planner_types::post_asap::ExactKind::Sum, + planner_types::post_asap::ExactParams::Sum, + ), + window_secs: 60, + spatial_filter: String::new(), + grouping: vec!["zone".to_string()], + item_label: None, + aggregation_input: AggregationInput::Raw, + }; let cfg = BackendStageConfig { aggregations: vec![a], readouts: vec![BackendReadout { @@ -488,8 +493,13 @@ mod tests { .iter() .next() .expect("one materialization"); - assert!(m.kind.is_exact()); - assert_eq!(m.kind, SummaryKind::Sum); + assert!(matches!( + m.family, + planner_types::post_asap::SummaryFamilyType::ExactAggregate( + planner_types::post_asap::ExactKind::Sum, + planner_types::post_asap::ExactParams::Sum + ) + )); let entry = &plan.routing[0]; assert_eq!(entry.satisfies, Capability::ExactAgg(AggregationType::Sum)); diff --git a/control_plane/src/backend_plan/mod.rs b/control_plane/src/backend_plan/mod.rs index 8c958080..ca546ee1 100644 --- a/control_plane/src/backend_plan/mod.rs +++ b/control_plane/src/backend_plan/mod.rs @@ -7,8 +7,9 @@ //! `proto/backend_plan.proto`), and the conversions between them. //! //! Deliberately reuses this deployment's existing canonical vocabulary -//! rather than re-encoding it: `planner_types::post_asap::{SummaryKind, SummaryParams}` -//! for the materialization payload (no separate `ExactAggregate` variant — +//! rather than re-encoding it: `planner_types::post_asap::SummaryFamilyType` +//! for the materialization payload (including canonical `ExactKind` and +//! `SketchKind` choices; no backend-owned summary-family enum — //! see the design doc §3 for why), `asap_ir`/`crate::intent_algebra`'s //! `Source`/`ColumnRef`/`WindowKind` for the L3 IR fragments, //! `crate::physical::runtime_capability::Capability` for routing, and @@ -33,14 +34,18 @@ pub use from_stage_config::from_stage_config; use std::collections::HashMap; +pub use asap_types::StorageBackend; use asap_types::{AggregationType, MonitorSpec, PolicyFingerprint}; -use asap_types::{SummaryKind, SummaryParams}; use prost::Message as _; -use serde::{Deserialize, Serialize}; use thiserror::Error; -use crate::physical::runtime_capability::{Capability, SketchKindHandle}; +use crate::physical::runtime_capability::Capability; use asap_types::enums::WindowKind; +use planner_types::post_asap::{ + EvaluationSchedule, ExactKind, ExactParams, GroupingStrategy, OutputRepresentation, + SketchAlgorithm, SketchKind, SketchParams, SummaryFamilyType, SummaryMaintenanceLifecycle, + SummaryMaintenanceLifecycleGuarantee, SummaryMaintenanceMode, +}; use planner_types::pre_asap::{ColumnRef, Source}; /// Errors decoding a `BackendPlan` (or one of its parts) from its proto @@ -202,59 +207,69 @@ impl TryFrom for ColumnRef { } } -// ── SummaryKind / SummaryParams ───────────────────────────────────────────── +// ── SummaryFamilyType wire adapter ────────────────────────────────────────── // -// `SummaryParams`'s wire form is a single self-describing `oneof` -- -// `SummaryKind` is always recoverable from which arm is set, so encoding -// only ever needs `&SummaryParams`. Decoding produces the `(SummaryKind, -// SummaryParams)` pair `Materialization` needs. +// The legacy `SummaryParams` protobuf is a single self-describing `oneof`. +// Encoding and decoding keep that wire compatibility at the boundary while +// the domain model carries Planner's canonical `SummaryFamilyType`. -impl From<&SummaryParams> for proto::SummaryParams { - fn from(p: &SummaryParams) -> Self { +impl From<&SummaryFamilyType> for proto::SummaryParams { + fn from(family: &SummaryFamilyType) -> Self { use proto::summary_params::Params as Wire; - let params = match p { - SummaryParams::Sum => Wire::Sum(true), - SummaryParams::Count => Wire::Count(true), - SummaryParams::MinMax => Wire::MinMax(true), - SummaryParams::Increase => Wire::Increase(true), - SummaryParams::Rate => Wire::Rate(true), - SummaryParams::Kll { k } => Wire::Kll(proto::KllParams { k: *k }), - SummaryParams::Cms { width, depth } => Wire::Cms(proto::CmsParams { - width: *width, - depth: *depth, - }), - SummaryParams::Hll { precision } => Wire::Hll(proto::HllParams { - precision: *precision as u32, - }), - SummaryParams::DDSketch { alpha } => { - Wire::Ddsketch(proto::DdSketchParams { alpha: *alpha }) + let params = match family { + SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) => Wire::Sum(true), + SummaryFamilyType::ExactAggregate(ExactKind::Count, ExactParams::Count) => { + Wire::Count(true) } - SummaryParams::CmsWithHeap { - width, - depth, - heap_size, - } => Wire::CmsWithHeap(proto::CmsWithHeapParams { - width: *width, - depth: *depth, - heap_size: *heap_size, - }), - SummaryParams::Kmv { k } => Wire::Kmv(proto::KmvParams { k: *k }), - SummaryParams::Theta { k } => Wire::Theta(proto::ThetaParams { k: *k }), - SummaryParams::CountSketch { width, depth } => { - Wire::CountSketch(proto::CountSketchParams { + SummaryFamilyType::ExactAggregate(ExactKind::MinMax, ExactParams::MinMax) => { + Wire::MinMax(true) + } + SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase) => { + Wire::Increase(true) + } + SummaryFamilyType::ExactAggregate(ExactKind::Rate, ExactParams::Rate) => { + Wire::Rate(true) + } + SummaryFamilyType::Sketch(kind, _) => match kind.params() { + SketchParams::Kll { k } => Wire::Kll(proto::KllParams { k: *k }), + SketchParams::Cms { width, depth } => Wire::Cms(proto::CmsParams { width: *width, depth: *depth, - }) - } - SummaryParams::CountSketchWithHeap { - width, - depth, - heap_size, - } => Wire::CountSketchWithHeap(proto::CountSketchWithHeapParams { - width: *width, - depth: *depth, - heap_size: *heap_size, - }), + }), + SketchParams::Hll { precision } => Wire::Hll(proto::HllParams { + precision: *precision as u32, + }), + SketchParams::DDSketch { alpha } => { + Wire::Ddsketch(proto::DdSketchParams { alpha: *alpha }) + } + SketchParams::CmsWithHeap { + width, + depth, + heap_size, + } => Wire::CmsWithHeap(proto::CmsWithHeapParams { + width: *width, + depth: *depth, + heap_size: *heap_size, + }), + SketchParams::Kmv { k } => Wire::Kmv(proto::KmvParams { k: *k }), + SketchParams::Theta { k } => Wire::Theta(proto::ThetaParams { k: *k }), + SketchParams::CountSketch { width, depth } => { + Wire::CountSketch(proto::CountSketchParams { + width: *width, + depth: *depth, + }) + } + SketchParams::CountSketchWithHeap { + width, + depth, + heap_size, + } => Wire::CountSketchWithHeap(proto::CountSketchWithHeapParams { + width: *width, + depth: *depth, + heap_size: *heap_size, + }), + }, + other => panic!("BackendPlan cannot encode unsupported summary family {other:?}"), }; proto::SummaryParams { params: Some(params), @@ -262,59 +277,61 @@ impl From<&SummaryParams> for proto::SummaryParams { } } -/// Decode a wire `SummaryParams` back to the `(SummaryKind, SummaryParams)` -/// pair. Not a `TryFrom` impl because the single wire message decodes to -/// TWO domain values (`SummaryKind` is implied, not carried separately on -/// the wire) — see this module's doc. -pub fn decode_summary_params( - p: proto::SummaryParams, -) -> Result<(SummaryKind, SummaryParams), DecodeError> { +/// Decode the legacy wire `SummaryParams` into Planner's canonical family. +pub fn decode_summary_params(p: proto::SummaryParams) -> Result { use proto::summary_params::Params as Wire; let params = p.params.ok_or(DecodeError::MissingOneof("SummaryParams"))?; + let exact = |kind, params| SummaryFamilyType::ExactAggregate(kind, params); + let sketch = |algorithm, params| { + SummaryFamilyType::Sketch( + SketchKind::new(algorithm, params), + GroupingStrategy::PerSubpopulationInstance, + ) + }; Ok(match params { - Wire::Sum(_) => (SummaryKind::Sum, SummaryParams::Sum), - Wire::Count(_) => (SummaryKind::Count, SummaryParams::Count), - Wire::MinMax(_) => (SummaryKind::MinMax, SummaryParams::MinMax), - Wire::Increase(_) => (SummaryKind::Increase, SummaryParams::Increase), - Wire::Rate(_) => (SummaryKind::Rate, SummaryParams::Rate), - Wire::Kll(k) => (SummaryKind::Kll, SummaryParams::Kll { k: k.k }), - Wire::Cms(c) => ( - SummaryKind::Cms, - SummaryParams::Cms { + Wire::Sum(_) => exact(ExactKind::Sum, ExactParams::Sum), + Wire::Count(_) => exact(ExactKind::Count, ExactParams::Count), + Wire::MinMax(_) => exact(ExactKind::MinMax, ExactParams::MinMax), + Wire::Increase(_) => exact(ExactKind::Increase, ExactParams::Increase), + Wire::Rate(_) => exact(ExactKind::Rate, ExactParams::Rate), + Wire::Kll(k) => sketch(SketchAlgorithm::Kll, SketchParams::Kll { k: k.k }), + Wire::Cms(c) => sketch( + SketchAlgorithm::Cms, + SketchParams::Cms { width: c.width, depth: c.depth, }, ), - Wire::Hll(h) => ( - SummaryKind::Hll, - SummaryParams::Hll { + Wire::Hll(h) => sketch( + SketchAlgorithm::Hll, + SketchParams::Hll { precision: h.precision as u8, }, ), - Wire::Ddsketch(d) => ( - SummaryKind::DDSketch, - SummaryParams::DDSketch { alpha: d.alpha }, + Wire::Ddsketch(d) => sketch( + SketchAlgorithm::DDSketch, + SketchParams::DDSketch { alpha: d.alpha }, ), - Wire::CmsWithHeap(c) => ( - SummaryKind::CmsWithHeap, - SummaryParams::CmsWithHeap { + Wire::CmsWithHeap(c) => sketch( + SketchAlgorithm::CmsWithHeap, + SketchParams::CmsWithHeap { width: c.width, depth: c.depth, heap_size: c.heap_size, }, ), - Wire::Kmv(k) => (SummaryKind::Kmv, SummaryParams::Kmv { k: k.k }), - Wire::Theta(t) => (SummaryKind::Theta, SummaryParams::Theta { k: t.k }), - Wire::CountSketch(c) => ( - SummaryKind::CountSketch, - SummaryParams::CountSketch { + Wire::Kmv(k) => sketch(SketchAlgorithm::Kmv, SketchParams::Kmv { k: k.k }), + Wire::Theta(t) => sketch(SketchAlgorithm::Theta, SketchParams::Theta { k: t.k }), + Wire::CountSketch(c) => sketch( + SketchAlgorithm::CountSketch, + SketchParams::CountSketch { width: c.width, depth: c.depth, }, ), - Wire::CountSketchWithHeap(c) => ( - SummaryKind::CountSketchWithHeap, - SummaryParams::CountSketchWithHeap { + Wire::CountSketchWithHeap(c) => sketch( + SketchAlgorithm::CountSketchWithHeap, + SketchParams::CountSketchWithHeap { width: c.width, depth: c.depth, heap_size: c.heap_size, @@ -323,37 +340,37 @@ pub fn decode_summary_params( }) } -// ── SketchKindHandle / AggregationType / Capability ───────────────────────── +// ── SketchAlgorithm / AggregationType / Capability ───────────────────────── -impl From for proto::SketchKindHandle { - fn from(h: SketchKindHandle) -> Self { +impl From> for proto::SketchKindHandle { + fn from(h: Option) -> Self { match h { - SketchKindHandle::DDSketch => proto::SketchKindHandle::Ddsketch, - SketchKindHandle::Kll => proto::SketchKindHandle::Kll, - SketchKindHandle::Hll => proto::SketchKindHandle::Hll, - SketchKindHandle::CountSketch => proto::SketchKindHandle::CountSketch, - SketchKindHandle::CountMin => proto::SketchKindHandle::CountMin, - SketchKindHandle::CmsWithHeap => proto::SketchKindHandle::CmsWithHeap, - SketchKindHandle::CountSketchWithHeap => proto::SketchKindHandle::CountSketchWithHeap, - SketchKindHandle::Any => proto::SketchKindHandle::Any, + Some(SketchAlgorithm::DDSketch) => Self::Ddsketch, + Some(SketchAlgorithm::Kll) => Self::Kll, + Some(SketchAlgorithm::Hll) => Self::Hll, + Some(SketchAlgorithm::CountSketch) => Self::CountSketch, + Some(SketchAlgorithm::Cms) => Self::CountMin, + Some(SketchAlgorithm::CmsWithHeap) => Self::CmsWithHeap, + Some(SketchAlgorithm::CountSketchWithHeap) => Self::CountSketchWithHeap, + Some(SketchAlgorithm::Kmv | SketchAlgorithm::Theta) | None => Self::Any, } } } -impl TryFrom for SketchKindHandle { +impl TryFrom for Option { type Error = DecodeError; fn try_from(h: proto::SketchKindHandle) -> Result { match h { - proto::SketchKindHandle::Ddsketch => Ok(SketchKindHandle::DDSketch), - proto::SketchKindHandle::Kll => Ok(SketchKindHandle::Kll), - proto::SketchKindHandle::Hll => Ok(SketchKindHandle::Hll), - proto::SketchKindHandle::CountSketch => Ok(SketchKindHandle::CountSketch), - proto::SketchKindHandle::CountMin => Ok(SketchKindHandle::CountMin), - proto::SketchKindHandle::CmsWithHeap => Ok(SketchKindHandle::CmsWithHeap), + proto::SketchKindHandle::Ddsketch => Ok(Some(SketchAlgorithm::DDSketch)), + proto::SketchKindHandle::Kll => Ok(Some(SketchAlgorithm::Kll)), + proto::SketchKindHandle::Hll => Ok(Some(SketchAlgorithm::Hll)), + proto::SketchKindHandle::CountSketch => Ok(Some(SketchAlgorithm::CountSketch)), + proto::SketchKindHandle::CountMin => Ok(Some(SketchAlgorithm::Cms)), + proto::SketchKindHandle::CmsWithHeap => Ok(Some(SketchAlgorithm::CmsWithHeap)), proto::SketchKindHandle::CountSketchWithHeap => { - Ok(SketchKindHandle::CountSketchWithHeap) + Ok(Some(SketchAlgorithm::CountSketchWithHeap)) } - proto::SketchKindHandle::Any => Ok(SketchKindHandle::Any), + proto::SketchKindHandle::Any => Ok(None), proto::SketchKindHandle::Unspecified => Err(DecodeError::UnknownEnumValue { field: "SketchKindHandle", value: proto::SketchKindHandle::Unspecified as i32, @@ -424,14 +441,14 @@ impl From<&Capability> for proto::Capability { use proto::capability::Capability as Wire; let capability = match c { Capability::QuantileApprox(h) => { - Wire::QuantileApprox(proto::SketchKindHandle::from(*h) as i32) + Wire::QuantileApprox(proto::SketchKindHandle::from(h.clone()) as i32) } Capability::CardinalityApprox => Wire::CardinalityApprox(true), Capability::FrequencyEstimate(h) => { - Wire::FrequencyEstimate(proto::SketchKindHandle::from(*h) as i32) + Wire::FrequencyEstimate(proto::SketchKindHandle::from(h.clone()) as i32) } Capability::FrequencyTopk(h) => { - Wire::FrequencyTopk(proto::SketchKindHandle::from(*h) as i32) + Wire::FrequencyTopk(proto::SketchKindHandle::from(h.clone()) as i32) } Capability::ExactAgg(a) => Wire::ExactAgg(proto::AggregationType::from(*a) as i32), }; @@ -446,7 +463,7 @@ impl TryFrom for Capability { fn try_from(c: proto::Capability) -> Result { use proto::capability::Capability as Wire; let decode_handle = - |v: i32, field: &'static str| -> Result { + |v: i32, field: &'static str| -> Result, DecodeError> { proto::SketchKindHandle::try_from(v) .map_err(|_| DecodeError::UnknownEnumValue { field, value: v })? .try_into() @@ -486,16 +503,6 @@ impl TryFrom for Capability { // mirrors -- see that type's own doc and asap_types::MonitorSpec's for // the same constraint) ─────────────────────────────────────────────────── -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] -#[serde(rename_all = "snake_case")] -pub enum StorageBackend { - #[default] - SketchStore, - GorillaObjectStore, - DoubleWrite, - PrometheusRemote, -} - impl From for proto::StorageBackend { fn from(b: StorageBackend) -> Self { match b { @@ -594,40 +601,73 @@ pub struct Materialization { pub group_by: Vec, pub rollup: Vec, pub spatial_filter: String, - pub kind: SummaryKind, - pub params: SummaryParams, + pub family: SummaryFamilyType, pub col: ColumnRef, pub retention: Option, - pub lifecycle: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct SummaryMaintenanceLifecycle { - pub kind: String, - pub maintenance_mode: String, - pub evaluation_schedule: String, - pub output_representation: String, + pub lifecycle: Option, } -impl From<&SummaryMaintenanceLifecycle> for proto::SummaryMaintenanceLifecycle { - fn from(value: &SummaryMaintenanceLifecycle) -> Self { +impl From<&SummaryMaintenanceLifecycleGuarantee> for proto::SummaryMaintenanceLifecycle { + fn from(value: &SummaryMaintenanceLifecycleGuarantee) -> Self { Self { - kind: value.kind.clone(), - maintenance_mode: value.maintenance_mode.clone(), - evaluation_schedule: value.evaluation_schedule.clone(), - output_representation: value.output_representation.clone(), + kind: match value.summary_maintenance_lifecycle { + SummaryMaintenanceLifecycle::Ephemeral => "ephemeral", + SummaryMaintenanceLifecycle::Prepared { .. } => "prepared", + SummaryMaintenanceLifecycle::Shared { .. } => "shared", + SummaryMaintenanceLifecycle::ContinuouslyMaintained => "continuously_maintained", + } + .into(), + maintenance_mode: value.summary_maintenance_mode.as_str().into(), + evaluation_schedule: match value.evaluation_schedule { + EvaluationSchedule::OneShot => "one_shot", + EvaluationSchedule::PerUpdate => "per_update", + EvaluationSchedule::OnRead => "on_read", + } + .into(), + output_representation: match value.output_representation { + OutputRepresentation::PlainRows => "plain_rows", + OutputRepresentation::SummaryState => "summary_state", + OutputRepresentation::FinalizedValue => "finalized_value", + } + .into(), } } } -impl From for SummaryMaintenanceLifecycle { - fn from(value: proto::SummaryMaintenanceLifecycle) -> Self { - Self { - kind: value.kind, - maintenance_mode: value.maintenance_mode, - evaluation_schedule: value.evaluation_schedule, - output_representation: value.output_representation, - } +impl TryFrom for SummaryMaintenanceLifecycleGuarantee { + type Error = DecodeError; + + fn try_from(value: proto::SummaryMaintenanceLifecycle) -> Result { + let summary_maintenance_lifecycle = match value.kind.as_str() { + "ephemeral" => SummaryMaintenanceLifecycle::Ephemeral, + "continuously_maintained" => SummaryMaintenanceLifecycle::ContinuouslyMaintained, + // Prepared/Shared carry timestamps/retention that the v1 wire message cannot + // represent. Reject them instead of manufacturing a lossy Planner value. + other => return Err(DecodeError::UnsupportedLifecycle(other.into())), + }; + let summary_maintenance_mode = match value.maintenance_mode.as_str() { + "direct_build" => SummaryMaintenanceMode::DirectBuild, + "incremental" => SummaryMaintenanceMode::Incremental, + other => return Err(DecodeError::UnsupportedLifecycle(other.into())), + }; + let evaluation_schedule = match value.evaluation_schedule.as_str() { + "one_shot" => EvaluationSchedule::OneShot, + "per_update" => EvaluationSchedule::PerUpdate, + "on_read" => EvaluationSchedule::OnRead, + other => return Err(DecodeError::UnsupportedLifecycle(other.into())), + }; + let output_representation = match value.output_representation.as_str() { + "plain_rows" => OutputRepresentation::PlainRows, + "summary_state" => OutputRepresentation::SummaryState, + "finalized_value" => OutputRepresentation::FinalizedValue, + other => return Err(DecodeError::UnsupportedLifecycle(other.into())), + }; + Ok(Self { + summary_maintenance_lifecycle, + summary_maintenance_mode, + evaluation_schedule, + output_representation, + }) } } @@ -640,7 +680,7 @@ impl From<&Materialization> for proto::Materialization { group_by: m.group_by.clone(), rollup: m.rollup.clone(), spatial_filter: m.spatial_filter.clone(), - params: Some((&m.params).into()), + params: Some((&m.family).into()), col: Some((&m.col).into()), retention: m.retention.as_ref().map(Into::into), lifecycle: m.lifecycle.as_ref().map(Into::into), @@ -651,26 +691,11 @@ impl From<&Materialization> for proto::Materialization { impl TryFrom for Materialization { type Error = DecodeError; fn try_from(m: proto::Materialization) -> Result { - let (kind, params) = decode_summary_params( + let family = decode_summary_params( m.params .ok_or(DecodeError::MissingOneof("Materialization.params"))?, )?; - let lifecycle = m.lifecycle.map(SummaryMaintenanceLifecycle::from); - if let Some(lifecycle) = &lifecycle { - if lifecycle.kind != "continuously_maintained" - || lifecycle.maintenance_mode != "incremental" - || lifecycle.evaluation_schedule != "per_update" - || lifecycle.output_representation != "summary_state" - { - return Err(DecodeError::UnsupportedLifecycle(format!( - "{}/{}/{}/{}", - lifecycle.kind, - lifecycle.maintenance_mode, - lifecycle.evaluation_schedule, - lifecycle.output_representation - ))); - } - } + let lifecycle = m.lifecycle.map(TryInto::try_into).transpose()?; Ok(Materialization { fingerprint: PolicyFingerprint(m.fingerprint), source: m @@ -684,8 +709,7 @@ impl TryFrom for Materialization { group_by: m.group_by, rollup: m.rollup, spatial_filter: m.spatial_filter, - kind, - params, + family, col: m .col .ok_or(DecodeError::MissingOneof("Materialization.col"))? @@ -821,7 +845,7 @@ impl BackendPlan { if materialization.window.slide_ms == Some(0) { return Err(ValidationError::ZeroSlide { fingerprint: key.0 }); } - if !kind_params_match(&materialization.kind, &materialization.params) { + if !summary_family_is_valid(&materialization.family) { return Err(ValidationError::KindParamsMismatch { fingerprint: key.0 }); } } @@ -841,46 +865,49 @@ impl BackendPlan { } } -fn kind_params_match(kind: &SummaryKind, params: &SummaryParams) -> bool { +fn summary_family_is_valid(family: &SummaryFamilyType) -> bool { matches!( - (kind, params), - (SummaryKind::Sum, SummaryParams::Sum) - | (SummaryKind::Count, SummaryParams::Count) - | (SummaryKind::MinMax, SummaryParams::MinMax) - | (SummaryKind::Increase, SummaryParams::Increase) - | (SummaryKind::Rate, SummaryParams::Rate) - | (SummaryKind::Kll, SummaryParams::Kll { .. }) - | (SummaryKind::Cms, SummaryParams::Cms { .. }) - | (SummaryKind::Hll, SummaryParams::Hll { .. }) - | (SummaryKind::DDSketch, SummaryParams::DDSketch { .. }) - | (SummaryKind::CmsWithHeap, SummaryParams::CmsWithHeap { .. }) - | (SummaryKind::Kmv, SummaryParams::Kmv { .. }) - | (SummaryKind::Theta, SummaryParams::Theta { .. }) - | (SummaryKind::CountSketch, SummaryParams::CountSketch { .. }) - | ( - SummaryKind::CountSketchWithHeap, - SummaryParams::CountSketchWithHeap { .. } - ) + family, + SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) + | SummaryFamilyType::ExactAggregate(ExactKind::Count, ExactParams::Count) + | SummaryFamilyType::ExactAggregate(ExactKind::MinMax, ExactParams::MinMax) + | SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase) + | SummaryFamilyType::ExactAggregate(ExactKind::Rate, ExactParams::Rate) + | SummaryFamilyType::Sketch(..) ) } fn materialization_satisfies(required: &Capability, m: &Materialization) -> bool { - let available = match m.kind { - SummaryKind::DDSketch => Capability::QuantileApprox(SketchKindHandle::DDSketch), - SummaryKind::Kll => Capability::QuantileApprox(SketchKindHandle::Kll), - SummaryKind::Hll => Capability::CardinalityApprox, - SummaryKind::Cms => Capability::FrequencyEstimate(SketchKindHandle::CountMin), - SummaryKind::CountSketch => Capability::FrequencyEstimate(SketchKindHandle::CountSketch), - SummaryKind::CmsWithHeap => Capability::FrequencyTopk(SketchKindHandle::CmsWithHeap), - SummaryKind::CountSketchWithHeap => { - Capability::FrequencyTopk(SketchKindHandle::CountSketchWithHeap) + let available = match &m.family { + SummaryFamilyType::Sketch(kind, _) => match kind.algorithm() { + SketchAlgorithm::DDSketch => { + Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch)) + } + SketchAlgorithm::Kll => Capability::QuantileApprox(Some(SketchAlgorithm::Kll)), + SketchAlgorithm::Hll => Capability::CardinalityApprox, + SketchAlgorithm::Cms => Capability::FrequencyEstimate(Some(SketchAlgorithm::Cms)), + SketchAlgorithm::CountSketch => { + Capability::FrequencyEstimate(Some(SketchAlgorithm::CountSketch)) + } + SketchAlgorithm::CmsWithHeap => { + Capability::FrequencyTopk(Some(SketchAlgorithm::CmsWithHeap)) + } + SketchAlgorithm::CountSketchWithHeap => { + Capability::FrequencyTopk(Some(SketchAlgorithm::CountSketchWithHeap)) + } + SketchAlgorithm::Kmv | SketchAlgorithm::Theta => return false, + }, + SummaryFamilyType::ExactAggregate(ExactKind::Sum, _) => { + Capability::ExactAgg(AggregationType::Sum) + } + SummaryFamilyType::ExactAggregate(ExactKind::MinMax, _) => { + Capability::ExactAgg(AggregationType::MinMax) } - SummaryKind::Sum => Capability::ExactAgg(AggregationType::Sum), - SummaryKind::MinMax => Capability::ExactAgg(AggregationType::MinMax), - SummaryKind::Increase => Capability::ExactAgg(AggregationType::Increase), - SummaryKind::Count | SummaryKind::Rate | SummaryKind::Kmv | SummaryKind::Theta => { - return false + SummaryFamilyType::ExactAggregate(ExactKind::Increase, _) => { + Capability::ExactAgg(AggregationType::Increase) } + SummaryFamilyType::ExactAggregate(ExactKind::Count | ExactKind::Rate, _) => return false, + _ => return false, }; required.is_satisfied_by(&available) } @@ -890,11 +917,7 @@ mod tests { use super::*; use std::time::Duration; - fn sample_materialization( - fingerprint: u64, - kind: SummaryKind, - params: SummaryParams, - ) -> Materialization { + fn sample_materialization(fingerprint: u64, family: SummaryFamilyType) -> Materialization { Materialization { fingerprint: PolicyFingerprint(fingerprint), source: Source::TimeSeries { @@ -908,33 +931,46 @@ mod tests { group_by: vec!["zone".to_string()], rollup: vec![], spatial_filter: String::new(), - kind, - params, + family, col: ColumnRef::SampleValue, retention: Some(RetentionPolicy { num_aggregates_to_retain: Some(1000), }), - lifecycle: Some(SummaryMaintenanceLifecycle { - kind: "continuously_maintained".into(), - maintenance_mode: "incremental".into(), - evaluation_schedule: "per_update".into(), - output_representation: "summary_state".into(), + lifecycle: Some(SummaryMaintenanceLifecycleGuarantee { + summary_maintenance_lifecycle: SummaryMaintenanceLifecycle::ContinuouslyMaintained, + summary_maintenance_mode: SummaryMaintenanceMode::Incremental, + evaluation_schedule: EvaluationSchedule::PerUpdate, + output_representation: OutputRepresentation::SummaryState, }), } } + fn exact(kind: ExactKind, params: ExactParams) -> SummaryFamilyType { + SummaryFamilyType::ExactAggregate(kind, params) + } + + fn sketch(algorithm: SketchAlgorithm, params: SketchParams) -> SummaryFamilyType { + SummaryFamilyType::Sketch( + SketchKind::new(algorithm, params), + GroupingStrategy::PerSubpopulationInstance, + ) + } + fn sample_plan() -> BackendPlan { let mut materializations = HashMap::new(); // Approximate sketch. materializations.insert( PolicyFingerprint(1), - sample_materialization(1, SummaryKind::Kll, SummaryParams::Kll { k: 200 }), + sample_materialization( + 1, + sketch(SketchAlgorithm::Kll, SketchParams::Kll { k: 200 }), + ), ); // Exact accumulator -- same `SummaryKind`/`SummaryParams` vocabulary, // no separate wire representation (design doc §3). materializations.insert( PolicyFingerprint(2), - sample_materialization(2, SummaryKind::Sum, SummaryParams::Sum), + sample_materialization(2, exact(ExactKind::Sum, ExactParams::Sum)), ); BackendPlan { @@ -943,7 +979,7 @@ mod tests { materializations, routing: vec![ RoutingEntry { - satisfies: Capability::QuantileApprox(SketchKindHandle::Any), + satisfies: Capability::QuantileApprox(None), materialization: PolicyFingerprint(1), storage_backend: StorageBackend::SketchStore, }, @@ -968,7 +1004,7 @@ mod tests { } #[test] - fn unsupported_lifecycle_fails_closed_on_decode() { + fn ephemeral_lifecycle_round_trips() { let mut plan = sample_plan(); plan.materializations .values_mut() @@ -977,11 +1013,9 @@ mod tests { .lifecycle .as_mut() .unwrap() - .kind = "ephemeral".into(); - assert!(matches!( - BackendPlan::decode(&plan.encode_to_vec()), - Err(DecodeError::UnsupportedLifecycle(_)) - )); + .summary_maintenance_lifecycle = SummaryMaintenanceLifecycle::Ephemeral; + let decoded = BackendPlan::decode(&plan.encode_to_vec()).expect("decode"); + assert_eq!(decoded, plan); } #[test] @@ -994,87 +1028,87 @@ mod tests { #[test] fn kll_survives_round_trip_with_kind_and_params_agreeing() { - let m = sample_materialization(1, SummaryKind::Kll, SummaryParams::Kll { k: 200 }); + let m = sample_materialization( + 1, + sketch(SketchAlgorithm::Kll, SketchParams::Kll { k: 200 }), + ); let wire = proto::Materialization::from(&m); let back = Materialization::try_from(wire).expect("decode must succeed"); - assert_eq!(back.kind, SummaryKind::Kll); - assert_eq!(back.params, SummaryParams::Kll { k: 200 }); + assert_eq!(back.family, m.family); assert_eq!(back, m); } #[test] fn sum_is_exact_and_carries_no_tuning_parameters() { - let m = sample_materialization(2, SummaryKind::Sum, SummaryParams::Sum); + let m = sample_materialization(2, exact(ExactKind::Sum, ExactParams::Sum)); let wire = proto::Materialization::from(&m); let back = Materialization::try_from(wire).expect("decode must succeed"); - assert!(back.kind.is_exact()); + assert!(matches!(back.family, SummaryFamilyType::ExactAggregate(..))); assert_eq!(back, m); } #[test] fn every_summary_kind_round_trips() { - let cases = [ - (SummaryKind::Sum, SummaryParams::Sum), - (SummaryKind::Count, SummaryParams::Count), - (SummaryKind::MinMax, SummaryParams::MinMax), - (SummaryKind::Increase, SummaryParams::Increase), - (SummaryKind::Rate, SummaryParams::Rate), - (SummaryKind::Kll, SummaryParams::Kll { k: 200 }), - ( - SummaryKind::Cms, - SummaryParams::Cms { + let cases = vec![ + exact(ExactKind::Sum, ExactParams::Sum), + exact(ExactKind::Count, ExactParams::Count), + exact(ExactKind::MinMax, ExactParams::MinMax), + exact(ExactKind::Increase, ExactParams::Increase), + exact(ExactKind::Rate, ExactParams::Rate), + sketch(SketchAlgorithm::Kll, SketchParams::Kll { k: 200 }), + sketch( + SketchAlgorithm::Cms, + SketchParams::Cms { width: 64, depth: 4, }, ), - (SummaryKind::Hll, SummaryParams::Hll { precision: 14 }), - ( - SummaryKind::DDSketch, - SummaryParams::DDSketch { alpha: 0.01 }, + sketch(SketchAlgorithm::Hll, SketchParams::Hll { precision: 14 }), + sketch( + SketchAlgorithm::DDSketch, + SketchParams::DDSketch { alpha: 0.01 }, ), - ( - SummaryKind::CmsWithHeap, - SummaryParams::CmsWithHeap { + sketch( + SketchAlgorithm::CmsWithHeap, + SketchParams::CmsWithHeap { width: 64, depth: 4, heap_size: 10, }, ), - (SummaryKind::Kmv, SummaryParams::Kmv { k: 1024 }), - (SummaryKind::Theta, SummaryParams::Theta { k: 1024 }), - ( - SummaryKind::CountSketch, - SummaryParams::CountSketch { + sketch(SketchAlgorithm::Kmv, SketchParams::Kmv { k: 1024 }), + sketch(SketchAlgorithm::Theta, SketchParams::Theta { k: 1024 }), + sketch( + SketchAlgorithm::CountSketch, + SketchParams::CountSketch { width: 64, depth: 4, }, ), - ( - SummaryKind::CountSketchWithHeap, - SummaryParams::CountSketchWithHeap { + sketch( + SketchAlgorithm::CountSketchWithHeap, + SketchParams::CountSketchWithHeap { width: 64, depth: 4, heap_size: 10, }, ), ]; - for (kind, params) in cases { - let wire = proto::SummaryParams::from(¶ms); - let (decoded_kind, decoded_params) = - decode_summary_params(wire).expect("decode must succeed"); - assert_eq!(decoded_kind, kind, "kind mismatch for {params:?}"); - assert_eq!(decoded_params, params); + for family in cases { + let wire = proto::SummaryParams::from(&family); + let decoded = decode_summary_params(wire).expect("decode must succeed"); + assert_eq!(decoded, family); } } #[test] fn every_capability_variant_round_trips() { let cases = [ - Capability::QuantileApprox(SketchKindHandle::DDSketch), - Capability::QuantileApprox(SketchKindHandle::Any), + Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch)), + Capability::QuantileApprox(None), Capability::CardinalityApprox, - Capability::FrequencyEstimate(SketchKindHandle::CountMin), - Capability::FrequencyTopk(SketchKindHandle::CmsWithHeap), + Capability::FrequencyEstimate(Some(SketchAlgorithm::Cms)), + Capability::FrequencyTopk(Some(SketchAlgorithm::CmsWithHeap)), Capability::ExactAgg(AggregationType::Sum), Capability::ExactAgg(AggregationType::MultipleSubpopulation), ]; diff --git a/control_plane/src/emit/backend_push.rs b/control_plane/src/emit/backend_push.rs index 079e3920..e8becfe4 100644 --- a/control_plane/src/emit/backend_push.rs +++ b/control_plane/src/emit/backend_push.rs @@ -685,19 +685,25 @@ mod tests { AggregationInput, BackendAggregation, BackendReadout, }; use planner_types::post_asap::SketchQuery; - use planner_types::post_asap::{SketchAlgorithm, SketchParams}; + use planner_types::post_asap::{ + GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SummaryFamilyType, + }; BackendStageConfig { aggregations: vec![BackendAggregation { aggregation_id: agg_id.to_string(), metric_name: metric.to_string(), - sketch_kind: SketchAlgorithm::DDSketch.into(), - sketch_params: SketchParams::DDSketch { alpha: 0.01 }.into(), + family: SummaryFamilyType::Sketch( + SketchKind::new( + SketchAlgorithm::DDSketch, + SketchParams::DDSketch { alpha: 0.01 }, + ), + GroupingStrategy::PerSubpopulationInstance, + ), grouping: vec![], item_label: None, spatial_filter: String::new(), window_secs: 60, aggregation_input: AggregationInput::SketchEnvelope, - agg_type_override: None, }], readouts: vec![BackendReadout { aggregation_id: agg_id.to_string(), diff --git a/control_plane/src/emit/mod.rs b/control_plane/src/emit/mod.rs index ae96ce55..b4731ae1 100644 --- a/control_plane/src/emit/mod.rs +++ b/control_plane/src/emit/mod.rs @@ -258,7 +258,7 @@ fn apply_cold_format_from_env(edge_cfg: &mut EdgeStageConfig) { // HashMap — the typed bootstrap / replan paths emitted single-pipeline // YAML and the routing-connector path stayed dormant. // -// `extract_root_sketch_kind` walks a `PhysicalExpr` tree and returns the +// `extract_root_sketch_algorithm` walks a `PhysicalExpr` tree and returns the // committed sketch family — looking through `SketchEstimate`, // `SketchAgg`, `SketchMerge`, `LetBinding`, and `RawAtEdgeSketchAtBackend`. // `SketchAgg::sketch_type` is the canonical source of truth (the typed @@ -284,7 +284,7 @@ fn apply_cold_format_from_env(edge_cfg: &mut EdgeStageConfig) { /// (`Logical`-only, unresolved `Ref`, raw Mode-3 archive). These map /// onto the raw-passthrough default pipeline in the routing emitter, /// which is correct. -pub fn extract_root_sketch_kind(expr: &PhysicalExpr) -> Option { +pub fn extract_root_sketch_algorithm(expr: &PhysicalExpr) -> Option { match expr { PhysicalExpr::Committed(plan) => extract_from_plan(plan), PhysicalExpr::RawAtEdgeSketchAtBackend { family, .. } => Some(family.clone()), @@ -394,7 +394,7 @@ pub fn collect_metric_to_family( else { continue; }; - if let Some(kind) = extract_root_sketch_kind(&deployment_expr) { + if let Some(kind) = extract_root_sketch_algorithm(&deployment_expr) { out.entry(entry.metric_name.clone()) .or_default() .insert(kind); @@ -772,7 +772,7 @@ mod runtime_tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "ddsketch".to_string(), - sketch_kind: SketchAlgorithm::DDSketch, + sketch_algorithm: SketchAlgorithm::DDSketch, sketch_params: SketchParams::DDSketch { alpha: 0.01 }, aggregation_id: "agg0".to_string(), }], diff --git a/control_plane/src/emit/monitor.rs b/control_plane/src/emit/monitor.rs index d93f6f69..1e35bbd3 100644 --- a/control_plane/src/emit/monitor.rs +++ b/control_plane/src/emit/monitor.rs @@ -16,43 +16,9 @@ //! planner stage gains a monitor-intent slot. See //! `ASAPCollector/docs/continuous-monitoring-tumbling-cost-analysis.md`. +pub use asap_types::MonitorFunctional as Functional; use serde_yaml::{Mapping, Value}; -/// Which additive readout to threshold. Mirrors the Go `monitor.Functional` -/// and the edge `ThresholdConfig.functional` string values. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Functional { - Sum, - CmsPoint, - LinearBuckets, - /// Whole-sketch second frequency moment F2 = ‖f‖₂². Monitors the entire - /// sketch's L2 mass (no per-point key) so any future point query stays within - /// ε — see `data_plane::monitor` module docs for the whole-sketch-vs-point - /// decision rule. The edge reports its local F2 = Σ_x f_i(x)² as the value. - F2, -} - -impl Functional { - pub fn as_str(self) -> &'static str { - match self { - Functional::Sum => "sum", - Functional::CmsPoint => "cms_point", - Functional::LinearBuckets => "linear_buckets", - Functional::F2 => "f2", - } - } - - /// Parse a functional name (workload-spec value); unknown/empty → Sum. - pub fn from_name(s: &str) -> Functional { - match s { - "cms_point" => Functional::CmsPoint, - "linear_buckets" => Functional::LinearBuckets, - "f2" | "l2" => Functional::F2, - _ => Functional::Sum, - } - } -} - /// One monitored standing-query intent: "alert when the global Σ of `metric`'s /// `functional` crosses `tau`". τ/ε/window are authoritative at the coordinator; /// the edge copies are advisory. diff --git a/control_plane/src/emit/otap.rs b/control_plane/src/emit/otap.rs index 9dc24d0f..95a13f35 100644 --- a/control_plane/src/emit/otap.rs +++ b/control_plane/src/emit/otap.rs @@ -295,7 +295,7 @@ fn build_asap_sketches_config(sp: &EdgeSketchProcessor, window_secs: Option // the DAG walk; it just doesn't reach the wire here. m.insert( "sketch_kind".into(), - Value::String(sketch_kind_tag(&sp.sketch_kind).into()), + Value::String(sketch_algorithm_tag(&sp.sketch_algorithm).into()), ); match &sp.sketch_params { SketchParams::Kll { k } => { @@ -335,7 +335,7 @@ fn build_asap_sketches_config(sp: &EdgeSketchProcessor, window_secs: Option Value::Mapping(m) } -fn sketch_kind_tag(kind: &SketchAlgorithm) -> &'static str { +fn sketch_algorithm_tag(kind: &SketchAlgorithm) -> &'static str { match kind { SketchAlgorithm::Kll => "kll", SketchAlgorithm::DDSketch => "ddsketch", @@ -404,7 +404,7 @@ mod tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "ddsketch".to_string(), - sketch_kind: SketchAlgorithm::DDSketch, + sketch_algorithm: SketchAlgorithm::DDSketch, sketch_params: SketchParams::DDSketch { alpha: 0.01 }, aggregation_id: "agg0".to_string(), }], diff --git a/control_plane/src/emit/stage_config.rs b/control_plane/src/emit/stage_config.rs index 3ef831e9..b9ff0181 100644 --- a/control_plane/src/emit/stage_config.rs +++ b/control_plane/src/emit/stage_config.rs @@ -63,12 +63,13 @@ use crate::physical::colored_dag::emitter::{ // unused-import warning on every non-test build, so they're scoped into the // test module's `use super::*` instead (P2-5). use crate::physical::colored_dag::stage_id::StageId; -use planner_types::post_asap::{SketchAlgorithm, SketchParams, SketchQuery}; +use planner_types::post_asap::{ + ExactKind, SketchAlgorithm, SketchParams, SketchQuery, SummaryFamilyType, +}; use planner_types::pre_asap::ColumnRef; -// `BackendAggregation.sketch_kind`/`.sketch_params` span both exact +// `BackendAggregation.sketch_algorithm`/`.sketch_params` span both exact // accumulators and approximate sketches -- see // `physical::colored_dag::emitter`'s `use asap_types::{...}` note. -use asap_types::{SummaryKind, SummaryParams}; // ── YAML structural types ───────────────────────────────────────────────────── // @@ -878,23 +879,26 @@ pub fn emit_backend_storage_routing_with_prometheus_for_tenant( /// where each `` is either `{ "engine": , "applies_to_query_shape": [...] }` /// or `{ "engine": }` for the default slot. fn build_routing_entry(metric_name: &str, cfg: &BackendStageConfig) -> JsonValue { - let kinds: Vec = cfg + let algorithms: Vec<&SketchAlgorithm> = cfg .aggregations .iter() - .map(|a| a.sketch_kind.clone()) + .filter_map(|a| match &a.family { + SummaryFamilyType::Sketch(kind, _) => Some(kind.algorithm()), + _ => None, + }) .collect(); // Sketch-eligible shapes — the ASAP tier serves these natively // because we planned a sketch for them. let mut warm_shapes: Vec<&'static str> = Vec::new(); - let has_quantile_sketch = kinds + let has_quantile_sketch = algorithms .iter() - .any(|k| matches!(k, SummaryKind::DDSketch | SummaryKind::Kll)); + .any(|k| matches!(k, SketchAlgorithm::DDSketch | SketchAlgorithm::Kll)); if has_quantile_sketch { warm_shapes.push("quantile"); warm_shapes.push("quantile_over_time"); } - let has_hll = kinds.iter().any(|k| matches!(k, SummaryKind::Hll)); + let has_hll = algorithms.iter().any(|k| matches!(k, SketchAlgorithm::Hll)); if has_hll { warm_shapes.push("count"); } @@ -902,18 +906,18 @@ fn build_routing_entry(metric_name: &str, cfg: &BackendStageConfig) -> JsonValue // `physical::post_asap::SummaryKind`) promotes `with_heap` to a distinct // identity variant, but a topk-bound Count-Sketch/CMS aggregation // still needs to register here exactly as it did before the split. - let has_count_sketch = kinds.iter().any(|k| { + let has_count_sketch = algorithms.iter().any(|k| { matches!( k, - SummaryKind::CountSketch | SummaryKind::CountSketchWithHeap + SketchAlgorithm::CountSketch | SketchAlgorithm::CountSketchWithHeap ) }); if has_count_sketch { warm_shapes.push("topk"); } - let has_cms = kinds + let has_cms = algorithms .iter() - .any(|k| matches!(k, SummaryKind::Cms | SummaryKind::CmsWithHeap)); + .any(|k| matches!(k, SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap)); if has_cms { // CMS's `Estimate` readout serves point-count / count queries. // If HLL also planned, `count` is already in the list — push @@ -926,7 +930,7 @@ fn build_routing_entry(metric_name: &str, cfg: &BackendStageConfig) -> JsonValue // ranges — every sketch family the planner emits also tracks the // range aggregation needed to answer these from the ASAP tier // (the gateway merge processor produces a windowed accumulator). - if !kinds.is_empty() { + if !algorithms.is_empty() { warm_shapes.push("rate"); warm_shapes.push("sum"); warm_shapes.push("avg"); @@ -1103,14 +1107,14 @@ fn emit_edge_yaml_5sketch_routing( // `with_heap` params flag). let mut family_to_proc: HashMap = HashMap::new(); for sp in &cfg.sketch_processors { - family_to_proc.insert(base_family(&sp.sketch_kind), sp); + family_to_proc.insert(base_family(&sp.sketch_algorithm), sp); } for kind in FAMILY_ORDER { if !needed_families.contains(&kind) { continue; } - let processor_name = sketch_kind_to_processor_name(&kind); + let processor_name = sketch_algorithm_to_processor_name(&kind); let metric_name_hint = cfg .metric_to_family .iter() @@ -1397,7 +1401,7 @@ fn emit_edge_yaml_5sketch_routing( let pipelines: Vec<&str> = FAMILY_ORDER .iter() .filter(|k| families.contains(*k)) - .map(sketch_kind_to_pipeline_name) + .map(sketch_algorithm_to_pipeline_name) .collect(); if pipelines.is_empty() { continue; @@ -1564,8 +1568,8 @@ fn emit_edge_yaml_5sketch_routing( if !needed_families.contains(&kind) { continue; } - let proc_name = sketch_kind_to_processor_name(&kind); - let pipeline_name = sketch_kind_to_pipeline_name(&kind); + let proc_name = sketch_algorithm_to_processor_name(&kind); + let pipeline_name = sketch_algorithm_to_pipeline_name(&kind); // Sort per-family keep-processor list deterministically so YAML // output is stable across runs (HashMap iteration is not // order-stable). Empty list when no metrics in the family have @@ -1628,7 +1632,7 @@ fn emit_edge_yaml_5sketch_routing( /// routing-connector path — the fused processor takes a lower-case /// family discriminant per entry, matching the hand-written contract in /// `asap-otel-agent-b6-asap-single-sketch.yaml`. -fn sketch_kind_to_asap_edge_family(kind: &SketchAlgorithm) -> &'static str { +fn sketch_algorithm_to_asap_edge_family(kind: &SketchAlgorithm) -> &'static str { match kind { SketchAlgorithm::DDSketch => "ddsketch", SketchAlgorithm::Kll => "kll", @@ -1639,7 +1643,7 @@ fn sketch_kind_to_asap_edge_family(kind: &SketchAlgorithm) -> &'static str { // list (heap-bearing kinds normalize through `base_family` // before reaching here), and no Bind* rule in this repo // produces the exact-accumulator / Kmv / Theta kinds at all. - other => unreachable!("sketch_kind_to_asap_edge_family: unexpected kind {other:?}"), + other => unreachable!("sketch_algorithm_to_asap_edge_family: unexpected kind {other:?}"), } } @@ -1886,7 +1890,7 @@ fn emit_edge_yaml_asap_edge( // mapped a family with no enumerated processor. let mut family_to_proc: HashMap = HashMap::new(); for sp in &cfg.sketch_processors { - family_to_proc.insert(base_family(&sp.sketch_kind), sp); + family_to_proc.insert(base_family(&sp.sketch_algorithm), sp); } const FAMILY_ORDER: [SketchAlgorithm; 5] = [ SketchAlgorithm::DDSketch, @@ -1910,7 +1914,7 @@ fn emit_edge_yaml_asap_edge( e.insert("metric".into(), Value::String((*metric).clone())); e.insert( "family".into(), - Value::String(sketch_kind_to_asap_edge_family(kind).to_string()), + Value::String(sketch_algorithm_to_asap_edge_family(kind).to_string()), ); // aggregate_by: emit this metric's workload grouping_labels so each // sketch is one-per-group (e.g. per zone), mirroring the Sum path @@ -2071,9 +2075,9 @@ fn emit_edge_yaml_asap_edge( // Heap-bearing-ness now lives on `sketch_kind`, not a params // flag — read it off the processor's kind before matching // its params. - let mut countsketch_with_heap = family_to_proc - .get(kind) - .is_some_and(|sp| matches!(sp.sketch_kind, SketchAlgorithm::CountSketchWithHeap)); + let mut countsketch_with_heap = family_to_proc.get(kind).is_some_and(|sp| { + matches!(sp.sketch_algorithm, SketchAlgorithm::CountSketchWithHeap) + }); match family_to_proc.get(kind).map(|sp| &sp.sketch_params) { Some(SketchParams::DDSketch { alpha }) => { e.insert("relative_accuracy".into(), Value::Number((*alpha).into())); @@ -2456,7 +2460,7 @@ tsdb_block_duration: {window_secs}s\n", /// Map a `SketchAlgorithm` to the OTel processor name registered by the /// patched contrib build's factory. Keep in sync with /// `crate::physical::colored_dag::emitter::edge_processor_name`. -fn sketch_kind_to_processor_name(kind: &SketchAlgorithm) -> &'static str { +fn sketch_algorithm_to_processor_name(kind: &SketchAlgorithm) -> &'static str { match kind { SketchAlgorithm::DDSketch => "ddsketch", SketchAlgorithm::Kll => "KLL", @@ -2464,13 +2468,13 @@ fn sketch_kind_to_processor_name(kind: &SketchAlgorithm) -> &'static str { SketchAlgorithm::CountSketch => "countsketch", SketchAlgorithm::Cms => "countmin", // Callers only ever pass a bare `FAMILY_ORDER` entry. - other => unreachable!("sketch_kind_to_processor_name: unexpected kind {other:?}"), + other => unreachable!("sketch_algorithm_to_processor_name: unexpected kind {other:?}"), } } /// Map a `SketchAlgorithm` to its per-family pipeline name in the routing /// connector layout. -fn sketch_kind_to_pipeline_name(kind: &SketchAlgorithm) -> &'static str { +fn sketch_algorithm_to_pipeline_name(kind: &SketchAlgorithm) -> &'static str { match kind { SketchAlgorithm::DDSketch => "metrics/ddsketch_path", SketchAlgorithm::Kll => "metrics/kll_path", @@ -2478,7 +2482,7 @@ fn sketch_kind_to_pipeline_name(kind: &SketchAlgorithm) -> &'static str { SketchAlgorithm::CountSketch => "metrics/countsketch_path", SketchAlgorithm::Cms => "metrics/countminsketch_path", // Callers only ever pass a bare `FAMILY_ORDER` entry. - other => unreachable!("sketch_kind_to_pipeline_name: unexpected kind {other:?}"), + other => unreachable!("sketch_algorithm_to_pipeline_name: unexpected kind {other:?}"), } } @@ -2659,10 +2663,10 @@ fn build_default_edge_processor_block( other => unreachable!("build_default_edge_processor_block: unexpected kind {other:?}"), }; let synthetic = EdgeSketchProcessor { - processor_name: sketch_kind_to_processor_name(kind).to_string(), - sketch_kind: stored_kind, + processor_name: sketch_algorithm_to_processor_name(kind).to_string(), + sketch_algorithm: stored_kind, sketch_params: params, - aggregation_id: format!("agg_default_{}", sketch_kind_tag(kind)), + aggregation_id: format!("agg_default_{}", sketch_algorithm_tag(kind)), }; build_edge_processor_block(&synthetic, window_secs, &[], metric_name_hint, sample_p) } @@ -2891,7 +2895,7 @@ fn insert_sample_p(m: &mut Mapping, sample_p: Option) { /// `countsketchmerge`. We map the kind to the family-specific name /// here so the emitted YAML round-trips through the patched build. fn gateway_merge_processor_name(mp: &GatewayMergeProcessor) -> String { - match mp.sketch_kind { + match mp.sketch_algorithm { SketchAlgorithm::Kll => "kllmerge".to_string(), SketchAlgorithm::DDSketch => "ddsketchmerge".to_string(), SketchAlgorithm::Hll => "hllmerge".to_string(), @@ -2916,7 +2920,7 @@ fn build_gateway_merge_block(mp: &GatewayMergeProcessor) -> Value { ); m.insert( "sketch_kind".into(), - Value::String(sketch_kind_tag(&mp.sketch_kind).to_string()), + Value::String(sketch_algorithm_tag(&mp.sketch_algorithm).to_string()), ); Value::Mapping(m) } @@ -2954,12 +2958,23 @@ pub(crate) fn build_backend_aggregation_json(agg: &BackendAggregation) -> JsonVa // synthesizes for non-sketch (Sum-shaped) workloads. The // `sketch_kind` / `sketch_params` fields carry sentinel values // in this case and are not emitted on the wire. - let (aggregation_type, mut parameters) = match &agg.agg_type_override { - Some(s) => (s.clone(), json!({})), - None => ( - sketch_kind_to_backend_type(&agg.sketch_kind, &agg.sketch_params).to_string(), - sketch_params_to_json(&agg.sketch_params), + let (aggregation_type, mut parameters) = match &agg.family { + SummaryFamilyType::ExactAggregate(kind, _) => ( + match kind { + ExactKind::Sum => "Sum", + ExactKind::Count => "Count", + ExactKind::MinMax => "MinMax", + ExactKind::Increase => "Increase", + ExactKind::Rate => "Rate", + } + .to_string(), + json!({}), ), + SummaryFamilyType::Sketch(kind, _) => ( + sketch_algorithm_to_backend_type(kind.algorithm()).to_string(), + sketch_params_to_json(kind.params()), + ), + other => panic!("backend emitter cannot encode summary family {other:?}"), }; // Carry the per-item dimension (e.g. "endpoint"/"service") into the // policy parameters so the data-plane ingest can record it on the CMS @@ -3084,26 +3099,17 @@ fn base_family(kind: &SketchAlgorithm) -> SketchAlgorithm { /// lets the backend's `policy_capability` lookup return /// `FrequencyTopk(*WithHeap)` for heap-bearing aggregations — required /// for `topk(...)` queries to bind to the right sids. -fn sketch_kind_to_backend_type(kind: &SummaryKind, _params: &SummaryParams) -> &'static str { +fn sketch_algorithm_to_backend_type(kind: &SketchAlgorithm) -> &'static str { match kind { - SummaryKind::DDSketch => "DDSketch", - SummaryKind::Kll => "DatasketchesKLL", - SummaryKind::Hll => "HLL", - SummaryKind::CountSketchWithHeap => "CountSketchWithHeap", - SummaryKind::CountSketch => "CountSketch", - SummaryKind::CmsWithHeap => "CountMinSketchWithHeap", - SummaryKind::Cms => "CountMinSketch", - // Exact accumulators never reach here -- the caller takes the - // `agg_type_override` branch for them instead (see the call - // site's comment). - SummaryKind::Sum - | SummaryKind::Count - | SummaryKind::MinMax - | SummaryKind::Increase - | SummaryKind::Rate - | SummaryKind::Kmv - | SummaryKind::Theta => unreachable!( - "sketch_kind_to_backend_type: non-sketch or unsupported SummaryKind; \ + SketchAlgorithm::DDSketch => "DDSketch", + SketchAlgorithm::Kll => "DatasketchesKLL", + SketchAlgorithm::Hll => "HLL", + SketchAlgorithm::CountSketchWithHeap => "CountSketchWithHeap", + SketchAlgorithm::CountSketch => "CountSketch", + SketchAlgorithm::CmsWithHeap => "CountMinSketchWithHeap", + SketchAlgorithm::Cms => "CountMinSketch", + SketchAlgorithm::Kmv | SketchAlgorithm::Theta => unreachable!( + "sketch_algorithm_to_backend_type: unsupported SketchAlgorithm; \ no Bind* rule in this repo produces one" ), } @@ -3114,7 +3120,7 @@ fn sketch_kind_to_backend_type(kind: &SummaryKind, _params: &SummaryParams) -> & /// without round-tripping through serde. Heap-bearing kinds reuse their /// bare counterpart's tag — this field never distinguished `with_heap` /// even before `SketchAlgorithm` split it into its own variant. -fn sketch_kind_tag(kind: &SketchAlgorithm) -> &'static str { +fn sketch_algorithm_tag(kind: &SketchAlgorithm) -> &'static str { match kind { SketchAlgorithm::Kll => "kll", SketchAlgorithm::DDSketch => "ddsketch", @@ -3122,7 +3128,7 @@ fn sketch_kind_tag(kind: &SketchAlgorithm) -> &'static str { SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap => "cms", SketchAlgorithm::CountSketch | SketchAlgorithm::CountSketchWithHeap => "count_sketch", SketchAlgorithm::Kmv | SketchAlgorithm::Theta => unreachable!( - "sketch_kind_tag: non-sketch or unsupported SketchAlgorithm; \ + "sketch_algorithm_tag: non-sketch or unsupported SketchAlgorithm; \ no Bind* rule in this repo produces one" ), } @@ -3131,38 +3137,32 @@ fn sketch_kind_tag(kind: &SketchAlgorithm) -> &'static str { /// Serialize a `SketchParams` payload to a flat JSON object the backend /// can read directly without round-tripping through the controller's /// internally-tagged enum form. -fn sketch_params_to_json(p: &SummaryParams) -> JsonValue { +fn sketch_params_to_json(p: &SketchParams) -> JsonValue { match p { - SummaryParams::Kll { k } => json!({ "k": k }), - SummaryParams::DDSketch { alpha } => json!({ "alpha": alpha }), - SummaryParams::Hll { precision } => json!({ "precision": precision }), + SketchParams::Kll { k } => json!({ "k": k }), + SketchParams::DDSketch { alpha } => json!({ "alpha": alpha }), + SketchParams::Hll { precision } => json!({ "precision": precision }), // Cms/CmsWithHeap: matches the pre-split shape exactly — the old // `SketchParams::Cms` arm never emitted `with_heap` in JSON even // though `CmsParams.with_heap` existed as a field; heap-bearing // and bare CMS produced identical wire JSON. `heap_size` is a // new field with no wire representation here (nothing on the // real backend wire path reads it — see `bind_cms_topk.rs`). - SummaryParams::Cms { width, depth } | SummaryParams::CmsWithHeap { width, depth, .. } => { + SketchParams::Cms { width, depth } | SketchParams::CmsWithHeap { width, depth, .. } => { json!({ "w": width, "d": depth }) } // CountSketch/CountSketchWithHeap: the old arm always emitted // `with_heap` (from `CountSketchParams.with_heap: bool`); // that boolean is now the kind identity itself. - SummaryParams::CountSketch { width, depth } => { + SketchParams::CountSketch { width, depth } => { json!({ "w": width, "d": depth, "with_heap": false }) } - SummaryParams::CountSketchWithHeap { width, depth, .. } => { + SketchParams::CountSketchWithHeap { width, depth, .. } => { json!({ "w": width, "d": depth, "with_heap": true }) } // Exact accumulators never reach here -- see // `sketch_kind_to_backend_type`'s doc. - SummaryParams::Sum - | SummaryParams::Count - | SummaryParams::MinMax - | SummaryParams::Increase - | SummaryParams::Rate - | SummaryParams::Kmv { .. } - | SummaryParams::Theta { .. } => unreachable!( + SketchParams::Kmv { .. } | SketchParams::Theta { .. } => unreachable!( "sketch_params_to_json: non-sketch or unsupported SummaryParams; \ no Bind* rule in this repo produces one" ), @@ -3178,6 +3178,29 @@ mod tests { // module; gating them here keeps the non-test build free of the // unused-import warning they previously triggered at module scope. use crate::physical::colored_dag::emitter::{ArchiveTierMetric, PrometheusArchiveMetric}; + + fn backend_sketch_aggregation( + aggregation_id: &str, + metric_name: &str, + algorithm: SketchAlgorithm, + params: SketchParams, + aggregation_input: AggregationInput, + ) -> BackendAggregation { + BackendAggregation { + aggregation_id: aggregation_id.into(), + metric_name: metric_name.into(), + family: SummaryFamilyType::Sketch( + planner_types::post_asap::SketchKind::new(algorithm, params), + planner_types::post_asap::GroupingStrategy::PerSubpopulationInstance, + ), + window_secs: 60, + spatial_filter: String::new(), + grouping: Vec::new(), + item_label: None, + aggregation_input, + } + } + fn ddsketch_edge_cfg() -> EdgeStageConfig { EdgeStageConfig { source_metric: Some("http_request_duration_seconds".to_string()), @@ -3185,7 +3208,7 @@ mod tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "ddsketch".to_string(), - sketch_kind: SketchAlgorithm::DDSketch, + sketch_algorithm: SketchAlgorithm::DDSketch, sketch_params: SketchParams::DDSketch { alpha: 0.01 }, aggregation_id: "agg0".to_string(), }], @@ -3246,7 +3269,7 @@ mod tests { "missing label matcher value\n{yaml}" ); assert!( - !yaml.contains("aggregation_id:") && !yaml.contains("sketch_kind:"), + !yaml.contains("aggregation_id:") && !yaml.contains("sketch_algorithm:"), "edge processor config must not emit planning-only fields rejected by OTel configs\n{yaml}" ); @@ -3270,7 +3293,7 @@ mod tests { let mut cfg = ddsketch_edge_cfg(); cfg.sketch_processors[0] = EdgeSketchProcessor { processor_name: "KLL".to_string(), - sketch_kind: SketchAlgorithm::Kll, + sketch_algorithm: SketchAlgorithm::Kll, sketch_params: SketchParams::Kll { k: 200 }, aggregation_id: "agg7".to_string(), }; @@ -3335,7 +3358,7 @@ mod tests { let mut cfg = ddsketch_edge_cfg(); cfg.sketch_processors[0] = EdgeSketchProcessor { processor_name: processor_name.to_string(), - sketch_kind: kind, + sketch_algorithm: kind, sketch_params: params, aggregation_id: "agg-delta".to_string(), }; @@ -3354,7 +3377,7 @@ mod tests { cfg.source_metric = Some("endpoint_request_freq".to_string()); cfg.sketch_processors[0] = EdgeSketchProcessor { processor_name: "countmin".to_string(), - sketch_kind: SketchAlgorithm::Cms, + sketch_algorithm: SketchAlgorithm::Cms, sketch_params: SketchParams::Cms { width: 4096, depth: 4, @@ -3387,7 +3410,7 @@ mod tests { otlp_receiver_port: 4317, merge_processors: vec![GatewayMergeProcessor { processor_name: "sketchmergeprocessor".to_string(), - sketch_kind: SketchAlgorithm::DDSketch, + sketch_algorithm: SketchAlgorithm::DDSketch, aggregation_id: "agg0".to_string(), }], exporter_target: ExportTarget::Stage(StageId::Backend), @@ -3431,12 +3454,12 @@ mod tests { merge_processors: vec![ GatewayMergeProcessor { processor_name: "x".into(), - sketch_kind: SketchAlgorithm::Kll, + sketch_algorithm: SketchAlgorithm::Kll, aggregation_id: "agg0".into(), }, GatewayMergeProcessor { processor_name: "x".into(), - sketch_kind: SketchAlgorithm::Hll, + sketch_algorithm: SketchAlgorithm::Hll, aggregation_id: "agg1".into(), }, ], @@ -3490,30 +3513,20 @@ mod tests { fn backend_json_round_trips_aggregations_and_readouts() { let cfg = BackendStageConfig { aggregations: vec![ - BackendAggregation { - item_label: None, - aggregation_id: "agg0".into(), - metric_name: "http_latency_ms".into(), - sketch_kind: SummaryKind::DDSketch, - sketch_params: SummaryParams::DDSketch { alpha: 0.01 }, - window_secs: 60, - spatial_filter: String::new(), - grouping: Vec::new(), - aggregation_input: AggregationInput::SketchEnvelope, - agg_type_override: None, - }, - BackendAggregation { - item_label: None, - aggregation_id: "agg1".into(), - metric_name: "http_requests_total".into(), - sketch_kind: SummaryKind::Hll, - sketch_params: SummaryParams::Hll { precision: 14 }, - window_secs: 60, - spatial_filter: String::new(), - grouping: Vec::new(), - aggregation_input: AggregationInput::SketchEnvelope, - agg_type_override: None, - }, + backend_sketch_aggregation( + "agg0", + "http_latency_ms", + SketchAlgorithm::DDSketch, + SketchParams::DDSketch { alpha: 0.01 }, + AggregationInput::SketchEnvelope, + ), + backend_sketch_aggregation( + "agg1", + "http_requests_total", + SketchAlgorithm::Hll, + SketchParams::Hll { precision: 14 }, + AggregationInput::SketchEnvelope, + ), ], readouts: vec![ BackendReadout { @@ -3556,37 +3569,27 @@ mod tests { fn backend_json_handles_topk_and_pointcount_readouts() { let cfg = BackendStageConfig { aggregations: vec![ - BackendAggregation { - item_label: None, - aggregation_id: "agg0".into(), - metric_name: "endpoint_count".into(), - sketch_kind: SummaryKind::CountSketchWithHeap, - sketch_params: SummaryParams::CountSketchWithHeap { + backend_sketch_aggregation( + "agg0", + "endpoint_count", + SketchAlgorithm::CountSketchWithHeap, + SketchParams::CountSketchWithHeap { width: 2048, depth: 5, heap_size: 10, }, - window_secs: 60, - spatial_filter: String::new(), - grouping: Vec::new(), - aggregation_input: AggregationInput::SketchEnvelope, - agg_type_override: None, - }, - BackendAggregation { - item_label: None, - aggregation_id: "agg1".into(), - metric_name: "endpoint_hits".into(), - sketch_kind: SummaryKind::Cms, - sketch_params: SummaryParams::Cms { + AggregationInput::SketchEnvelope, + ), + backend_sketch_aggregation( + "agg1", + "endpoint_hits", + SketchAlgorithm::Cms, + SketchParams::Cms { width: 4096, depth: 4, }, - window_secs: 60, - spatial_filter: String::new(), - grouping: Vec::new(), - aggregation_input: AggregationInput::SketchEnvelope, - agg_type_override: None, - }, + AggregationInput::SketchEnvelope, + ), ], readouts: vec![ BackendReadout { @@ -3665,18 +3668,13 @@ mod tests { other => unreachable!("backend_cfg_with_kind: unsupported test fixture kind {other:?}"), }; BackendStageConfig { - aggregations: vec![BackendAggregation { - item_label: None, - aggregation_id: "agg0".into(), - metric_name: "test_metric".into(), - sketch_kind: stored_kind.into(), - sketch_params: params.into(), - window_secs: 60, - spatial_filter: String::new(), - grouping: Vec::new(), - aggregation_input: AggregationInput::SketchEnvelope, - agg_type_override: None, - }], + aggregations: vec![backend_sketch_aggregation( + "agg0", + "test_metric", + stored_kind, + params, + AggregationInput::SketchEnvelope, + )], readouts: vec![BackendReadout { aggregation_id: "agg0".into(), op: match kind { @@ -4061,13 +4059,11 @@ mod tests { "CountSketchWithHeap", ), ]; - for (kind, params, expected) in cases { - let kind: SummaryKind = kind.into(); - let params: SummaryParams = params.into(); + for (kind, _params, expected) in cases { assert_eq!( - sketch_kind_to_backend_type(&kind, ¶ms), + sketch_algorithm_to_backend_type(&kind), expected, - "sketch_kind_to_backend_type({kind:?}, {params:?}) drift — backend FromStr will reject" + "sketch_algorithm_to_backend_type({kind:?}) drift — backend FromStr will reject" ); } } @@ -4081,16 +4077,15 @@ mod tests { fn backend_json_emits_grouping_under_labels() { let cfg = BackendStageConfig { aggregations: vec![BackendAggregation { - item_label: None, - aggregation_id: "agg0".into(), - metric_name: "http_latency_ms".into(), - sketch_kind: SummaryKind::DDSketch, - sketch_params: SummaryParams::DDSketch { alpha: 0.01 }, - window_secs: 30, - spatial_filter: String::new(), grouping: vec!["zone".into(), "service".into()], - aggregation_input: AggregationInput::SketchEnvelope, - agg_type_override: None, + window_secs: 30, + ..backend_sketch_aggregation( + "agg0", + "http_latency_ms", + SketchAlgorithm::DDSketch, + SketchParams::DDSketch { alpha: 0.01 }, + AggregationInput::SketchEnvelope, + ) }], readouts: vec![], }; @@ -4126,18 +4121,13 @@ mod tests { #[test] fn phase_b_backend_json_aggregation_readout_alias_snapshot() { let cfg = BackendStageConfig { - aggregations: vec![BackendAggregation { - item_label: None, - aggregation_id: "phase_b_agg0".into(), - metric_name: "phase_b_metric".into(), - sketch_kind: SummaryKind::Kll, - sketch_params: SummaryParams::Kll { k: 200 }, - window_secs: 60, - spatial_filter: String::new(), - grouping: Vec::new(), - aggregation_input: AggregationInput::SketchEnvelope, - agg_type_override: None, - }], + aggregations: vec![backend_sketch_aggregation( + "phase_b_agg0", + "phase_b_metric", + SketchAlgorithm::Kll, + SketchParams::Kll { k: 200 }, + AggregationInput::SketchEnvelope, + )], readouts: vec![BackendReadout { aggregation_id: "phase_b_agg0".into(), op: SketchQuery::Quantile { q: 0.99 }, @@ -4174,18 +4164,13 @@ mod tests { #[test] fn phase_eps1_mode1_aggregation_input_is_sketch_envelope() { let cfg = BackendStageConfig { - aggregations: vec![BackendAggregation { - item_label: None, - aggregation_id: "agg0".into(), - metric_name: "test_metric".into(), - sketch_kind: SummaryKind::DDSketch, - sketch_params: SummaryParams::DDSketch { alpha: 0.01 }, - window_secs: 60, - spatial_filter: String::new(), - grouping: Vec::new(), - aggregation_input: AggregationInput::SketchEnvelope, - agg_type_override: None, - }], + aggregations: vec![backend_sketch_aggregation( + "agg0", + "test_metric", + SketchAlgorithm::DDSketch, + SketchParams::DDSketch { alpha: 0.01 }, + AggregationInput::SketchEnvelope, + )], readouts: vec![], }; let v = emit_backend_streaming_config_json(&cfg, &[]).expect("emit ok"); @@ -4199,18 +4184,13 @@ mod tests { #[test] fn phase_eps1_mode2_aggregation_input_is_raw() { let cfg = BackendStageConfig { - aggregations: vec![BackendAggregation { - item_label: None, - aggregation_id: "agg0".into(), - metric_name: "test_metric".into(), - sketch_kind: SummaryKind::DDSketch, - sketch_params: SummaryParams::DDSketch { alpha: 0.01 }, - window_secs: 60, - spatial_filter: String::new(), - grouping: Vec::new(), - aggregation_input: AggregationInput::Raw, - agg_type_override: None, - }], + aggregations: vec![backend_sketch_aggregation( + "agg0", + "test_metric", + SketchAlgorithm::DDSketch, + SketchParams::DDSketch { alpha: 0.01 }, + AggregationInput::Raw, + )], readouts: vec![], }; let v = emit_backend_streaming_config_json(&cfg, &[]).expect("emit ok"); @@ -5911,16 +5891,15 @@ mod tests { let cfg = BackendStageConfig { aggregations: vec![BackendAggregation { - item_label: None, - aggregation_id: "agg0".to_string(), - metric_name: "http_requests_total_latency_ms".to_string(), - sketch_kind: SummaryKind::DDSketch, - sketch_params: SummaryParams::DDSketch { alpha: 0.01 }, window_secs: 300, // pre-clamp 5m - spatial_filter: String::new(), grouping: vec!["zone".to_string()], - aggregation_input: AggregationInput::SketchEnvelope, - agg_type_override: None, + ..backend_sketch_aggregation( + "agg0", + "http_requests_total_latency_ms", + SketchAlgorithm::DDSketch, + SketchParams::DDSketch { alpha: 0.01 }, + AggregationInput::SketchEnvelope, + ) }], readouts: vec![BackendReadout { aggregation_id: "agg0".to_string(), @@ -6114,25 +6093,25 @@ mod tests { let sketch_processors = vec![ EdgeSketchProcessor { processor_name: "ddsketch".into(), - sketch_kind: SketchAlgorithm::DDSketch, + sketch_algorithm: SketchAlgorithm::DDSketch, sketch_params: SketchParams::DDSketch { alpha: 0.01 }, aggregation_id: "agg0".into(), }, EdgeSketchProcessor { processor_name: "KLL".into(), - sketch_kind: SketchAlgorithm::Kll, + sketch_algorithm: SketchAlgorithm::Kll, sketch_params: SketchParams::Kll { k: 200 }, aggregation_id: "agg1".into(), }, EdgeSketchProcessor { processor_name: "HLL".into(), - sketch_kind: SketchAlgorithm::Hll, + sketch_algorithm: SketchAlgorithm::Hll, sketch_params: SketchParams::Hll { precision: 14 }, aggregation_id: "agg2".into(), }, EdgeSketchProcessor { processor_name: "countsketch".into(), - sketch_kind: SketchAlgorithm::CountSketchWithHeap, + sketch_algorithm: SketchAlgorithm::CountSketchWithHeap, sketch_params: SketchParams::CountSketchWithHeap { width: 2048, depth: 5, @@ -6142,7 +6121,7 @@ mod tests { }, EdgeSketchProcessor { processor_name: "countmin".into(), - sketch_kind: SketchAlgorithm::Cms, + sketch_algorithm: SketchAlgorithm::Cms, sketch_params: SketchParams::Cms { width: 2048, depth: 5, @@ -6594,7 +6573,7 @@ mod tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "HLL".into(), - sketch_kind: SketchAlgorithm::Hll, + sketch_algorithm: SketchAlgorithm::Hll, sketch_params: SketchParams::Hll { precision: 14 }, aggregation_id: "agg0".into(), }], @@ -6691,7 +6670,7 @@ mod tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "HLL".into(), - sketch_kind: SketchAlgorithm::Hll, + sketch_algorithm: SketchAlgorithm::Hll, sketch_params: SketchParams::Hll { precision: 14 }, aggregation_id: "agg0".into(), }], @@ -6765,7 +6744,7 @@ mod tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "HLL".into(), - sketch_kind: SketchAlgorithm::Hll, + sketch_algorithm: SketchAlgorithm::Hll, sketch_params: SketchParams::Hll { precision: 14 }, aggregation_id: "agg0".into(), }], @@ -6832,13 +6811,13 @@ mod tests { sketch_processors: vec![ EdgeSketchProcessor { processor_name: "ddsketch".into(), - sketch_kind: SketchAlgorithm::DDSketch, + sketch_algorithm: SketchAlgorithm::DDSketch, sketch_params: SketchParams::DDSketch { alpha: 0.01 }, aggregation_id: "agg0".into(), }, EdgeSketchProcessor { processor_name: "KLL".into(), - sketch_kind: SketchAlgorithm::Kll, + sketch_algorithm: SketchAlgorithm::Kll, sketch_params: SketchParams::Kll { k: 200 }, aggregation_id: "agg1".into(), }, @@ -7320,7 +7299,7 @@ mod tests { for &(w, d) in &[(2048u32, 5u32), (1024, 4), (4096, 6), (2, 1), (256, 3)] { let sp = EdgeSketchProcessor { processor_name: "countsketch".into(), - sketch_kind: SketchAlgorithm::CountSketch, + sketch_algorithm: SketchAlgorithm::CountSketch, sketch_params: SketchParams::CountSketch { width: w, depth: d }, aggregation_id: "agg-cs".into(), }; @@ -7345,7 +7324,7 @@ mod tests { // params as `{ "w", "d", "with_heap" }`. The fingerprint keys off // `parameters["w"]`, which must equal the agent-derived width. let backend_json = - sketch_params_to_json(&SummaryParams::CountSketch { width: w, depth: d }); + sketch_params_to_json(&SketchParams::CountSketch { width: w, depth: d }); let backend_w = backend_json["w"].as_u64().expect("backend w present"); let backend_d = backend_json["d"].as_u64().expect("backend d present"); @@ -7434,7 +7413,7 @@ mod tests { .insert("top_endpoint_qps".into(), one(SketchAlgorithm::CountSketch)); cfg.sketch_processors = vec![EdgeSketchProcessor { processor_name: "countsketch".into(), - sketch_kind: SketchAlgorithm::CountSketchWithHeap, + sketch_algorithm: SketchAlgorithm::CountSketchWithHeap, sketch_params: SketchParams::CountSketchWithHeap { width: 2048, depth: 5, diff --git a/control_plane/src/emit/telegraf.rs b/control_plane/src/emit/telegraf.rs index 421ce556..6fdd3a64 100644 --- a/control_plane/src/emit/telegraf.rs +++ b/control_plane/src/emit/telegraf.rs @@ -165,7 +165,7 @@ fn emit_processors_allsketches( // internal emit plumbing; it just doesn't reach the wire here. out.push_str(&format!( " sketch_kind = \"{}\"\n", - sketch_kind_tag(&sp.sketch_kind) + sketch_algorithm_tag(&sp.sketch_algorithm) )); match &sp.sketch_params { SketchParams::Kll { k } => { @@ -207,7 +207,7 @@ fn emit_processors_allsketches( out.push('\n'); } -fn sketch_kind_tag(kind: &SketchAlgorithm) -> &'static str { +fn sketch_algorithm_tag(kind: &SketchAlgorithm) -> &'static str { match kind { SketchAlgorithm::Kll => "kll", SketchAlgorithm::DDSketch => "ddsketch", @@ -327,7 +327,7 @@ mod tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "ddsketch".to_string(), - sketch_kind: SketchAlgorithm::DDSketch, + sketch_algorithm: SketchAlgorithm::DDSketch, sketch_params: SketchParams::DDSketch { alpha: 0.01 }, aggregation_id: "agg0".to_string(), }], @@ -536,7 +536,7 @@ mod tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "KLL".to_string(), - sketch_kind: SketchAlgorithm::Kll, + sketch_algorithm: SketchAlgorithm::Kll, sketch_params: SketchParams::Kll { k: 200 }, aggregation_id: "agg0".to_string(), }], diff --git a/control_plane/src/physical/colored_dag/emitter.rs b/control_plane/src/physical/colored_dag/emitter.rs index f2f95221..026e1c7a 100644 --- a/control_plane/src/physical/colored_dag/emitter.rs +++ b/control_plane/src/physical/colored_dag/emitter.rs @@ -35,19 +35,10 @@ use crate::physical::colored_dag::dag::ColoredDag; use crate::physical::colored_dag::stage_id::{StageId, Topology}; use crate::physical::post_asap::deployment_expr::{PhysicalExpr, PostAsapPlan}; use crate::types_v2::BindingName; -use planner_types::post_asap::{SketchAlgorithm, SketchParams, SketchQuery, SummaryExpr}; -// `BackendAggregation.sketch_kind`/`.sketch_params` (below) span BOTH -// exact accumulators (Sum/Count/MinMax/Increase/Rate, via -// `agg_type_override`) and approximate sketches -- unlike -// `EdgeSketchProcessor`/`GatewayMergeProcessor`, which are always real -// materialized sketches. They use the vendored flat -// `asap_types::{SummaryKind, SummaryParams}` (same 14-variant shape as -// ASAPController's pre-ASAPPlanner#218 flat `SummaryKind` -- see -// `crates/asap_types/src/accumulator_spec.rs`'s module doc and -// control_plane/docs/design-asapplanner-pin-migration.md), aliased here -// so this file's existing `BackendAggregation`-adjacent code needs no -// further changes. -use asap_types::{SummaryKind as BackendSketchKind, SummaryParams as BackendSketchParams}; +use planner_types::post_asap::{ + ExactKind, ExactParams, GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, + SketchQuery, SummaryExpr, SummaryFamilyType, +}; /// Flattened view of one [`ColoredNode`](crate::physical::colored_dag::dag::ColoredNode)'s /// `PhysicalExpr`, for the tuple-style `(&node.expr, node.stage)` matching @@ -582,7 +573,7 @@ pub struct EdgeSketchProcessor { /// `countmin`, etc. Maps 1:1 from `SketchAlgorithm`. pub processor_name: String, /// Sketch family (mirror of the `SketchAgg::sketch_type` field). - pub sketch_kind: SketchAlgorithm, + pub sketch_algorithm: SketchAlgorithm, /// Sketch parameters (mirror of the `SketchAgg::params` field). pub sketch_params: SketchParams, /// Internal emitter plumbing — threads `EdgeSketchProcessor` → @@ -625,7 +616,7 @@ pub struct GatewayMergeProcessor { pub processor_name: String, /// Sketch family being merged. All inputs to the merge agree on /// this (L4 type checker enforces it; design.md §6.4). - pub sketch_kind: SketchAlgorithm, + pub sketch_algorithm: SketchAlgorithm, /// Aggregation id — matches the upstream edge's /// `EdgeSketchProcessor::aggregation_id` so the gateway routes /// streams correctly. @@ -672,7 +663,7 @@ pub struct BackendStageConfig { /// payload is built by `emit::stage_config::build_backend_aggregation_json` /// (a hand-written JSON builder reading these fields), never a whole-struct /// serialize. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq)] pub struct BackendAggregation { /// Internal-only id (see struct doc). Not on the wire. pub aggregation_id: String, @@ -680,14 +671,10 @@ pub struct BackendAggregation { /// `http_requests_total_latency_ms`). Required by the backend's /// `AggregationConfig` parser. pub metric_name: String, - /// Accumulator family -- exact (Sum/Count/MinMax/Increase/Rate) or - /// approximate sketch (see this file's `use asap_types::{...}` note - /// above for why this is the flat type, not `SketchAlgorithm`). - pub sketch_kind: BackendSketchKind, - /// Accumulator parameters — the backend uses these to build its - /// per-aggregation instance (KLL with the right `k`, DDSketch with - /// the right `alpha`, etc.). - pub sketch_params: BackendSketchParams, + /// Planner-owned committed summary identity. Sketch entries carry a + /// validated `SketchKind` (category + algorithm + params); exact entries + /// carry the matching `ExactKind`/`ExactParams` pair. + pub family: SummaryFamilyType, /// Tumbling window size in seconds. Required by the backend; the /// parser rejects zero-window aggregations. pub window_secs: u64, @@ -722,27 +709,6 @@ pub struct BackendAggregation { /// backend's `StreamingConfig` consumer interprets the field — /// Phase ε.2 implements the raw-input ingest path. pub aggregation_input: AggregationInput, - - /// Option B (post-PR-#287) — when `Some(s)`, the wire-side - /// `aggregationType` is `s` (e.g. `"Sum"`, `"Increase"`, - /// `"MinMax"`) and the `parameters` object is emitted as `{}`, - /// bypassing the sketch-kind → backend-type mapping that runs - /// for the regular sketched aggregations. - /// - /// Why: the typed `bind_workload_typed` rule chain only knows - /// how to lower sketch-shaped statistics (Quantile / Cardinality - /// / Frequency / TopK). Sum/Rate/Count workloads — `sum by - /// (zone) (http_requests_total)`, `rate(metric[5m])`, - /// `count(metric)` — currently decline binding (return None) so - /// the typed L5 stage-split emits nothing for them. Under the - /// Option B unification, the Replanner falls back to this - /// override shape to emit an `ExactAgg(Sum)` (or Increase / - /// Count) row into the cumulative streaming-config so the data - /// plane recognises the metric and `sum by (zone) (…)` queries - /// resolve. `sketch_kind` / `sketch_params` carry sentinel - /// values when the override is in effect (their emitted form is - /// suppressed in `build_backend_aggregation_json`). - pub agg_type_override: Option, } /// Phase ε.1 — what wire shape the backend ingests for an aggregation. @@ -926,7 +892,7 @@ impl Emitter for ThreeStageEmitter { .unwrap_or_else(|| format!("agg{}", node.id.0)); edge.sketch_processors.push(EdgeSketchProcessor { processor_name, - sketch_kind: sketch_type.clone(), + sketch_algorithm: sketch_type.clone(), sketch_params: params.clone(), aggregation_id: aggregation_id.clone(), }); @@ -934,8 +900,10 @@ impl Emitter for ThreeStageEmitter { item_label: None, aggregation_id, metric_name: edge.source_metric.clone().unwrap_or_default(), - sketch_kind: sketch_type.clone().into(), - sketch_params: params.clone().into(), + family: SummaryFamilyType::Sketch( + SketchKind::new(sketch_type.clone(), params.clone()), + GroupingStrategy::PerSubpopulationInstance, + ), window_secs: edge.window_secs.unwrap_or(0), spatial_filter: spatial_filter_from_label_filters(&edge.label_filters), // Populated post-emit by the caller (handle_plan) @@ -944,8 +912,6 @@ impl Emitter for ThreeStageEmitter { grouping: Vec::new(), // Mode 1 — sketch built at edge, ships envelope. aggregation_input: AggregationInput::SketchEnvelope, - // Regular sketch path — no override. - agg_type_override: None, }); } // Gateway: SketchMerge over edge sketches → one merge @@ -960,7 +926,7 @@ impl Emitter for ThreeStageEmitter { { gateway_processors.push(GatewayMergeProcessor { processor_name: "sketchmergeprocessor".into(), - sketch_kind: kind, + sketch_algorithm: kind, aggregation_id: aid, }); } @@ -1018,15 +984,15 @@ impl Emitter for ThreeStageEmitter { item_label: None, aggregation_id: aid, metric_name: edge.source_metric.clone().unwrap_or_default(), - sketch_kind: family.clone().into(), - sketch_params: params.clone().into(), + family: SummaryFamilyType::Sketch( + SketchKind::new(family.clone(), params.clone()), + GroupingStrategy::PerSubpopulationInstance, + ), window_secs: edge.window_secs.unwrap_or(0), spatial_filter: spatial_filter_from_label_filters(&edge.label_filters), grouping: Vec::new(), // Mode 2 — backend builds sketch from raw OTLP. aggregation_input: AggregationInput::Raw, - // Regular sketch path — no override. - agg_type_override: None, }); } _ => {} diff --git a/control_plane/src/physical/colored_dag/tests.rs b/control_plane/src/physical/colored_dag/tests.rs index a203fe8a..8257711d 100644 --- a/control_plane/src/physical/colored_dag/tests.rs +++ b/control_plane/src/physical/colored_dag/tests.rs @@ -381,7 +381,10 @@ fn emitter_edge_config_has_correct_processor_kll() { StageConfig::Edge(e) => { assert_eq!(e.sketch_processors.len(), 1); assert_eq!(e.sketch_processors[0].processor_name, "KLL"); - assert_eq!(e.sketch_processors[0].sketch_kind, SketchAlgorithm::Kll); + assert_eq!( + e.sketch_processors[0].sketch_algorithm, + SketchAlgorithm::Kll + ); assert_eq!( e.source_metric.as_deref(), Some("http_request_duration_seconds") @@ -428,7 +431,11 @@ fn emitter_backend_config_routes_aggregation_id() { StageConfig::Backend(b) => { assert_eq!(b.aggregations.len(), 1); assert_eq!(b.aggregations[0].aggregation_id, edge_aid); - assert_eq!(b.aggregations[0].sketch_kind, SketchAlgorithm::Kll.into()); + assert!(matches!( + &b.aggregations[0].family, + SummaryFamilyType::Sketch(kind, _) + if kind.algorithm() == &SketchAlgorithm::Kll + )); assert_eq!(b.readouts.len(), 1); assert_eq!(b.readouts[0].aggregation_id, edge_aid); // `SketchQuery` has no `PartialEq` upstream — destructure @@ -511,7 +518,7 @@ fn end_to_end_quantile_workload() { StageConfig::Gateway(g) => { assert!(!g.merge_processors.is_empty()); assert_eq!(g.merge_processors[0].processor_name, "sketchmergeprocessor"); - assert_eq!(g.merge_processors[0].sketch_kind, SketchAlgorithm::Kll); + assert_eq!(g.merge_processors[0].sketch_algorithm, SketchAlgorithm::Kll); } _ => unreachable!(), } diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index ebbbe475..2592b806 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -15,7 +15,8 @@ use asap_aware_mapping::{ }; use planner_types::post_asap::{ CompositionOperator, EvaluationSchedule, OutputRepresentation, SketchQuery, SummaryExpr, - SummaryFamilyType, SummaryMaintenanceLifecycle, SummaryMaintenanceMode, SummaryNode, + SummaryFamilyType, SummaryMaintenanceLifecycle, SummaryMaintenanceLifecycleGuarantee, + SummaryMaintenanceMode, SummaryNode, }; use planner_types::pre_asap::QueryExpr; use planner_types::workload::{ @@ -253,19 +254,18 @@ impl PhysicalCompiler { } }; let aggregation_id = format!("{}:{}", query.query_id, metric); - let kind = asap_types::SummaryKind::from(selected.kind.clone()); - let params = asap_types::SummaryParams::from(selected.params.clone()); aggregations.push(BackendAggregation { aggregation_id: aggregation_id.clone(), metric_name: metric.clone(), - sketch_kind: kind, - sketch_params: params, + family: SummaryFamilyType::Sketch( + selected.kind.clone(), + planner_types::post_asap::GroupingStrategy::PerSubpopulationInstance, + ), window_secs: query.window_secs, spatial_filter: String::new(), grouping: query.group_by.clone(), item_label: None, aggregation_input: AggregationInput::SketchEnvelope, - agg_type_override: None, }); readouts.push(BackendReadout { aggregation_id, @@ -301,11 +301,11 @@ impl PhysicalCompiler { environment.observed_at_unix_ms, )?; for materialization in backend_plan.materializations.values_mut() { - materialization.lifecycle = Some(backend_plan::SummaryMaintenanceLifecycle { - kind: "continuously_maintained".into(), - maintenance_mode: "incremental".into(), - evaluation_schedule: "per_update".into(), - output_representation: "summary_state".into(), + materialization.lifecycle = Some(SummaryMaintenanceLifecycleGuarantee { + summary_maintenance_lifecycle: SummaryMaintenanceLifecycle::ContinuouslyMaintained, + summary_maintenance_mode: SummaryMaintenanceMode::Incremental, + evaluation_schedule: EvaluationSchedule::PerUpdate, + output_representation: OutputRepresentation::SummaryState, }); } let collector_plans = environment @@ -649,8 +649,8 @@ mod tests { .lifecycle .as_ref() .unwrap() - .kind, - "continuously_maintained" + .summary_maintenance_lifecycle, + SummaryMaintenanceLifecycle::ContinuouslyMaintained ); assert_eq!(bundle.backend_plan.routing.len(), 1); for plan in &bundle.collector_plans { diff --git a/control_plane/src/physical/deployment_cost/sketch_capability.rs b/control_plane/src/physical/deployment_cost/sketch_capability.rs index 3f9ade57..66022ba6 100644 --- a/control_plane/src/physical/deployment_cost/sketch_capability.rs +++ b/control_plane/src/physical/deployment_cost/sketch_capability.rs @@ -42,8 +42,6 @@ pub struct SketchCapability { pub cpu_micros_per_insert: f64, /// Transmission size per flush (bytes). pub transmission_bytes: u64, - /// Which logical aggregation intents this sketch supports. - pub supported_intents: Vec, /// Whether the sketch supports merge (`sketch(A∪B) = merge(sketch(A), sketch(B))`). pub mergeable: bool, /// Whether the sketch supports delta encoding. @@ -52,18 +50,6 @@ pub struct SketchCapability { pub supports_sliding_window: bool, } -/// A logical aggregation intent that a sketch can serve. Used in -/// [`SketchCapability::supported_intents`] to declare per-sketch -/// coverage; the optimizer reads this when deciding which family to -/// bind to an `AggIntent`. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum SupportedIntent { - Quantile, - Cardinality, - Frequency, - Extrema, -} - // ── YAML override loader ───────────────────────────────────────────────────── /// YAML-serialisable capability profile (matches `sketch_capabilities.yml`). @@ -74,7 +60,6 @@ struct SketchCapabilityYaml { memory_bytes_per_series: u64, cpu_micros_per_insert: f64, transmission_bytes: u64, - supported_intents: Vec, mergeable: bool, supports_delta: bool, supports_sliding_window: bool, @@ -82,24 +67,12 @@ struct SketchCapabilityYaml { impl SketchCapabilityYaml { fn to_capability(&self) -> SketchCapability { - let intents = self - .supported_intents - .iter() - .filter_map(|s| match s.as_str() { - "quantile" => Some(SupportedIntent::Quantile), - "cardinality" => Some(SupportedIntent::Cardinality), - "frequency" => Some(SupportedIntent::Frequency), - "extrema" => Some(SupportedIntent::Extrema), - _ => None, - }) - .collect(); SketchCapability { insert_throughput: self.insert_throughput, query_throughput: self.query_throughput, memory_bytes_per_series: self.memory_bytes_per_series, cpu_micros_per_insert: self.cpu_micros_per_insert, transmission_bytes: self.transmission_bytes, - supported_intents: intents, mergeable: self.mergeable, supports_delta: self.supports_delta, supports_sliding_window: self.supports_sliding_window, @@ -133,7 +106,6 @@ pub fn default_capability_table() -> HashMap memory_bytes_per_series: 4_096, cpu_micros_per_insert: 0.1, transmission_bytes: 4_096, - supported_intents: vec![SupportedIntent::Quantile, SupportedIntent::Extrema], mergeable: true, supports_delta: true, supports_sliding_window: false, @@ -147,7 +119,6 @@ pub fn default_capability_table() -> HashMap memory_bytes_per_series: 8_192, cpu_micros_per_insert: 0.2, transmission_bytes: 8_192, - supported_intents: vec![SupportedIntent::Quantile, SupportedIntent::Extrema], mergeable: true, supports_delta: false, supports_sliding_window: false, @@ -161,7 +132,6 @@ pub fn default_capability_table() -> HashMap memory_bytes_per_series: 16_384, cpu_micros_per_insert: 0.05, transmission_bytes: 16_384, - supported_intents: vec![SupportedIntent::Cardinality], mergeable: true, supports_delta: true, supports_sliding_window: false, @@ -175,7 +145,6 @@ pub fn default_capability_table() -> HashMap memory_bytes_per_series: 80_000, cpu_micros_per_insert: 0.5, transmission_bytes: 80_000, - supported_intents: vec![SupportedIntent::Frequency], mergeable: true, supports_delta: true, supports_sliding_window: false, @@ -189,7 +158,6 @@ pub fn default_capability_table() -> HashMap memory_bytes_per_series: 80_000, cpu_micros_per_insert: 0.5, transmission_bytes: 80_000, - supported_intents: vec![SupportedIntent::Frequency], mergeable: true, supports_delta: true, supports_sliding_window: false, @@ -236,20 +204,17 @@ mod tests { } #[test] - fn default_table_ddsketch_serves_quantile_intent() { + fn default_table_ddsketch_has_runtime_cost_profile() { let t = default_capability_table(); let cap = t.get(&SketchAlgorithm::DDSketch).unwrap(); - assert!(cap.supported_intents.contains(&SupportedIntent::Quantile)); assert!(cap.mergeable); } #[test] - fn default_table_hll_serves_cardinality_intent() { + fn default_table_hll_has_runtime_cost_profile() { let t = default_capability_table(); let cap = t.get(&SketchAlgorithm::Hll).unwrap(); - assert!(cap - .supported_intents - .contains(&SupportedIntent::Cardinality)); + assert!(cap.query_throughput > 0.0); } #[test] @@ -258,8 +223,7 @@ mod tests { let defaults = default_capability_table(); // Same set of keys, same defaults — we don't assert byte // equality on the SketchCapability values because they don't - // impl PartialEq, but they share the same supported_intents - // set per kind. + // impl PartialEq. assert_eq!(loaded.len(), defaults.len()); for k in defaults.keys() { assert!(loaded.contains_key(k)); diff --git a/control_plane/src/physical/deployment_cost/wire.rs b/control_plane/src/physical/deployment_cost/wire.rs index f4c3ad76..dee12c6e 100644 --- a/control_plane/src/physical/deployment_cost/wire.rs +++ b/control_plane/src/physical/deployment_cost/wire.rs @@ -123,8 +123,8 @@ impl WireCostTable { // "exact accumulators have no sketch wire-state cost" panic, are // unreachable by construction now instead of at runtime; see // control_plane/docs/design-asapplanner-pin-migration.md). - pub const fn for_kind(&self, kind: &SketchAlgorithm) -> SketchWireCost { - match kind { + pub const fn for_algorithm(&self, algorithm: &SketchAlgorithm) -> SketchWireCost { + match algorithm { SketchAlgorithm::DDSketch => self.ddsketch_delta, SketchAlgorithm::Kll => self.kll_full, SketchAlgorithm::Hll => self.hll_delta, diff --git a/control_plane/src/physical/post_asap/cost_model.rs b/control_plane/src/physical/post_asap/cost_model.rs index 7fe82369..eb8a093f 100644 --- a/control_plane/src/physical/post_asap/cost_model.rs +++ b/control_plane/src/physical/post_asap/cost_model.rs @@ -157,7 +157,7 @@ impl ControlPlaneCostModel { }; let table = WireCostTable::default(); let mut ranked: Vec = allowed.to_vec(); - ranked.sort_by_key(|k| table.for_kind(k).per_flush()); + ranked.sort_by_key(|algorithm| table.for_algorithm(algorithm).per_flush()); ranked } diff --git a/control_plane/src/physical/post_asap/matcher.rs b/control_plane/src/physical/post_asap/matcher.rs index e8aeba27..fadd444e 100644 --- a/control_plane/src/physical/post_asap/matcher.rs +++ b/control_plane/src/physical/post_asap/matcher.rs @@ -91,11 +91,11 @@ impl Matcher for SummaryFamilyMatcher { /// `Sketch` arm, exposed directly for callers that only have bare kinds /// (no [`planner_types::post_asap::SketchParams`]) to compare. /// `control_plane::physical::runtime_capability::Capability::is_satisfied_by` -/// is the first such caller: its `SketchKindHandle` query-side dispatch +/// is the first such caller: its `SketchAlgorithm` query-side dispatch /// tag never carries params, so constructing a full /// `Implementation::Sketch{kind, params}` just to discard the params /// would mean fabricating meaningless param values. See that module's -/// doc for why `Capability`/`SketchKindHandle` themselves aren't deleted +/// doc for why `Capability`/`SketchAlgorithm` themselves aren't deleted /// outright (`scratchpad/artifacts/enum-unification-plan.md` §8 Step 4). pub fn sketch_family_satisfied(required: &SketchAlgorithm, available: &SketchAlgorithm) -> bool { let req_family = summary_family(required); diff --git a/control_plane/src/physical/post_asap/tests.rs b/control_plane/src/physical/post_asap/tests.rs index 46ac6237..324f710c 100644 --- a/control_plane/src/physical/post_asap/tests.rs +++ b/control_plane/src/physical/post_asap/tests.rs @@ -321,8 +321,10 @@ fn bind_cms_topk_tight_recall_picks_countsketch() { fn bind_cms_topk_picks_cost_min_meeting_sla() { use crate::physical::deployment_cost::wire::WireCostTable; let table = WireCostTable::default(); - let cms = table.for_kind(&SketchAlgorithm::Cms).per_flush(); - let cs = table.for_kind(&SketchAlgorithm::CountSketch).per_flush(); + let cms = table.for_algorithm(&SketchAlgorithm::Cms).per_flush(); + let cs = table + .for_algorithm(&SketchAlgorithm::CountSketch) + .per_flush(); assert!( cms < cs, "CMS-heap ({cms} B) must be cheaper than CountSketch ({cs} B) on the wire" @@ -667,7 +669,7 @@ fn phase_b_e2e_quantile_over_time_binds_to_quantile_sketch() { "quantile_over_time(0.99, http_request_duration_seconds[5m])", AccuracyTarget::Epsilon(0.01), ); - let kind = crate::emit::extract_root_sketch_kind(&bound); + let kind = crate::emit::extract_root_sketch_algorithm(&bound); assert!( matches!( kind, @@ -684,7 +686,7 @@ fn phase_b_e2e_quantile_over_time_binds_to_quantile_sketch() { /// `sum_over_time.yaml` — the legacy planner produces an exact-sum /// aggregation row (no summary). Control plane path: `Aggregate{Sum}` over /// `Window` → binds to an exact accumulator (`SummaryAgg{Sum}`), which is -/// neither an approximate summary (so `extract_root_sketch_kind`, which +/// neither an approximate summary (so `extract_root_sketch_algorithm`, which /// excludes exact accumulators — see its doc comment — returns `None`) /// nor archive-routed. #[test] @@ -694,7 +696,7 @@ fn phase_b_e2e_sum_over_time_falls_through_to_logical() { AccuracyTarget::Epsilon(0.01), ); assert!( - crate::emit::extract_root_sketch_kind(&bound).is_none(), + crate::emit::extract_root_sketch_algorithm(&bound).is_none(), "sum_over_time should not produce an approximate summary" ); assert!( @@ -717,7 +719,7 @@ fn phase_b_e2e_sum_by_preserves_grouping_label() { AccuracyTarget::Epsilon(0.01), ); // No approximate summary family for plain Sum. - assert!(crate::emit::extract_root_sketch_kind(&bound).is_none()); + assert!(crate::emit::extract_root_sketch_algorithm(&bound).is_none()); // The end shape may carry `Logical(Aggregate{by, ...})` beneath a // `SummaryAgg{Sum}` wrapper, or `Logical(Window{...})` when the // ParsedQuery → QueryExpr lowering drops the Aggregate (legacy @@ -750,7 +752,7 @@ fn phase_b_e2e_rate_falls_through_to_logical() { "rate(http_requests_total[5m])", AccuracyTarget::Epsilon(0.01), ); - assert!(crate::emit::extract_root_sketch_kind(&bound).is_none()); + assert!(crate::emit::extract_root_sketch_algorithm(&bound).is_none()); assert!( !binding_is_archive(&bound), "Rate is ASAP-tier, not archive" @@ -803,7 +805,7 @@ fn phase_b_e2e_archive_only_e2e_binding() { "archive-only intent must surface archive flag through L4 binding" ); // No approximate summary fires for archive-only intents. - assert!(crate::emit::extract_root_sketch_kind(&bound).is_none()); + assert!(crate::emit::extract_root_sketch_algorithm(&bound).is_none()); } /// Cross-cutting: every Phase β archive-only intent reaches diff --git a/control_plane/src/physical/runtime_capability.rs b/control_plane/src/physical/runtime_capability.rs index 30225cea..ee46c529 100644 --- a/control_plane/src/physical/runtime_capability.rs +++ b/control_plane/src/physical/runtime_capability.rs @@ -3,12 +3,12 @@ //! Step 2a of the architectural refactor originally consolidated four //! overlapping capability tables into this module. The performance / //! cost-model half of that consolidation — [`SketchCapability`] / -//! `SupportedIntent` / `default_capability_table` / `load_capability_overrides` +//! `default_capability_table` / `load_capability_overrides` //! — moved to `crate::physical::deployment_cost::sketch_capability` (Stage 4 of the //! `physical::post_asap` re-layering): it's a cost-model concern read by the //! optimizer and physical planner, not L4 IR. What's left here: //! -//! - [`Capability`] / [`SketchKindHandle`] — query-side capability tag, +//! - [`Capability`] / [`SketchAlgorithm`] — query-side capability tag, //! used by the ASAP-tier reducer in `asap-query-engine` to dispatch //! PromQL → per-Capability sketch evaluation. //! - [`capability_for`] — the **semantic** intent → ASAP-tier dispatch @@ -22,13 +22,13 @@ use crate::physical::post_asap::matcher::sketch_family_satisfied; use crate::types_v2::AccuracyTarget; use asap_types::AggregationType; -use planner_types::post_asap::SketchAlgorithm; +pub use planner_types::post_asap::SketchAlgorithm; use planner_types::pre_asap::AggIntent; // ── Query-side capability tag ──────────────────────────────────────────────── /// Warm-tier capability tag. One variant per logical query family the -/// ASAP tier can answer. The inner [`SketchKindHandle`] is the +/// ASAP tier can answer. The inner [`SketchAlgorithm`] is the /// implementation choice (e.g. DDSketch vs KLL for `QuantileApprox`). /// Query routing keys on the variant, not the implementation, so two /// CMS instances and one CountSketch instance for the same metric-and- @@ -44,7 +44,7 @@ pub enum Capability { /// Approximate quantile via DDSketch / KLL / t-digest. The handle's /// `Any` variant means "any quantile-family sketch satisfies"; a /// concrete handle means "must be exactly this family". - QuantileApprox(SketchKindHandle), + QuantileApprox(Option), /// Approximate cardinality via HLL / theta-sketch / linear-counting. /// No inner handle — cardinality has a single canonical family /// today (HLL). @@ -56,12 +56,12 @@ pub enum Capability { /// additional info layered on top of the sketch matrix), so /// `is_satisfied_by` allows {CountMin, CountSketch, CmsWithHeap, /// CountSketchWithHeap} on the available side. - FrequencyEstimate(SketchKindHandle), + FrequencyEstimate(Option), /// Heavy-hitter top-k via CMS-with-heap or CountSketch-with-heap. /// Heap-BEARING — only handles that carry an item universe in their /// wire format can answer this. `Any` required matches either /// `CmsWithHeap` or `CountSketchWithHeap`. - FrequencyTopk(SketchKindHandle), + FrequencyTopk(Option), /// Exact-aggregation ASAP-tier state — Sum / Count / MinMax / Avg / /// Rate / Increase / SetAggregator etc. Backed by a per-accumulator /// payload (`AggPayload::ExactAgg` in the data plane). One variant @@ -293,36 +293,10 @@ impl OuterAgg { } } -/// Compact, hashable handle for sketch implementation choice. Mirrors -/// `planner_types::post_asap::SummaryKind` but adds the `Any` query-side wildcard -/// (not a sketch family — a dispatch hint). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum SketchKindHandle { - DDSketch, - Kll, - Hll, - CountSketch, - CountMin, - /// CMS paired with a Misra-Gries / heavy-hitter heap. Distinct from - /// `CountMin` because vanilla CMS carries no item universe — the - /// heap is what lets the ASAP-tier reducer enumerate top-k items - /// without an external item list. - CmsWithHeap, - /// CountSketch paired with a heavy-hitter heap. Same role as - /// `CmsWithHeap` but on the CountSketch substrate (balanced / - /// zero-mean error instead of CMS's one-sided bias). - CountSketchWithHeap, - /// "Any implementation that satisfies the family". Analysis-time - /// wildcard, never indexed against a concrete sketch instance. - /// Consumed by [`Capability::is_satisfied_by`]. - Any, -} - impl Capability { /// True when an indexed sketch instance's capability satisfies the - /// query's required capability. `SketchKindHandle::Any` on the - /// query side is a wildcard that matches any concrete handle in - /// the same family. + /// query's required capability. `None` on the query side means any + /// algorithm in that capability family. /// /// `self` is the **required** capability (from the analyzer); /// `indexed` is the **available** capability (from the sketch @@ -341,7 +315,7 @@ impl Capability { pub fn is_satisfied_by(&self, indexed: &Capability) -> bool { match (self, indexed) { (Capability::QuantileApprox(req), Capability::QuantileApprox(have)) => { - sketch_kinds_compatible(*req, SketchAlgorithm::Kll, *have, SketchAlgorithm::Kll) + sketch_algorithms_compatible(req, SketchAlgorithm::Kll, have, SketchAlgorithm::Kll) } // Cardinality has no inner handle; family match is total. (Capability::CardinalityApprox, Capability::CardinalityApprox) => true, @@ -350,10 +324,10 @@ impl Capability { // stand-in would let a heap-less available sketch wrongly // satisfy a top-k requirement (see `resolve_handle`'s doc). (Capability::FrequencyTopk(req), Capability::FrequencyTopk(have)) => { - sketch_kinds_compatible( - *req, + sketch_algorithms_compatible( + req, SketchAlgorithm::CmsWithHeap, - *have, + have, SketchAlgorithm::CmsWithHeap, ) } @@ -364,13 +338,13 @@ impl Capability { // the sketch matrix). A heap-bearing `FrequencyTopk` indexed // capability ALSO satisfies a bare-frequency required capability. (Capability::FrequencyEstimate(req), Capability::FrequencyEstimate(have)) => { - sketch_kinds_compatible(*req, SketchAlgorithm::Cms, *have, SketchAlgorithm::Cms) + sketch_algorithms_compatible(req, SketchAlgorithm::Cms, have, SketchAlgorithm::Cms) } (Capability::FrequencyEstimate(req), Capability::FrequencyTopk(have)) => { - sketch_kinds_compatible( - *req, + sketch_algorithms_compatible( + req, SketchAlgorithm::Cms, - *have, + have, SketchAlgorithm::CmsWithHeap, ) } @@ -398,68 +372,18 @@ impl Capability { } } -/// Map a concrete [`SketchKindHandle`] to its [`SketchAlgorithm`] -/// equivalent. `Any` has no single equivalent by design — resolve it to -/// a concrete per-family stand-in via [`resolve_handle`] before calling -/// this. -fn to_summary_kind(h: SketchKindHandle) -> Option { - match h { - SketchKindHandle::DDSketch => Some(SketchAlgorithm::DDSketch), - SketchKindHandle::Kll => Some(SketchAlgorithm::Kll), - SketchKindHandle::Hll => Some(SketchAlgorithm::Hll), - SketchKindHandle::CountSketch => Some(SketchAlgorithm::CountSketch), - SketchKindHandle::CountMin => Some(SketchAlgorithm::Cms), - SketchKindHandle::CmsWithHeap => Some(SketchAlgorithm::CmsWithHeap), - SketchKindHandle::CountSketchWithHeap => Some(SketchAlgorithm::CountSketchWithHeap), - // Defensive: `Any` should never reach this function directly — - // every call site resolves it via `resolve_handle` first. `None` - // here means "does not satisfy anything", the safe default. - SketchKindHandle::Any => None, - } -} - -/// Resolve a [`SketchKindHandle`] to the [`SketchAlgorithm`] fed into -/// [`sketch_family_satisfied`]. `Any` (the query-side "any -/// implementation in this family satisfies" wildcard) resolves to -/// `any_stand_in` — a concrete per-family placeholder — because -/// `SketchAlgorithm` has no wildcard concept of its own; -/// `sketch_family_satisfied`'s same-family-satisfies rule already -/// treats every member of a family as interchangeable, so picking ANY -/// concrete family member as the stand-in reproduces the wildcard's -/// effect (`enum-unification-plan.md` §8 Step 4's investigation note). -/// -/// The one place this needs care: [`Capability::FrequencyTopk`]'s stand-in -/// must be the heap-bearing `CmsWithHeap`, never bare `Cms` — bare `Cms` -/// and `CmsWithHeap` are the SAME family (`Frequency`/`FrequencyTopk` are -/// related by the asymmetric "heap satisfies bare" rule, not equal), so a -/// bare stand-in would let a heap-less available sketch wrongly satisfy a -/// top-k requirement. Every `FrequencyTopk` call site in this module -/// passes `SketchAlgorithm::CmsWithHeap` as `any_stand_in` for exactly this -/// reason. -fn resolve_handle(h: SketchKindHandle, any_stand_in: SketchAlgorithm) -> Option { - match h { - SketchKindHandle::Any => Some(any_stand_in), - other => to_summary_kind(other), - } -} - -/// Resolve both sides of a handle comparison and delegate to -/// [`sketch_family_satisfied`]. `false` if either side fails to resolve -/// (only possible today via the defensive `to_summary_kind` fallback, -/// since `resolve_handle` always resolves `Any`). -fn sketch_kinds_compatible( - required: SketchKindHandle, +/// Resolve optional query-side algorithm constraints and delegate family +/// compatibility to the ASAPPlanner algorithm taxonomy. `None` means any +/// algorithm in the capability's category; it is not a fake algorithm. +fn sketch_algorithms_compatible( + required: &Option, required_any_stand_in: SketchAlgorithm, - available: SketchKindHandle, + available: &Option, available_any_stand_in: SketchAlgorithm, ) -> bool { - match ( - resolve_handle(required, required_any_stand_in), - resolve_handle(available, available_any_stand_in), - ) { - (Some(r), Some(a)) => sketch_family_satisfied(&r, &a), - _ => false, - } + let required = required.as_ref().unwrap_or(&required_any_stand_in); + let available = available.as_ref().unwrap_or(&available_any_stand_in); + sketch_family_satisfied(required, available) } /// True when `available` is the multi-population equivalent of @@ -543,7 +467,7 @@ pub fn capability_for(intent: &AggIntent) -> Option { // CountSketchWithHeap works (the heap is additional // info that the FrequencyTopk path uses). `Any` here // means the optimizer picks the cheapest indexed sid. - Some(Capability::FrequencyEstimate(SketchKindHandle::Any)) + Some(Capability::FrequencyEstimate(None)) }; } match intent { @@ -555,16 +479,16 @@ pub fn capability_for(intent: &AggIntent) -> Option { Some(Capability::ExactAgg(AggregationType::Increase)) } AggIntent::Quantile { accuracy, .. } if !is_exact(accuracy) => { - Some(Capability::QuantileApprox(SketchKindHandle::Any)) + Some(Capability::QuantileApprox(None)) } AggIntent::Cardinality { accuracy, .. } if !is_exact(accuracy) => { Some(Capability::CardinalityApprox) } AggIntent::TopK { accuracy, .. } if !is_exact(accuracy) => { - Some(Capability::FrequencyTopk(SketchKindHandle::Any)) + Some(Capability::FrequencyTopk(None)) } AggIntent::Count { accuracy } if !is_exact(accuracy) => { - Some(Capability::FrequencyEstimate(SketchKindHandle::Any)) + Some(Capability::FrequencyEstimate(None)) } _ => None, } @@ -581,7 +505,6 @@ fn is_exact(accuracy: &AccuracyTarget) -> bool { #[cfg(test)] mod tests { use super::*; - use std::time::Duration; // ── capability_for: AggIntent → Capability bridge ──────────────────── @@ -594,7 +517,7 @@ mod tests { }; assert_eq!( capability_for(&intent), - Some(Capability::QuantileApprox(SketchKindHandle::Any)) + Some(Capability::QuantileApprox(None)) ); } @@ -653,7 +576,7 @@ mod tests { }; assert_eq!( capability_for(&intent), - Some(Capability::FrequencyEstimate(SketchKindHandle::Any)) + Some(Capability::FrequencyEstimate(None)) ); } @@ -733,7 +656,7 @@ mod tests { }; assert_eq!( capability_for(&intent), - Some(Capability::FrequencyTopk(SketchKindHandle::Any)) + Some(Capability::FrequencyTopk(None)) ); } @@ -752,7 +675,7 @@ mod tests { let intent = crate::planner_selection::frequency(AccuracyTarget::Epsilon(0.01), None); assert_eq!( capability_for(&intent), - Some(Capability::FrequencyEstimate(SketchKindHandle::Any)) + Some(Capability::FrequencyEstimate(None)) ); } @@ -767,7 +690,7 @@ mod tests { ); assert_eq!( capability_for(&intent), - Some(Capability::FrequencyEstimate(SketchKindHandle::Any)) + Some(Capability::FrequencyEstimate(None)) ); } @@ -798,9 +721,9 @@ mod tests { #[test] fn is_satisfied_by_any_wildcard_matches_concrete() { - let required = Capability::QuantileApprox(SketchKindHandle::Any); - let indexed_dd = Capability::QuantileApprox(SketchKindHandle::DDSketch); - let indexed_kll = Capability::QuantileApprox(SketchKindHandle::Kll); + let required = Capability::QuantileApprox(None); + let indexed_dd = Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch)); + let indexed_kll = Capability::QuantileApprox(Some(SketchAlgorithm::Kll)); assert!(required.is_satisfied_by(&indexed_dd)); assert!(required.is_satisfied_by(&indexed_kll)); } @@ -818,15 +741,15 @@ mod tests { // to spell out `Any` or a concrete kind. Harmless in practice: // `capability_for` never emits a concrete `QuantileApprox` handle // (always `Any`), so this path is exercised only defensively. - let required = Capability::QuantileApprox(SketchKindHandle::DDSketch); - let indexed_dd = Capability::QuantileApprox(SketchKindHandle::DDSketch); - let indexed_kll = Capability::QuantileApprox(SketchKindHandle::Kll); + let required = Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch)); + let indexed_dd = Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch)); + let indexed_kll = Capability::QuantileApprox(Some(SketchAlgorithm::Kll)); assert!(required.is_satisfied_by(&indexed_dd)); assert!(required.is_satisfied_by(&indexed_kll)); // Still cross-family-incompatible: a concrete quantile requirement // is never satisfied by a cardinality-family available handle. - let indexed_hll = Capability::QuantileApprox(SketchKindHandle::Hll); + let indexed_hll = Capability::QuantileApprox(Some(SketchAlgorithm::Hll)); assert!(!required.is_satisfied_by(&indexed_hll)); } @@ -839,20 +762,20 @@ mod tests { #[test] fn is_satisfied_by_different_families_are_incompatible() { - let required = Capability::QuantileApprox(SketchKindHandle::Any); + let required = Capability::QuantileApprox(None); let indexed = Capability::CardinalityApprox; assert!(!required.is_satisfied_by(&indexed)); - let required = Capability::FrequencyTopk(SketchKindHandle::CmsWithHeap); - let indexed = Capability::QuantileApprox(SketchKindHandle::DDSketch); + let required = Capability::FrequencyTopk(Some(SketchAlgorithm::CmsWithHeap)); + let indexed = Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch)); assert!(!required.is_satisfied_by(&indexed)); } #[test] fn is_satisfied_by_topk_handles_must_match() { - let required = Capability::FrequencyTopk(SketchKindHandle::CmsWithHeap); - let indexed_with_heap = Capability::FrequencyTopk(SketchKindHandle::CmsWithHeap); - let indexed_no_heap = Capability::FrequencyTopk(SketchKindHandle::CountMin); + let required = Capability::FrequencyTopk(Some(SketchAlgorithm::CmsWithHeap)); + let indexed_with_heap = Capability::FrequencyTopk(Some(SketchAlgorithm::CmsWithHeap)); + let indexed_no_heap = Capability::FrequencyTopk(Some(SketchAlgorithm::Cms)); assert!(required.is_satisfied_by(&indexed_with_heap)); assert!(!required.is_satisfied_by(&indexed_no_heap)); } @@ -863,10 +786,10 @@ mod tests { // capability declares itself as `FrequencyTopk(CountMin)` (an // ill-formed catalog entry), the satisfaction check must reject // it — top-k cannot enumerate items off a heap-less sketch. - let required_any = Capability::FrequencyTopk(SketchKindHandle::Any); - let required_concrete = Capability::FrequencyTopk(SketchKindHandle::CmsWithHeap); - let indexed_heapless = Capability::FrequencyTopk(SketchKindHandle::CountMin); - let indexed_heapless_cs = Capability::FrequencyTopk(SketchKindHandle::CountSketch); + let required_any = Capability::FrequencyTopk(None); + let required_concrete = Capability::FrequencyTopk(Some(SketchAlgorithm::CmsWithHeap)); + let indexed_heapless = Capability::FrequencyTopk(Some(SketchAlgorithm::Cms)); + let indexed_heapless_cs = Capability::FrequencyTopk(Some(SketchAlgorithm::CountSketch)); assert!(!required_any.is_satisfied_by(&indexed_heapless)); assert!(!required_any.is_satisfied_by(&indexed_heapless_cs)); assert!(!required_concrete.is_satisfied_by(&indexed_heapless)); @@ -875,9 +798,9 @@ mod tests { #[test] fn is_satisfied_by_frequency_topk_any_matches_either_heap() { // `Any` required for top-k accepts either heap-bearing handle. - let required = Capability::FrequencyTopk(SketchKindHandle::Any); - let cms_heap = Capability::FrequencyTopk(SketchKindHandle::CmsWithHeap); - let cs_heap = Capability::FrequencyTopk(SketchKindHandle::CountSketchWithHeap); + let required = Capability::FrequencyTopk(None); + let cms_heap = Capability::FrequencyTopk(Some(SketchAlgorithm::CmsWithHeap)); + let cs_heap = Capability::FrequencyTopk(Some(SketchAlgorithm::CountSketchWithHeap)); assert!(required.is_satisfied_by(&cms_heap)); assert!(required.is_satisfied_by(&cs_heap)); } @@ -888,11 +811,11 @@ mod tests { // frequency-family sketch — heap-less AND heap-bearing both work // (the heap is additional metadata; the underlying CMS / CS // matrix answers the point query either way). - let required = Capability::FrequencyEstimate(SketchKindHandle::Any); - let cms = Capability::FrequencyEstimate(SketchKindHandle::CountMin); - let cs = Capability::FrequencyEstimate(SketchKindHandle::CountSketch); - let cms_heap = Capability::FrequencyTopk(SketchKindHandle::CmsWithHeap); - let cs_heap = Capability::FrequencyTopk(SketchKindHandle::CountSketchWithHeap); + let required = Capability::FrequencyEstimate(None); + let cms = Capability::FrequencyEstimate(Some(SketchAlgorithm::Cms)); + let cs = Capability::FrequencyEstimate(Some(SketchAlgorithm::CountSketch)); + let cms_heap = Capability::FrequencyTopk(Some(SketchAlgorithm::CmsWithHeap)); + let cs_heap = Capability::FrequencyTopk(Some(SketchAlgorithm::CountSketchWithHeap)); assert!(required.is_satisfied_by(&cms)); assert!(required.is_satisfied_by(&cs)); assert!(required.is_satisfied_by(&cms_heap)); @@ -901,13 +824,13 @@ mod tests { #[test] fn is_satisfied_by_frequency_estimate_rejects_non_frequency_family() { - let required = Capability::FrequencyEstimate(SketchKindHandle::Any); + let required = Capability::FrequencyEstimate(None); // QuantileApprox / CardinalityApprox don't answer frequency. - let q = Capability::QuantileApprox(SketchKindHandle::DDSketch); + let q = Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch)); let c = Capability::CardinalityApprox; // FrequencyEstimate with a non-frequency-family handle on the // available side is also rejected (defensive). - let bad = Capability::FrequencyEstimate(SketchKindHandle::Hll); + let bad = Capability::FrequencyEstimate(Some(SketchAlgorithm::Hll)); assert!(!required.is_satisfied_by(&q)); assert!(!required.is_satisfied_by(&c)); assert!(!required.is_satisfied_by(&bad)); @@ -939,18 +862,20 @@ mod tests { // with QuantileApprox / CardinalityApprox / FrequencyEstimate / // FrequencyTopk. let required = Capability::ExactAgg(AggregationType::Sum); - assert!(!required.is_satisfied_by(&Capability::QuantileApprox(SketchKindHandle::DDSketch))); - assert!(!required.is_satisfied_by(&Capability::CardinalityApprox)); assert!( - !required.is_satisfied_by(&Capability::FrequencyEstimate(SketchKindHandle::CountMin)) + !required.is_satisfied_by(&Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch))) ); + assert!(!required.is_satisfied_by(&Capability::CardinalityApprox)); assert!( - !required.is_satisfied_by(&Capability::FrequencyTopk(SketchKindHandle::CmsWithHeap)) + !required.is_satisfied_by(&Capability::FrequencyEstimate(Some(SketchAlgorithm::Cms))) ); + assert!(!required.is_satisfied_by(&Capability::FrequencyTopk(Some( + SketchAlgorithm::CmsWithHeap + )))); // And the reverse — a sketch-family required capability must // not match an ExactAgg-backed sid. - let sketch_required = Capability::QuantileApprox(SketchKindHandle::Any); + let sketch_required = Capability::QuantileApprox(None); let exact_indexed = Capability::ExactAgg(AggregationType::DatasketchesKLL); assert!(!sketch_required.is_satisfied_by(&exact_indexed)); } @@ -1059,11 +984,12 @@ mod tests { // `CountSketchWithHeap` is the CountSketch counterpart to // `CmsWithHeap`. Construct a `FrequencyTopk` capability around // it and verify it satisfies an `Any`-required top-k. - let cap = Capability::FrequencyTopk(SketchKindHandle::CountSketchWithHeap); - let required = Capability::FrequencyTopk(SketchKindHandle::Any); + let cap = Capability::FrequencyTopk(Some(SketchAlgorithm::CountSketchWithHeap)); + let required = Capability::FrequencyTopk(None); assert!(required.is_satisfied_by(&cap)); // And the concrete-against-concrete (same handle) case matches. - let required_concrete = Capability::FrequencyTopk(SketchKindHandle::CountSketchWithHeap); + let required_concrete = + Capability::FrequencyTopk(Some(SketchAlgorithm::CountSketchWithHeap)); assert!(required_concrete.is_satisfied_by(&cap)); // Intentional broadening vs. this module's pre-`sketch_family_satisfied` // behavior: CMS and CountSketch are the SAME frequency family in @@ -1076,11 +1002,11 @@ mod tests { // `capability_for` always emits `Any` for `FrequencyTopk`, never a // concrete handle, so this exact combination never arises from a // real query. - let required_cms = Capability::FrequencyTopk(SketchKindHandle::CmsWithHeap); + let required_cms = Capability::FrequencyTopk(Some(SketchAlgorithm::CmsWithHeap)); assert!(required_cms.is_satisfied_by(&cap)); // Still cross-family-incompatible: a heap-bearing requirement is // never satisfied by a bare (heap-less) available handle. - let bare = Capability::FrequencyTopk(SketchKindHandle::CountMin); + let bare = Capability::FrequencyTopk(Some(SketchAlgorithm::Cms)); assert!(!required_cms.is_satisfied_by(&bare)); } diff --git a/control_plane/src/physical/workload_planner.rs b/control_plane/src/physical/workload_planner.rs index 66d0ba9c..5ab88c7f 100644 --- a/control_plane/src/physical/workload_planner.rs +++ b/control_plane/src/physical/workload_planner.rs @@ -469,7 +469,7 @@ mod tests { /// statistic just to make an incompatible family appear valid. #[test] fn incompatible_override_does_not_rewrite_query_semantics() { - use crate::emit::extract_root_sketch_kind; + use crate::emit::extract_root_sketch_algorithm; use planner_types::post_asap::SketchAlgorithm; for (ov, expect) in [ (SketchType::DDSketch, SketchAlgorithm::DDSketch), @@ -483,7 +483,7 @@ mod tests { let pe = bind_workload_typed(&w) .unwrap_or_else(|| panic!("bind declined for override {ov:?}")); assert_eq!( - extract_root_sketch_kind(&pe), + extract_root_sketch_algorithm(&pe), Some(expect.clone()), "override {ov:?} must not change Quantile semantics", ); @@ -616,9 +616,9 @@ mod tests { /// Walk the L4 binding output and pull out the approximate sketch /// family. Returns `None` if no sketch node is present (raw / pure /// logical pass-through, or an exact accumulator — see - /// `emit::extract_root_sketch_kind`, whose logic this mirrors). + /// `emit::extract_root_sketch_algorithm`, whose logic this mirrors). fn extract_family(expr: &PhysicalExpr) -> Option { - crate::emit::extract_root_sketch_kind(expr) + crate::emit::extract_root_sketch_algorithm(expr) } /// Pull the `SketchQuery` out of a bound `PhysicalExpr`'s top-level diff --git a/control_plane/src/query_planning.rs b/control_plane/src/query_planning.rs index 48205b09..568a6d03 100644 --- a/control_plane/src/query_planning.rs +++ b/control_plane/src/query_planning.rs @@ -129,7 +129,7 @@ where fn planned_capability( node: &planner_types::post_asap::SummaryNode, ) -> Option<(String, Capability)> { - use crate::physical::runtime_capability::SketchKindHandle; + use crate::physical::runtime_capability::SketchAlgorithm; use planner_types::post_asap::{SketchQuery, SummaryExpr, SummaryFamilyType}; fn metric(node: &planner_types::post_asap::SummaryNode) -> Option { @@ -164,19 +164,22 @@ fn planned_capability( } } - fn handle(family: &SummaryFamilyType) -> Option { + fn handle(family: &SummaryFamilyType) -> Option { let SummaryFamilyType::Sketch(kind, _) = family else { return None; }; - Some(match asap_types::SummaryKind::from(kind.clone()) { - asap_types::SummaryKind::DDSketch => SketchKindHandle::DDSketch, - asap_types::SummaryKind::Kll => SketchKindHandle::Kll, - asap_types::SummaryKind::Hll => SketchKindHandle::Hll, - asap_types::SummaryKind::Cms => SketchKindHandle::CountMin, - asap_types::SummaryKind::CmsWithHeap => SketchKindHandle::CmsWithHeap, - asap_types::SummaryKind::CountSketch => SketchKindHandle::CountSketch, - asap_types::SummaryKind::CountSketchWithHeap => SketchKindHandle::CountSketchWithHeap, - _ => return None, + Some(match kind.algorithm() { + planner_types::post_asap::SketchAlgorithm::DDSketch => SketchAlgorithm::DDSketch, + planner_types::post_asap::SketchAlgorithm::Kll => SketchAlgorithm::Kll, + planner_types::post_asap::SketchAlgorithm::Hll => SketchAlgorithm::Hll, + planner_types::post_asap::SketchAlgorithm::Cms => SketchAlgorithm::Cms, + planner_types::post_asap::SketchAlgorithm::CmsWithHeap => SketchAlgorithm::CmsWithHeap, + planner_types::post_asap::SketchAlgorithm::CountSketch => SketchAlgorithm::CountSketch, + planner_types::post_asap::SketchAlgorithm::CountSketchWithHeap => { + SketchAlgorithm::CountSketchWithHeap + } + planner_types::post_asap::SketchAlgorithm::Kmv + | planner_types::post_asap::SketchAlgorithm::Theta => return None, }) } @@ -190,21 +193,26 @@ fn planned_capability( return None; }; match query { - SketchQuery::Quantile { .. } => Capability::QuantileApprox(handle(family)?), + SketchQuery::Quantile { .. } => Capability::QuantileApprox(Some(handle(family)?)), SketchQuery::Cardinality => Capability::CardinalityApprox, - SketchQuery::PointCount { .. } => Capability::FrequencyEstimate(handle(family)?), - SketchQuery::TopK { .. } => Capability::FrequencyTopk(handle(family)?), + SketchQuery::PointCount { .. } => { + Capability::FrequencyEstimate(Some(handle(family)?)) + } + SketchQuery::TopK { .. } => Capability::FrequencyTopk(Some(handle(family)?)), } } SummaryExpr::SummaryAgg { family: SummaryFamilyType::ExactAggregate(kind, _), .. } => { - let agg = match asap_types::SummaryKind::from(kind.clone()) { - asap_types::SummaryKind::Sum => asap_types::AggregationType::Sum, - asap_types::SummaryKind::Increase => asap_types::AggregationType::Increase, - asap_types::SummaryKind::MinMax => asap_types::AggregationType::MinMax, - _ => return None, + let agg = match kind { + planner_types::post_asap::ExactKind::Sum + | planner_types::post_asap::ExactKind::Count => asap_types::AggregationType::Sum, + planner_types::post_asap::ExactKind::Increase + | planner_types::post_asap::ExactKind::Rate => { + asap_types::AggregationType::Increase + } + planner_types::post_asap::ExactKind::MinMax => asap_types::AggregationType::MinMax, }; Capability::ExactAgg(agg) } diff --git a/control_plane/src/replan.rs b/control_plane/src/replan.rs index c6631fc1..19a86813 100644 --- a/control_plane/src/replan.rs +++ b/control_plane/src/replan.rs @@ -650,25 +650,39 @@ impl Replanner { AggRole::Other => None, AggRole::Quantile | AggRole::Topk => None, }; - let agg_type_override = agg_type_override?.to_string(); + let exact_kind = match agg_type_override? { + "Sum" => planner_types::post_asap::ExactKind::Sum, + "Count" => planner_types::post_asap::ExactKind::Count, + "MinMax" => planner_types::post_asap::ExactKind::MinMax, + "Increase" | "Rate" => planner_types::post_asap::ExactKind::Increase, + _ => return None, + }; + let exact_params = match &exact_kind { + planner_types::post_asap::ExactKind::Sum => planner_types::post_asap::ExactParams::Sum, + planner_types::post_asap::ExactKind::Count => { + planner_types::post_asap::ExactParams::Count + } + planner_types::post_asap::ExactKind::MinMax => { + planner_types::post_asap::ExactParams::MinMax + } + planner_types::post_asap::ExactKind::Increase => { + planner_types::post_asap::ExactParams::Increase + } + planner_types::post_asap::ExactKind::Rate => { + planner_types::post_asap::ExactParams::Rate + } + }; use crate::physical::colored_dag::emitter::{ AggregationInput, BackendAggregation, BackendStageConfig, }; - use planner_types::post_asap::{SketchAlgorithm, SketchParams}; + use planner_types::post_asap::SummaryFamilyType; let window_secs = workload.time_window.as_secs().max(1); Some(BackendStageConfig { aggregations: vec![BackendAggregation { item_label: None, aggregation_id: format!("exact-{}-{}", workload.metric_name, role), metric_name: workload.metric_name.clone(), - // Sentinel sketch_kind / sketch_params — `agg_type_override` - // takes precedence in `build_backend_aggregation_json`, so - // these are not emitted on the wire. DDSketch is the - // chosen sentinel because every backend that recognises - // `AggregationType::FromStr` also accepts DDSketch (and - // we don't have a `SketchAlgorithm::None` variant today). - sketch_kind: SketchAlgorithm::DDSketch.into(), - sketch_params: SketchParams::DDSketch { alpha: 0.01 }.into(), + family: SummaryFamilyType::ExactAggregate(exact_kind, exact_params), window_secs, spatial_filter: String::new(), grouping: workload.group_by_labels.clone(), @@ -676,7 +690,6 @@ impl Replanner { // ships counter samples; the backend's // SumAccumulator integrates them). aggregation_input: AggregationInput::Raw, - agg_type_override: Some(agg_type_override), }], // No readout entries — ExactAgg produces the answer // directly; the readout dispatch happens at PromQL eval @@ -1250,7 +1263,6 @@ mod tests { use crate::physical::colored_dag::emitter::{ AggregationInput, BackendAggregation, BackendStageConfig, }; - use planner_types::post_asap::{SketchAlgorithm, SketchParams}; let (url, hits) = start_repost_mock().await; let client = StdArc::new(BackendClient::new(url)); @@ -1300,13 +1312,14 @@ mod tests { item_label: None, aggregation_id: "exact-http_requests_total-sum".to_string(), metric_name: "http_requests_total".to_string(), - sketch_kind: SketchAlgorithm::DDSketch.into(), - sketch_params: SketchParams::DDSketch { alpha: 0.01 }.into(), + family: planner_types::post_asap::SummaryFamilyType::ExactAggregate( + planner_types::post_asap::ExactKind::Sum, + planner_types::post_asap::ExactParams::Sum, + ), grouping: vec!["zone".to_string()], spatial_filter: String::new(), window_secs: 60, aggregation_input: AggregationInput::Raw, - agg_type_override: Some("Sum".to_string()), }], readouts: Vec::new(), }, diff --git a/control_plane/src/sketch_selection.rs b/control_plane/src/sketch_selection.rs index 8030c5c3..eac452c9 100644 --- a/control_plane/src/sketch_selection.rs +++ b/control_plane/src/sketch_selection.rs @@ -16,14 +16,14 @@ //! * `FrequencyTopk` → CountSketch | CountMinSketch (heap layered on the same matrix) //! * `ExactAgg` → ∅ (served by the exact-aggregation / archive path, not a sketch) //! -//! When a capability binds a *concrete* `SketchKindHandle` (not `Any`), the +//! When a capability binds a *concrete* `SketchAlgorithm` (not `Any`), the //! result is exactly that one family; `Any` expands to the full candidate set. //! //! Moved out of `physical::post_asap` (Stage 4 of the `physical::post_asap` //! re-layering) — this is a query-planning concern (its one caller is //! [`crate::query_planning`]), not L4 IR. -use crate::physical::runtime_capability::{Capability, SketchKindHandle}; +use crate::physical::runtime_capability::{Capability, SketchAlgorithm}; use crate::types::SketchType; /// Map a sketch handle to the allocatable control-plane [`SketchType`]. @@ -33,18 +33,16 @@ use crate::types::SketchType; /// heap is an allocation detail layered on the same sketch, not a distinct /// allocatable family. `Any` is an analysis-time wildcard with no single /// concrete family, so it returns `None`. -pub fn sketch_type_for_handle(h: SketchKindHandle) -> Option { +pub fn sketch_type_for_algorithm(h: SketchAlgorithm) -> Option { match h { - SketchKindHandle::DDSketch => Some(SketchType::DDSketch), - SketchKindHandle::Kll => Some(SketchType::KLL), - SketchKindHandle::Hll => Some(SketchType::HLL), - SketchKindHandle::CountSketch | SketchKindHandle::CountSketchWithHeap => { + SketchAlgorithm::DDSketch => Some(SketchType::DDSketch), + SketchAlgorithm::Kll => Some(SketchType::KLL), + SketchAlgorithm::Hll => Some(SketchType::HLL), + SketchAlgorithm::CountSketch | SketchAlgorithm::CountSketchWithHeap => { Some(SketchType::CountSketch) } - SketchKindHandle::CountMin | SketchKindHandle::CmsWithHeap => { - Some(SketchType::CountMinSketch) - } - SketchKindHandle::Any => None, + SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap => Some(SketchType::CountMinSketch), + SketchAlgorithm::Kmv | SketchAlgorithm::Theta => None, } } @@ -56,10 +54,10 @@ pub fn sketch_type_for_handle(h: SketchKindHandle) -> Option { /// no-op (the caller routes it to cold/exact instead). pub fn sketch_families_for_capability(cap: &Capability) -> Vec { match cap { - Capability::QuantileApprox(h) => concrete_or(*h, &[SketchType::DDSketch, SketchType::KLL]), + Capability::QuantileApprox(h) => concrete_or(h, &[SketchType::DDSketch, SketchType::KLL]), Capability::CardinalityApprox => vec![SketchType::HLL], Capability::FrequencyEstimate(h) | Capability::FrequencyTopk(h) => { - concrete_or(*h, &[SketchType::CountSketch, SketchType::CountMinSketch]) + concrete_or(h, &[SketchType::CountSketch, SketchType::CountMinSketch]) } Capability::ExactAgg(_) => Vec::new(), } @@ -84,8 +82,8 @@ where out } -fn concrete_or(h: SketchKindHandle, any_set: &[SketchType]) -> Vec { - match sketch_type_for_handle(h) { +fn concrete_or(h: &Option, any_set: &[SketchType]) -> Vec { + match h.clone().and_then(sketch_type_for_algorithm) { Some(t) => vec![t], None => any_set.to_vec(), } @@ -98,19 +96,20 @@ mod tests { #[test] fn quantile_any_expands_to_ddsketch_and_kll() { - let fams = - sketch_families_for_capability(&Capability::QuantileApprox(SketchKindHandle::Any)); + let fams = sketch_families_for_capability(&Capability::QuantileApprox(None)); assert_eq!(fams, vec![SketchType::DDSketch, SketchType::KLL]); } #[test] fn quantile_concrete_handle_pins_one_family() { assert_eq!( - sketch_families_for_capability(&Capability::QuantileApprox(SketchKindHandle::DDSketch)), + sketch_families_for_capability(&Capability::QuantileApprox(Some( + SketchAlgorithm::DDSketch + ))), vec![SketchType::DDSketch] ); assert_eq!( - sketch_families_for_capability(&Capability::QuantileApprox(SketchKindHandle::Kll)), + sketch_families_for_capability(&Capability::QuantileApprox(Some(SketchAlgorithm::Kll))), vec![SketchType::KLL] ); } @@ -127,20 +126,20 @@ mod tests { fn frequency_families_and_heap_collapse() { // bare frequency: Any -> both matrix families assert_eq!( - sketch_families_for_capability(&Capability::FrequencyEstimate(SketchKindHandle::Any)), + sketch_families_for_capability(&Capability::FrequencyEstimate(None)), vec![SketchType::CountSketch, SketchType::CountMinSketch] ); // heap-bearing handles collapse to their matrix family assert_eq!( - sketch_families_for_capability(&Capability::FrequencyTopk( - SketchKindHandle::CmsWithHeap - )), + sketch_families_for_capability(&Capability::FrequencyTopk(Some( + SketchAlgorithm::CmsWithHeap + ))), vec![SketchType::CountMinSketch] ); assert_eq!( - sketch_families_for_capability(&Capability::FrequencyTopk( - SketchKindHandle::CountSketchWithHeap - )), + sketch_families_for_capability(&Capability::FrequencyTopk(Some( + SketchAlgorithm::CountSketchWithHeap + ))), vec![SketchType::CountSketch] ); } @@ -157,9 +156,9 @@ mod tests { // a query set needing {quantile, cardinality, quantile-again} -> // DDSketch, KLL, HLL with no duplicate DDSketch/KLL. let caps = vec![ - Capability::QuantileApprox(SketchKindHandle::Any), + Capability::QuantileApprox(None), Capability::CardinalityApprox, - Capability::QuantileApprox(SketchKindHandle::DDSketch), + Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch)), ]; let got = required_sketches_for_capabilities(&caps); assert_eq!( diff --git a/control_plane/src/types_v2.rs b/control_plane/src/types_v2.rs index 86b18ae8..0797c118 100644 --- a/control_plane/src/types_v2.rs +++ b/control_plane/src/types_v2.rs @@ -17,8 +17,7 @@ //! an L4 rule engine to pivot on `AccuracyTarget` and a stage allocator //! that respects `QueryShape::Streaming`. -// Several types in this module (`BindingName`, `WorkloadPlan`, -// `QueryExprPlaceholder`, the `new` / `as_str` helpers on `QueryId` +// Several types in this module (`BindingName`, the `new` / `as_str` helpers on `QueryId` // and `BindingName`) are intentionally part of the public surface but // have no in-tree consumers yet — they're targets for the downstream // PR that wires the planner to consume the typed schema. Suppress the @@ -201,47 +200,6 @@ impl std::fmt::Display for BindingName { // ── WorkloadPlan ────────────────────────────────────────────────────────────── -/// Multi-root DAG container, one level above `QueryExpr` (`design.md` §6 -/// "Multi-root DAGs live one level above `QueryExpr`"). `QueryExpr` -/// stays single-root; this struct holds N roots plus the hoisted -/// bindings the CSE pass shares between them. -/// -/// **Container only — no CSE pass yet.** This type is defined so the -/// `analyzer::QuerySpec.id` field has a target to dock against and so -/// the downstream planner can grow into a `WorkloadPlan` consumer -/// without another schema rev. The `bindings` and `roots` payloads use -/// `String` placeholders for the `QueryExpr` slot; the real `QueryExpr` -/// from `algebra/expr.rs` lacks `Serialize` today, and the design's L3 -/// rewrites — `LetBinding` / `Ref` / sketch-binding split — haven't -/// landed in `algebra/`. When they do, the placeholder becomes a -/// `QueryExpr` and the surrounding plumbing stays put. -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -pub struct WorkloadPlan { - /// Named shared producers, hoisted out of individual queries by - /// the CSE pass. Each binding is referenced by ≥2 roots via - /// `QueryExpr::Ref` once that lowering exists. - pub bindings: Vec<(BindingName, QueryExprPlaceholder)>, - /// One root per `QuerySpec` in the workload, in input order. - pub roots: Vec<(QueryId, QueryExprPlaceholder)>, -} - -/// Placeholder for `QueryExpr` until `algebra::expr::QueryExpr` gets a -/// `Serialize` impl + the L3 rewrites that `WorkloadPlan` consumers -/// expect (CTE-style `LetBinding` / `Ref`, sketch-binding split). Today -/// it's a string carrying the source-level query text or a debug -/// `format!("{qe:?}")` of the algebra tree — enough for the control plane -/// to round-trip a `WorkloadPlan` through JSON without losing identity, -/// but not yet enough for L4 to consume. -#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)] -#[serde(transparent)] -pub struct QueryExprPlaceholder(pub String); - -impl QueryExprPlaceholder { - pub fn new(text: impl Into) -> Self { - QueryExprPlaceholder(text.into()) - } -} - // ── Tests ───────────────────────────────────────────────────────────────────── #[cfg(test)] @@ -337,16 +295,4 @@ mod tests { assert_eq!(id, back); assert_eq!(id.as_str(), "metric-x@0.99"); } - - #[test] - fn workload_plan_default_is_empty() { - let wp = WorkloadPlan::default(); - assert!(wp.bindings.is_empty()); - assert!(wp.roots.is_empty()); - // Round-trip an empty WorkloadPlan as JSON. - let json = serde_json::to_string(&wp).unwrap(); - let back: WorkloadPlan = serde_json::from_str(&json).unwrap(); - assert!(back.bindings.is_empty()); - assert!(back.roots.is_empty()); - } } diff --git a/crates/asap_types/src/accumulator_spec.rs b/crates/asap_types/src/accumulator_spec.rs index e57f23c2..3c9e4071 100644 --- a/crates/asap_types/src/accumulator_spec.rs +++ b/crates/asap_types/src/accumulator_spec.rs @@ -1,15 +1,8 @@ -//! `AccumulatorSpec` — data_plane's replacement for the -//! `AggregationType` + `aggregation_sub_type: String` + untyped -//! `parameters: HashMap` triple. +//! Typed accumulator dispatch derived from legacy streaming config. //! -//! **Step 5 of the sketch-identity unification** (see -//! `scratchpad/artifacts/enum-unification-plan.md`, §7-8). Converges -//! accumulator *identity* onto ASAPController's `planner_types::post_asap::SummaryKind` -//! / `SummaryParams` — the same representation `control_plane` already -//! uses as of Stage 3 (merged) — extended with the one axis that -//! representation doesn't have: keyed-vs-unkeyed grouping, which -//! `AggregationType` wrongly folded into identity (`Sum` vs -//! `MultipleSum`, etc.) instead of modeling as a sibling field. +//! The semantic identity is ASAPPlanner's [`SummaryFamilyType`]. This module +//! only adds the backend execution concern of keyed versus unkeyed state and +//! adapts the stable legacy wire fields into that canonical representation. //! //! ## This is an additive representation, not a replacement (yet) //! @@ -46,37 +39,18 @@ //! unaffected — nothing here changes how `AggregationConfig::from_yaml` //! / `from_json` parse or how `serialize_to_json` emits. //! -//! ## What doesn't fit `SummaryKind`/`SummaryParams` +//! Backend-specific execution details remain deliberately separate: //! -//! `SummaryKind`/`SummaryParams` were ASAPController's (now ASAPPlanner's) -//! types, not ours to extend from this repo, until ASAPPlanner split them -//! into a per-family `(ExactKind, SketchKind, SamplingKind, ...)` union -//! (ASAPPlanner#218) with no single flat type spanning both exact and -//! approximate accumulator identity anymore — see -//! `control_plane/docs/design-asapplanner-pin-migration.md`. This -//! module's own `AccumulatorSpec::kind`/`params` dispatch (below, and its -//! ~25 call sites in `data_plane::precompute_engine::accumulator_factory`) -//! never needed that pre-ASAP/post-ASAP distinction in the first place — -//! it's a flat "which concrete Rust accumulator struct to construct" -//! question, entirely internal to this workspace. Rather than thread a -//! two-level `Exact(ExactKind) | Sketch(SketchKind)` wrapper through every -//! one of those call sites for a distinction they don't care about, -//! `SummaryKind`/`SummaryParams` are now vendored here as local types, -//! same 14-variant shape as before the split (mirrors the same call this -//! workspace made for `WindowKind` — see `enums.rs`). -//! -//! Two data_plane-specific details don't fit them: -//! -//! - **Min/max direction.** `SummaryParams::MinMax` carries no fields — +//! - **Min/max direction.** Planner's `ExactParams::MinMax` carries no fields — //! upstream doesn't model a direction axis. `accumulator_factory.rs` //! keeps reading `AggregationConfig::aggregation_sub_type` directly //! for this one bit (`eq_ignore_ascii_case("max")`), exactly as it did //! before this refactor. -//! - **HydraKLL's `(row, col)` tiling.** `SummaryParams::Kll` carries +//! - **HydraKLL's `(row, col)` tiling.** `SketchParams::Kll` carries //! only `k` — upstream has no concept of the CMS-like grid-of-KLL-cells //! layout `HydraKllSketchAccumulator` uses to parallelize a keyed KLL //! across many populations. `accumulator_factory.rs` calls -//! [`cms_params`] directly for the `(SummaryKind::Kll, keyed=true)` +//! [`cms_params`] directly for keyed KLL execution //! arm, same extraction the plain CMS arms use, because `w`/`d` are //! genuinely the same wire keys for both. //! - **Top-k ranking mode (`weight_mode`).** Not a sketch structural @@ -89,260 +63,10 @@ use serde_json::Value; use crate::aggregation_config::AggregationConfig; use crate::key_by_label_names::KeyByLabelNames; use crate::AggregationType; - -/// Which accumulator family to run — identity only (no keyed/unkeyed -/// axis, no heap-vs-bare ambiguity: heap-bearing sketches are their own -/// variant, e.g. `CmsWithHeap` vs `Cms`). Vendored (see module doc): same -/// 14-variant shape ASAPController's pre-split `asap_sketch::SummaryKind` -/// had. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum SummaryKind { - Sum, - Count, - MinMax, - Increase, - Rate, - Kll, - Cms, - Hll, - DDSketch, - CmsWithHeap, - Kmv, - Theta, - CountSketch, - CountSketchWithHeap, -} - -impl SummaryKind { - /// True for the exact, zero-error mergeable-accumulator kinds (no - /// tuning parameters, upstream's old `ExactKind` set); false for the - /// approximate sketch families (upstream's old `SketchKind` set). - pub fn is_exact(&self) -> bool { - matches!( - self, - SummaryKind::Sum - | SummaryKind::Count - | SummaryKind::MinMax - | SummaryKind::Increase - | SummaryKind::Rate - ) - } -} - -/// Typed tuning parameters matching a [`SummaryKind`]. Vendored -/// alongside it (see module doc) — same shape as ASAPController's -/// pre-split `asap_sketch::SummaryParams`. -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum SummaryParams { - Sum, - Count, - MinMax, - Increase, - Rate, - Kll { - k: u32, - }, - Cms { - width: u32, - depth: u32, - }, - Hll { - precision: u8, - }, - DDSketch { - alpha: f64, - }, - CmsWithHeap { - width: u32, - depth: u32, - heap_size: u32, - }, - Kmv { - k: u32, - }, - Theta { - k: u32, - }, - CountSketch { - width: u32, - depth: u32, - }, - CountSketchWithHeap { - width: u32, - depth: u32, - heap_size: u32, - }, -} - -/// Widen an upstream (post-ASAPPlanner#218) `ExactKind` into this crate's -/// flat [`SummaryKind`]. See the module doc for why the flat type exists. -impl From for SummaryKind { - fn from(k: planner_types::post_asap::ExactKind) -> Self { - use planner_types::post_asap::ExactKind as K; - match k { - K::Sum => SummaryKind::Sum, - K::Count => SummaryKind::Count, - K::MinMax => SummaryKind::MinMax, - K::Increase => SummaryKind::Increase, - K::Rate => SummaryKind::Rate, - } - } -} - -impl From for SummaryParams { - fn from(p: planner_types::post_asap::ExactParams) -> Self { - use planner_types::post_asap::ExactParams as P; - match p { - P::Sum => SummaryParams::Sum, - P::Count => SummaryParams::Count, - P::MinMax => SummaryParams::MinMax, - P::Increase => SummaryParams::Increase, - P::Rate => SummaryParams::Rate, - } - } -} - -/// Widen an upstream (post-ASAPPlanner#218) `SketchKind` -- always a real -/// approximate sketch, never exact -- into this crate's flat -/// [`SummaryKind`]. Used at the boundary where a wire/config type needs -/// to represent both exact and approximate accumulators in one field -/// (e.g. `control_plane`'s `BackendAggregation`) but the value in hand is -/// known-sketch. See the module doc for why the flat type exists. -impl From for SummaryKind { - fn from(k: planner_types::post_asap::SketchKind) -> Self { - use planner_types::post_asap::SketchAlgorithm as K; - match k.algorithm() { - K::Kll => SummaryKind::Kll, - K::Cms => SummaryKind::Cms, - K::Hll => SummaryKind::Hll, - K::DDSketch => SummaryKind::DDSketch, - K::CmsWithHeap => SummaryKind::CmsWithHeap, - K::Kmv => SummaryKind::Kmv, - K::Theta => SummaryKind::Theta, - K::CountSketch => SummaryKind::CountSketch, - K::CountSketchWithHeap => SummaryKind::CountSketchWithHeap, - } - } -} - -impl From for SummaryKind { - fn from(k: planner_types::post_asap::SketchAlgorithm) -> Self { - use planner_types::post_asap::SketchAlgorithm as K; - match k { - K::Kll => SummaryKind::Kll, - K::Cms => SummaryKind::Cms, - K::Hll => SummaryKind::Hll, - K::DDSketch => SummaryKind::DDSketch, - K::CmsWithHeap => SummaryKind::CmsWithHeap, - K::Kmv => SummaryKind::Kmv, - K::Theta => SummaryKind::Theta, - K::CountSketch => SummaryKind::CountSketch, - K::CountSketchWithHeap => SummaryKind::CountSketchWithHeap, - } - } -} - -/// Narrow this crate's flat [`SummaryKind`] back down to an upstream -/// `SketchKind`, when it identifies a real approximate sketch. `None` for -/// the exact-accumulator variants (Sum/Count/MinMax/Increase/Rate), -/// which have no `SketchKind` equivalent -- the inverse of the widening -/// [`From`] impl above, fallible because that direction isn't total. -impl SummaryKind { - pub fn as_sketch_kind(&self) -> Option { - use planner_types::post_asap::SketchAlgorithm as K; - Some(match self { - SummaryKind::Kll => K::Kll, - SummaryKind::Cms => K::Cms, - SummaryKind::Hll => K::Hll, - SummaryKind::DDSketch => K::DDSketch, - SummaryKind::CmsWithHeap => K::CmsWithHeap, - SummaryKind::Kmv => K::Kmv, - SummaryKind::Theta => K::Theta, - SummaryKind::CountSketch => K::CountSketch, - SummaryKind::CountSketchWithHeap => K::CountSketchWithHeap, - SummaryKind::Sum - | SummaryKind::Count - | SummaryKind::MinMax - | SummaryKind::Increase - | SummaryKind::Rate => return None, - }) - } -} - -/// Same narrowing as [`SummaryKind::as_sketch_kind`], for the paired -/// params. `None` whenever `self` isn't a sketch-family variant. -impl SummaryParams { - pub fn as_sketch_params(&self) -> Option { - use planner_types::post_asap::SketchParams as P; - Some(match self.clone() { - SummaryParams::Kll { k } => P::Kll { k }, - SummaryParams::Cms { width, depth } => P::Cms { width, depth }, - SummaryParams::Hll { precision } => P::Hll { precision }, - SummaryParams::DDSketch { alpha } => P::DDSketch { alpha }, - SummaryParams::CmsWithHeap { - width, - depth, - heap_size, - } => P::CmsWithHeap { - width, - depth, - heap_size, - }, - SummaryParams::Kmv { k } => P::Kmv { k }, - SummaryParams::Theta { k } => P::Theta { k }, - SummaryParams::CountSketch { width, depth } => P::CountSketch { width, depth }, - SummaryParams::CountSketchWithHeap { - width, - depth, - heap_size, - } => P::CountSketchWithHeap { - width, - depth, - heap_size, - }, - SummaryParams::Sum - | SummaryParams::Count - | SummaryParams::MinMax - | SummaryParams::Increase - | SummaryParams::Rate => return None, - }) - } -} - -impl From for SummaryParams { - fn from(p: planner_types::post_asap::SketchParams) -> Self { - use planner_types::post_asap::SketchParams as P; - match p { - P::Kll { k } => SummaryParams::Kll { k }, - P::Cms { width, depth } => SummaryParams::Cms { width, depth }, - P::Hll { precision } => SummaryParams::Hll { precision }, - P::DDSketch { alpha } => SummaryParams::DDSketch { alpha }, - P::CmsWithHeap { - width, - depth, - heap_size, - } => SummaryParams::CmsWithHeap { - width, - depth, - heap_size, - }, - P::Kmv { k } => SummaryParams::Kmv { k }, - P::Theta { k } => SummaryParams::Theta { k }, - P::CountSketch { width, depth } => SummaryParams::CountSketch { width, depth }, - P::CountSketchWithHeap { - width, - depth, - heap_size, - } => SummaryParams::CountSketchWithHeap { - width, - depth, - heap_size, - }, - } - } -} +use planner_types::post_asap::{ + ExactKind, ExactParams, GroupingStrategy, HydraKind, HydraParams, SketchAlgorithm, SketchKind, + SketchParams, SummaryFamilyType, +}; /// Data_plane's typed replacement for /// `(aggregation_type, aggregation_sub_type, parameters)`: which @@ -355,15 +79,10 @@ impl From for SummaryParams { /// feed [`crate::policy_fingerprint::PolicyFingerprint`]. #[derive(Debug, Clone, PartialEq)] pub struct AccumulatorSpec { - /// Which accumulator family — identity only (no keyed/unkeyed axis, - /// no heap-vs-bare ambiguity: heap-bearing sketches are their own - /// `SummaryKind` variant, e.g. `CmsWithHeap` vs `Cms`). - pub kind: SummaryKind, - /// Typed tuning parameters matching `kind` (no `HashMap` lookups — - /// see the module doc for the handful of details that still need - /// one, kept in `accumulator_factory.rs` since `SummaryParams` has - /// no field for them). - pub params: SummaryParams, + /// Planner-owned committed summary identity. For sketches this is a + /// validated `SketchKind` (category + algorithm + params), following the + /// ASAP-aware-mapping vocabulary. + pub family: SummaryFamilyType, /// `Some(labels)` for a keyed (multi-population) accumulator, /// `None` for a single-population one. This is the axis /// `AggregationType` wrongly folded into identity (`Sum` vs @@ -390,8 +109,8 @@ pub enum AccumulatorSpecError { /// than the `SingleSubpopulation` case). UnknownMultipleSubpopulationSubType(String), /// `aggregation_type` itself has no accumulator-dispatch mapping. - /// Today this is only ever `AggregationType::HLL` — it's a real - /// `SummaryKind::Hll` identity and `control_plane` can emit + /// Today this is only ever `AggregationType::HLL` — it maps to the real + /// `SketchAlgorithm::Hll` identity and `control_plane` can emit /// `aggregationType: HLL` on the wire, but /// `accumulator_factory::create_accumulator_updater` never grew a /// real HLL arm (HLL accumulators are built via the SketchEnvelope @@ -439,36 +158,72 @@ impl AggregationConfig { let sub_type = self.aggregation_sub_type.as_str(); - let (kind, params, keyed): (SummaryKind, SummaryParams, bool) = match self.aggregation_type - { - Sum => (SummaryKind::Sum, SummaryParams::Sum, false), - Increase => (SummaryKind::Increase, SummaryParams::Increase, false), - MinMax => (SummaryKind::MinMax, SummaryParams::MinMax, false), + let independent_sketch = |algorithm, params| { + SummaryFamilyType::Sketch( + SketchKind::new(algorithm, params), + GroupingStrategy::PerSubpopulationInstance, + ) + }; + let (family, keyed) = match self.aggregation_type { + Sum => ( + SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + false, + ), + Increase => ( + SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), + false, + ), + MinMax => ( + SummaryFamilyType::ExactAggregate(ExactKind::MinMax, ExactParams::MinMax), + false, + ), DatasketchesKLL => ( - SummaryKind::Kll, - SummaryParams::Kll { - k: kll_k_param(self) as u32, - }, + independent_sketch( + SketchAlgorithm::Kll, + SketchParams::Kll { + k: kll_k_param(self) as u32, + }, + ), false, ), - MultipleSum => (SummaryKind::Sum, SummaryParams::Sum, true), - MultipleIncrease => (SummaryKind::Increase, SummaryParams::Increase, true), - MultipleMinMax => (SummaryKind::MinMax, SummaryParams::MinMax, true), - HydraKLL => ( - SummaryKind::Kll, - SummaryParams::Kll { - k: kll_k_param(self) as u32, - }, + MultipleSum => ( + SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), true, ), + MultipleIncrease => ( + SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), + true, + ), + MultipleMinMax => ( + SummaryFamilyType::ExactAggregate(ExactKind::MinMax, ExactParams::MinMax), + true, + ), + HydraKLL => { + let k = kll_k_param(self) as u32; + ( + SummaryFamilyType::Sketch( + SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k }), + GroupingStrategy::SharedMultiSubpopulation { + kind: HydraKind::HydraKll, + params: HydraParams::HydraKll { + k, + shared_buckets: k, + }, + }, + ), + true, + ) + } CountMinSketch => { let (row_num, col_num) = cms_params(self); ( - SummaryKind::Cms, - SummaryParams::Cms { - width: col_num as u32, - depth: row_num as u32, - }, + independent_sketch( + SketchAlgorithm::Cms, + SketchParams::Cms { + width: col_num as u32, + depth: row_num as u32, + }, + ), true, ) } @@ -476,23 +231,27 @@ impl AggregationConfig { let (row_num, col_num) = cms_params(self); let heap_size = heap_size_param(self); ( - SummaryKind::CmsWithHeap, - SummaryParams::CmsWithHeap { - width: col_num as u32, - depth: row_num as u32, - heap_size: heap_size as u32, - }, + independent_sketch( + SketchAlgorithm::CmsWithHeap, + SketchParams::CmsWithHeap { + width: col_num as u32, + depth: row_num as u32, + heap_size: heap_size as u32, + }, + ), true, ) } CountSketch => { let (row_num, col_num) = cms_params(self); ( - SummaryKind::CountSketch, - SummaryParams::CountSketch { - width: col_num as u32, - depth: row_num as u32, - }, + independent_sketch( + SketchAlgorithm::CountSketch, + SketchParams::CountSketch { + width: col_num as u32, + depth: row_num as u32, + }, + ), true, ) } @@ -500,33 +259,47 @@ impl AggregationConfig { let (row_num, col_num) = cms_params(self); let heap_size = heap_size_param(self); ( - SummaryKind::CountSketchWithHeap, - SummaryParams::CountSketchWithHeap { - width: col_num as u32, - depth: row_num as u32, - heap_size: heap_size as u32, - }, + independent_sketch( + SketchAlgorithm::CountSketchWithHeap, + SketchParams::CountSketchWithHeap { + width: col_num as u32, + depth: row_num as u32, + heap_size: heap_size as u32, + }, + ), true, ) } DDSketch => ( - SummaryKind::DDSketch, - SummaryParams::DDSketch { - alpha: ddsketch_alpha_param(self), - }, + independent_sketch( + SketchAlgorithm::DDSketch, + SketchParams::DDSketch { + alpha: ddsketch_alpha_param(self), + }, + ), false, ), HLL => return Err(AccumulatorSpecError::UnmappedAggregationType(HLL)), SingleSubpopulation => match sub_type { - "Sum" | "sum" => (SummaryKind::Sum, SummaryParams::Sum, false), - "Min" | "min" => (SummaryKind::MinMax, SummaryParams::MinMax, false), - "Max" | "max" => (SummaryKind::MinMax, SummaryParams::MinMax, false), - "Increase" | "increase" => (SummaryKind::Increase, SummaryParams::Increase, false), + "Sum" | "sum" => ( + SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + false, + ), + "Min" | "min" | "Max" | "max" => ( + SummaryFamilyType::ExactAggregate(ExactKind::MinMax, ExactParams::MinMax), + false, + ), + "Increase" | "increase" => ( + SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), + false, + ), "DatasketchesKLL" | "datasketches_kll" | "KLL" | "kll" => ( - SummaryKind::Kll, - SummaryParams::Kll { - k: kll_k_param(self) as u32, - }, + independent_sketch( + SketchAlgorithm::Kll, + SketchParams::Kll { + k: kll_k_param(self) as u32, + }, + ), false, ), other => { @@ -536,26 +309,38 @@ impl AggregationConfig { } }, MultipleSubpopulation => match sub_type { - "Sum" | "sum" => (SummaryKind::Sum, SummaryParams::Sum, true), - "Min" | "min" => (SummaryKind::MinMax, SummaryParams::MinMax, true), - "Max" | "max" => (SummaryKind::MinMax, SummaryParams::MinMax, true), - "Increase" | "increase" => (SummaryKind::Increase, SummaryParams::Increase, true), + "Sum" | "sum" => ( + SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + true, + ), + "Min" | "min" | "Max" | "max" => ( + SummaryFamilyType::ExactAggregate(ExactKind::MinMax, ExactParams::MinMax), + true, + ), + "Increase" | "increase" => ( + SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), + true, + ), "CountMinSketch" | "count_min_sketch" | "CMS" | "cms" => { let (row_num, col_num) = cms_params(self); ( - SummaryKind::Cms, - SummaryParams::Cms { - width: col_num as u32, - depth: row_num as u32, - }, + independent_sketch( + SketchAlgorithm::Cms, + SketchParams::Cms { + width: col_num as u32, + depth: row_num as u32, + }, + ), true, ) } "HydraKLL" | "hydra_kll" => ( - SummaryKind::Kll, - SummaryParams::Kll { - k: kll_k_param(self) as u32, - }, + independent_sketch( + SketchAlgorithm::Kll, + SketchParams::Kll { + k: kll_k_param(self) as u32, + }, + ), true, ), other => { @@ -572,11 +357,7 @@ impl AggregationConfig { None }; - Ok(AccumulatorSpec { - kind, - params, - grouping, - }) + Ok(AccumulatorSpec { family, grouping }) } } @@ -668,6 +449,27 @@ mod tests { use crate::key_by_label_names::KeyByLabelNames; use std::collections::HashMap; + fn assert_exact(spec: &AccumulatorSpec, expected: ExactKind) { + assert!(matches!( + &spec.family, + SummaryFamilyType::ExactAggregate(kind, _) if kind == &expected + )); + } + + fn assert_sketch( + spec: &AccumulatorSpec, + expected_algorithm: planner_types::post_asap::SketchAlgorithm, + expected_params: planner_types::post_asap::SketchParams, + ) { + match &spec.family { + SummaryFamilyType::Sketch(kind, _) => { + assert_eq!(kind.algorithm(), &expected_algorithm); + assert_eq!(kind.params(), &expected_params); + } + other => panic!("expected sketch family, got {other:?}"), + } + } + #[allow(clippy::too_many_arguments)] fn make_config( agg_type: AggregationType, @@ -700,8 +502,7 @@ mod tests { fn sum_is_unkeyed_sum() { let cfg = make_config(AggregationType::Sum, "", HashMap::new(), vec![]); let spec = cfg.accumulator_spec().expect("resolves"); - assert_eq!(spec.kind, SummaryKind::Sum); - assert_eq!(spec.params, SummaryParams::Sum); + assert_exact(&spec, ExactKind::Sum); assert!(spec.grouping.is_none()); } @@ -714,7 +515,7 @@ mod tests { vec!["zone"], ); let spec = cfg.accumulator_spec().expect("resolves"); - assert_eq!(spec.kind, SummaryKind::Sum); + assert_exact(&spec, ExactKind::Sum); assert_eq!( spec.grouping, Some(KeyByLabelNames::new(vec!["zone".to_string()])) @@ -727,8 +528,11 @@ mod tests { params.insert("k".to_string(), serde_json::json!(128)); let cfg = make_config(AggregationType::DatasketchesKLL, "", params, vec![]); let spec = cfg.accumulator_spec().expect("resolves"); - assert_eq!(spec.kind, SummaryKind::Kll); - assert_eq!(spec.params, SummaryParams::Kll { k: 128 }); + assert_sketch( + &spec, + planner_types::post_asap::SketchAlgorithm::Kll, + planner_types::post_asap::SketchParams::Kll { k: 128 }, + ); assert!(spec.grouping.is_none()); } @@ -738,8 +542,11 @@ mod tests { params.insert("k".to_string(), serde_json::json!(64)); let cfg = make_config(AggregationType::HydraKLL, "", params, vec!["host"]); let spec = cfg.accumulator_spec().expect("resolves"); - assert_eq!(spec.kind, SummaryKind::Kll); - assert_eq!(spec.params, SummaryParams::Kll { k: 64 }); + assert_sketch( + &spec, + planner_types::post_asap::SketchAlgorithm::Kll, + planner_types::post_asap::SketchParams::Kll { k: 64 }, + ); assert!(spec.grouping.is_some()); } @@ -750,13 +557,13 @@ mod tests { params.insert("w".to_string(), serde_json::json!(2048)); let cfg = make_config(AggregationType::CountMinSketch, "", params, vec!["host"]); let spec = cfg.accumulator_spec().expect("resolves"); - assert_eq!(spec.kind, SummaryKind::Cms); - assert_eq!( - spec.params, - SummaryParams::Cms { + assert_sketch( + &spec, + planner_types::post_asap::SketchAlgorithm::Cms, + planner_types::post_asap::SketchParams::Cms { width: 2048, - depth: 7 - } + depth: 7, + }, ); assert!(spec.grouping.is_some()); } @@ -774,19 +581,19 @@ mod tests { vec!["host"], ); let spec = cfg.accumulator_spec().expect("resolves"); - assert_eq!(spec.kind, SummaryKind::CmsWithHeap); - assert_eq!( - spec.params, - SummaryParams::CmsWithHeap { + assert_sketch( + &spec, + planner_types::post_asap::SketchAlgorithm::CmsWithHeap, + planner_types::post_asap::SketchParams::CmsWithHeap { width: 256, depth: 4, - heap_size: 8 - } + heap_size: 8, + }, ); } /// Documented existing quirk (see `accumulator_factory.rs`): bare - /// `CountSketch` gets its own `SummaryKind` identity here, but + /// `CountSketch` gets its own `SketchAlgorithm` identity here, but /// `accumulator_factory` routes it through the same /// `CmsAccumulatorUpdater` as bare CMS — no dedicated heap-less /// Count-Sketch accumulator exists. This test locks in the *identity* @@ -796,7 +603,11 @@ mod tests { fn count_sketch_gets_its_own_kind_identity() { let cfg = make_config(AggregationType::CountSketch, "", HashMap::new(), vec!["h"]); let spec = cfg.accumulator_spec().expect("resolves"); - assert_eq!(spec.kind, SummaryKind::CountSketch); + assert!(matches!( + &spec.family, + SummaryFamilyType::Sketch(kind, _) + if kind.algorithm() == &planner_types::post_asap::SketchAlgorithm::CountSketch + )); } #[test] @@ -805,8 +616,11 @@ mod tests { params.insert("relativeAccuracy".to_string(), serde_json::json!(0.02)); let cfg = make_config(AggregationType::DDSketch, "", params, vec![]); let spec = cfg.accumulator_spec().expect("resolves"); - assert_eq!(spec.kind, SummaryKind::DDSketch); - assert_eq!(spec.params, SummaryParams::DDSketch { alpha: 0.02 }); + assert_sketch( + &spec, + planner_types::post_asap::SketchAlgorithm::DDSketch, + planner_types::post_asap::SketchParams::DDSketch { alpha: 0.02 }, + ); assert!(spec.grouping.is_none()); } @@ -832,7 +646,7 @@ mod tests { vec![], ); let spec = cfg.accumulator_spec().expect("resolves"); - assert_eq!(spec.kind, SummaryKind::Sum); + assert_exact(&spec, ExactKind::Sum); assert!(spec.grouping.is_none()); } } @@ -847,7 +661,11 @@ mod tests { vec!["host"], ); let spec = cfg.accumulator_spec().expect("resolves"); - assert_eq!(spec.kind, SummaryKind::Cms); + assert!(matches!( + &spec.family, + SummaryFamilyType::Sketch(kind, _) + if kind.algorithm() == &planner_types::post_asap::SketchAlgorithm::Cms + )); assert!(spec.grouping.is_some()); } } @@ -862,7 +680,11 @@ mod tests { vec!["host"], ); let spec = cfg.accumulator_spec().expect("resolves"); - assert_eq!(spec.kind, SummaryKind::Kll); + assert!(matches!( + &spec.family, + SummaryFamilyType::Sketch(kind, _) + if kind.algorithm() == &planner_types::post_asap::SketchAlgorithm::Kll + )); assert!(spec.grouping.is_some()); } } diff --git a/crates/asap_types/src/lib.rs b/crates/asap_types/src/lib.rs index 8b14c401..4b8b2054 100644 --- a/crates/asap_types/src/lib.rs +++ b/crates/asap_types/src/lib.rs @@ -8,16 +8,18 @@ pub mod policy_fingerprint; pub mod policy_registry; pub mod query_requirements; pub mod routing_index; +pub mod storage_backend; pub mod traits; pub mod utils; -pub use accumulator_spec::{AccumulatorSpec, AccumulatorSpecError, SummaryKind, SummaryParams}; +pub use accumulator_spec::{AccumulatorSpec, AccumulatorSpecError}; pub use aggregation_config::*; pub use aggregation_type::AggregationType; pub use enums::*; pub use key_by_label_names::KeyByLabelNames; -pub use monitor_spec::MonitorSpec; +pub use monitor_spec::{MonitorFunctional, MonitorSpec}; pub use policy_fingerprint::PolicyFingerprint; pub use policy_registry::PolicyRegistry; pub use query_requirements::*; pub use routing_index::RoutingIndex; +pub use storage_backend::*; diff --git a/crates/asap_types/src/monitor_spec.rs b/crates/asap_types/src/monitor_spec.rs index 2300ca12..535ec3ef 100644 --- a/crates/asap_types/src/monitor_spec.rs +++ b/crates/asap_types/src/monitor_spec.rs @@ -1,5 +1,36 @@ use serde::{Deserialize, Serialize}; +/// Continuous-monitoring readout shared by the control and data planes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum MonitorFunctional { + #[default] + Sum, + CmsPoint, + LinearBuckets, + F2, +} + +impl MonitorFunctional { + pub const fn as_str(self) -> &'static str { + match self { + Self::Sum => "sum", + Self::CmsPoint => "cms_point", + Self::LinearBuckets => "linear_buckets", + Self::F2 => "f2", + } + } + + pub fn from_name(value: &str) -> Self { + match value.trim().to_ascii_lowercase().as_str() { + "cms_point" | "cms" => Self::CmsPoint, + "linear_buckets" | "linear" => Self::LinearBuckets, + "f2" | "l2" => Self::F2, + _ => Self::Sum, + } + } +} + /// One continuous-monitoring (CDM) threshold spec. The data-plane monitor /// coordinator owns the AUTHORITATIVE `tau`/`epsilon`/`window_ms` (the edge /// copy is advisory), keyed by the same content-addressed `agg_id` the edge and diff --git a/crates/asap_types/src/storage_backend.rs b/crates/asap_types/src/storage_backend.rs new file mode 100644 index 00000000..92818ba4 --- /dev/null +++ b/crates/asap_types/src/storage_backend.rs @@ -0,0 +1,37 @@ +use serde::{Deserialize, Serialize}; + +pub const ENGINE_ID_ASAP_QUERY: &str = "asap_query"; +pub const ENGINE_ID_THANOS_QUERY: &str = "thanos_query"; +pub const CANONICAL_QUERY_ENGINE_IDS: &[&str] = &[ENGINE_ID_ASAP_QUERY, ENGINE_ID_THANOS_QUERY]; + +/// Backend-owned physical storage target shared by both runtime planes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum StorageBackend { + #[default] + SketchStore, + GorillaObjectStore, + DoubleWrite, + PrometheusRemote, +} + +impl StorageBackend { + pub const fn data_source_id(self) -> &'static str { + match self { + Self::SketchStore => ENGINE_ID_ASAP_QUERY, + Self::GorillaObjectStore => ENGINE_ID_THANOS_QUERY, + Self::DoubleWrite => "double_write", + Self::PrometheusRemote => "prometheus_remote", + } + } +} + +pub fn parse_storage_backend_engine_id(value: &str) -> Option { + match value { + ENGINE_ID_ASAP_QUERY => Some(StorageBackend::SketchStore), + ENGINE_ID_THANOS_QUERY => Some(StorageBackend::GorillaObjectStore), + "double_write" => Some(StorageBackend::DoubleWrite), + "prometheus_remote" => Some(StorageBackend::PrometheusRemote), + _ => None, + } +} diff --git a/data_plane/benches/sketch_db.rs b/data_plane/benches/sketch_db.rs index 845e574f..78c770cb 100644 --- a/data_plane/benches/sketch_db.rs +++ b/data_plane/benches/sketch_db.rs @@ -35,8 +35,8 @@ use prost::Message; use data_plane::precompute_engine::operators::SumAccumulator; use data_plane::storage_engines::sketch_db::data::{ - AccuracyBound, AggKind, AggregationType, Capability, SketchConfig, SketchEncoding, - SketchKindHandle, + AccuracyBound, AggKind, AggregationType, Capability, SketchAlgorithm, SketchConfig, + SketchEncoding, }; use data_plane::storage_engines::sketch_db::index::{SketchInstanceMetadata, SketchStore}; use data_plane::storage_engines::SketchSampleState; @@ -105,18 +105,22 @@ fn sample(bytes: Vec) -> SketchSampleState { // ── Metadata builders ─────────────────────────────────────────────────────── -fn sketch_meta(sid: u64, kind: SketchKindHandle, config: SketchConfig) -> SketchInstanceMetadata { +fn sketch_meta( + sid: u64, + algorithm: SketchAlgorithm, + config: SketchConfig, +) -> SketchInstanceMetadata { SketchInstanceMetadata { sid, metric_name: "bench_metric".into(), group_by_keys: BTreeSet::new(), - capability: Some(match kind { - SketchKindHandle::Hll => Capability::CardinalityApprox, - _ => Capability::QuantileApprox(kind), + capability: Some(match algorithm { + SketchAlgorithm::Hll => Capability::CardinalityApprox, + _ => Capability::QuantileApprox(Some(algorithm.clone())), }), accuracy: Some(AccuracyBound::from_config(&config)), agg_kind: AggKind::Sketch { - kind, + algorithm, config, spatial_filter_canonical: String::new(), }, @@ -146,10 +150,10 @@ fn precompute_meta(sid: u64, metric: &str, agg_type: AggregationType) -> SketchI } } -#[derive(Clone, Copy)] +#[derive(Clone)] struct KindCase { name: &'static str, - kind: SketchKindHandle, + algorithm: SketchAlgorithm, build: fn() -> (SketchConfig, Vec), } @@ -176,17 +180,17 @@ fn hll_small() -> (SketchConfig, Vec) { const KINDS: &[KindCase] = &[ KindCase { name: "DDSketch", - kind: SketchKindHandle::DDSketch, + algorithm: SketchAlgorithm::DDSketch, build: dd_small, }, KindCase { name: "KLL", - kind: SketchKindHandle::Kll, + algorithm: SketchAlgorithm::Kll, build: kll_small, }, KindCase { name: "HLL", - kind: SketchKindHandle::Hll, + algorithm: SketchAlgorithm::Hll, build: hll_small, }, ]; @@ -215,7 +219,11 @@ fn bench_append_sample(c: &mut Criterion) { || { let store = SketchStore::new(); for sid in 0..num_sids as u64 { - store.register(sketch_meta(sid + 1, kind.kind, cfg.clone())); + store.register(sketch_meta( + sid + 1, + kind.algorithm.clone(), + cfg.clone(), + )); } (store, 0u64) }, @@ -284,10 +292,10 @@ fn bench_append_precompute(c: &mut Criterion) { // ── query_range ───────────────────────────────────────────────────────────── -fn build_populated_store(depth: u64, kind: KindCase, sid: u64) -> SketchStore { +fn build_populated_store(depth: u64, kind: &KindCase, sid: u64) -> SketchStore { let store = SketchStore::new(); let (cfg, payload_bytes) = (kind.build)(); - store.register(sketch_meta(sid, kind.kind, cfg)); + store.register(sketch_meta(sid, kind.algorithm.clone(), cfg)); for i in 0..depth { let win = (1_000 + i * 10, 1_000 + i * 10 + 10); store.append_sample(sid, BTreeMap::new(), win, sample(payload_bytes.clone())); @@ -302,7 +310,7 @@ fn bench_query_range(c: &mut Criterion) { let sid = 1u64; for kind in KINDS { for depth in [10u64, 100, 1_000] { - let store = build_populated_store(depth, *kind, sid); + let store = build_populated_store(depth, kind, sid); let full_end = 1_000 + depth * 10 + 10; let cases = [ ("w=1", (1_000u64, 1_010u64)), diff --git a/data_plane/examples/sketch_db_diag.rs b/data_plane/examples/sketch_db_diag.rs index c9b1beb9..fdf27f6b 100644 --- a/data_plane/examples/sketch_db_diag.rs +++ b/data_plane/examples/sketch_db_diag.rs @@ -24,7 +24,7 @@ use asap_sketchlib::DdSketch; use prost::Message; use data_plane::storage_engines::sketch_db::data::{ - AccuracyBound, AggKind, Capability, SketchConfig, SketchEncoding, SketchKindHandle, + AccuracyBound, AggKind, Capability, SketchAlgorithm, SketchConfig, SketchEncoding, SketchSampleState, }; use data_plane::storage_engines::sketch_db::index::{SketchInstanceMetadata, SketchStore}; @@ -54,10 +54,10 @@ fn dd_meta(sid: u64) -> SketchInstanceMetadata { sid, metric_name: "bench_metric".into(), group_by_keys: BTreeSet::new(), - capability: Some(Capability::QuantileApprox(SketchKindHandle::DDSketch)), + capability: Some(Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch))), accuracy: Some(AccuracyBound::from_config(&cfg)), agg_kind: AggKind::Sketch { - kind: SketchKindHandle::DDSketch, + algorithm: SketchAlgorithm::DDSketch, config: cfg, spatial_filter_canonical: String::new(), }, diff --git a/data_plane/src/drivers/ingest/otel.rs b/data_plane/src/drivers/ingest/otel.rs index 0160f6db..1c4f1fd0 100644 --- a/data_plane/src/drivers/ingest/otel.rs +++ b/data_plane/src/drivers/ingest/otel.rs @@ -39,6 +39,7 @@ use asap_sketchlib::proto::sketchlib::{sketch_envelope, SketchEnvelope}; use asap_sketchlib::MessagePackCodec; use axum::{body::Bytes, extract::State, routing::post, Json, Router}; use flate2::read::GzDecoder; +use planner_types::post_asap::SketchAlgorithm; use prost::Message; use std::sync::Arc; use std::time::Instant; @@ -594,7 +595,7 @@ fn flush_barrier_drops(_state: &IngestState, drops: &HashMap, driver_t /// `reconcile_from_streaming_config` derives from the same config; the /// modified-OTLP first-class sketch path takes a different sid- /// resolution route inside `route_modified_otlp_sketches_to_precompute` -/// because it carries per-DP `(SketchKindHandle, SketchConfig)` and +/// because it carries per-DP `(SketchAlgorithm, SketchConfig)` and /// must distinguish (e.g.) DDSketch vs Kll over the same series. fn resolve_bucket_sid_for_agg_config( ingest_state: &Arc, @@ -948,7 +949,7 @@ async fn route_modified_otlp_sketches_to_precompute( d.data_points .iter() .map(|dp| ModifiedOtlpSketchDp { - kind: SketchKind::DdSketch, + algorithm: SketchAlgorithm::DDSketch, attrs: merge_point_attributes(&base_labels, &dp.attributes), time_unix_nano: dp.time_unix_nano, sketch: dp.sketch.clone(), @@ -965,7 +966,7 @@ async fn route_modified_otlp_sketches_to_precompute( k.data_points .iter() .map(|dp| ModifiedOtlpSketchDp { - kind: SketchKind::Kll, + algorithm: SketchAlgorithm::Kll, attrs: merge_point_attributes(&base_labels, &dp.attributes), time_unix_nano: dp.time_unix_nano, sketch: dp.sketch.clone(), @@ -985,7 +986,7 @@ async fn route_modified_otlp_sketches_to_precompute( c.data_points .iter() .map(|dp| ModifiedOtlpSketchDp { - kind: SketchKind::CountSketch, + algorithm: SketchAlgorithm::CountSketch, attrs: merge_point_attributes(&base_labels, &dp.attributes), time_unix_nano: dp.time_unix_nano, sketch: dp.sketch.clone(), @@ -1005,7 +1006,7 @@ async fn route_modified_otlp_sketches_to_precompute( c.data_points .iter() .map(|dp| ModifiedOtlpSketchDp { - kind: SketchKind::CountMin, + algorithm: SketchAlgorithm::Cms, attrs: merge_point_attributes(&base_labels, &dp.attributes), time_unix_nano: dp.time_unix_nano, sketch: dp.sketch.clone(), @@ -1023,7 +1024,7 @@ async fn route_modified_otlp_sketches_to_precompute( h.data_points .iter() .map(|dp| ModifiedOtlpSketchDp { - kind: SketchKind::Hll, + algorithm: SketchAlgorithm::Hll, attrs: merge_point_attributes(&base_labels, &dp.attributes), time_unix_nano: dp.time_unix_nano, sketch: dp.sketch.clone(), @@ -1052,7 +1053,8 @@ async fn route_modified_otlp_sketches_to_precompute( // determines the suffix for the whole metric. let canonical_name: String = match dps.first() { Some(first) => { - canonical_sketch_metric_name(&metric.name, first.kind).to_string() + canonical_sketch_metric_name(&metric.name, first.algorithm.clone()) + .to_string() } None => metric.name.clone(), }; @@ -1143,7 +1145,7 @@ async fn route_modified_otlp_sketches_to_precompute( // Build the canonical AggKind string for this DP so // the resolver's cache key is `(metric, fp, agg_kind)`. // Two DPs over the same (metric, attrs) but different - // sketch kinds/configs (e.g. DDSketch vs Kll, or two + // sketch algorithms/configs (e.g. DDSketch vs Kll, or two // DDSketches at different relative_accuracy) get // SEPARATE sids — matching the identity model the // retired `compute_sketch_sid` hashed over. For the @@ -1161,9 +1163,9 @@ async fn route_modified_otlp_sketches_to_precompute( // frames would mint distinct sids and the upgrade // could never fire (the analyzer would also see two // candidates for one logical series). - let kind_for_sid = base_sketch_kind_handle(sketch_kind_handle_for(&dp)); + let algorithm_for_sid = base_sketch_algorithm(sketch_algorithm_for(&dp)); let agg_kind = crate::storage_engines::sketch_db::data::AggKind::Sketch { - kind: kind_for_sid, + algorithm: algorithm_for_sid, config: dp.container_config.clone(), // OTel-ingest path: no per-DP spatial filter applies // at this layer (the agent has already filtered @@ -1219,25 +1221,25 @@ async fn route_modified_otlp_sketches_to_precompute( // and its key set IS the group-by KEY set. { use crate::storage_engines::sketch_db::index::{ - AccuracyBound, Capability, SketchEncoding, SketchInstanceMetadata, - SketchKindHandle, SketchSampleState, + AccuracyBound, Capability, SketchAlgorithm, SketchEncoding, + SketchInstanceMetadata, SketchSampleState, }; use std::collections::{BTreeMap, BTreeSet}; if ingest_state.sketch_index.instance(sid).is_none() { - let kind = sketch_kind_handle_for(&dp); - let cap = match kind { - SketchKindHandle::DDSketch | SketchKindHandle::Kll => { - Capability::QuantileApprox(kind) + let algorithm = sketch_algorithm_for(&dp); + let cap = match algorithm { + SketchAlgorithm::DDSketch | SketchAlgorithm::Kll => { + Capability::QuantileApprox(Some(algorithm.clone())) } - SketchKindHandle::Hll => Capability::CardinalityApprox, + SketchAlgorithm::Hll => Capability::CardinalityApprox, // Heap-LESS frequency sketches answer bare // frequency point queries (no top-k); index // them as FrequencyEstimate so a `topk(...)` // query routes to archive (or to a different // sid that carries a heap-bearing variant). - SketchKindHandle::CountSketch | SketchKindHandle::CountMin => { - Capability::FrequencyEstimate(kind) + SketchAlgorithm::CountSketch | SketchAlgorithm::Cms => { + Capability::FrequencyEstimate(Some(algorithm.clone())) } // Heap-BEARING frequency sketches answer // both point-frequency AND top-k. We register @@ -1246,19 +1248,13 @@ async fn route_modified_otlp_sketches_to_precompute( // `is_satisfied_by` for FrequencyEstimate // explicitly accepts heap-bearing variants, // so bare-frequency queries still route here. - SketchKindHandle::CmsWithHeap - | SketchKindHandle::CountSketchWithHeap => { - Capability::FrequencyTopk(kind) + SketchAlgorithm::CmsWithHeap + | SketchAlgorithm::CountSketchWithHeap => { + Capability::FrequencyTopk(Some(algorithm.clone())) + } + SketchAlgorithm::Kmv | SketchAlgorithm::Theta => { + Capability::CardinalityApprox } - // `Any` is the control-plane-side analysis- - // time wildcard — it should never appear - // on the ingest path (which detects a - // concrete sketch kind from the OTLP - // wire variant). Default defensively to - // QuantileApprox so a stray `Any` - // doesn't panic; the analyzer's - // `is_satisfied_by` rejects mismatches. - SketchKindHandle::Any => Capability::QuantileApprox(kind), }; let group_by_keys: BTreeSet = dp.attrs.keys().cloned().collect(); @@ -1280,7 +1276,7 @@ async fn route_modified_otlp_sketches_to_precompute( let policy_fp = derive_sketch_policy_fp( ingest_state, &canonical_name, - kind, + algorithm.clone(), &cfg, &group_by_keys, ); @@ -1308,7 +1304,7 @@ async fn route_modified_otlp_sketches_to_precompute( capability: Some(cap), agg_kind: crate::storage_engines::sketch_db::index::AggKind::Sketch { - kind, + algorithm, config: cfg.clone(), spatial_filter_canonical: String::new(), }, @@ -1333,26 +1329,27 @@ async fn route_modified_otlp_sketches_to_precompute( // top-k queries here. Never downgrades: we only // act when the current cap is heap-LESS frequency // and the incoming frame actually carries a heap. - let incoming_kind = sketch_kind_handle_for(&dp); - let upgrade_to = match (&existing.capability, incoming_kind) { + let incoming_algorithm = sketch_algorithm_for(&dp); + let upgrade_to = match (&existing.capability, incoming_algorithm) { ( - Some(Capability::FrequencyEstimate(SketchKindHandle::CountMin)), - SketchKindHandle::CmsWithHeap, - ) => Some(SketchKindHandle::CmsWithHeap), + Some(Capability::FrequencyEstimate(Some(SketchAlgorithm::Cms))), + SketchAlgorithm::CmsWithHeap, + ) => Some(SketchAlgorithm::CmsWithHeap), ( - Some(Capability::FrequencyEstimate( - SketchKindHandle::CountSketch, - )), - SketchKindHandle::CountSketchWithHeap, - ) => Some(SketchKindHandle::CountSketchWithHeap), + Some(Capability::FrequencyEstimate(Some( + SketchAlgorithm::CountSketch, + ))), + SketchAlgorithm::CountSketchWithHeap, + ) => Some(SketchAlgorithm::CountSketchWithHeap), _ => None, }; - if let Some(new_kind) = upgrade_to { + if let Some(new_algorithm) = upgrade_to { let mut upgraded = existing; - upgraded.capability = Some(Capability::FrequencyTopk(new_kind)); + upgraded.capability = + Some(Capability::FrequencyTopk(Some(new_algorithm.clone()))); upgraded.agg_kind = crate::storage_engines::sketch_db::index::AggKind::Sketch { - kind: new_kind, + algorithm: new_algorithm.clone(), config: dp.container_config.clone(), spatial_filter_canonical: String::new(), }; @@ -1365,8 +1362,8 @@ async fn route_modified_otlp_sketches_to_precompute( "OTLP sketch sid {} upgraded {:?} -> FrequencyTopk({:?}) \ on heap-bearing frame (metric={}, encoding={})", sid, - SketchKindHandle::CountMin, - new_kind, + SketchAlgorithm::Cms, + new_algorithm, metric.name, dp.encoding ); @@ -1449,7 +1446,7 @@ async fn route_modified_otlp_sketches_to_precompute( // unchanged "drop until the next full frame" // behavior. match empty_accumulator_for_delta_bootstrap( - dp.kind, + dp.algorithm.clone(), &dp.container_config, dp.encoding, ) { @@ -1460,7 +1457,7 @@ async fn route_modified_otlp_sketches_to_precompute( accumulator (metric={}, \ series_key={}, kind={:?}, \ encoding={})", - metric.name, series_key, dp.kind, dp.encoding + metric.name, series_key, dp.algorithm, dp.encoding ); // Treat the freshly-minted empty base // as belonging to THIS delta's window @@ -1483,7 +1480,7 @@ async fn route_modified_otlp_sketches_to_precompute( (metric={}, series_key={}, kind={:?}); \ dropping — agent must resend the next full \ frame", - metric.name, series_key, dp.kind + metric.name, series_key, dp.algorithm ); continue; } @@ -1505,7 +1502,7 @@ async fn route_modified_otlp_sketches_to_precompute( merged.reset_to_empty(); } if let Err(e) = apply_modified_otlp_delta_bytes( - dp.kind, + dp.algorithm.clone(), dp.encoding, &mut merged, &dp.sketch, @@ -1521,7 +1518,7 @@ async fn route_modified_otlp_sketches_to_precompute( bytes={}): {} — falling through to §5.2 \ fallback", metric.name, - dp.kind, + dp.algorithm, dp.encoding, dp.sketch.len(), e @@ -1541,7 +1538,11 @@ async fn route_modified_otlp_sketches_to_precompute( ingest_state.note_window_and_sweep(dp.start_time_unix_nano); merged } else { - match decode_modified_otlp_sketch_bytes(dp.kind, dp.encoding, &dp.sketch) { + match decode_modified_otlp_sketch_bytes( + dp.algorithm.clone(), + dp.encoding, + &dp.sketch, + ) { Ok(acc) => { ingest_state.sketch_snapshots.insert( series_key.clone(), @@ -1574,7 +1575,7 @@ async fn route_modified_otlp_sketches_to_precompute( validation (metric={}, kind={:?}, \ encoding={}, bytes={}): {}", metric.name, - dp.kind, + dp.algorithm, dp.encoding, dp.sketch.len(), msg @@ -1586,7 +1587,7 @@ async fn route_modified_otlp_sketches_to_precompute( bytes={}): {} — falling through to §5.2 \ fallback", metric.name, - dp.kind, + dp.algorithm, dp.encoding, dp.sketch.len(), msg @@ -1714,9 +1715,9 @@ async fn route_modified_otlp_sketches_to_precompute( } } -/// Map `SketchKindHandle` to the corresponding wire-format +/// Map `SketchAlgorithm` to the corresponding wire-format /// `AggregationType`. Inverse direction is in -/// `sketch_kind_handle_for` above. Used by +/// `sketch_algorithm_for` above. Used by /// [`derive_sketch_policy_fp`] to find the policy whose /// `AggregationConfig.aggregation_type` matches a freshly-ingested /// sketch. @@ -1724,20 +1725,20 @@ async fn route_modified_otlp_sketches_to_precompute( /// `Any` is a control-plane analysis-time wildcard — it doesn't /// appear on the ingest path. Returns `None` so the policy lookup /// fails the (rare) defensive path explicitly. -fn aggregation_type_for_sketch_handle( - handle: crate::storage_engines::sketch_db::index::SketchKindHandle, +fn aggregation_type_for_sketch_algorithm( + handle: crate::storage_engines::sketch_db::index::SketchAlgorithm, ) -> Option { - use crate::storage_engines::sketch_db::index::SketchKindHandle; + use crate::storage_engines::sketch_db::index::SketchAlgorithm; use asap_types::AggregationType; match handle { - SketchKindHandle::DDSketch => Some(AggregationType::DDSketch), - SketchKindHandle::Kll => Some(AggregationType::DatasketchesKLL), - SketchKindHandle::Hll => Some(AggregationType::HLL), - SketchKindHandle::CountSketch => Some(AggregationType::CountSketch), - SketchKindHandle::CountSketchWithHeap => Some(AggregationType::CountSketchWithHeap), - SketchKindHandle::CountMin => Some(AggregationType::CountMinSketch), - SketchKindHandle::CmsWithHeap => Some(AggregationType::CountMinSketchWithHeap), - SketchKindHandle::Any => None, + SketchAlgorithm::DDSketch => Some(AggregationType::DDSketch), + SketchAlgorithm::Kll => Some(AggregationType::DatasketchesKLL), + SketchAlgorithm::Hll => Some(AggregationType::HLL), + SketchAlgorithm::CountSketch => Some(AggregationType::CountSketch), + SketchAlgorithm::CountSketchWithHeap => Some(AggregationType::CountSketchWithHeap), + SketchAlgorithm::Cms => Some(AggregationType::CountMinSketch), + SketchAlgorithm::CmsWithHeap => Some(AggregationType::CountMinSketchWithHeap), + SketchAlgorithm::Kmv | SketchAlgorithm::Theta => None, } } @@ -1792,7 +1793,7 @@ fn sketch_config_to_params( /// `PolicyRegistry`, and asks `find_policy_by_content` for the /// fingerprint of a policy whose contents match. Returns /// `PolicyFingerprint::UNSET` when: -/// 1. The `SketchKindHandle::Any` wildcard reached this path +/// 1. An unsupported planner algorithm reached this path /// (defensive — shouldn't happen). /// 2. No policy in the registry matches. /// 3. Multiple policies match (would-have-been-a-bug case; @@ -1805,11 +1806,11 @@ fn sketch_config_to_params( fn derive_sketch_policy_fp( ingest_state: &IngestState, metric: &str, - kind: crate::storage_engines::sketch_db::index::SketchKindHandle, + kind: crate::storage_engines::sketch_db::index::SketchAlgorithm, cfg: &crate::storage_engines::sketch_db::data::SketchConfig, group_by_keys: &std::collections::BTreeSet, ) -> asap_types::PolicyFingerprint { - let Some(agg_type) = aggregation_type_for_sketch_handle(kind) else { + let Some(agg_type) = aggregation_type_for_sketch_algorithm(kind) else { return asap_types::PolicyFingerprint::UNSET; }; let params = sketch_config_to_params(cfg); @@ -1821,7 +1822,7 @@ fn derive_sketch_policy_fp( } /// Phase 5 helper — map a `ModifiedOtlpSketchDp` to the matching -/// `SketchKindHandle` so registration and capability classification +/// `SketchAlgorithm` so registration and capability classification /// share one source of truth. /// /// CMS-with-heap detection: the OTLP `CountMinSketch` wire struct @@ -1859,7 +1860,7 @@ fn derive_sketch_policy_fp( /// /// Per `docs/design_docs/series-identity.md`, the summary *family* /// is a wire-level attribute (carried here in `agg_kind` / -/// [`SketchKindHandle`]), NOT a name suffix; storage + query must be +/// [`SketchAlgorithm`]), NOT a name suffix; storage + query must be /// keyed on the raw SDK metric name. This helper applies that /// canonicalization at the ingest seam so the backend resolves /// correctly regardless of whether the deployed agent still suffixes. @@ -1868,13 +1869,16 @@ fn derive_sketch_policy_fp( /// sketch kind, so a metric a user legitimately named `foo_hll` that /// arrives as a KLL sketch is left untouched, and the operation is a /// no-op (and therefore safe / idempotent) once agents stop suffixing. -fn canonical_sketch_metric_name<'a>(name: &'a str, kind: SketchKind) -> &'a str { - let suffix: &str = match kind { - SketchKind::DdSketch => "_ddsketch", - SketchKind::Kll => "_kll", - SketchKind::Hll => "_hll", - SketchKind::CountSketch => "_countsketch", - SketchKind::CountMin => "_countminsketch", +fn canonical_sketch_metric_name<'a>(name: &'a str, algorithm: SketchAlgorithm) -> &'a str { + let suffix: &str = match algorithm { + SketchAlgorithm::DDSketch => "_ddsketch", + SketchAlgorithm::Kll => "_kll", + SketchAlgorithm::Hll => "_hll", + SketchAlgorithm::CountSketch => "_countsketch", + SketchAlgorithm::Cms => "_countminsketch", + // These algorithms do not currently have modified-OTLP + // containers in this receiver, so there is no suffix to strip. + _ => return name, }; // Only strip when there's a non-empty base left over (so a metric // literally named `_kll` is never collapsed to the empty string). @@ -1923,26 +1927,26 @@ fn dp_carries_heap(dp: &ModifiedOtlpSketchDp) -> bool { /// heap is enrichment on the same substrate, not a different series). The /// CAPABILITY still tracks the heap via the metadata upgrade path. All /// other handles pass through unchanged. -fn base_sketch_kind_handle( - kind: crate::storage_engines::sketch_db::index::SketchKindHandle, -) -> crate::storage_engines::sketch_db::index::SketchKindHandle { - use crate::storage_engines::sketch_db::index::SketchKindHandle; +fn base_sketch_algorithm( + kind: crate::storage_engines::sketch_db::index::SketchAlgorithm, +) -> crate::storage_engines::sketch_db::index::SketchAlgorithm { + use crate::storage_engines::sketch_db::index::SketchAlgorithm; match kind { - SketchKindHandle::CmsWithHeap => SketchKindHandle::CountMin, - SketchKindHandle::CountSketchWithHeap => SketchKindHandle::CountSketch, + SketchAlgorithm::CmsWithHeap => SketchAlgorithm::Cms, + SketchAlgorithm::CountSketchWithHeap => SketchAlgorithm::CountSketch, other => other, } } -fn sketch_kind_handle_for( +fn sketch_algorithm_for( dp: &ModifiedOtlpSketchDp, -) -> crate::storage_engines::sketch_db::index::SketchKindHandle { - use crate::storage_engines::sketch_db::index::SketchKindHandle; - match dp.kind { - SketchKind::DdSketch => SketchKindHandle::DDSketch, - SketchKind::Kll => SketchKindHandle::Kll, - SketchKind::Hll => SketchKindHandle::Hll, - SketchKind::CountSketch => { +) -> crate::storage_engines::sketch_db::index::SketchAlgorithm { + use crate::storage_engines::sketch_db::index::SketchAlgorithm; + match dp.algorithm.clone() { + SketchAlgorithm::DDSketch => SketchAlgorithm::DDSketch, + SketchAlgorithm::Kll => SketchAlgorithm::Kll, + SketchAlgorithm::Hll => SketchAlgorithm::Hll, + SketchAlgorithm::CountSketch => { // Mirror the CountMin branch: CountSketch-with-heap // payloads share the same outer msgpack envelope // (`CountMinSketchWithHeapSerialized` — see the comment @@ -1953,11 +1957,11 @@ fn sketch_kind_handle_for( // decode AND the heap is non-empty; otherwise stay with // vanilla `CountSketch`. if dp_carries_heap(dp) { - return SketchKindHandle::CountSketchWithHeap; + return SketchAlgorithm::CountSketchWithHeap; } - SketchKindHandle::CountSketch + SketchAlgorithm::CountSketch } - SketchKind::CountMin => { + SketchAlgorithm::Cms => { // Try a no-cost peek: msgpack-encoded CMS-with-heap payloads // round-trip through asap_sketchlib's // `CountMinSketchWithHeap::deserialize_msgpack`. If the @@ -1966,10 +1970,11 @@ fn sketch_kind_handle_for( // CmsWithHeap so ASAP-tier `topk` can read the heap. // Otherwise stay with vanilla `CountMin`. if dp_carries_heap(dp) { - return SketchKindHandle::CmsWithHeap; + return SketchAlgorithm::CmsWithHeap; } - SketchKindHandle::CountMin + SketchAlgorithm::Cms } + other => other, } } @@ -1990,22 +1995,11 @@ fn encoding_to_handle( } } -/// Sketch family carried by a modified-OTLP `*SketchDataPoint`. Used by -/// the encoding dispatcher in `decode_modified_otlp_sketch_bytes`. -#[derive(Debug, Clone, Copy)] -pub(crate) enum SketchKind { - DdSketch, - Kll, - CountSketch, - CountMin, - Hll, -} - /// A single modified-OTLP sketch data point flattened across the five /// per-variant data-point types so the routing loop can treat them /// uniformly. struct ModifiedOtlpSketchDp { - kind: SketchKind, + algorithm: SketchAlgorithm, attrs: HashMap, time_unix_nano: u64, sketch: Vec, @@ -2028,15 +2022,15 @@ struct ModifiedOtlpSketchDp { /// Decode the typed `sketch` bytes from a modified-OTLP /// `*SketchDataPoint` into a concrete `AggregateCore`. /// -/// Dispatches on the `(SketchKind, encoding)` pair. For each -/// `(kind, _ENCODING_PROTO)` pair we call the matching accumulator's +/// Dispatches on the `(SketchAlgorithm, encoding)` pair. For each +/// `(algorithm, _ENCODING_PROTO)` pair we call the matching accumulator's /// `from_sketchlib_proto_bytes` constructor. Variants without a /// constructor today return `Err`; the caller falls through to §5.2 /// fallback so the user still gets a correct answer. Per-variant /// decoders are tracked in PR C (task #8) and PR I (task #14, for /// `_ENCODING_MSGPACK` parity). fn decode_modified_otlp_sketch_bytes( - kind: SketchKind, + algorithm: SketchAlgorithm, encoding: i32, bytes: &[u8], ) -> Result, Box> { @@ -2057,7 +2051,7 @@ fn decode_modified_otlp_sketch_bytes( // 4 — ENCODING_MSGPACK_DELTA (MSGPACK diff; not yet wired) match encoding { - ENCODING_PROTO => match kind { + ENCODING_PROTO => match algorithm { // Phase 3 step 3: DDSketch and KLL envelope-parsing / // sketch reconstruction route through the shared // `edge_runtime_adapter`, which delegates to @@ -2070,7 +2064,7 @@ fn decode_modified_otlp_sketch_bytes( // tracked under ProjectASAP/ASAPCollector#243 — until it // lands those three sketches keep using the backend's // existing per-accumulator decoder. - SketchKind::DdSketch => { + SketchAlgorithm::DDSketch => { use crate::precompute_engine::operators::edge_runtime_adapter::{ reconstruct_via_runtime, ReconstructedSketch, SketchType as RtSketchType, }; @@ -2098,7 +2092,7 @@ fn decode_modified_otlp_sketch_bytes( )?)), } } - SketchKind::Kll => { + SketchAlgorithm::Kll => { use crate::precompute_engine::operators::edge_runtime_adapter::{ reconstruct_via_runtime, ReconstructedSketch, SketchType as RtSketchType, }; @@ -2123,27 +2117,30 @@ fn decode_modified_otlp_sketch_bytes( )), } } - SketchKind::CountMin => Ok(Box::new( + SketchAlgorithm::Cms => Ok(Box::new( CountMinSketchAccumulator::from_sketchlib_proto_bytes(bytes)?, )), - SketchKind::CountSketch => Ok(Box::new( + SketchAlgorithm::CountSketch => Ok(Box::new( CountSketchAccumulator::from_sketchlib_proto_bytes(bytes)?, )), - SketchKind::Hll => Ok(Box::new(HllSketchAccumulator::from_sketchlib_proto_bytes( + SketchAlgorithm::Hll => Ok(Box::new(HllSketchAccumulator::from_sketchlib_proto_bytes( bytes, )?)), + other => { + Err(format!("modified-OTLP PROTO decoding is not implemented for {other:?}").into()) + } }, - ENCODING_MSGPACK => match kind { - SketchKind::CountMin => Ok(Box::new(CountMinSketchAccumulator::from_msgpack_bytes( + ENCODING_MSGPACK => match algorithm { + SketchAlgorithm::Cms => Ok(Box::new(CountMinSketchAccumulator::from_msgpack_bytes( bytes, )?)), - SketchKind::CountSketch => { + SketchAlgorithm::CountSketch => { // Heap-bearing CountSketch full frame: the bytes are the // `{sketch,topk_heap,heap_size}` envelope (a DIFFERENT inner // field order than the plain CountSketch msgpack), so // `CountSketch::from_msgpack` can't parse it. Try the heap // decode FIRST when the heap is non-empty (the same promotion - // gate `sketch_kind_handle_for` uses); cache THAT heap + // gate `sketch_algorithm_for` uses); cache THAT heap // accumulator as the per-series base so a later MSGPACK_DELTA // frame applies its matrix delta + heap onto a heap // accumulator. Fall back to the plain CountSketch decode for @@ -2167,11 +2164,17 @@ fn decode_modified_otlp_sketch_bytes( } Ok(Box::new(CountSketchAccumulator::from_msgpack_bytes(bytes)?)) } - SketchKind::Kll => Ok(Box::new(DatasketchesKLLAccumulator::from_msgpack_bytes( + SketchAlgorithm::Kll => Ok(Box::new(DatasketchesKLLAccumulator::from_msgpack_bytes( bytes, )?)), - SketchKind::DdSketch => Ok(Box::new(DDSketchAccumulator::from_msgpack_bytes(bytes)?)), - SketchKind::Hll => Ok(Box::new(HllSketchAccumulator::from_msgpack_bytes(bytes)?)), + SketchAlgorithm::DDSketch => { + Ok(Box::new(DDSketchAccumulator::from_msgpack_bytes(bytes)?)) + } + SketchAlgorithm::Hll => Ok(Box::new(HllSketchAccumulator::from_msgpack_bytes(bytes)?)), + other => Err(format!( + "modified-OTLP MSGPACK decoding is not implemented for {other:?}" + ) + .into()), }, ENCODING_PROTO_DELTA => Err(format!( "sketch encoding PROTO_DELTA (2) is not standalone-decodable — \ @@ -2214,7 +2217,7 @@ fn decode_modified_otlp_sketch_bytes( /// deltas. Those keep the unchanged "drop until the next full frame" /// behavior. fn empty_accumulator_for_delta_bootstrap( - kind: SketchKind, + algorithm: SketchAlgorithm, config: &crate::storage_engines::sketch_db::index::SketchConfig, encoding: i32, ) -> Option> { @@ -2224,8 +2227,8 @@ fn empty_accumulator_for_delta_bootstrap( }; use crate::storage_engines::sketch_db::index::SketchConfig; - match (kind, config) { - (SketchKind::Hll, SketchConfig::Hll { precision }) => { + match (algorithm, config) { + (SketchAlgorithm::Hll, SketchConfig::Hll { precision }) => { use asap_sketchlib::HllVariant; // Regular is the default agent variant; HLL's additive delta // merge tolerates an empty same-precision base. @@ -2234,10 +2237,10 @@ fn empty_accumulator_for_delta_bootstrap( *precision, ))) } - (SketchKind::CountMin, SketchConfig::CountMin { rows, cols }) => Some(Box::new( + (SketchAlgorithm::Cms, SketchConfig::CountMin { rows, cols }) => Some(Box::new( CountMinSketchAccumulator::new(*rows as usize, *cols as usize), )), - (SketchKind::CountSketch, SketchConfig::CountSketch { rows, cols }) => { + (SketchAlgorithm::CountSketch, SketchConfig::CountSketch { rows, cols }) => { // A heap-bearing DELTA-HEAP frame must reconstruct onto a heap // accumulator (the apply path downcasts to // `CountSketchWithHeapAccumulator`); a plain matrix delta @@ -2287,7 +2290,7 @@ const ENCODING_MSGPACK_DELTA: i32 = 4; /// KLL/CountSketch/CountMinSketch deltas are deferred to follow-ups /// as their delta codecs land. pub(crate) fn apply_modified_otlp_delta_bytes( - kind: SketchKind, + algorithm: SketchAlgorithm, encoding: i32, existing: &mut Box, bytes: &[u8], @@ -2297,8 +2300,8 @@ pub(crate) fn apply_modified_otlp_delta_bytes( DDSketchAccumulator, HllSketchAccumulator, }; - match (encoding, kind) { - (ENCODING_PROTO_DELTA, SketchKind::DdSketch) => { + match (encoding, algorithm) { + (ENCODING_PROTO_DELTA, SketchAlgorithm::DDSketch) => { let dd = existing .as_any_mut() .downcast_mut::() @@ -2308,7 +2311,7 @@ pub(crate) fn apply_modified_otlp_delta_bytes( )?; dd.apply_proto_delta_bytes(bytes) } - (ENCODING_PROTO_DELTA, SketchKind::Hll) => { + (ENCODING_PROTO_DELTA, SketchAlgorithm::Hll) => { let hll = existing .as_any_mut() .downcast_mut::() @@ -2318,7 +2321,7 @@ pub(crate) fn apply_modified_otlp_delta_bytes( )?; hll.apply_proto_delta_bytes(bytes) } - (ENCODING_PROTO_DELTA, SketchKind::CountSketch) => { + (ENCODING_PROTO_DELTA, SketchAlgorithm::CountSketch) => { let cs = existing .as_any_mut() .downcast_mut::() @@ -2328,7 +2331,7 @@ pub(crate) fn apply_modified_otlp_delta_bytes( )?; cs.apply_proto_delta_bytes(bytes) } - (ENCODING_PROTO_DELTA, SketchKind::CountMin) => { + (ENCODING_PROTO_DELTA, SketchAlgorithm::Cms) => { let cms = existing .as_any_mut() .downcast_mut::() @@ -2343,7 +2346,7 @@ pub(crate) fn apply_modified_otlp_delta_bytes( DDSketch / HLL / CountSketch / CountMin are wired" ) .into()), - (ENCODING_MSGPACK_DELTA, SketchKind::CountSketch) => { + (ENCODING_MSGPACK_DELTA, SketchAlgorithm::CountSketch) => { // DELTA-HEAP frame for the heap-bearing CountSketch: a sparse // signed matrix delta + the full top-k heap. The cached base is // a heap accumulator (window-1 full frame decoded via @@ -2723,30 +2726,36 @@ mod canonical_metric_name_tests { #[test] fn strips_matching_family_suffix() { assert_eq!( - canonical_sketch_metric_name("request_size_bytes_kll", SketchKind::Kll), + canonical_sketch_metric_name("request_size_bytes_kll", SketchAlgorithm::Kll), "request_size_bytes" ); assert_eq!( - canonical_sketch_metric_name("http_requests_total_latency_ms_kll", SketchKind::Kll), + canonical_sketch_metric_name( + "http_requests_total_latency_ms_kll", + SketchAlgorithm::Kll + ), "http_requests_total_latency_ms" ); assert_eq!( - canonical_sketch_metric_name("unique_users_per_min_hll", SketchKind::Hll), + canonical_sketch_metric_name("unique_users_per_min_hll", SketchAlgorithm::Hll), "unique_users_per_min" ); assert_eq!( - canonical_sketch_metric_name("top_endpoint_qps_countsketch", SketchKind::CountSketch), + canonical_sketch_metric_name( + "top_endpoint_qps_countsketch", + SketchAlgorithm::CountSketch + ), "top_endpoint_qps" ); assert_eq!( canonical_sketch_metric_name( "endpoint_request_freq_countminsketch", - SketchKind::CountMin + SketchAlgorithm::Cms ), "endpoint_request_freq" ); assert_eq!( - canonical_sketch_metric_name("latency_ddsketch", SketchKind::DdSketch), + canonical_sketch_metric_name("latency_ddsketch", SketchAlgorithm::DDSketch), "latency" ); } @@ -2756,11 +2765,11 @@ mod canonical_metric_name_tests { // Once agents stop suffixing (series-identity // Phase 1), the strip must be a no-op. assert_eq!( - canonical_sketch_metric_name("request_size_bytes", SketchKind::Kll), + canonical_sketch_metric_name("request_size_bytes", SketchAlgorithm::Kll), "request_size_bytes" ); assert_eq!( - canonical_sketch_metric_name("unique_users_per_min", SketchKind::Hll), + canonical_sketch_metric_name("unique_users_per_min", SketchAlgorithm::Hll), "unique_users_per_min" ); } @@ -2772,7 +2781,7 @@ mod canonical_metric_name_tests { // actual sketch kind, so we never collapse a legitimately-named // metric onto a different one. assert_eq!( - canonical_sketch_metric_name("my_metric_hll", SketchKind::Kll), + canonical_sketch_metric_name("my_metric_hll", SketchAlgorithm::Kll), "my_metric_hll" ); } @@ -2782,7 +2791,7 @@ mod canonical_metric_name_tests { // A metric literally named `_kll` (base would be empty) is left // intact rather than emptied. assert_eq!( - canonical_sketch_metric_name("_kll", SketchKind::Kll), + canonical_sketch_metric_name("_kll", SketchAlgorithm::Kll), "_kll" ); } @@ -2901,7 +2910,7 @@ mod series_key_roundtrip_tests { mod policy_fp_lookup_tests { use super::*; use crate::storage_engines::sketch_db::data::SketchConfig; - use crate::storage_engines::sketch_db::index::SketchKindHandle; + use crate::storage_engines::sketch_db::index::SketchAlgorithm; use asap_types::AggregationType; #[test] @@ -2910,32 +2919,32 @@ mod policy_fp_lookup_tests { // Drift surfaces as policy lookups that silently miss because // the handle resolves to an `AggregationType` no policy uses. assert_eq!( - aggregation_type_for_sketch_handle(SketchKindHandle::DDSketch), + aggregation_type_for_sketch_algorithm(SketchAlgorithm::DDSketch), Some(AggregationType::DDSketch) ); assert_eq!( - aggregation_type_for_sketch_handle(SketchKindHandle::Kll), + aggregation_type_for_sketch_algorithm(SketchAlgorithm::Kll), Some(AggregationType::DatasketchesKLL) ); assert_eq!( - aggregation_type_for_sketch_handle(SketchKindHandle::Hll), + aggregation_type_for_sketch_algorithm(SketchAlgorithm::Hll), Some(AggregationType::HLL) ); assert_eq!( - aggregation_type_for_sketch_handle(SketchKindHandle::CountMin), + aggregation_type_for_sketch_algorithm(SketchAlgorithm::Cms), Some(AggregationType::CountMinSketch) ); assert_eq!( - aggregation_type_for_sketch_handle(SketchKindHandle::CmsWithHeap), + aggregation_type_for_sketch_algorithm(SketchAlgorithm::CmsWithHeap), Some(AggregationType::CountMinSketchWithHeap) ); assert_eq!( - aggregation_type_for_sketch_handle(SketchKindHandle::CountSketch), + aggregation_type_for_sketch_algorithm(SketchAlgorithm::CountSketch), Some(AggregationType::CountSketch) ); // `Any` is a control-plane wildcard, not a real DP shape. assert_eq!( - aggregation_type_for_sketch_handle(SketchKindHandle::Any), + aggregation_type_for_sketch_algorithm(SketchAlgorithm::Kmv), None ); } @@ -3005,7 +3014,7 @@ mod dispatcher_tests { .encode_to_vec(); apply_modified_otlp_delta_bytes( - SketchKind::DdSketch, + SketchAlgorithm::DDSketch, ENCODING_PROTO_DELTA, &mut acc, &bytes, @@ -3037,8 +3046,13 @@ mod dispatcher_tests { } .encode_to_vec(); - apply_modified_otlp_delta_bytes(SketchKind::Hll, ENCODING_PROTO_DELTA, &mut acc, &bytes) - .expect("apply ok"); + apply_modified_otlp_delta_bytes( + SketchAlgorithm::Hll, + ENCODING_PROTO_DELTA, + &mut acc, + &bytes, + ) + .expect("apply ok"); let hll = acc.as_any().downcast_ref::().unwrap(); assert_eq!(hll.inner.registers, vec![4, 5, 6, 7]); @@ -3049,7 +3063,7 @@ mod dispatcher_tests { let mut acc: Box = Box::new(HllSketchAccumulator::new(HllVariant::Regular, 2)); let err = apply_modified_otlp_delta_bytes( - SketchKind::DdSketch, + SketchAlgorithm::DDSketch, ENCODING_PROTO_DELTA, &mut acc, &[0u8; 4], @@ -3062,17 +3076,21 @@ mod dispatcher_tests { #[test] fn apply_rejects_full_state_encoding() { let mut acc: Box = Box::new(DDSketchAccumulator::new(0.01)); - let err = - apply_modified_otlp_delta_bytes(SketchKind::DdSketch, ENCODING_PROTO, &mut acc, &[]) - .expect_err("expected full-state-rejection error") - .to_string(); + let err = apply_modified_otlp_delta_bytes( + SketchAlgorithm::DDSketch, + ENCODING_PROTO, + &mut acc, + &[], + ) + .expect_err("expected full-state-rejection error") + .to_string(); assert!(err.contains("full-state frame")); } #[test] fn decode_rejects_delta_encoding_with_helpful_message() { let err = match decode_modified_otlp_sketch_bytes( - SketchKind::DdSketch, + SketchAlgorithm::DDSketch, ENCODING_PROTO_DELTA, &[], ) { @@ -3339,7 +3357,7 @@ mod sid_resolution_tests { let mut attrs = HashMap::new(); attrs.insert("zone".to_string(), "z0".to_string()); let series_key = format_series_key( - canonical_sketch_metric_name("http_latency_ms", SketchKind::DdSketch), + canonical_sketch_metric_name("http_latency_ms", SketchAlgorithm::DDSketch), &attrs, ); @@ -3674,7 +3692,7 @@ mod sid_resolution_tests { let mut attrs = HashMap::new(); attrs.insert("svc".to_string(), "auth".to_string()); let series_key = format_series_key( - canonical_sketch_metric_name("frequency_metric", SketchKind::CountMin), + canonical_sketch_metric_name("frequency_metric", SketchAlgorithm::Cms), &attrs, ); { @@ -3746,7 +3764,7 @@ mod sid_resolution_tests { let mut attrs = HashMap::new(); attrs.insert("svc".to_string(), "auth".to_string()); let series_key = format_series_key( - canonical_sketch_metric_name("cardinality_metric", SketchKind::Hll), + canonical_sketch_metric_name("cardinality_metric", SketchAlgorithm::Hll), &attrs, ); { @@ -3810,7 +3828,7 @@ mod sid_resolution_tests { let mut attrs = HashMap::new(); attrs.insert("zone".to_string(), "z0".to_string()); let series_key = format_series_key( - canonical_sketch_metric_name("dd_latency_ms", SketchKind::DdSketch), + canonical_sketch_metric_name("dd_latency_ms", SketchAlgorithm::DDSketch), &attrs, ); assert!( @@ -3838,7 +3856,7 @@ mod sid_resolution_tests { /// frame arrives for the same sid. One-way; never downgrades. #[tokio::test] async fn heap_bearing_frame_upgrades_cms_sid_capability() { - use crate::storage_engines::sketch_db::index::{Capability, SketchKindHandle}; + use crate::storage_engines::sketch_db::index::{Capability, SketchAlgorithm}; use asap_sketchlib::{CountMinSketchWithHeap, MessagePackCodec}; let (state, drain) = make_state().await; @@ -3864,7 +3882,7 @@ mod sid_resolution_tests { let meta1 = state.sketch_index.instance(sid).expect("sid registered"); assert_eq!( meta1.capability, - Some(Capability::FrequencyEstimate(SketchKindHandle::CountMin)), + Some(Capability::FrequencyEstimate(Some(SketchAlgorithm::Cms))), "heap-less first frame registers FrequencyEstimate(CountMin)" ); @@ -3895,7 +3913,9 @@ mod sid_resolution_tests { .expect("sid still registered"); assert_eq!( meta2.capability, - Some(Capability::FrequencyTopk(SketchKindHandle::CmsWithHeap)), + Some(Capability::FrequencyTopk(Some( + SketchAlgorithm::CmsWithHeap + ))), "heap-bearing frame upgrades the sid to FrequencyTopk(CmsWithHeap)" ); // Still one instance — the upgrade is an in-place overwrite, not a @@ -3921,7 +3941,9 @@ mod sid_resolution_tests { .expect("sid still registered"); assert_eq!( meta3.capability, - Some(Capability::FrequencyTopk(SketchKindHandle::CmsWithHeap)), + Some(Capability::FrequencyTopk(Some( + SketchAlgorithm::CmsWithHeap + ))), "a later heap-less frame never downgrades the capability" ); diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index 72af0b27..ddc8e1ee 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -1319,7 +1319,7 @@ async fn process_via_router( // metrics; for `GorillaObjectStore`-only deploys the dispatch // is a function of `metric_storage` alone. let stat = Statistic::Sum; - let accuracy = AccuracyTarget::Approximate; + let accuracy = AccuracyTarget::Epsilon(0.01); let router_result = state .query_router @@ -1840,7 +1840,7 @@ async fn process_range_query_request( // an `Exact` range query will route straight to the archive via the // shared policy table. let stat = Statistic::Sum; - let accuracy = AccuracyTarget::Approximate; + let accuracy = AccuracyTarget::Epsilon(0.01); // Warm-vs-archive routing fix: split by the warm-retention boundary // rather than "archive-on ⇒ everything to archive". When the @@ -3739,7 +3739,7 @@ aggregations: // `ColdJsonlFallback` last-resort slot; the surviving // failover surface is ASAP-tier sketch ↔ Gorilla-S3 archive. // The HTTP handler dispatches with default - // `(Statistic::Sum, AccuracyTarget::Approximate)`, so for a + // `(Statistic::Sum, AccuracyTarget::Epsilon(_))`, so for a // `DoubleWrite` metric the compatibility list is // `[SketchStore, GorillaObjectStore]` and the ASAP-tier // mock answers first. The archive must NOT be hit (no diff --git a/data_plane/src/monitor/coordinator.rs b/data_plane/src/monitor/coordinator.rs index 091e8a4b..f41520d5 100644 --- a/data_plane/src/monitor/coordinator.rs +++ b/data_plane/src/monitor/coordinator.rs @@ -39,25 +39,7 @@ use super::sampling_alloc::epsilon_sample_floor; /// for identity/config-schema compatibility (a monitor is still keyed by /// `(agg_id, key)`, and `key` is only meaningful for `CmsPoint`) even though /// no functional-specific THRESHOLDING happens here anymore. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] -pub enum Functional { - #[default] - Sum, - CmsPoint, - LinearBuckets, -} - -impl Functional { - /// Parse the pushed-config functional string (`streaming_config` / - /// `emit/monitor.rs` use the same names). Unknown ⇒ `Sum`. - pub fn from_name(s: &str) -> Self { - match s.trim().to_ascii_lowercase().as_str() { - "cms_point" | "cms" => Functional::CmsPoint, - "linear_buckets" | "linear" => Functional::LinearBuckets, - _ => Functional::Sum, - } - } -} +pub use asap_types::MonitorFunctional as Functional; /// Static configuration for one monitor, sourced from the streaming-config /// `monitors:` section. diff --git a/data_plane/src/precompute_engine/accumulator_factory.rs b/data_plane/src/precompute_engine/accumulator_factory.rs index 2e96eec8..cec14adf 100644 --- a/data_plane/src/precompute_engine/accumulator_factory.rs +++ b/data_plane/src/precompute_engine/accumulator_factory.rs @@ -10,19 +10,19 @@ use crate::storage_engines::types::{ use asap_types::aggregation_config::AggregationConfig; // Step 5 (sketch-identity unification, see // scratchpad/artifacts/enum-unification-plan.md): dispatch below is -// driven by `AccumulatorSpec` (SummaryKind + typed SummaryParams + +// driven by `AccumulatorSpec` (SummaryFamilyType + typed family parameters + // keyed-axis grouping) instead of raw `AggregationType` + // `aggregation_sub_type` string matching. Numeric params come straight -// off `spec.params` (typed, no HashMap lookups) except `cms_params`, -// kept as a raw-`parameters` read for the one case `SummaryParams` has -// no field for: HydraKLL's `(row, col)` tiling grid (see +// off the committed family's typed params (no HashMap lookups) except +// `cms_params`, kept as a raw-`parameters` read for the one case Planner's +// family parameters have no field for: HydraKLL's `(row, col)` tiling grid (see // `asap_types::accumulator_spec`'s module doc for why). `cms_params` // now lives there — the only place that still needs the other three // former local helpers (`kll_k_param`, `heap_size_param`, // `ddsketch_alpha_param`) is that module's own `AccumulatorSpec` // construction, so they aren't re-imported here. use asap_types::accumulator_spec::{cms_params, AccumulatorSpecError}; -use asap_types::{SummaryKind, SummaryParams}; +use planner_types::post_asap::{ExactKind, SketchAlgorithm, SketchParams, SummaryFamilyType}; /// Generate the two boilerplate clone-based `AccumulatorUpdater` methods /// for updaters whose inner `acc` field implements `Clone + AggregateCore`. @@ -688,12 +688,12 @@ impl AccumulatorUpdater for CmsHeapAccumulatorUpdater { /// Keyed point-frequency updater backed by a real `asap_sketchlib::CountSketch` /// (signed rows, median-of-rows estimator) — distinct math from /// `CmsAccumulatorUpdater`'s CMS (min-of-rows). Closes, on the raw-metric -/// ingest path, the conflation bug where `SummaryKind::CountSketch` silently +/// ingest path, the conflation bug where `SketchAlgorithm::CountSketch` silently /// shared `CmsAccumulatorUpdater` with bare CMS. /// /// As with bare CMS, each raw Prometheus sample contributes its `value`. /// Unit event counting must be selected explicitly by a future typed plan -/// contract rather than being implied by `SummaryKind::CountSketch`. +/// contract rather than being implied by `SketchAlgorithm::CountSketch`. pub struct CountSketchAccumulatorUpdater { acc: CountSketchAccumulator, row_num: usize, @@ -904,73 +904,74 @@ fn topk_weight_param(config: &AggregationConfig) -> TopkWeight { // Factory function // --------------------------------------------------------------------------- -/// Read the KLL `k` out of `SummaryParams::Kll`. `accumulator_spec()` -/// always pairs `SummaryKind::Kll` with `SummaryParams::Kll`, so the +/// Read the KLL `k` out of `SketchParams::Kll`. `accumulator_spec()` +/// always builds a `SketchKind` whose `SketchAlgorithm::Kll` is paired with +/// `SketchParams::Kll`, so the /// other arm is unreachable from a `spec` this module builds itself. -fn kll_k(params: &SummaryParams) -> u16 { +fn kll_k(params: &SketchParams) -> u16 { match params { // Lossless: `accumulator_spec()` only ever stores a value that // already fit in `u16` (via `kll_k_param`'s own `u16::try_from` // fallback) widened to `u32`. - SummaryParams::Kll { k } => *k as u16, + SketchParams::Kll { k } => *k as u16, other => unreachable!( - "accumulator_spec() paired SummaryKind::Kll with non-Kll params: {other:?}" + "accumulator_spec() paired SketchAlgorithm::Kll with non-Kll params: {other:?}" ), } } -/// Read `(width, depth)` out of `SummaryParams::Cms` or `::CountSketch` +/// Read `(width, depth)` out of `SketchParams::Cms` or `::CountSketch` /// — same shape, different variant per bare-sketch identity. -fn cms_dims(params: &SummaryParams) -> (usize, usize) { +fn cms_dims(params: &SketchParams) -> (usize, usize) { match params { - SummaryParams::Cms { width, depth } | SummaryParams::CountSketch { width, depth } => { + SketchParams::Cms { width, depth } | SketchParams::CountSketch { width, depth } => { (*width as usize, *depth as usize) } other => unreachable!( - "accumulator_spec() paired SummaryKind::Cms/CountSketch with unexpected params: {other:?}" + "accumulator_spec() paired SketchAlgorithm::Cms/CountSketch with unexpected params: {other:?}" ), } } -/// Read `(width, depth, heap_size)` out of `SummaryParams::CmsWithHeap` +/// Read `(width, depth, heap_size)` out of `SketchParams::CmsWithHeap` /// or `::CountSketchWithHeap`. -fn cms_heap_dims(params: &SummaryParams) -> (usize, usize, usize) { +fn cms_heap_dims(params: &SketchParams) -> (usize, usize, usize) { match params { - SummaryParams::CmsWithHeap { + SketchParams::CmsWithHeap { width, depth, heap_size, } - | SummaryParams::CountSketchWithHeap { + | SketchParams::CountSketchWithHeap { width, depth, heap_size, } => (*width as usize, *depth as usize, *heap_size as usize), other => unreachable!( - "accumulator_spec() paired a WithHeap SummaryKind with unexpected params: {other:?}" + "accumulator_spec() paired a WithHeap SketchAlgorithm with unexpected params: {other:?}" ), } } -/// Read the DDSketch relative-accuracy `alpha` out of `SummaryParams::DDSketch`. -fn ddsketch_alpha(params: &SummaryParams) -> f64 { +/// Read the DDSketch relative-accuracy `alpha` out of `SketchParams::DDSketch`. +fn ddsketch_alpha(params: &SketchParams) -> f64 { match params { - SummaryParams::DDSketch { alpha } => *alpha, + SketchParams::DDSketch { alpha } => *alpha, other => unreachable!( - "accumulator_spec() paired SummaryKind::DDSketch with non-DDSketch params: {other:?}" + "accumulator_spec() paired SketchAlgorithm::DDSketch with non-DDSketch params: {other:?}" ), } } /// Create an appropriate `AccumulatorUpdater` from an `AggregationConfig`. /// -/// Dispatches on [`asap_types::AccumulatorSpec`] — `SummaryKind` identity +/// Dispatches on [`asap_types::AccumulatorSpec`] — `SummaryFamilyType` identity /// plus the keyed/unkeyed `grouping` axis — instead of the pre-Step-5 /// `AggregationType` + `aggregation_sub_type` string combo. See /// `asap_types::accumulator_spec`'s module doc for why min/max direction, /// HydraKLL's `(row, col)` tiling, and top-k `weight_mode` still read -/// `config` directly rather than going through `SummaryParams` — none of -/// those three have a field in ASAPController's upstream type to live in. +/// `config` directly rather than going through Planner family parameters — +/// none of those three have a field in the Planner-owned types. pub fn create_accumulator_updater(config: &AggregationConfig) -> Box { let spec = match config.accumulator_spec() { Ok(spec) => spec, @@ -1003,39 +1004,53 @@ pub fn create_accumulator_updater(config: &AggregationConfig) -> Box Box::new(SumAccumulatorUpdater::new()), - (SummaryKind::Sum, true) => Box::new(MultipleSumAccumulatorUpdater::new()), + match (&spec.family, keyed) { + (SummaryFamilyType::ExactAggregate(ExactKind::Sum, _), false) => { + Box::new(SumAccumulatorUpdater::new()) + } + (SummaryFamilyType::ExactAggregate(ExactKind::Sum, _), true) => { + Box::new(MultipleSumAccumulatorUpdater::new()) + } - // Min/max direction isn't part of `SummaryParams::MinMax` + // Min/max direction isn't part of `ExactParams::MinMax` // (upstream models no direction axis) — read straight off // `aggregation_sub_type`, exactly as the pre-Step-5 dispatch did // for the direct `AggregationType::MinMax`/`MultipleMinMax` // arms. `accumulator_spec()` only resolves a wrapper's sub_type - // to `SummaryKind::MinMax` for an exact "Min"/"min"/"Max"/"max" + // to `ExactKind::MinMax` for an exact "Min"/"min"/"Max"/"max" // match, so re-deriving via `eq_ignore_ascii_case("max")` here // reproduces the same true/false split for that path too. - (SummaryKind::MinMax, false) => Box::new(MinMaxAccumulatorUpdater::new( - config.aggregation_sub_type.eq_ignore_ascii_case("max"), - )), - (SummaryKind::MinMax, true) => Box::new(MultipleMinMaxAccumulatorUpdater::new( - config.aggregation_sub_type.eq_ignore_ascii_case("max"), - )), + (SummaryFamilyType::ExactAggregate(ExactKind::MinMax, _), false) => Box::new( + MinMaxAccumulatorUpdater::new(config.aggregation_sub_type.eq_ignore_ascii_case("max")), + ), + (SummaryFamilyType::ExactAggregate(ExactKind::MinMax, _), true) => { + Box::new(MultipleMinMaxAccumulatorUpdater::new( + config.aggregation_sub_type.eq_ignore_ascii_case("max"), + )) + } - (SummaryKind::Increase, false) => Box::new(IncreaseAccumulatorUpdater::new()), - (SummaryKind::Increase, true) => Box::new(MultipleIncreaseAccumulatorUpdater::new()), + (SummaryFamilyType::ExactAggregate(ExactKind::Increase, _), false) => { + Box::new(IncreaseAccumulatorUpdater::new()) + } + (SummaryFamilyType::ExactAggregate(ExactKind::Increase, _), true) => { + Box::new(MultipleIncreaseAccumulatorUpdater::new()) + } - (SummaryKind::Kll, false) => Box::new(KllAccumulatorUpdater::new(kll_k(&spec.params))), + (SummaryFamilyType::Sketch(kind, _), false) + if kind.algorithm() == &SketchAlgorithm::Kll => + { + Box::new(KllAccumulatorUpdater::new(kll_k(kind.params()))) + } // HydraKLL: `k` comes off the typed params like the unkeyed case, - // but the `(row, col)` tiling grid has no `SummaryParams::Kll` + // but the `(row, col)` tiling grid has no `SketchParams::Kll` // field to live in (see `asap_types::accumulator_spec`'s module // doc) — read it the same way bare CMS does, via `cms_params`. - (SummaryKind::Kll, true) => { + (SummaryFamilyType::Sketch(kind, _), true) if kind.algorithm() == &SketchAlgorithm::Kll => { let (row_num, col_num) = cms_params(config); Box::new(HydraKllAccumulatorUpdater::new( row_num, col_num, - kll_k(&spec.params), + kll_k(kind.params()), )) } @@ -1043,8 +1058,8 @@ pub fn create_accumulator_updater(config: &AggregationConfig) -> Box { - let (row_num, col_num) = cms_dims(&spec.params); + (SummaryFamilyType::Sketch(kind, _), _) if kind.algorithm() == &SketchAlgorithm::Cms => { + let (row_num, col_num) = cms_dims(kind.params()); Box::new(CmsAccumulatorUpdater::new(row_num, col_num)) } @@ -1052,8 +1067,10 @@ pub fn create_accumulator_updater(config: &AggregationConfig) -> Box { - let (row_num, col_num) = cms_dims(&spec.params); + (SummaryFamilyType::Sketch(kind, _), _) + if kind.algorithm() == &SketchAlgorithm::CountSketch => + { + let (row_num, col_num) = cms_dims(kind.params()); Box::new(CountSketchAccumulatorUpdater::new(row_num, col_num)) } @@ -1065,8 +1082,10 @@ pub fn create_accumulator_updater(config: &AggregationConfig) -> Box { - let (row_num, col_num, heap_size) = cms_heap_dims(&spec.params); + (SummaryFamilyType::Sketch(kind, _), _) + if kind.algorithm() == &SketchAlgorithm::CmsWithHeap => + { + let (row_num, col_num, heap_size) = cms_heap_dims(kind.params()); Box::new(CmsHeapAccumulatorUpdater::new( row_num, col_num, @@ -1078,8 +1097,10 @@ pub fn create_accumulator_updater(config: &AggregationConfig) -> Box { - let (row_num, col_num, heap_size) = cms_heap_dims(&spec.params); + (SummaryFamilyType::Sketch(kind, _), _) + if kind.algorithm() == &SketchAlgorithm::CountSketchWithHeap => + { + let (row_num, col_num, heap_size) = cms_heap_dims(kind.params()); Box::new(CountSketchWithHeapAccumulatorUpdater::new( row_num, col_num, @@ -1088,23 +1109,27 @@ pub fn create_accumulator_updater(config: &AggregationConfig) -> Box Box::new(DDSketchAccumulatorUpdater::new(ddsketch_alpha( - &spec.params, - ))), + (SummaryFamilyType::Sketch(kind, _), _) + if kind.algorithm() == &SketchAlgorithm::DDSketch => + { + Box::new(DDSketchAccumulatorUpdater::new(ddsketch_alpha( + kind.params(), + ))) + } - // `SummaryKind::Hll` / `Count` / `Rate` / `Kmv` / `Theta`: no + // unsupported HLL, Count, Rate, Kmv, and Theta families: no // `AggregationType` resolves to one of these via // `accumulator_spec()`'s `Ok` path today — HLL is caught by // `AccumulatorSpecError::UnmappedAggregationType` above (see its // doc for why: a pre-existing gap, not introduced here), and the // other four have no `AggregationType` counterpart at all. Kept // as an explicit warning fallback rather than `unreachable!()` - // so a future `SummaryKind` this dispatch doesn't yet know how + // so a future `SummaryFamilyType` this dispatch doesn't yet know how // to build fails safe instead of panicking. - (other_kind, keyed) => { + (other_family, keyed) => { tracing::warn!( - "SummaryKind {:?} (keyed={}) has no accumulator_factory mapping, defaulting to Sum", - other_kind, + "SummaryFamilyType {:?} (keyed={}) has no accumulator_factory mapping, defaulting to Sum", + other_family, keyed ); Box::new(SumAccumulatorUpdater::new()) @@ -1452,7 +1477,7 @@ mod tests { } /// Same as `ranked_topk`, but for the real `CountSketchWithHeapAccumulator` - /// (median-of-signed-rows) built by `SummaryKind::CountSketchWithHeap` — + /// (median-of-signed-rows) built by `SketchAlgorithm::CountSketchWithHeap` — /// no longer conflated with the CMS-family accumulator above. fn ranked_topk_cs(acc: &dyn AggregateCore) -> Vec<(String, f64)> { let heap = acc diff --git a/data_plane/src/precompute_engine/ingest_handler.rs b/data_plane/src/precompute_engine/ingest_handler.rs index e905300d..15810ecb 100644 --- a/data_plane/src/precompute_engine/ingest_handler.rs +++ b/data_plane/src/precompute_engine/ingest_handler.rs @@ -368,10 +368,11 @@ mod tests { /// hand-merged sequence of full sketches. #[tokio::test] async fn delta_path_reconstitutes_cumulative_state() { - use crate::drivers::ingest::otel::{apply_modified_otlp_delta_bytes, SketchKind}; + use crate::drivers::ingest::otel::apply_modified_otlp_delta_bytes; use crate::precompute_engine::operators::DDSketchAccumulator; use asap_otel_proto::sketchlib::v1::{DdSketchBucketDelta, DdSketchDelta as PbDelta}; use asap_sketchlib::DdSketch; + use planner_types::post_asap::SketchAlgorithm; use prost::Message; const ENCODING_PROTO_DELTA: i32 = 2; @@ -416,8 +417,13 @@ mod tests { .unwrap() .core .clone_boxed_core(); - apply_modified_otlp_delta_bytes(SketchKind::DdSketch, ENCODING_PROTO_DELTA, &mut acc1, &d1) - .expect("apply first delta"); + apply_modified_otlp_delta_bytes( + SketchAlgorithm::DDSketch, + ENCODING_PROTO_DELTA, + &mut acc1, + &d1, + ) + .expect("apply first delta"); state.sketch_snapshots.insert( series_key.to_string(), SnapshotCacheEntry { @@ -441,8 +447,13 @@ mod tests { .unwrap() .core .clone_boxed_core(); - apply_modified_otlp_delta_bytes(SketchKind::DdSketch, ENCODING_PROTO_DELTA, &mut acc2, &d2) - .expect("apply second delta"); + apply_modified_otlp_delta_bytes( + SketchAlgorithm::DDSketch, + ENCODING_PROTO_DELTA, + &mut acc2, + &d2, + ) + .expect("apply second delta"); let final_dd = acc2.as_any().downcast_ref::().unwrap(); // Base [1,2,3] + d1 [+10 on 0, +20 on 2] = [11,2,23]; diff --git a/data_plane/src/precompute_engine/operators/count_sketch_with_heap_accumulator.rs b/data_plane/src/precompute_engine/operators/count_sketch_with_heap_accumulator.rs index e4cc527e..7c8de44a 100644 --- a/data_plane/src/precompute_engine/operators/count_sketch_with_heap_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/count_sketch_with_heap_accumulator.rs @@ -9,7 +9,7 @@ //! this session's `delta_apply.rs`/`decoders.rs` fix on the read side. //! Before this file existed, `accumulator_factory.rs`'s raw-metric //! ingest dispatch built a `CountMinSketchWithHeapAccumulator` (CMS math) -//! for `SummaryKind::CountSketchWithHeap` sids -- the same conflation bug +//! for `SketchAlgorithm::CountSketchWithHeap` sids -- the same conflation bug //! already fixed on the read side, now closed on the write side too. use crate::storage_engines::types::{ diff --git a/data_plane/src/precompute_engine/operators/sum_accumulator.rs b/data_plane/src/precompute_engine/operators/sum_accumulator.rs index 870841e7..c1069d2a 100644 --- a/data_plane/src/precompute_engine/operators/sum_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/sum_accumulator.rs @@ -51,7 +51,7 @@ impl SumAccumulator { /// self-contained fixed layout. It decodes into the SAME /// `AggregationType::Sum` accumulator as a plain-OTLP Sum, so the SumAgg /// envelope and a plain Sum land on one identity (`exact_agg:Sum`) with no - /// new SketchKindHandle. `count` is decoded but not retained + /// new SketchAlgorithm. `count` is decoded but not retained /// (SumAccumulator tracks the scalar sum only; Sum is never sample_p-thinned /// so no 1/p rescale is needed). pub fn from_sum_bytes(buffer: &[u8]) -> Result> { diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index d85c8034..d4cd924a 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -1235,7 +1235,7 @@ mod asap_tier_classify_tests { use crate::query_engines::routing::query_engine_routing::QueryEngine as _; use crate::query_engines::EngineError; use crate::storage_engines::sketch_db::index::{ - AccuracyBound, Capability, SketchConfig, SketchInstanceMetadata, SketchKindHandle, + AccuracyBound, Capability, SketchAlgorithm, SketchConfig, SketchInstanceMetadata, SketchSampleState, SketchStore, }; use crate::storage_engines::types::{CleanupPolicy, HotReloadStreamingConfig}; @@ -1258,9 +1258,9 @@ mod asap_tier_classify_tests { .iter() .map(|s| s.to_string()) .collect::>(), - capability: Some(Capability::QuantileApprox(SketchKindHandle::DDSketch)), + capability: Some(Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch))), agg_kind: crate::storage_engines::sketch_db::index::AggKind::Sketch { - kind: SketchKindHandle::DDSketch, + algorithm: SketchAlgorithm::DDSketch, config: cfg.clone(), spatial_filter_canonical: String::new(), }, @@ -1532,9 +1532,9 @@ mod asap_tier_classify_tests { sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), - capability: Some(Capability::QuantileApprox(SketchKindHandle::Kll)), + capability: Some(Capability::QuantileApprox(Some(SketchAlgorithm::Kll))), agg_kind: crate::storage_engines::sketch_db::index::AggKind::Sketch { - kind: SketchKindHandle::Kll, + algorithm: SketchAlgorithm::Kll, config: cfg.clone(), spatial_filter_canonical: String::new(), }, @@ -1572,7 +1572,7 @@ mod asap_tier_classify_tests { group_by_keys: BTreeSet::new(), capability: Some(Capability::CardinalityApprox), agg_kind: crate::storage_engines::sketch_db::index::AggKind::Sketch { - kind: SketchKindHandle::Hll, + algorithm: SketchAlgorithm::Hll, config: cfg.clone(), spatial_filter_canonical: String::new(), }, @@ -1720,7 +1720,7 @@ mod asap_tier_classify_tests { for (sid, items) in [(8200u64, &a_items), (8201u64, &b_items)] { let mut meta = hll_meta(sid, "unique_users_global"); meta.agg_kind = crate::storage_engines::sketch_db::index::AggKind::Sketch { - kind: SketchKindHandle::Hll, + algorithm: SketchAlgorithm::Hll, config: SketchConfig::Hll { precision }, spatial_filter_canonical: String::new(), }; @@ -2413,9 +2413,9 @@ mod asap_tier_classify_tests { .iter() .map(|s| s.to_string()) .collect::>(), - capability: Some(Capability::FrequencyEstimate(SketchKindHandle::CountMin)), + capability: Some(Capability::FrequencyEstimate(Some(SketchAlgorithm::Cms))), agg_kind: crate::storage_engines::sketch_db::index::AggKind::Sketch { - kind: SketchKindHandle::CountMin, + algorithm: SketchAlgorithm::Cms, config: cfg.clone(), spatial_filter_canonical: spatial_filter.to_string(), }, @@ -2510,14 +2510,12 @@ mod asap_tier_classify_tests { } #[tokio::test] - async fn keyed_cms_frequency_fails_over_instead_of_misleading_total() { - // P2-6: a per-item selector `cms_metric{item="X"}` against a - // FrequencyEstimate sid would silently get the per-window bucket - // TOTAL (all items), not item X's count. The engine must - // capability-miss to archive rather than return that misleading - // total. The sid here is registered with an EMPTY spatial filter, - // so the `{item="X"}` matcher is an additional per-item selector - // not baked into the sketch. + async fn keyed_cms_frequency_reads_point_estimate_instead_of_total() { + use crate::query_engines::query_result::QueryResult; + // A per-item selector is bound to Planner's typed PointCount readout. + // This synthetic matrix stores 600 only in cell zero; item X hashes + // elsewhere, so its estimate is 0. Returning 600 would prove the old + // misleading whole-bucket-total behavior had regressed. let now = now_ms_for_test(); let idx = Arc::new(SketchStore::new()); register_cms_freq_sid(&idx, 7100, "cms_metric", &[], "", 600, now); @@ -2525,16 +2523,29 @@ mod asap_tier_classify_tests { let engine = build_engine_with_index(idx); let result = engine .execute("count_over_time(cms_metric{item=\"X\"}[5m])") + .await + .expect("keyed CMS frequency should execute through PointCount"); + let QueryResult::Vector(vector) = result else { + panic!("expected instant vector") + }; + assert_eq!(vector.values.len(), 1); + assert_eq!(vector.values[0].value, 0.0); + } + + #[tokio::test] + async fn ordinary_label_filter_is_not_bound_as_frequency_item_key() { + let now = now_ms_for_test(); + let idx = Arc::new(SketchStore::new()); + register_cms_freq_sid(&idx, 7150, "cms_metric", &[], "", 600, now); + + let engine = build_engine_with_index(idx); + let result = engine + .execute("count_over_time(cms_metric{region=\"west\"}[5m])") .await; - match result { - Err(EngineError::CapabilityMiss { detail, .. }) => { - assert!( - detail.contains("KeyedFrequency"), - "expected the Planner-DAG keyed-frequency safe-miss, got: {detail}" - ); - } - other => panic!("keyed CMS frequency must fail over to archive (P2-6), got {other:?}"), - } + assert!( + matches!(result, Err(EngineError::CapabilityMiss { .. })), + "a spatial label must not be reinterpreted as a sketch item key: {result:?}" + ); } #[tokio::test] @@ -2575,8 +2586,8 @@ mod outer_agg_integration_tests { use crate::query_engines::routing::query_engine_routing::QueryEngine as _; use crate::query_engines::EngineError; use crate::storage_engines::sketch_db::index::{ - AccuracyBound, Capability, SketchConfig, SketchEncoding, SketchInstanceMetadata, - SketchKindHandle, SketchSampleState, SketchStore, + AccuracyBound, Capability, SketchAlgorithm, SketchConfig, SketchEncoding, + SketchInstanceMetadata, SketchSampleState, SketchStore, }; use crate::storage_engines::types::HotReloadStreamingConfig; use asap_sketchlib::DdSketch; @@ -2611,9 +2622,9 @@ mod outer_agg_integration_tests { .iter() .map(|s| s.to_string()) .collect::>(), - capability: Some(Capability::QuantileApprox(SketchKindHandle::DDSketch)), + capability: Some(Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch))), agg_kind: crate::storage_engines::sketch_db::index::AggKind::Sketch { - kind: SketchKindHandle::DDSketch, + algorithm: SketchAlgorithm::DDSketch, config: cfg.clone(), spatial_filter_canonical: String::new(), }, @@ -2928,8 +2939,8 @@ mod range_stitch_tests { use crate::query_engines::routing::query_engine_routing::{EngineCapabilities, QueryEngine}; use crate::query_engines::EngineError; use crate::storage_engines::sketch_db::index::{ - AccuracyBound, Capability, SketchConfig, SketchEncoding, SketchInstanceMetadata, - SketchKindHandle, SketchSampleState, SketchStore, + AccuracyBound, Capability, SketchAlgorithm, SketchConfig, SketchEncoding, + SketchInstanceMetadata, SketchSampleState, SketchStore, }; use crate::storage_engines::types::{HotReloadStreamingConfig, KeyByLabelValues}; use async_trait::async_trait; @@ -2998,9 +3009,9 @@ mod range_stitch_tests { sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), - capability: Some(Capability::FrequencyEstimate(SketchKindHandle::CountMin)), + capability: Some(Capability::FrequencyEstimate(Some(SketchAlgorithm::Cms))), agg_kind: crate::storage_engines::sketch_db::index::AggKind::Sketch { - kind: SketchKindHandle::CountMin, + algorithm: SketchAlgorithm::Cms, config: cfg.clone(), spatial_filter_canonical: String::new(), }, diff --git a/data_plane/src/query_engines/asap_query_engine/live_serve.rs b/data_plane/src/query_engines/asap_query_engine/live_serve.rs index 49a50376..8032b0d0 100644 --- a/data_plane/src/query_engines/asap_query_engine/live_serve.rs +++ b/data_plane/src/query_engines/asap_query_engine/live_serve.rs @@ -33,7 +33,30 @@ use crate::storage_engines::sketch_db::query::ASAPTierResult; /// query can't be served either way). `data_plane` doesn't carry a /// per-workload `AccuracyTarget` today (see the design doc's "Rollout" /// section). -const LIVE_ACCURACY: AccuracyTarget = AccuracyTarget::Epsilon(0.01); +const DEFAULT_LIVE_EPSILON: f64 = 0.01; + +fn live_accuracy() -> AccuracyTarget { + live_accuracy_from_values( + std::env::var("ASAP_SUMMARY_EXECUTOR_EPSILON") + .ok() + .as_deref(), + std::env::var("ASAP_SUMMARY_EXECUTOR_DELTA").ok().as_deref(), + ) +} + +fn live_accuracy_from_values(epsilon: Option<&str>, delta: Option<&str>) -> AccuracyTarget { + let epsilon = epsilon + .and_then(|value| value.trim().parse::().ok()) + .filter(|value| value.is_finite() && *value > 0.0 && *value < 1.0) + .unwrap_or(DEFAULT_LIVE_EPSILON); + match delta + .and_then(|value| value.trim().parse::().ok()) + .filter(|value| value.is_finite() && *value > 0.0 && *value < 1.0) + { + Some(delta) => AccuracyTarget::EpsilonDelta { epsilon, delta }, + None => AccuracyTarget::Epsilon(epsilon), + } +} /// Whether the serving cutover is enabled for this process. The env var /// stays as a kill switch (`ASAP_SUMMARY_EXECUTOR_LIVE=0`/`false`/`off`) — @@ -50,12 +73,14 @@ const LIVE_ACCURACY: AccuracyTarget = AccuracyTarget::Epsilon(0.01); /// (empty-`by` ambiguity) is resolved via the real `Reduction::{Reduce, /// PerEntity}` IR signal, not a heuristic. pub fn summary_executor_live_enabled() -> bool { - std::env::var("ASAP_SUMMARY_EXECUTOR_LIVE") - .map(|v| { - let v = v.trim(); - !(v == "0" || v.eq_ignore_ascii_case("false") || v.eq_ignore_ascii_case("off")) - }) - .unwrap_or(true) + let value = std::env::var("ASAP_SUMMARY_EXECUTOR_LIVE").ok(); + summary_executor_live_value(value.as_deref()) +} + +fn summary_executor_live_value(value: Option<&str>) -> bool { + value.map(str::trim).is_none_or(|v| { + !(v == "0" || v.eq_ignore_ascii_case("false") || v.eq_ignore_ascii_case("off")) + }) } /// Try to serve `query` entirely from `SummaryExecutor`. Returns `None` @@ -88,7 +113,7 @@ pub fn try_serve_from_summary_executor( t0_ms, t1_ms, is_cumulative, - LIVE_ACCURACY, + live_accuracy(), backend_plan, ) .map_err(|skip| { @@ -140,7 +165,7 @@ pub fn serve_instant_from_summary_executor( return Err(LoweringSkip::Disabled); } let (outcome, t0_ms) = - execute_post_asap_instant(index, query, now_ms, LIVE_ACCURACY, backend_plan)?; + execute_post_asap_instant(index, query, now_ms, live_accuracy(), backend_plan)?; Ok(( ASAPTierResult { series: outcome.series, @@ -157,40 +182,9 @@ mod tests { use crate::storage_engines::sketch_db::data::{AggKind, SketchConfig}; use crate::storage_engines::sketch_db::index::{ - AccuracyBound, Capability, SketchInstanceMetadata, SketchKindHandle, SketchSampleState, + AccuracyBound, Capability, SketchAlgorithm, SketchInstanceMetadata, SketchSampleState, }; - /// Mirrors `shadow_compare.rs`'s `ENV_VAR_LOCK`/`ShadowEnvGuard` - /// pattern exactly, own env var — `std::env::set_var`/`remove_var` - /// mutate process-global state and `cargo test` runs this module's - /// tests on multiple threads in the same process. - static ENV_VAR_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); - - #[allow(dead_code)] - struct LiveEnvGuard(std::sync::MutexGuard<'static, ()>); - - impl Drop for LiveEnvGuard { - fn drop(&mut self) { - std::env::remove_var("ASAP_SUMMARY_EXECUTOR_LIVE"); - } - } - - fn set_live_env(value: &str) -> LiveEnvGuard { - let guard = ENV_VAR_LOCK - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); - std::env::set_var("ASAP_SUMMARY_EXECUTOR_LIVE", value); - LiveEnvGuard(guard) - } - - fn clear_live_env() -> LiveEnvGuard { - let guard = ENV_VAR_LOCK - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); - std::env::remove_var("ASAP_SUMMARY_EXECUTOR_LIVE"); - LiveEnvGuard(guard) - } - fn ddsketch_fixture() -> SketchStore { let idx = SketchStore::new(); let cfg = SketchConfig::DDSketch { @@ -200,9 +194,9 @@ mod tests { sid: 1, metric_name: "latency_ms".to_string(), group_by_keys: std::collections::BTreeSet::new(), - capability: Some(Capability::QuantileApprox(SketchKindHandle::DDSketch)), + capability: Some(Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch))), agg_kind: AggKind::Sketch { - kind: SketchKindHandle::DDSketch, + algorithm: SketchAlgorithm::DDSketch, config: cfg.clone(), spatial_filter_canonical: String::new(), }, @@ -239,7 +233,7 @@ mod tests { group_by_keys, capability: Some(Capability::CardinalityApprox), agg_kind: AggKind::Sketch { - kind: SketchKindHandle::Hll, + algorithm: SketchAlgorithm::Hll, config: cfg.clone(), spatial_filter_canonical: String::new(), }, @@ -268,20 +262,10 @@ mod tests { } #[test] - fn flag_explicitly_off_never_serves() { - // Kill switch: an explicit off-spelling still disables live-serve - // even though the default (unset) is now on. - let _guard = set_live_env("0"); - let idx = ddsketch_fixture(); - let result = try_serve_from_summary_executor( - &idx, - "quantile_over_time(0.99, latency_ms[1m])", - 1_000, - 2_000, - true, - None, - ); - assert!(result.is_none(), "flag explicitly off must never serve"); + fn flag_off_spellings_disable_live_serve() { + assert!(!summary_executor_live_value(Some("0"))); + assert!(!summary_executor_live_value(Some(" false "))); + assert!(!summary_executor_live_value(Some("OFF"))); } #[test] @@ -289,7 +273,7 @@ mod tests { // Default flipped to on (design-target-architecture.md §4/Part A) // -- an unset env var must serve, not fall back to the legacy // path, for a shape this executor already proves safe. - let _guard = clear_live_env(); + assert!(summary_executor_live_value(None)); let idx = ddsketch_fixture(); let result = try_serve_from_summary_executor( &idx, @@ -302,9 +286,23 @@ mod tests { assert!(result.is_some(), "unset flag must default to serving"); } + #[test] + fn live_accuracy_accepts_explicit_valid_epsilon_delta() { + assert_eq!( + live_accuracy_from_values(Some("0.02"), Some("0.04")), + AccuracyTarget::EpsilonDelta { + epsilon: 0.02, + delta: 0.04, + } + ); + assert_eq!( + live_accuracy_from_values(Some("invalid"), Some("1.0")), + AccuracyTarget::Epsilon(DEFAULT_LIVE_EPSILON) + ); + } + #[test] fn flag_on_safe_shape_serves() { - let _guard = set_live_env("1"); let idx = ddsketch_fixture(); let result = try_serve_from_summary_executor( &idx, @@ -328,7 +326,6 @@ mod tests { // executor resolves it -- `count(...)` lowers to `Reduce([])`, both // sids share one group key, and the new path serves the correctly // merged answer instead of falling back. - let _guard = set_live_env("1"); let idx = SketchStore::new(); register_hll(&idx, 1, "svc-a", &["a", "b", "c"]); register_hll(&idx, 2, "svc-b", &["d", "e", "f"]); @@ -352,7 +349,6 @@ mod tests { #[test] fn flag_on_unservable_query_falls_back() { - let _guard = set_live_env("1"); let idx = SketchStore::new(); let result = try_serve_from_summary_executor( &idx, diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_planner.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_planner.rs index aac5a685..ab4aeea6 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_planner.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_planner.rs @@ -14,7 +14,7 @@ //! a genuine planning decision, and planning already made it once, for //! real, when this metric's workload was planned — that decision is what //! `data_plane`'s ingest path actually registered in the `SketchStore` -//! (`AggKind::Sketch { kind, config, .. }`). Serving time must reproduce +//! (`AggKind::Sketch { algorithm: kind, config, .. }`). Serving time must reproduce //! THAT decision, not independently re-derive a fresh one from a //! hardcoded accuracy target: doing so picks whatever family/params an //! accuracy-driven cost model prefers in the abstract (e.g. DDSketch @@ -44,7 +44,6 @@ use control_plane::physical::post_asap::cost_model::ObservedFamilyCostModel; use control_plane::physical::post_asap::{ bind_query_expr_with_cost_model, BindingError, PhysicalExpr, PostAsapPlan, }; -use control_plane::physical::runtime_capability::SketchKindHandle; use control_plane::types_v2::AccuracyTarget; use crate::query_engines::asap_query_engine::summary_executor::find_metric_in_query_expr; @@ -119,11 +118,11 @@ pub enum LoweringSkip { // The semantic tree returned by ASAPPlanner is `post_asap::SummaryNode`; this // module does not claim or recreate an ASAPPlanner "L4" IR. -/// Map a registered sid's `(SketchKindHandle, SketchConfig)` — the +/// Map a registered sid's `(SketchAlgorithm, SketchConfig)` — the /// durable record of what planning actually decided for this metric — to /// the `(SketchAlgorithm, SketchParams)` pair `ObservedFamilyCostModel` /// needs to reproduce that decision exactly. `None` for shapes this -/// deployment doesn't map (e.g. `SketchKindHandle::Any`, which is an +/// deployment doesn't map (e.g. an unsupported algorithm, which is an /// analysis-time wildcard that's never actually registered on a sid). /// /// Heap-bearing kinds (`CmsWithHeap`/`CountSketchWithHeap`) reuse their @@ -133,34 +132,34 @@ pub enum LoweringSkip { /// only compares `width`/`depth` for these kinds, so it doesn't affect /// matching. fn observed_summary_params( - kind: SketchKindHandle, + kind: SketchAlgorithm, config: &SketchConfig, ) -> Option<(SketchAlgorithm, SketchParams)> { const PLACEHOLDER_HEAP_SIZE: u32 = 100; match (kind, config) { - (SketchKindHandle::DDSketch, SketchConfig::DDSketch { relative_accuracy }) => Some(( + (SketchAlgorithm::DDSketch, SketchConfig::DDSketch { relative_accuracy }) => Some(( SketchAlgorithm::DDSketch, SketchParams::DDSketch { alpha: *relative_accuracy, }, )), - (SketchKindHandle::Kll, SketchConfig::Kll { k }) => { + (SketchAlgorithm::Kll, SketchConfig::Kll { k }) => { Some((SketchAlgorithm::Kll, SketchParams::Kll { k: *k })) } - (SketchKindHandle::Hll, SketchConfig::Hll { precision }) => Some(( + (SketchAlgorithm::Hll, SketchConfig::Hll { precision }) => Some(( SketchAlgorithm::Hll, SketchParams::Hll { precision: *precision as u8, }, )), - (SketchKindHandle::CountMin, SketchConfig::CountMin { rows, cols }) => Some(( + (SketchAlgorithm::Cms, SketchConfig::CountMin { rows, cols }) => Some(( SketchAlgorithm::Cms, SketchParams::Cms { width: *cols as u32, depth: *rows as u32, }, )), - (SketchKindHandle::CmsWithHeap, SketchConfig::CountMin { rows, cols }) => Some(( + (SketchAlgorithm::CmsWithHeap, SketchConfig::CountMin { rows, cols }) => Some(( SketchAlgorithm::CmsWithHeap, SketchParams::CmsWithHeap { width: *cols as u32, @@ -168,23 +167,21 @@ fn observed_summary_params( heap_size: PLACEHOLDER_HEAP_SIZE, }, )), - (SketchKindHandle::CountSketch, SketchConfig::CountSketch { rows, cols }) => Some(( + (SketchAlgorithm::CountSketch, SketchConfig::CountSketch { rows, cols }) => Some(( SketchAlgorithm::CountSketch, SketchParams::CountSketch { width: *cols as u32, depth: *rows as u32, }, )), - (SketchKindHandle::CountSketchWithHeap, SketchConfig::CountSketch { rows, cols }) => { - Some(( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width: *cols as u32, - depth: *rows as u32, - heap_size: PLACEHOLDER_HEAP_SIZE, - }, - )) - } + (SketchAlgorithm::CountSketchWithHeap, SketchConfig::CountSketch { rows, cols }) => Some(( + SketchAlgorithm::CountSketchWithHeap, + SketchParams::CountSketchWithHeap { + width: *cols as u32, + depth: *rows as u32, + heap_size: PLACEHOLDER_HEAP_SIZE, + }, + )), _ => None, } } @@ -204,7 +201,11 @@ fn observed_family_for_metric( ) -> Option<(SketchAlgorithm, SketchParams)> { for sid in index.instances_matching(metric, &Default::default()) { let found = index.with_instance(sid, |m| match &m.agg_kind { - AggKind::Sketch { kind, config, .. } => observed_summary_params(*kind, config), + AggKind::Sketch { + algorithm: kind, + config, + .. + } => observed_summary_params(kind.clone(), config), AggKind::ExactAgg { .. } => None, }); if let Some(Some(observed)) = found { @@ -217,8 +218,8 @@ fn observed_family_for_metric( /// Look up what family/params `plan` says is materialized for `metric` — /// the `BackendPlan`-sourced sibling of [`observed_family_for_metric`]. /// Unlike that function, no reconstruction is needed: -/// `Materialization.kind`/`.params` already ARE the pair this needs, -/// straight off the wire the control plane pushed. Returns the first +/// `Materialization.family` already carries the canonical `SketchKind` this +/// needs, straight off the wire the control plane pushed. Returns the first /// matching materialization found (mirrors /// `observed_family_for_metric`'s "first sketch-typed one found" /// semantics); `None` when the plan has no materialization for this @@ -243,12 +244,12 @@ fn observed_families_for_metric_from_plan( { return None; } - // `Materialization.kind`/`.params` are the flat type (spans - // exact accumulators too) -- narrow to the sketch-only pair - // this function returns, skipping exact-accumulator - // materializations (mirrors `observed_family_for_metric`'s - // "first sketch-typed one found" semantics). - Some((m.kind.as_sketch_kind()?, m.params.as_sketch_params()?)) + match &m.family { + planner_types::post_asap::SummaryFamilyType::Sketch(kind, _) => { + Some((kind.algorithm().clone(), kind.params().clone())) + } + _ => None, + } }).collect() } @@ -289,27 +290,20 @@ pub fn resolve_materializations_for_post_asap( .ok_or_else(|| { LoweringSkip::NoWarmRoute("summary has no time-series source".into()) })?; - let (kind, params): (asap_types::SummaryKind, asap_types::SummaryParams) = - match family { - planner_types::post_asap::SummaryFamilyType::ExactAggregate( - kind, - params, - ) => (kind.clone().into(), params.clone().into()), - planner_types::post_asap::SummaryFamilyType::Sketch(kind, _) => { - (kind.clone().into(), kind.params().clone().into()) - } - _ => { - return Err(LoweringSkip::NoWarmRoute(format!( - "unsupported maintained family for metric `{metric}`" - ))) - } - }; + if !matches!( + family, + planner_types::post_asap::SummaryFamilyType::ExactAggregate(..) + | planner_types::post_asap::SummaryFamilyType::Sketch(..) + ) { + return Err(LoweringSkip::NoWarmRoute(format!( + "unsupported maintained family for metric `{metric}`" + ))); + } let matches: Vec<_> = plan.routing.iter().filter_map(|route| { (route.storage_backend == control_plane::backend_plan::StorageBackend::SketchStore && plan.materializations.get(&route.materialization).is_some_and(|m| { matches!(&m.source, planner_types::pre_asap::Source::TimeSeries { metric: mm } if mm == &metric) - && m.kind == kind - && m.params == params + && &m.family == family && m.spatial_filter == spatial_filter && required_groups.iter().all(|key| m.group_by.contains(key)) && m.window.size_ms <= query_window_ms @@ -318,7 +312,7 @@ pub fn resolve_materializations_for_post_asap( }).collect(); if matches.is_empty() { return Err(LoweringSkip::NoWarmRoute(format!( - "no warm BackendPlan route for metric `{metric}` and family `{kind:?}`" + "no warm BackendPlan route for metric `{metric}` and family `{family:?}`" ))); } resolved.extend(matches); @@ -583,13 +577,9 @@ fn ensure_warm_runtime_support( } ensure_warm_runtime_support(child, source_has_filter) } - SummaryExpr::SummaryEstimate { - summary_input: _, - query: SketchQuery::PointCount { value: Some(_), .. }, - } => Err(LoweringSkip::KeyedFrequency), SummaryExpr::SummaryEstimate { summary_input, - query: SketchQuery::PointCount { .. }, + query: SketchQuery::PointCount { value: None, .. }, } if source_has_filter => Err(LoweringSkip::KeyedFrequency), SummaryExpr::SummaryEstimate { summary_input, .. } => { ensure_warm_runtime_support(summary_input, source_has_filter) @@ -607,6 +597,36 @@ fn ensure_warm_runtime_support( } } +/// Bind the conventional PromQL `item="..."` equality matcher to a frequency +/// point readout. The Planner DAG owns the `SketchQuery`; this adapter only +/// supplies the literal value that the PromQL frontend currently leaves as +/// `None`. +fn bind_point_count_filter(node: &mut Rc, key: &str, value: &str) -> bool { + let node = Rc::make_mut(node); + match &mut node.expr { + SummaryExpr::SummaryEstimate { + query: + planner_types::post_asap::SketchQuery::PointCount { + key: point_key, + value: point_value, + }, + .. + } if point_value.is_none() => { + *point_key = planner_types::pre_asap::ColumnRef::Named(key.to_string()); + *point_value = Some(value.to_string()); + true + } + SummaryExpr::SummaryAgg { child, .. } => bind_point_count_filter(child, key, value), + SummaryExpr::SummaryEstimate { summary_input, .. } => { + bind_point_count_filter(summary_input, key, value) + } + SummaryExpr::SummaryMerge { children } => children + .iter_mut() + .any(|child| bind_point_count_filter(child, key, value)), + _ => false, + } +} + /// Lower a raw PromQL query string to the `SummaryNode` tree /// `crate::query_engines::asap_query_engine::summary_exec::execute`/`SummaryExecutor` needs — the actual /// serving cutover (`live_serve.rs`). Returns `Err` for any shape serving @@ -630,8 +650,7 @@ pub fn plan_promql_to_post_asap( // independently re-derive one -- see this module's docs. Prefer // reading it straight off an installed `BackendPlan`'s // materializations when one covers this metric -- - // `Materialization.kind`/`.params` already ARE the - // `(SketchAlgorithm, SketchParams)` pair this needs, no + // `Materialization.family` already carries the canonical `SketchKind`, no // `AggregationConfig` reconstruction required (design-backend-plan-wire-format.md // §5). Otherwise fall back to the `SketchStore`-reconstruction path // (`observed_family_for_metric`), which is `None` when this metric @@ -681,10 +700,19 @@ pub fn plan_promql_to_post_asap( }; match physical { - PhysicalExpr::Committed(PostAsapPlan::Summary(node)) => { + PhysicalExpr::Committed(PostAsapPlan::Summary(mut node)) => { if matches!(node.expr, SummaryExpr::KeepPreAsap(_)) { last_skip = LoweringSkip::NotRealized; } else { + if let Ok(parsed) = + control_plane::query_parser::parse_query(query, accuracy.clone()) + { + if parsed.label_filters.len() == 1 { + if let Some(value) = parsed.label_filters.get("item") { + bind_point_count_filter(&mut node, "item", value); + } + } + } if let Err(skip) = ensure_warm_runtime_support(&node, source_has_filter) { last_skip = skip; continue; @@ -865,7 +893,7 @@ mod tests { mod backend_plan_cutover { use super::*; use crate::storage_engines::sketch_db::index::{ - AccuracyBound, Capability, SketchInstanceMetadata, SketchKindHandle, + AccuracyBound, Capability, SketchAlgorithm, SketchInstanceMetadata, }; use asap_types::enums::WindowKind; use control_plane::backend_plan::{ @@ -880,9 +908,9 @@ mod tests { sid: 1, metric_name: metric.to_string(), group_by_keys: Default::default(), - capability: Some(Capability::QuantileApprox(SketchKindHandle::Kll)), + capability: Some(Capability::QuantileApprox(Some(SketchAlgorithm::Kll))), agg_kind: AggKind::Sketch { - kind: SketchKindHandle::Kll, + algorithm: SketchAlgorithm::Kll, config: cfg.clone(), spatial_filter_canonical: String::new(), }, @@ -912,14 +940,16 @@ mod tests { group_by: Vec::new(), rollup: Vec::new(), spatial_filter: String::new(), - // `Materialization.kind`/`.params` span both exact - // accumulators and sketches -- the flat - // `asap_types::SummaryKind`, not this file's own - // `planner_types::post_asap::SketchAlgorithm` import (see - // `physical::colored_dag::emitter`'s `use - // asap_types::{...}` note in control_plane). - kind: asap_types::SummaryKind::DDSketch, - params: asap_types::SummaryParams::DDSketch { alpha: 0.01 }, + // `Materialization.family` is Planner's canonical + // `SummaryFamilyType`; its sketch branch carries a + // validated `SketchKind` (category + algorithm + params). + family: planner_types::post_asap::SummaryFamilyType::Sketch( + planner_types::post_asap::SketchKind::new( + SketchAlgorithm::DDSketch, + SketchParams::DDSketch { alpha: 0.01 }, + ), + planner_types::post_asap::GroupingStrategy::PerSubpopulationInstance, + ), col: ColumnRef::SampleValue, retention: None, lifecycle: None, @@ -930,7 +960,7 @@ mod tests { generated_at_unix_ms: 0, materializations, routing: vec![RoutingEntry { - satisfies: Capability::QuantileApprox(SketchKindHandle::DDSketch), + satisfies: Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch)), materialization: fingerprint, storage_backend: StorageBackend::SketchStore, }], diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index c8fd1845..439ca8ec 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -212,7 +212,7 @@ mod tests { use super::*; use crate::storage_engines::sketch_db::data::{AggKind, SketchConfig}; use crate::storage_engines::sketch_db::index::{ - AccuracyBound, Capability, SketchInstanceMetadata, SketchKindHandle, SketchSampleState, + AccuracyBound, Capability, SketchAlgorithm, SketchInstanceMetadata, SketchSampleState, }; fn accuracy() -> AccuracyTarget { @@ -235,7 +235,7 @@ mod tests { group_by_keys, capability: Some(Capability::CardinalityApprox), agg_kind: AggKind::Sketch { - kind: SketchKindHandle::Hll, + algorithm: SketchAlgorithm::Hll, config: cfg.clone(), spatial_filter_canonical: String::new(), }, @@ -275,9 +275,9 @@ mod tests { sid: 1, metric_name: "latency_ms".to_string(), group_by_keys: std::collections::BTreeSet::new(), - capability: Some(Capability::QuantileApprox(SketchKindHandle::DDSketch)), + capability: Some(Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch))), agg_kind: AggKind::Sketch { - kind: SketchKindHandle::DDSketch, + algorithm: SketchAlgorithm::DDSketch, config: cfg.clone(), spatial_filter_canonical: String::new(), }, diff --git a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs index 0be2b334..1c374624 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs @@ -42,7 +42,7 @@ //! `Some(_)`) — reported rather than silently guessed at. //! //! `find_candidates`/`fetch_state`/`merge_states` ALSO recognize -//! `AggKind::ExactAgg` sids for `SummaryKind::{Sum, Increase}` (see +//! `AggKind::ExactAgg` sids for `ExactKind::{Sum, Increase}` (see //! `exact_agg_kind_match`'s doc for why `MinMax`/`Count`/`Rate` aren't //! matched) — one sid is one aggregation, read out directly, with no //! special-casing of exact-vs-approximate at the `find_candidates`/merge @@ -64,18 +64,11 @@ use std::rc::Rc; use std::sync::Arc; use crate::query_engines::asap_query_engine::summary_exec::SummaryExecutor; -use planner_types::post_asap::{SketchQuery, SummaryExpr, SummaryFamilyType, SummaryNode}; +use planner_types::post_asap::{ + ExactKind, ExactParams, SketchAlgorithm, SketchParams, SketchQuery, SummaryExpr, + SummaryFamilyType, SummaryNode, +}; use planner_types::pre_asap::{ColumnId, ColumnRef, QueryExpr, Reduction, Source}; -// This file's own flat `(SummaryKind, SummaryParams)` -- spans both exact -// accumulators and approximate sketches in one pair, matching every -// internal matcher below (`summary_params_match`/`exact_agg_kind_match`) -// -- vendored because ASAPPlanner split its old flat `SummaryKind` into -// per-family `SketchKind`/`ExactKind` (ASAPPlanner#218). See -// `crates/asap_types/src/accumulator_spec.rs`'s module doc and -// control_plane/docs/design-asapplanner-pin-migration.md. -use asap_types::{SummaryKind, SummaryParams}; - -use control_plane::physical::runtime_capability::SketchKindHandle; use crate::storage_engines::sketch_db::data::{AggKind, SketchConfig, SketchTimeSeries}; use crate::storage_engines::sketch_db::index::{SketchSampleState, SketchStore}; @@ -253,7 +246,7 @@ impl GroupState { #[derive(Debug)] pub enum SummaryExecutorError { /// No sid in the catalog matches the requested `(metric, by, - /// SummaryKind, SummaryParams)` — mirrors today's `CapabilityMiss` + /// SummaryFamilyType)` — mirrors today's `CapabilityMiss` /// contract; the caller fails over to archive. NoCandidates, /// Couldn't recover a metric name by walking the `SummaryAgg`'s @@ -263,7 +256,7 @@ pub enum SummaryExecutorError { /// A requested `by` `ColumnId` doesn't resolve to a name against the /// child's schema. UnresolvedColumn(ColumnId), - /// A candidate sid claims a `SummaryKind` this executor doesn't + /// A candidate sid claims a `SummaryFamilyType` this executor doesn't /// implement cross-sid merge for, or the sid's on-disk /// `SketchConfig` didn't decode into a `DeltaSketchKind`. UnsupportedFamily, @@ -340,29 +333,15 @@ impl<'a> SummaryExecutor for QueryExecutionContext<'a> { reduction: &Reduction, child: &SummaryNode, ) -> Result, Self::Error> { - // `SummaryAgg`'s `kind`/`params` collapsed into this one `family` - // field (ASAPPlanner#218 -- see this file's `use asap_types::{...}` - // note above); recover the flat `(SummaryKind, SummaryParams)` - // pair every matcher below still expects. `Plain`/`Sample`/ - // `Wavelet`/`StatModel` never occur on a real `SummaryAgg` (never - // `Plain` by construction; the others are unreachable via this - // deployment's own `CostModel` -- see - // the physical runtime-capability adapter - // doc for the same reasoning), so there's no candidate to find. - let (sketch, params): (SummaryKind, SummaryParams) = match family { - SummaryFamilyType::ExactAggregate(kind, params) => { - (kind.clone().into(), params.clone().into()) - } - SummaryFamilyType::Sketch(kind, _) => { - (kind.clone().into(), kind.params().clone().into()) - } + if matches!( + family, SummaryFamilyType::Plain(_) - | SummaryFamilyType::Sample(..) - | SummaryFamilyType::Wavelet(..) - | SummaryFamilyType::StatModel(..) => return Ok(Vec::new()), - }; - let sketch = &sketch; - let params = ¶ms; + | SummaryFamilyType::Sample(..) + | SummaryFamilyType::Wavelet(..) + | SummaryFamilyType::StatModel(..) + ) { + return Ok(Vec::new()); + } let metric = find_metric(child).ok_or(SummaryExecutorError::NoMetricFound)?; let by: &[ColumnId] = reduction.group_keys().map(|k| k.keys()).unwrap_or(&[]); @@ -401,14 +380,16 @@ impl<'a> SummaryExecutor for QueryExecutionContext<'a> { return None; } match &m.agg_kind { - AggKind::Sketch { kind, config, .. } => { - summary_params_match(sketch, params, *kind, config) - .then(|| to_delta_kind(*kind, config)) - .flatten() - .map(Candidate::Sketch) - } + AggKind::Sketch { + algorithm: kind, + config, + .. + } => summary_family_matches_sketch(family, kind.clone(), config) + .then(|| to_delta_kind(kind.clone(), config)) + .flatten() + .map(Candidate::Sketch), AggKind::ExactAgg { agg_type, .. } => { - exact_agg_kind_match(sketch, params, *agg_type) + summary_family_matches_exact(family, *agg_type) .then_some(Candidate::ExactAgg(*agg_type)) } } @@ -506,7 +487,7 @@ impl<'a> SummaryExecutor for QueryExecutionContext<'a> { entries.extend(more); } // `find_candidates`'s exact-match contract never produces a - // mixed group (a `(SummaryKind, SummaryParams)` query + // mixed group (a `(SummaryFamilyType)` query // matches either sketch-family sids or ExactAgg sids, never // both) -- defensive, not a real path. _ => return Err(SummaryExecutorError::UnsupportedFamily), @@ -720,109 +701,104 @@ fn topk_ranked(rs: &SummaryState, k: usize) -> Result, Summar Ok(items) } -/// Exact `(SummaryKind, SummaryParams)` match against a sid's own -/// `(SketchKindHandle, SketchConfig)` -- the check `find_candidates`'s +/// Exact canonical `SketchKind` match against a sid's own +/// `(SketchAlgorithm, SketchConfig)` -- the check `find_candidates`'s /// trait contract requires (not the looser family-only /// `Capability::is_satisfied_by` check the legacy analyzer path uses), /// so a `SummaryMerge`'s precondition (every child agrees on kind AND /// params) is guaranteed by construction for anything routed through /// this executor. -fn summary_params_match( - sketch: &SummaryKind, - params: &SummaryParams, - kind: SketchKindHandle, +fn summary_family_matches_sketch( + family: &SummaryFamilyType, + kind: SketchAlgorithm, config: &SketchConfig, ) -> bool { - // `SummaryParams::{Cms,CmsWithHeap,CountSketch,CountSketchWithHeap}` + // `SketchParams::{Cms,CmsWithHeap,CountSketch,CountSketchWithHeap}` // use width=cols/depth=rows (matches the control-plane wire // convention -- see `sketch_config_to_json`'s comment). `SketchConfig` // has no `heap_size` field at all (heap-bearing kinds reuse their // heap-less base's config shape for identity -- see - // `base_sketch_kind_handle`'s doc in `drivers/ingest/otel.rs`), so + // `base_sketch_algorithm`'s doc in `drivers/ingest/otel.rs`), so // heap_size can't be part of this match; width/depth are. - match (sketch, params, kind, config) { + let SummaryFamilyType::Sketch(sketch, _) = family else { + return false; + }; + match (sketch.algorithm(), sketch.params(), kind, config) { ( - SummaryKind::DDSketch, - SummaryParams::DDSketch { alpha }, - SketchKindHandle::DDSketch, + SketchAlgorithm::DDSketch, + SketchParams::DDSketch { alpha }, + SketchAlgorithm::DDSketch, SketchConfig::DDSketch { relative_accuracy }, ) => alpha == relative_accuracy, ( - SummaryKind::Kll, - SummaryParams::Kll { k }, - SketchKindHandle::Kll, + SketchAlgorithm::Kll, + SketchParams::Kll { k }, + SketchAlgorithm::Kll, SketchConfig::Kll { k: sid_k }, ) => k == sid_k, ( - SummaryKind::Hll, - SummaryParams::Hll { precision }, - SketchKindHandle::Hll, + SketchAlgorithm::Hll, + SketchParams::Hll { precision }, + SketchAlgorithm::Hll, SketchConfig::Hll { precision: sid_p }, ) => u32::from(*precision) == *sid_p, ( - SummaryKind::Cms, - SummaryParams::Cms { width, depth }, - SketchKindHandle::CountMin, + SketchAlgorithm::Cms, + SketchParams::Cms { width, depth }, + SketchAlgorithm::Cms, SketchConfig::CountMin { rows, cols }, ) => *depth as i32 == *rows && *width as i32 == *cols, ( - SummaryKind::CountSketch, - SummaryParams::CountSketch { width, depth }, - SketchKindHandle::CountSketch, + SketchAlgorithm::CountSketch, + SketchParams::CountSketch { width, depth }, + SketchAlgorithm::CountSketch, SketchConfig::CountSketch { rows, cols }, ) => *depth as i32 == *rows && *width as i32 == *cols, ( - SummaryKind::CmsWithHeap, - SummaryParams::CmsWithHeap { width, depth, .. }, - SketchKindHandle::CmsWithHeap, + SketchAlgorithm::CmsWithHeap, + SketchParams::CmsWithHeap { width, depth, .. }, + SketchAlgorithm::CmsWithHeap, SketchConfig::CountMin { rows, cols }, ) => *depth as i32 == *rows && *width as i32 == *cols, ( - SummaryKind::CountSketchWithHeap, - SummaryParams::CountSketchWithHeap { width, depth, .. }, - SketchKindHandle::CountSketchWithHeap, + SketchAlgorithm::CountSketchWithHeap, + SketchParams::CountSketchWithHeap { width, depth, .. }, + SketchAlgorithm::CountSketchWithHeap, SketchConfig::CountSketch { rows, cols }, ) => *depth as i32 == *rows && *width as i32 == *cols, _ => false, } } -/// Exact-agg analog of `summary_params_match`, for `AggKind::ExactAgg` -/// sids. `SummaryParams::{Sum, Count, MinMax, Increase, Rate}` are unit -/// variants (no tuning parameters — see `asap-sketch`'s `SummaryParams` -/// doc), so this is a pure kind-identity check against the sid's +/// Exact-aggregate analog of `summary_family_matches_sketch`, for +/// `AggKind::ExactAgg` sids. `ExactParams` variants carry no tuning +/// parameters, so this is a pure `ExactKind` identity check against the sid's /// `AggregationType`, mirroring the canonical `AggregationType -> -/// SummaryKind` mapping `asap_types::accumulator_spec` uses on the write -/// side (`Sum|MultipleSum -> SummaryKind::Sum`, `Increase|MultipleIncrease -/// -> SummaryKind::Increase` — confirmed against that module's own +/// ExactKind` mapping `asap_types::accumulator_spec` uses on the write +/// side (`Sum|MultipleSum -> ExactKind::Sum`, `Increase|MultipleIncrease +/// -> ExactKind::Increase` — confirmed against that module's own /// dispatch table rather than invented here). /// -/// `SummaryKind::MinMax` is deliberately NOT matched: `AggregationType` +/// `ExactKind::MinMax` is deliberately NOT matched: `AggregationType` /// carries no min-vs-max DIRECTION (that lives in the write-side /// `AggregationConfig::aggregation_sub_type` string, which this sid's /// `AggKind::ExactAgg` metadata doesn't retain), so there's no honest way /// for `GroupState::exact_value` to know which statistic to compute -- /// matching it here would force a later caller to silently guess a -/// direction. `SummaryKind::Count`/`Rate` are ALSO not matched: no +/// direction. `ExactKind::Count`/`Rate` are ALSO not matched: no /// `AggregationType` variant resolves to either today (mirrors /// `sketch_reducer.rs::evaluate_exact_agg`'s own `stat` mapping, which /// only handles `Sum`/`Increase` for the same reason); `Rate` in /// particular is "outer-agg-fold" territory the user has explicitly /// deferred pending a design conversation with ASAPController. -fn exact_agg_kind_match( - sketch: &SummaryKind, - params: &SummaryParams, - agg_type: AggregationType, -) -> bool { +fn summary_family_matches_exact(family: &SummaryFamilyType, agg_type: AggregationType) -> bool { matches!( - (sketch, params, agg_type), + (family, agg_type), ( - SummaryKind::Sum, - SummaryParams::Sum, + SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), AggregationType::Sum | AggregationType::MultipleSum, ) | ( - SummaryKind::Increase, - SummaryParams::Increase, + SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), AggregationType::Increase | AggregationType::MultipleIncrease, ) ) @@ -897,7 +873,7 @@ fn resolve_group_key( /// `SketchConfig` (data_plane's per-sid stored params) -> `DeltaSketchKind` /// (`delta_apply`'s decode/merge parameter carrier). -fn to_delta_kind(kind: SketchKindHandle, config: &SketchConfig) -> Option { +fn to_delta_kind(kind: SketchAlgorithm, config: &SketchConfig) -> Option { // Default heap_size when bootstrapping an empty Heap state for a // delta-from-empty leading window -- `SketchConfig` carries no // heap_size (see `summary_params_match`'s doc), so this only matters @@ -907,35 +883,35 @@ fn to_delta_kind(kind: SketchKindHandle, config: &SketchConfig) -> Option { + (SketchAlgorithm::DDSketch, SketchConfig::DDSketch { relative_accuracy }) => { Some(DeltaSketchKind::DDSketch { alpha: *relative_accuracy, }) } - (SketchKindHandle::Kll, SketchConfig::Kll { k }) => Some(DeltaSketchKind::Kll { k: *k }), - (SketchKindHandle::Hll, SketchConfig::Hll { precision }) => Some(DeltaSketchKind::Hll { + (SketchAlgorithm::Kll, SketchConfig::Kll { k }) => Some(DeltaSketchKind::Kll { k: *k }), + (SketchAlgorithm::Hll, SketchConfig::Hll { precision }) => Some(DeltaSketchKind::Hll { precision: *precision, }), - (SketchKindHandle::CountMin, SketchConfig::CountMin { rows, cols }) => { + (SketchAlgorithm::Cms, SketchConfig::CountMin { rows, cols }) => { Some(DeltaSketchKind::Cms { rows: *rows as usize, cols: *cols as usize, }) } - (SketchKindHandle::CountSketch, SketchConfig::CountSketch { rows, cols }) => { + (SketchAlgorithm::CountSketch, SketchConfig::CountSketch { rows, cols }) => { Some(DeltaSketchKind::CountSketch { rows: *rows as usize, cols: *cols as usize, }) } - (SketchKindHandle::CmsWithHeap, SketchConfig::CountMin { rows, cols }) => { + (SketchAlgorithm::CmsWithHeap, SketchConfig::CountMin { rows, cols }) => { Some(DeltaSketchKind::CmsWithHeap { rows: *rows as usize, cols: *cols as usize, heap_size: DEFAULT_HEAP_SIZE, }) } - (SketchKindHandle::CountSketchWithHeap, SketchConfig::CountSketch { rows, cols }) => { + (SketchAlgorithm::CountSketchWithHeap, SketchConfig::CountSketch { rows, cols }) => { Some(DeltaSketchKind::CountSketchWithHeap { rows: *rows as usize, cols: *cols as usize, @@ -1127,9 +1103,9 @@ mod tests { .iter() .map(|s| s.to_string()) .collect::>(), - capability: Some(Capability::QuantileApprox(SketchKindHandle::Kll)), + capability: Some(Capability::QuantileApprox(Some(SketchAlgorithm::Kll))), agg_kind: crate::storage_engines::sketch_db::index::AggKind::Sketch { - kind: SketchKindHandle::Kll, + algorithm: SketchAlgorithm::Kll, config: cfg.clone(), spatial_filter_canonical: String::new(), }, @@ -1149,7 +1125,7 @@ mod tests { group_by_keys: BTreeSet::new(), capability: Some(Capability::CardinalityApprox), agg_kind: crate::storage_engines::sketch_db::index::AggKind::Sketch { - kind: SketchKindHandle::Hll, + algorithm: SketchAlgorithm::Hll, config: cfg.clone(), spatial_filter_canonical: String::new(), }, @@ -1193,9 +1169,9 @@ mod tests { sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), - capability: Some(Capability::FrequencyEstimate(SketchKindHandle::CountMin)), + capability: Some(Capability::FrequencyEstimate(Some(SketchAlgorithm::Cms))), agg_kind: crate::storage_engines::sketch_db::index::AggKind::Sketch { - kind: SketchKindHandle::CountMin, + algorithm: SketchAlgorithm::Cms, config: cfg.clone(), spatial_filter_canonical: String::new(), }, @@ -1260,9 +1236,11 @@ mod tests { sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), - capability: Some(Capability::FrequencyTopk(SketchKindHandle::CmsWithHeap)), + capability: Some(Capability::FrequencyTopk(Some( + SketchAlgorithm::CmsWithHeap, + ))), agg_kind: crate::storage_engines::sketch_db::index::AggKind::Sketch { - kind: SketchKindHandle::CmsWithHeap, + algorithm: SketchAlgorithm::CmsWithHeap, config: cfg.clone(), spatial_filter_canonical: String::new(), }, @@ -2604,7 +2582,7 @@ mod tests { #[test] fn minmax_exactagg_sid_is_not_matched() { - // `SummaryKind::MinMax` is deliberately NOT matched against + // `ExactKind::MinMax` is deliberately NOT matched against // ExactAgg sids (see `exact_agg_kind_match`'s doc: no direction // info survives to `AggKind::ExactAgg`) -- must fail over as // NoCandidates, not silently guess a direction. diff --git a/data_plane/src/query_engines/routing/capability_matching.rs b/data_plane/src/query_engines/routing/capability_matching.rs index af19b0cd..716d55d9 100644 --- a/data_plane/src/query_engines/routing/capability_matching.rs +++ b/data_plane/src/query_engines/routing/capability_matching.rs @@ -12,27 +12,15 @@ //! shared-struct-field reason to exist. Exercised only by this crate's //! own [`super::query_engine_routing`]. -use asap_types::Statistic; -use serde::{Deserialize, Serialize}; - use crate::storage_engines::types::StorageBackend; +use asap_types::Statistic; +pub use planner_types::types::AccuracyTarget; /// Accuracy hint pushed by the controller at intent-binding time /// (`controller/docs/design.md` §6 `core::workload`). The Phase-5 capability /// router consults this to decide whether a metric configured for both warm- /// tier and Gorilla-S3 should answer from the archive (Exact) or the /// approximate ASAP-tier sketch. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] -#[serde(rename_all = "snake_case")] -pub enum AccuracyTarget { - /// Caller demands an exact answer; ASAP-tier sketches are not eligible - /// unless they happen to be exact accumulators (Sum, MinMax, Increase). - Exact, - /// Caller accepts ε/δ-bounded approximate answers. Default. - #[default] - Approximate, -} - /// Returns the storage backends that can serve a `(statistic, accuracy)` /// query when the metric is configured for `metric_storage_config`. /// @@ -65,7 +53,7 @@ pub enum AccuracyTarget { /// deploy. pub fn compatible_storage_backends( _stat: Statistic, - accuracy: AccuracyTarget, + accuracy: &AccuracyTarget, metric_storage_config: StorageBackend, ) -> Vec { match metric_storage_config { @@ -81,7 +69,7 @@ pub fn compatible_storage_backends( // Exact: archive only — the warm sketches are ε/δ-bounded. AccuracyTarget::Exact => vec![StorageBackend::GorillaObjectStore], // Approximate: ASAP-tier first, archive (Thanos) fallback. - AccuracyTarget::Approximate => vec![ + AccuracyTarget::Epsilon(_) | AccuracyTarget::EpsilonDelta { .. } => vec![ StorageBackend::SketchStore, StorageBackend::GorillaObjectStore, ], @@ -112,7 +100,7 @@ mod tests { StorageBackend::SketchStore, StorageBackend::DoubleWrite, ] { - let backends = compatible_storage_backends(Statistic::Sum, AccuracyTarget::Exact, cfg); + let backends = compatible_storage_backends(Statistic::Sum, &AccuracyTarget::Exact, cfg); assert_eq!( backends, vec![StorageBackend::GorillaObjectStore], @@ -132,8 +120,11 @@ mod tests { StorageBackend::GorillaObjectStore, StorageBackend::DoubleWrite, ] { - let backends = - compatible_storage_backends(Statistic::Quantile, AccuracyTarget::Approximate, cfg); + let backends = compatible_storage_backends( + Statistic::Quantile, + &AccuracyTarget::Epsilon(0.01), + cfg, + ); assert_eq!( backends, vec![ @@ -164,7 +155,7 @@ mod tests { Statistic::Quantile, Statistic::Topk, ]; - let accuracies = [AccuracyTarget::Exact, AccuracyTarget::Approximate]; + let accuracies = [AccuracyTarget::Exact, AccuracyTarget::Epsilon(0.01)]; let configs = [ StorageBackend::SketchStore, StorageBackend::GorillaObjectStore, @@ -173,7 +164,7 @@ mod tests { ]; for &stat in &stats { - for &acc in &accuracies { + for acc in &accuracies { for &cfg in &configs { let backends = compatible_storage_backends(stat, acc, cfg); assert!( @@ -198,7 +189,9 @@ mod tests { let expected_head = match (cfg, acc) { (StorageBackend::PrometheusRemote, _) => StorageBackend::PrometheusRemote, (_, AccuracyTarget::Exact) => StorageBackend::GorillaObjectStore, - (_, AccuracyTarget::Approximate) => StorageBackend::SketchStore, + (_, AccuracyTarget::Epsilon(_) | AccuracyTarget::EpsilonDelta { .. }) => { + StorageBackend::SketchStore + } }; assert_eq!( backends[0], expected_head, diff --git a/data_plane/src/query_engines/routing/query_engine_routing.rs b/data_plane/src/query_engines/routing/query_engine_routing.rs index 010abe9d..af88f54f 100644 --- a/data_plane/src/query_engines/routing/query_engine_routing.rs +++ b/data_plane/src/query_engines/routing/query_engine_routing.rs @@ -219,7 +219,7 @@ impl EngineRouter { accuracy: AccuracyTarget, metric_storage: StorageBackend, ) -> Result { - let backends = compatible_storage_backends(stat, accuracy, metric_storage); + let backends = compatible_storage_backends(stat, &accuracy, metric_storage); debug!( query = query, stat = ?stat, @@ -354,7 +354,7 @@ impl EngineRouter { step_ms: u64, tier: RangeTier, ) -> Result { - let mut backends = compatible_storage_backends(stat, accuracy, metric_storage); + let mut backends = compatible_storage_backends(stat, &accuracy, metric_storage); if matches!(tier, RangeTier::WarmOnly) { // Drop the archive leg: a range fully inside warm retention // must never be answered (empty) by the archive. @@ -517,7 +517,7 @@ mod tests { .execute( "sum_over_time(foo[5m])", Statistic::Sum, - AccuracyTarget::Approximate, + AccuracyTarget::Epsilon(0.01), StorageBackend::SketchStore, ) .await; @@ -574,7 +574,7 @@ mod tests { .execute( "sum_over_time(foo[5m])", Statistic::Sum, - AccuracyTarget::Approximate, + AccuracyTarget::Epsilon(0.01), StorageBackend::DoubleWrite, ) .await; @@ -593,7 +593,7 @@ mod tests { .execute( "sum_over_time(foo[5m])", Statistic::Sum, - AccuracyTarget::Approximate, + AccuracyTarget::Epsilon(0.01), StorageBackend::SketchStore, ) .await; @@ -627,7 +627,7 @@ mod tests { .execute( "sum_over_time(foo[5m])", Statistic::Sum, - AccuracyTarget::Approximate, + AccuracyTarget::Epsilon(0.01), StorageBackend::SketchStore, ) .await; @@ -685,7 +685,7 @@ mod tests { .execute( "sum_over_time(foo[5m])", Statistic::Sum, - AccuracyTarget::Approximate, + AccuracyTarget::Epsilon(0.01), StorageBackend::SketchStore, ) .await; @@ -771,7 +771,7 @@ mod tests { .execute_range( "rate(http_requests_total[1m])", Statistic::Sum, - AccuracyTarget::Approximate, + AccuracyTarget::Epsilon(0.01), StorageBackend::SketchStore, 1_700_000_000_000, 1_700_003_600_000, @@ -803,7 +803,7 @@ mod tests { .execute_range( "rate(http_requests_total[1m])", Statistic::Sum, - AccuracyTarget::Approximate, + AccuracyTarget::Epsilon(0.01), StorageBackend::SketchStore, 0, 1_000, @@ -885,7 +885,7 @@ mod tests { .execute_range_for_tier( "sum_over_time(http_requests_total[300s])", Statistic::Sum, - AccuracyTarget::Approximate, + AccuracyTarget::Epsilon(0.01), // Cold metric resolves to the archive axis, but the range // is recent so the HTTP layer marks it WarmOnly. StorageBackend::GorillaObjectStore, @@ -927,7 +927,7 @@ mod tests { .execute_range_for_tier( "quantile_over_time(0.99, latency_ms[300s])", Statistic::Sum, - AccuracyTarget::Approximate, + AccuracyTarget::Epsilon(0.01), StorageBackend::GorillaObjectStore, 1_700_000_000_000, 1_700_000_300_000, @@ -963,7 +963,7 @@ mod tests { .execute_range_for_tier( "sum_over_time(http_requests_total[300s])", Statistic::Sum, - AccuracyTarget::Approximate, + AccuracyTarget::Epsilon(0.01), StorageBackend::GorillaObjectStore, 1_600_000_000_000, 1_600_000_300_000, diff --git a/data_plane/src/storage_engines/mod.rs b/data_plane/src/storage_engines/mod.rs index 543d167c..b2bdb5c6 100644 --- a/data_plane/src/storage_engines/mod.rs +++ b/data_plane/src/storage_engines/mod.rs @@ -23,8 +23,8 @@ pub mod traits; pub mod types; pub use sketch_db::index::{ - AccuracyBound, Capability, SidLookup, SketchConfig, SketchEncoding, SketchInstanceMetadata, - SketchKindHandle, SketchSampleState, SketchStore, SketchTimeSeries, + AccuracyBound, Capability, SidLookup, SketchAlgorithm, SketchConfig, SketchEncoding, + SketchInstanceMetadata, SketchSampleState, SketchStore, SketchTimeSeries, }; pub use sketch_db::AggStatus; pub use traits::*; diff --git a/data_plane/src/storage_engines/sketch_db/data/mod.rs b/data_plane/src/storage_engines/sketch_db/data/mod.rs index 2de7decf..7e12e4a1 100644 --- a/data_plane/src/storage_engines/sketch_db/data/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/data/mod.rs @@ -39,7 +39,7 @@ //! //! ## Re-exports for callers //! -//! - [`Capability`] / [`SketchKindHandle`] — the canonical +//! - [`Capability`] / [`SketchAlgorithm`] — the canonical //! control-plane-side capability vocabulary. //! - [`AggregationType`] — the agg-type enum that //! `AggKind::ExactAgg` carries. @@ -56,7 +56,7 @@ // `is_satisfied_by` (used by the engine ASAP-tier hook) lives on the // control-plane-side `Capability` impl. -pub use control_plane::physical::runtime_capability::{Capability, SketchKindHandle}; +pub use control_plane::physical::runtime_capability::{Capability, SketchAlgorithm}; /// Re-export so callers don't need to depend on promql_utilities /// directly for the agg_type tag. @@ -96,7 +96,7 @@ pub enum AggKind { /// Payload at storage layer is an encoded byte string /// (`SketchSampleState`). Sketch { - kind: SketchKindHandle, + algorithm: SketchAlgorithm, config: SketchConfig, /// Canonical form of the policy's spatial-filter predicate, /// produced by [`asap_types::utils::normalize_spatial_filter`] @@ -170,13 +170,13 @@ impl AggKind { pub fn canonical_string(&self) -> String { match self { AggKind::Sketch { - kind, + algorithm: kind, config, spatial_filter_canonical, } => { format!( "sketch:{}:{}:filter={}", - sketch_kind_canonical(*kind), + sketch_algorithm_canonical(kind.clone()), sketch_config_canonical(config), spatial_filter_canonical, ) @@ -199,20 +199,21 @@ impl AggKind { } } -fn sketch_kind_canonical(k: SketchKindHandle) -> &'static str { +fn sketch_algorithm_canonical(k: SketchAlgorithm) -> &'static str { match k { - SketchKindHandle::DDSketch => "DDSketch", - SketchKindHandle::Kll => "Kll", - SketchKindHandle::Hll => "Hll", - SketchKindHandle::CountSketch => "CountSketch", - SketchKindHandle::CountMin => "CountMin", - SketchKindHandle::CmsWithHeap => "CmsWithHeap", - SketchKindHandle::CountSketchWithHeap => "CountSketchWithHeap", + SketchAlgorithm::DDSketch => "DDSketch", + SketchAlgorithm::Kll => "Kll", + SketchAlgorithm::Hll => "Hll", + SketchAlgorithm::CountSketch => "CountSketch", + SketchAlgorithm::Cms => "CountMin", + SketchAlgorithm::CmsWithHeap => "CmsWithHeap", + SketchAlgorithm::CountSketchWithHeap => "CountSketchWithHeap", // `Any` is the analysis-time wildcard; never reaches the // ingest path which detects a concrete kind from the OTLP // wire variant. Mapping it to a unique tag anyway keeps the // canonical form total. - SketchKindHandle::Any => "Any", + SketchAlgorithm::Kmv => "Kmv", + SketchAlgorithm::Theta => "Theta", } } diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index 513696f6..5fbe90ae 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -35,7 +35,7 @@ use crate::storage_engines::sketch_db::lifecycle::AggStatus; // during the reorg. pub use crate::storage_engines::sketch_db::data::{ canonical_parameters, AccuracyBound, AggKind, AggPayload, AggregationType, Capability, - SketchConfig, SketchEncoding, SketchKindHandle, SketchSampleState, SketchTimeSeries, + SketchAlgorithm, SketchConfig, SketchEncoding, SketchSampleState, SketchTimeSeries, }; fn now_ms() -> u64 { @@ -247,14 +247,16 @@ impl SketchInstanceMetadata { /// `Some(handle)` iff this sid is sketch-backed; `None` for /// exact-aggregation-backed sids. Consumers that only meaningfully /// run on sketches (e.g. the ASAP-tier reducer) `.expect` it. - pub fn sketch_kind(&self) -> Option { + pub fn sketch_algorithm(&self) -> Option { match &self.agg_kind { - AggKind::Sketch { kind, .. } => Some(*kind), + AggKind::Sketch { + algorithm: kind, .. + } => Some(kind.clone()), AggKind::ExactAgg { .. } => None, } } - /// Sketch-config accessor mirroring [`Self::sketch_kind`]. + /// Sketch-config accessor mirroring [`Self::sketch_algorithm`]. pub fn sketch_config(&self) -> Option<&SketchConfig> { match &self.agg_kind { AggKind::Sketch { config, .. } => Some(config), @@ -1291,7 +1293,7 @@ impl SketchStore { // ── KNOWN GAP — sketch-backed aggs return empty here ──────── // The `matches!` predicate below ONLY matches // `AggKind::ExactAgg`. Sketch-backed sids - // (`AggKind::Sketch { kind, config, .. }`, registered by + // (`AggKind::Sketch { algorithm: kind, config, .. }`, registered by // `route_modified_otlp_sketches_to_precompute` for every // OTLP DDSketch/KLL/HLL/CountSketch/CountMinSketch DP) are // NEVER picked up — and the agg-keyed precompute query @@ -2201,7 +2203,9 @@ impl crate::storage_engines::sketch_db::index::persistence::EpochSource for Sket .ok() .and_then(|g| g.get(&sid).cloned()) .and_then(|m| match &m.agg_kind { - AggKind::Sketch { kind, .. } => Some(format!("{:?}", kind)), + AggKind::Sketch { + algorithm: kind, .. + } => Some(format!("{:?}", kind)), AggKind::ExactAgg { .. } => None, }) }; @@ -2321,9 +2325,9 @@ mod tests { sid, metric_name: "m".into(), group_by_keys: BTreeSet::new(), - capability: Some(Capability::QuantileApprox(SketchKindHandle::DDSketch)), + capability: Some(Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch))), agg_kind: AggKind::Sketch { - kind: SketchKindHandle::DDSketch, + algorithm: SketchAlgorithm::DDSketch, config: cfg.clone(), spatial_filter_canonical: String::new(), }, @@ -3446,9 +3450,9 @@ mod tests { sid, metric_name: "http_latency".into(), group_by_keys: ["host".to_string()].into_iter().collect(), - capability: Some(Capability::QuantileApprox(SketchKindHandle::Kll)), + capability: Some(Capability::QuantileApprox(Some(SketchAlgorithm::Kll))), agg_kind: AggKind::Sketch { - kind: SketchKindHandle::Kll, + algorithm: SketchAlgorithm::Kll, config: cfg.clone(), spatial_filter_canonical: String::new(), }, @@ -3512,7 +3516,7 @@ mod tests { assert_eq!(meta.metric_name, "http_latency"); assert!(matches!( meta.capability, - Some(Capability::QuantileApprox(SketchKindHandle::Kll)) + Some(Capability::QuantileApprox(Some(SketchAlgorithm::Kll))) )); // (b) a range query over the EVICTED window returns the data. diff --git a/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs b/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs index d5c9ec55..ffe0c2a9 100644 --- a/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs +++ b/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs @@ -20,7 +20,7 @@ //! sweep can later drop it. //! //! Sketch-typed agg-configs are not yet covered: the -//! `AggregationConfig` shape doesn't carry a `SketchKindHandle` / +//! `AggregationConfig` shape doesn't carry a `SketchAlgorithm` / //! `SketchConfig` natively (control plane pushes them through a parallel //! capability-routing channel). For now the reconciler treats every //! agg-config as a precompute signature; sketch sids never compare @@ -210,23 +210,24 @@ fn build_live_signature_set(config: &StreamingConfig) -> HashSet> { } fn encode_agg_kind(agg_kind: &AggKind, buf: &mut Vec) { - use crate::storage_engines::sketch_db::data::{SketchConfig, SketchKindHandle}; + use crate::storage_engines::sketch_db::data::{SketchAlgorithm, SketchConfig}; match agg_kind { AggKind::Sketch { - kind, + algorithm: kind, config, spatial_filter_canonical, } => { buf.push(b'S'); buf.push(match kind { - SketchKindHandle::DDSketch => 1, - SketchKindHandle::Kll => 2, - SketchKindHandle::Hll => 3, - SketchKindHandle::CountSketch => 4, - SketchKindHandle::CountMin => 5, - SketchKindHandle::CmsWithHeap => 6, - SketchKindHandle::CountSketchWithHeap => 7, - SketchKindHandle::Any => 0, + SketchAlgorithm::DDSketch => 1, + SketchAlgorithm::Kll => 2, + SketchAlgorithm::Hll => 3, + SketchAlgorithm::CountSketch => 4, + SketchAlgorithm::Cms => 5, + SketchAlgorithm::CmsWithHeap => 6, + SketchAlgorithm::CountSketchWithHeap => 7, + SketchAlgorithm::Kmv => 8, + SketchAlgorithm::Theta => 9, }); match config { SketchConfig::DDSketch { relative_accuracy } => { @@ -415,7 +416,7 @@ mod tests { fn meta_sketch( sid: u64, metric: &str, - kind: crate::storage_engines::sketch_db::data::SketchKindHandle, + kind: crate::storage_engines::sketch_db::data::SketchAlgorithm, config: crate::storage_engines::sketch_db::data::SketchConfig, group_by: Vec<&str>, ) -> SketchInstanceMetadata { @@ -426,7 +427,7 @@ mod tests { group_by_keys, capability: None, agg_kind: AggKind::Sketch { - kind, + algorithm: kind, config, spatial_filter_canonical: String::new(), }, @@ -450,19 +451,19 @@ mod tests { // plane's eviction RPC's job, not this path's. #[test] fn sketch_sids_are_never_retired_by_signature_reconcile() { - use crate::storage_engines::sketch_db::data::{SketchConfig, SketchKindHandle}; + use crate::storage_engines::sketch_db::data::{SketchAlgorithm, SketchConfig}; let store = SketchStore::new(); store.register(meta_sketch( 1, "http_requests_total_latency_ms", - SketchKindHandle::Kll, + SketchAlgorithm::Kll, SketchConfig::Kll { k: 200 }, vec!["node", "pod", "zone"], )); store.register(meta_sketch( 2, "unique_users_per_min", - SketchKindHandle::Hll, + SketchAlgorithm::Hll, SketchConfig::Hll { precision: 12 }, vec!["zone"], )); diff --git a/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs b/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs index 464bd9c9..aecab91c 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs @@ -42,7 +42,7 @@ //! diagnosis. The record stores serializable PRIMITIVES — the //! `Capability` / `AccuracyBound` are DERIVED on load from `agg_kind` //! exactly as the ingest path derives them, so this module needs no -//! serde on the control-plane `Capability` / `SketchKindHandle` enums. +//! serde on the control-plane `Capability` / `SketchAlgorithm` enums. use std::collections::HashMap; use std::fs::{self, File, OpenOptions}; @@ -52,7 +52,7 @@ use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; use crate::storage_engines::sketch_db::data::{ - AccuracyBound, AggKind, Capability, SketchConfig, SketchKindHandle, + AccuracyBound, AggKind, Capability, SketchAlgorithm, SketchConfig, }; use crate::storage_engines::types::AggregationType; @@ -119,32 +119,37 @@ impl From<&SketchConfigRec> for SketchConfig { } } -/// Stable string form of a [`SketchKindHandle`] for the sidecar. Mirrors -/// `sketch_kind_canonical` but is owned by the persistence layer so the +/// Stable string form of a [`SketchAlgorithm`] for the sidecar. Mirrors +/// `sketch_algorithm_canonical` but is owned by the persistence layer so the /// on-disk vocabulary is stable independent of any upstream rename. -fn sketch_kind_to_str(k: SketchKindHandle) -> &'static str { +fn sketch_algorithm_to_str(k: SketchAlgorithm) -> &'static str { match k { - SketchKindHandle::DDSketch => "DDSketch", - SketchKindHandle::Kll => "Kll", - SketchKindHandle::Hll => "Hll", - SketchKindHandle::CountSketch => "CountSketch", - SketchKindHandle::CountMin => "CountMin", - SketchKindHandle::CmsWithHeap => "CmsWithHeap", - SketchKindHandle::CountSketchWithHeap => "CountSketchWithHeap", - SketchKindHandle::Any => "Any", + SketchAlgorithm::DDSketch => "DDSketch", + SketchAlgorithm::Kll => "Kll", + SketchAlgorithm::Hll => "Hll", + SketchAlgorithm::CountSketch => "CountSketch", + SketchAlgorithm::Cms => "CountMin", + SketchAlgorithm::CmsWithHeap => "CmsWithHeap", + SketchAlgorithm::CountSketchWithHeap => "CountSketchWithHeap", + SketchAlgorithm::Kmv => "Kmv", + SketchAlgorithm::Theta => "Theta", } } -fn sketch_kind_from_str(s: &str) -> Option { +fn sketch_algorithm_from_str(s: &str) -> Option { Some(match s { - "DDSketch" => SketchKindHandle::DDSketch, - "Kll" => SketchKindHandle::Kll, - "Hll" => SketchKindHandle::Hll, - "CountSketch" => SketchKindHandle::CountSketch, - "CountMin" => SketchKindHandle::CountMin, - "CmsWithHeap" => SketchKindHandle::CmsWithHeap, - "CountSketchWithHeap" => SketchKindHandle::CountSketchWithHeap, - "Any" => SketchKindHandle::Any, + "DDSketch" => SketchAlgorithm::DDSketch, + "Kll" => SketchAlgorithm::Kll, + "Hll" => SketchAlgorithm::Hll, + "CountSketch" => SketchAlgorithm::CountSketch, + "CountMin" => SketchAlgorithm::Cms, + "CmsWithHeap" => SketchAlgorithm::CmsWithHeap, + "CountSketchWithHeap" => SketchAlgorithm::CountSketchWithHeap, + "Kmv" => SketchAlgorithm::Kmv, + "Theta" => SketchAlgorithm::Theta, + // `Any` was never a valid stored implementation. Reject legacy + // sidecars that contain it instead of inventing an algorithm. + "Any" => return None, _ => return None, }) } @@ -169,11 +174,11 @@ impl From<&AggKind> for AggKindRec { fn from(a: &AggKind) -> Self { match a { AggKind::Sketch { - kind, + algorithm: kind, config, spatial_filter_canonical, } => AggKindRec::Sketch { - sketch_kind: sketch_kind_to_str(*kind).to_string(), + sketch_kind: sketch_algorithm_to_str(kind.clone()).to_string(), config: config.into(), spatial_filter_canonical: spatial_filter_canonical.clone(), }, @@ -201,7 +206,7 @@ impl AggKindRec { config, spatial_filter_canonical, } => AggKind::Sketch { - kind: sketch_kind_from_str(sketch_kind)?, + algorithm: sketch_algorithm_from_str(sketch_kind)?, config: config.into(), spatial_filter_canonical: spatial_filter_canonical.clone(), }, @@ -265,20 +270,20 @@ impl SidMetaRecord { pub fn capability(&self) -> Option { let agg_kind = self.agg_kind()?; Some(match agg_kind { - AggKind::Sketch { kind, .. } => match kind { - SketchKindHandle::DDSketch | SketchKindHandle::Kll => { - Capability::QuantileApprox(kind) + AggKind::Sketch { + algorithm: kind, .. + } => match kind { + SketchAlgorithm::DDSketch | SketchAlgorithm::Kll => { + Capability::QuantileApprox(Some(kind)) } - SketchKindHandle::Hll => Capability::CardinalityApprox, - SketchKindHandle::CountSketch | SketchKindHandle::CountMin => { - Capability::FrequencyEstimate(kind) + SketchAlgorithm::Hll => Capability::CardinalityApprox, + SketchAlgorithm::CountSketch | SketchAlgorithm::Cms => { + Capability::FrequencyEstimate(Some(kind)) } - SketchKindHandle::CmsWithHeap | SketchKindHandle::CountSketchWithHeap => { - Capability::FrequencyTopk(kind) + SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap => { + Capability::FrequencyTopk(Some(kind)) } - // Defensive: `Any` is a control-plane wildcard that should - // never reach the index; mirror the ingest fallback. - SketchKindHandle::Any => Capability::QuantileApprox(kind), + SketchAlgorithm::Kmv | SketchAlgorithm::Theta => Capability::CardinalityApprox, }, AggKind::ExactAgg { agg_type, .. } => Capability::ExactAgg(agg_type), }) @@ -411,7 +416,7 @@ mod tests { "http_latency".into(), vec!["host".into(), "zone".into()], &AggKind::Sketch { - kind: SketchKindHandle::Kll, + algorithm: SketchAlgorithm::Kll, config: SketchConfig::Kll { k: 200 }, spatial_filter_canonical: String::new(), }, @@ -475,7 +480,7 @@ mod tests { let kll = sketch_meta(1); assert!(matches!( kll.capability(), - Some(Capability::QuantileApprox(SketchKindHandle::Kll)) + Some(Capability::QuantileApprox(Some(SketchAlgorithm::Kll))) )); assert!(kll.accuracy().is_some()); diff --git a/data_plane/src/storage_engines/sketch_db/query/timeline.rs b/data_plane/src/storage_engines/sketch_db/query/timeline.rs index c0992f0f..44364613 100644 --- a/data_plane/src/storage_engines/sketch_db/query/timeline.rs +++ b/data_plane/src/storage_engines/sketch_db/query/timeline.rs @@ -42,7 +42,7 @@ use std::collections::{BTreeMap, BTreeSet}; use xxhash_rust::xxh64::xxh64; -use crate::storage_engines::sketch_db::data::{AggKind, SketchConfig, SketchKindHandle}; +use crate::storage_engines::sketch_db::data::{AggKind, SketchAlgorithm, SketchConfig}; use crate::storage_engines::sketch_db::index::{SketchInstanceMetadata, SketchStore}; use crate::storage_engines::sketch_db::lifecycle::AggStatus; @@ -228,12 +228,12 @@ impl AggSignatureGroup { fn encode_agg_kind(agg_kind: &AggKind, buf: &mut Vec) { match agg_kind { AggKind::Sketch { - kind, + algorithm: kind, config, spatial_filter_canonical, } => { buf.push(b'S'); - buf.push(sketch_kind_byte(*kind)); + buf.push(sketch_algorithm_byte(kind.clone())); encode_sketch_config(config, buf); buf.push(b'F'); buf.extend_from_slice(spatial_filter_canonical.as_bytes()); @@ -253,16 +253,17 @@ fn encode_agg_kind(agg_kind: &AggKind, buf: &mut Vec) { } } -fn sketch_kind_byte(k: SketchKindHandle) -> u8 { +fn sketch_algorithm_byte(k: SketchAlgorithm) -> u8 { match k { - SketchKindHandle::DDSketch => 1, - SketchKindHandle::Kll => 2, - SketchKindHandle::Hll => 3, - SketchKindHandle::CountSketch => 4, - SketchKindHandle::CountMin => 5, - SketchKindHandle::CmsWithHeap => 6, - SketchKindHandle::CountSketchWithHeap => 7, - SketchKindHandle::Any => 0, + SketchAlgorithm::DDSketch => 1, + SketchAlgorithm::Kll => 2, + SketchAlgorithm::Hll => 3, + SketchAlgorithm::CountSketch => 4, + SketchAlgorithm::Cms => 5, + SketchAlgorithm::CmsWithHeap => 6, + SketchAlgorithm::CountSketchWithHeap => 7, + SketchAlgorithm::Kmv => 8, + SketchAlgorithm::Theta => 9, } } @@ -318,7 +319,7 @@ mod tests { sid, metric_name: metric.into(), group_by_keys: BTreeSet::new(), - capability: Some(Capability::QuantileApprox(SketchKindHandle::DDSketch)), + capability: Some(Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch))), agg_kind, accuracy: None, first_seen_unix_ms: first_seen, diff --git a/data_plane/src/storage_engines/types/storage_backend.rs b/data_plane/src/storage_engines/types/storage_backend.rs index e7f4075d..6d961889 100644 --- a/data_plane/src/storage_engines/types/storage_backend.rs +++ b/data_plane/src/storage_engines/types/storage_backend.rs @@ -1,140 +1,4 @@ -use serde::{Deserialize, Serialize}; +//! Compatibility re-export for the backend-owned storage vocabulary shared by +//! control-plane emission and data-plane routing. -pub const ENGINE_ID_ASAP_QUERY: &str = "asap_query"; -pub const ENGINE_ID_THANOS_QUERY: &str = "thanos_query"; - -pub const CANONICAL_QUERY_ENGINE_IDS: &[&str] = &[ENGINE_ID_ASAP_QUERY, ENGINE_ID_THANOS_QUERY]; - -// --------------------------------------------------------------------------- -// Phase-5: storage-backend capability axis -// -// Matching on `(metric, statistic, sub_type, window_size, grouping_labels, -// spatial_filter)` alone has no axis for "which storage tier serves this -// query." The Phase-5 `GorillaQueryEngine` (PR #85) introduces a parallel -// exact tier; the planner / router needs to disambiguate between ASAP-tier -// sketches and Gorilla-S3 chunks. See `docs/design-gorilla-s3-cold-engine.md` -// §8. -// --------------------------------------------------------------------------- - -/// Which physical storage tier a query (or a metric configuration) routes to. -/// -/// `SketchStore` is the default — every existing `AggregationConfig` and -/// `StreamingConfig` decodes into this variant via `#[serde(default)]`, so -/// pre-Phase-5 deploys keep dispatching to `ASAPQueryEngine` unchanged. -/// -/// **Step-1 of the JSONL deprecation refactor** removed the -/// `ColdJsonlFallback` variant. The legacy local-FS JSONL leg -/// (`LocalFsColdStore`, `parse_jsonl`, the §5.2 raw-store -/// fallback) was deleted at the same commit; the surviving -/// failover surface is ASAP-tier sketch ↔ Thanos archive. -/// -/// Formerly `asap_types::capability_matching::StorageBackend` (then -/// `asap_types::storage_backend::StorageBackend`). Moved here alongside -/// `StreamingConfig` (see `scratchpad/artifacts/enum-unification-plan.md`) -/// once auditing real call sites showed `control_plane` never actually -/// depends on this type or `StreamingConfig` — it emits wire-compatible -/// JSON by hand via its own `StreamingConfigEmitter`, never importing -/// either. See [`super::streaming_config`]'s module doc for the fuller -/// story. The routing *policy* (`AccuracyTarget`, -/// `compatible_storage_backends`) already lived in -/// `data_plane::query_engines::routing::capability_matching`. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] -#[serde(rename_all = "snake_case")] -pub enum StorageBackend { - /// Warm-tier sketch DB (today's `SketchStore` + accumulators). - /// Served by `ASAPQueryEngine`. Default for unconfigured metrics. - #[default] - SketchStore, - - /// Thanos archive over MinIO/S3. The enum name is kept for - /// serde/back-compat with existing configs, but its canonical - /// query-engine identity is `thanos_query`. Gorilla is an - /// archive chunk format/storage detail, not a public query engine. - GorillaObjectStore, - - /// Double-write: the metric is written to both ASAP-tier sketches AND the - /// Gorilla-S3 archive. Capability matching surfaces both options and the - /// cost-aware dispatcher picks per query (typically ASAP-tier for low- - /// latency approximate, archive for exact). - DoubleWrite, - - /// Prometheus-remote: the metric's data is shipped raw to a - /// Prometheus instance via the native OTLP receiver. Phase ε.2 - /// registers a `PrometheusForwardEngine` (HTTP-forwarder to - /// Prometheus's `/api/v1/query`) under this slot so the - /// controller's `RawAtEdgePrometheusArchive` mode can route a - /// metric's queries to Prometheus directly. Mirrors the - /// `GorillaObjectStore` slot's "single backend, no failover" - /// semantics — there is no ASAP-tier sketch to fall back on for a - /// Prometheus-remote metric. - PrometheusRemote, -} - -impl StorageBackend { - /// Canonical string tag pinned for byte-comparable dispatch on the wire (mirrors - /// the `data_source: ` info-line on `QueryResult`). Engines - /// register themselves under these IDs in the router. - pub const fn data_source_id(self) -> &'static str { - match self { - StorageBackend::SketchStore => ENGINE_ID_ASAP_QUERY, - StorageBackend::GorillaObjectStore => ENGINE_ID_THANOS_QUERY, - StorageBackend::DoubleWrite => "double_write", - StorageBackend::PrometheusRemote => "prometheus_remote", - } - } -} - -pub fn parse_storage_backend_engine_id(s: &str) -> Option { - match s { - ENGINE_ID_ASAP_QUERY => Some(StorageBackend::SketchStore), - ENGINE_ID_THANOS_QUERY => Some(StorageBackend::GorillaObjectStore), - "double_write" => Some(StorageBackend::DoubleWrite), - "prometheus_remote" => Some(StorageBackend::PrometheusRemote), - _ => None, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn storage_backend_default_is_asap_tier() { - // `#[serde(default)]` on `StreamingConfig.storage_backend` (and on - // `StorageBackend::default()`) MUST be `SketchStore` so pre-Phase-5 - // configs decode without bumping deploys onto the archive. - assert_eq!(StorageBackend::default(), StorageBackend::SketchStore); - } - - #[test] - fn storage_backend_data_source_id_is_pinned() { - // The router registers engines by these strings; dashboards - // byte-compare them. Pin to catch accidental rename. - assert_eq!( - StorageBackend::SketchStore.data_source_id(), - ENGINE_ID_ASAP_QUERY - ); - assert_eq!( - StorageBackend::GorillaObjectStore.data_source_id(), - ENGINE_ID_THANOS_QUERY, - ); - assert_eq!(StorageBackend::DoubleWrite.data_source_id(), "double_write",); - assert_eq!( - StorageBackend::PrometheusRemote.data_source_id(), - "prometheus_remote", - ); - } - - #[test] - fn storage_backend_engine_id_parser_accepts_only_canonical_query_engines() { - assert_eq!( - parse_storage_backend_engine_id(ENGINE_ID_ASAP_QUERY), - Some(StorageBackend::SketchStore), - ); - assert_eq!( - parse_storage_backend_engine_id(ENGINE_ID_THANOS_QUERY), - Some(StorageBackend::GorillaObjectStore), - ); - assert_eq!(parse_storage_backend_engine_id("not_an_engine"), None); - } -} +pub use asap_types::storage_backend::*; diff --git a/data_plane/tests/all_sketches_process_oracle_e2e.rs b/data_plane/tests/all_sketches_process_oracle_e2e.rs index ce054a47..b8648fe4 100644 --- a/data_plane/tests/all_sketches_process_oracle_e2e.rs +++ b/data_plane/tests/all_sketches_process_oracle_e2e.rs @@ -21,18 +21,25 @@ use asap_otel_proto::tonic::metrics::v1::{ ScopeMetrics, }; use asap_sketchlib::proto::sketchlib::{HllVariant as ProtoHllVariant, HyperLogLogState, KllState}; -use asap_sketchlib::{ - CountMinSketchWithHeap, CountSketchWithHeap, HllSketch, HllVariant, MessagePackCodec, -}; +use asap_sketchlib::{CountMinSketch, CountSketch, HllSketch, HllVariant, MessagePackCodec}; use prost::Message; use serde_json::Value; const SERVICE: &str = "oracle-e2e"; -const K: u32 = 200; -const HLL_PRECISION: u32 = 10; -const ROWS: usize = 5; -const COLS: usize = 2048; -const HEAP_SIZE: usize = 16; +// These parameters satisfy the production query path's LIVE_ACCURACY +// (`epsilon = 0.01`). ASAPPlanner validates an observed `SketchKind` +// against that accuracy before committing it, so the fixture must use the +// same contract as a real control-plane-generated materialization. +const K: u32 = 269; +const HLL_PRECISION: u32 = 14; +const CMS_ROWS: usize = 5; +const CMS_COLS: usize = 2048; +// ASAPPlanner's CountSketch guarantee is L2-based: epsilon=sqrt(3/width), +// with an odd Hoeffding-median depth. The current runtime's packed sign hash +// limits this width to five rows, which satisfies the explicit delta=0.8 +// contract used only by the CountSketch child process below. +const COUNT_SKETCH_ROWS: usize = 5; +const COUNT_SKETCH_COLS: usize = 32_768; struct ChildGuard(Child); @@ -96,7 +103,7 @@ fn envelope(metric: &str, data: Data) -> ExportMetricsServiceRequest { } } -async fn start_backend(config_yaml: &str) -> Backend { +async fn start_backend(config_yaml: &str, live_delta: Option<&str>) -> Backend { let query_port = unused_port(); let otlp_http_port = unused_port(); let otlp_grpc_port = unused_port(); @@ -107,7 +114,8 @@ async fn start_backend(config_yaml: &str) -> Backend { .expect("write streaming config"); config.flush().expect("flush streaming config"); - let child = Command::new(env!("CARGO_BIN_EXE_data_plane")) + let mut command = Command::new(env!("CARGO_BIN_EXE_data_plane")); + command .arg("--streaming-config") .arg(config.path()) .arg("--http-port") @@ -124,9 +132,11 @@ async fn start_backend(config_yaml: &str) -> Backend { .arg("--precompute-flush-interval-ms") .arg("100") .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .expect("start production data-plane binary"); + .stderr(Stdio::null()); + if let Some(delta) = live_delta { + command.env("ASAP_SUMMARY_EXECUTOR_DELTA", delta); + } + let child = command.spawn().expect("start production data-plane binary"); let mut child = ChildGuard(child); let client = reqwest::Client::new(); let query_base = format!("http://127.0.0.1:{query_port}"); @@ -297,7 +307,7 @@ fn hll_export(metric: &str, timestamp_ns: u64, raw: &[&str]) -> ExportMetricsSer } fn cms_export(metric: &str, timestamp_ns: u64, raw: &[&str]) -> ExportMetricsServiceRequest { - let mut sketch = CountMinSketchWithHeap::new(ROWS, COLS, HEAP_SIZE); + let mut sketch = CountMinSketch::new(CMS_ROWS, CMS_COLS); for key in raw { sketch.update(key, 1.0); } @@ -314,8 +324,8 @@ fn cms_export(metric: &str, timestamp_ns: u64, raw: &[&str]) -> ExportMetricsSer series_id: 0, }], aggregation_temporality: 0, - rows: ROWS as i32, - cols: COLS as i32, + rows: CMS_ROWS as i32, + cols: CMS_COLS as i32, }), ) } @@ -325,7 +335,7 @@ fn count_sketch_export( timestamp_ns: u64, raw: &[&str], ) -> ExportMetricsServiceRequest { - let mut sketch = CountSketchWithHeap::new(ROWS, COLS, HEAP_SIZE); + let mut sketch = CountSketch::new(COUNT_SKETCH_ROWS, COUNT_SKETCH_COLS); for key in raw { sketch.update(key, 1.0); } @@ -342,8 +352,8 @@ fn count_sketch_export( series_id: 0, }], aggregation_temporality: 0, - rows: ROWS as i32, - cols: COLS as i32, + rows: COUNT_SKETCH_ROWS as i32, + cols: COUNT_SKETCH_COLS as i32, }), ) } @@ -357,37 +367,35 @@ fn exact_quantile(raw: &[f64], q: f64) -> f64 { sorted[low] + (sorted[high] - sorted[low]) * (rank - low as f64) } -fn raw_frequencies(raw: &[&str]) -> HashMap { - let mut counts = HashMap::new(); - for key in raw { - *counts.entry((*key).to_string()).or_insert(0) += 1; - } - counts +fn assert_frequency_total_oracle(response: &Value, raw: &[&str]) { + let samples = scalar_values(response); + assert_eq!(samples.len(), 1, "expected one grouped frequency total"); + assert_eq!( + samples[0].0.get("service").map(String::as_str), + Some(SERVICE) + ); + assert_eq!(samples[0].1.round() as usize, raw.len()); } -fn assert_topk_oracle(response: &Value, raw: &[&str], k: usize) { - let mut expected: Vec<_> = raw_frequencies(raw).into_iter().collect(); - expected.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); - expected.truncate(k); - let expected: HashMap<_, _> = expected.into_iter().collect(); - - let actual: HashMap = scalar_values(response) - .into_iter() - .map(|(labels, value)| { - assert_eq!(labels.get("service").map(String::as_str), Some(SERVICE)); - ( - labels.get("item").expect("topk item label").clone(), - value.round() as u64, - ) - }) - .collect(); - assert_eq!(actual, expected, "top-k differs from raw frequency oracle"); +fn assert_frequency_point_oracle(response: &Value, raw: &[&str], item: &str) { + let samples = scalar_values(response); + assert_eq!(samples.len(), 1, "expected one grouped frequency estimate"); + assert_eq!( + samples[0].0.get("service").map(String::as_str), + Some(SERVICE) + ); + let expected = raw.iter().filter(|value| **value == item).count(); + assert_eq!(samples[0].1.round() as usize, expected); } #[tokio::test] async fn production_kll_matches_raw_quantile_oracle() { let metric = "oracle_kll_latency"; - let backend = start_backend(&config(metric, "DatasketchesKLL", " K: 200")).await; + let backend = start_backend( + &config(metric, "DatasketchesKLL", &format!(" K: {K}")), + None, + ) + .await; let raw: Vec = (1..=101).map(f64::from).collect(); let timestamp = now_ns().saturating_sub(2_000_000_000); post(&backend, kll_export(metric, timestamp, &raw)).await; @@ -409,7 +417,11 @@ async fn production_kll_matches_raw_quantile_oracle() { #[tokio::test] async fn production_hll_matches_raw_distinct_oracle() { let metric = "oracle_hll_users"; - let backend = start_backend(&config(metric, "HLL", " precision: 10")).await; + let backend = start_backend( + &config(metric, "HLL", &format!(" precision: {HLL_PRECISION}")), + None, + ) + .await; let owned: Vec = (0..2_000).map(|i| format!("user-{i}")).collect(); let mut raw: Vec<&str> = owned.iter().map(String::as_str).collect(); raw.extend(owned.iter().take(500).map(String::as_str)); @@ -439,12 +451,10 @@ fn frequency_fixture() -> Vec<&'static str> { } #[tokio::test] -async fn production_cms_matches_raw_topk_oracle() { +async fn production_cms_matches_raw_frequency_oracle() { let metric = "oracle_cms_frequency"; - let params = format!( - " w: {COLS}\n d: {ROWS}\n heap_size: {HEAP_SIZE}\n with_heap: true" - ); - let backend = start_backend(&config(metric, "CountMinSketchWithHeap", ¶ms)).await; + let params = format!(" w: {CMS_COLS}\n d: {CMS_ROWS}"); + let backend = start_backend(&config(metric, "CountMinSketch", ¶ms), None).await; let raw = frequency_fixture(); let timestamp = now_ns().saturating_sub(2_000_000_000); post(&backend, cms_export(metric, timestamp, &raw)).await; @@ -453,17 +463,21 @@ async fn production_cms_matches_raw_topk_oracle() { cms_export(metric, timestamp + 1_000_000_000, &raw), ) .await; - let response = query(&backend, &format!("topk(3, {metric})")).await; - assert_topk_oracle(&response, &raw, 3); + let response = query(&backend, &format!("count_over_time({metric}[10s])")).await; + let mut merged_raw = raw.clone(); + merged_raw.extend_from_slice(&raw); + assert_frequency_total_oracle(&response, &merged_raw); } #[tokio::test] -async fn production_count_sketch_matches_raw_topk_oracle() { +async fn production_count_sketch_matches_raw_frequency_oracle() { let metric = "oracle_count_sketch_frequency"; - let params = format!( - " w: {COLS}\n d: {ROWS}\n heap_size: {HEAP_SIZE}\n with_heap: true" - ); - let backend = start_backend(&config(metric, "CountSketchWithHeap", ¶ms)).await; + let params = format!(" w: {COUNT_SKETCH_COLS}\n d: {COUNT_SKETCH_ROWS}"); + // Planner's CountSketch confidence model requires more rows than the + // current packed-hash runtime can execute at this width. Keep the default + // production SLA untouched and declare the weaker contract explicitly + // for this isolated algorithm-oracle process. + let backend = start_backend(&config(metric, "CountSketch", ¶ms), Some("0.8")).await; let raw = frequency_fixture(); let timestamp = now_ns().saturating_sub(2_000_000_000); post(&backend, count_sketch_export(metric, timestamp, &raw)).await; @@ -472,6 +486,12 @@ async fn production_count_sketch_matches_raw_topk_oracle() { count_sketch_export(metric, timestamp + 1_000_000_000, &raw), ) .await; - let response = query(&backend, &format!("topk(3, {metric})")).await; - assert_topk_oracle(&response, &raw, 3); + let response = query( + &backend, + &format!("count_over_time({metric}{{item=\"alpha\"}}[10s])"), + ) + .await; + let mut merged_raw = raw.clone(); + merged_raw.extend_from_slice(&raw); + assert_frequency_point_oracle(&response, &merged_raw, "alpha"); } diff --git a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs index 9de3b58f..054f59b8 100644 --- a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs +++ b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs @@ -932,7 +932,7 @@ async fn controller_plan_to_query_full_roundtrip_ddsketch() { // below carries `service` as an attribute (required to avoid the // receiver's "invalid wire shape" drop). The backend's // `derive_sketch_policy_fp` matches policies on - // `(metric, sketch_kind, config, group_by_keys)` — so the + // `(metric, sketch_algorithm, config, group_by_keys)` — so the // streaming-config's grouping MUST include "service" or the // fingerprint won't match and the registered sid stays orphaned // from any policy. @@ -1620,7 +1620,7 @@ async fn controller_plan_to_query_full_roundtrip_count_min_sketch() { // dispatches both variants through `decode_cms_with_heap_from_msgpack` // (see `sketch_reducer.rs` at the FrequencyTopk dispatch site). // -// On the ingest side, `sketch_kind_handle_for` peeks at incoming +// On the ingest side, `sketch_algorithm_for` peeks at incoming // CountMin / CountSketch DPs with `encoding=MSGPACK`; if the bytes // round-trip through the heap envelope AND the heap is non-empty, // the sid is auto-promoted to the corresponding `*WithHeap` variant @@ -1664,7 +1664,7 @@ fn extract_w_d_from_streaming_config(streaming_config_json: &JsonValue) -> (u32, /// OTLP `ExportMetricsServiceRequest` with a single `CountMinSketch` DP /// carrying msgpack-encoded heap-bearing bytes. `encoding=MSGPACK` (3) -/// triggers `sketch_kind_handle_for`'s auto-promotion to `CmsWithHeap`. +/// triggers `sketch_algorithm_for`'s auto-promotion to `CmsWithHeap`. /// `rows`/`cols` on the parent `CountMinSketch` MUST match the policy's /// `parameters.{d,w}` for the policy_fp content match to bind. fn build_cms_with_heap_msgpack_export( @@ -1720,7 +1720,7 @@ fn build_cms_with_heap_msgpack_export( /// OTLP `ExportMetricsServiceRequest` with a single `CountSketch` DP /// carrying msgpack-encoded heap-bearing bytes. `encoding=MSGPACK` (3) -/// triggers `sketch_kind_handle_for`'s auto-promotion to +/// triggers `sketch_algorithm_for`'s auto-promotion to /// `CountSketchWithHeap` (the heap envelope is identical to the CMS /// variant). `rows`/`cols` MUST match the policy's `parameters.{d,w}`. fn build_count_sketch_with_heap_msgpack_export( @@ -1783,7 +1783,7 @@ fn build_count_sketch_with_heap_msgpack_export( // Cormode & Muthukrishnan 2005). // // The OTLP DP carries a msgpack-encoded `CountMinSketchWithHeap` -// payload (`encoding=MSGPACK`); the receiver's `sketch_kind_handle_for` +// payload (`encoding=MSGPACK`); the receiver's `sketch_algorithm_for` // peeks at the bytes and auto-promotes the sid to `CmsWithHeap`, // registering it under `Capability::FrequencyTopk(CmsWithHeap)`. // @@ -1949,7 +1949,7 @@ async fn controller_plan_to_query_full_roundtrip_cms_with_heap_topk() { // outer DP type changes (`CountSketchDataPoint` instead of // `CountMinSketchDataPoint`). // -// `sketch_kind_handle_for` was extended in this PR to peek at +// `sketch_algorithm_for` was extended in this PR to peek at // CountSketch DPs the same way it does for CountMin — a // non-empty heap in a msgpack-encoded payload promotes the sid to // `CountSketchWithHeap`, which the analyzer's `is_satisfied_by`