diff --git a/Cargo.lock b/Cargo.lock index 2137860d..02889cab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -343,7 +343,7 @@ dependencies = [ [[package]] name = "asap-ir" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPController?rev=283b6ab97159233e2dfc221e575415973c73aaa7#283b6ab97159233e2dfc221e575415973c73aaa7" +source = "git+https://github.com/ProjectASAP/ASAPController?rev=150ef7d0786d24286b578dfae9dbbcefc8fbac3e#150ef7d0786d24286b578dfae9dbbcefc8fbac3e" dependencies = [ "serde", "serde_json", @@ -353,7 +353,7 @@ dependencies = [ [[package]] name = "asap-plan" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPController?rev=283b6ab97159233e2dfc221e575415973c73aaa7#283b6ab97159233e2dfc221e575415973c73aaa7" +source = "git+https://github.com/ProjectASAP/ASAPController?rev=150ef7d0786d24286b578dfae9dbbcefc8fbac3e#150ef7d0786d24286b578dfae9dbbcefc8fbac3e" dependencies = [ "asap-ir", "asap-sketch", @@ -373,7 +373,7 @@ dependencies = [ [[package]] name = "asap-sketch" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPController?rev=283b6ab97159233e2dfc221e575415973c73aaa7#283b6ab97159233e2dfc221e575415973c73aaa7" +source = "git+https://github.com/ProjectASAP/ASAPController?rev=150ef7d0786d24286b578dfae9dbbcefc8fbac3e#150ef7d0786d24286b578dfae9dbbcefc8fbac3e" dependencies = [ "asap-ir", ] diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index 69310fb7..ae3c41b1 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -40,13 +40,14 @@ asap_types.workspace = true # tagged releases yet. Re-pin as ASAPController's IR evolves; move to a tag # once one exists. # -# Bumped to 283b6ab (merge of PR #141, "convert Implementation::is_satisfied_by -# into a Matcher trait") to pick up asap-plan::Matcher/Implementation and -# asap-sketch::{SummaryKind, SummaryParams} for the sketch-identity -# unification work (see scratchpad/artifacts/enum-unification-plan.md). -asap-ir = { git = "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/ProjectASAP/ASAPController", rev = "283b6ab97159233e2dfc221e575415973c73aaa7" } -asap-sketch = { git = "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/ProjectASAP/ASAPController", rev = "283b6ab97159233e2dfc221e575415973c73aaa7" } -asap-plan = { git = "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/ProjectASAP/ASAPController", rev = "283b6ab97159233e2dfc221e575415973c73aaa7" } +# Bumped to 150ef7d (merge of PR #142, "derive PartialOrd/Ord for +# SummaryKind") for the sketch-identity unification work (see +# scratchpad/artifacts/enum-unification-plan.md) -- SummaryKind needs Ord +# for the BTreeSet deterministic-emission-order contract in +# control_plane::physical::colored_dag::emitter::EdgeStageConfig::metric_to_family. +asap-ir = { git = "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/ProjectASAP/ASAPController", rev = "150ef7d0786d24286b578dfae9dbbcefc8fbac3e" } +asap-sketch = { git = "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/ProjectASAP/ASAPController", rev = "150ef7d0786d24286b578dfae9dbbcefc8fbac3e" } +asap-plan = { git = "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/ProjectASAP/ASAPController", rev = "150ef7d0786d24286b578dfae9dbbcefc8fbac3e" } [dev-dependencies] tokio = { version = "1", features = ["full", "test-util"] } diff --git a/control_plane/src/emit/backend_push.rs b/control_plane/src/emit/backend_push.rs index 2059a602..7c066b20 100644 --- a/control_plane/src/emit/backend_push.rs +++ b/control_plane/src/emit/backend_push.rs @@ -658,14 +658,14 @@ mod tests { use crate::physical::colored_dag::emitter::{ AggregationInput, BackendAggregation, BackendReadout, }; - use crate::sketch_algebra::params::{DDSketchParams, SketchKind, SketchParams}; use crate::sketch_algebra::physical_expr::EstimateOp; + use asap_sketch::{SummaryKind, SummaryParams}; BackendStageConfig { aggregations: vec![BackendAggregation { aggregation_id: agg_id.to_string(), metric_name: metric.to_string(), - sketch_kind: SketchKind::DDSketch, - sketch_params: SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), + sketch_kind: SummaryKind::DDSketch, + sketch_params: SummaryParams::DDSketch { alpha: 0.01 }, grouping: vec![], item_label: None, spatial_filter: String::new(), diff --git a/control_plane/src/emit/mod.rs b/control_plane/src/emit/mod.rs index 5f6345ea..f559f0a1 100644 --- a/control_plane/src/emit/mod.rs +++ b/control_plane/src/emit/mod.rs @@ -51,10 +51,10 @@ pub use trait_def::{ pub use crate::workload::WorkloadRegistry; use crate::physical::colored_dag::emitter::EdgeStageConfig; -use crate::sketch_algebra::params::SketchKind; use crate::sketch_algebra::PhysicalExpr; use crate::store::WorkloadStore; use anyhow::Result; +use asap_sketch::SummaryKind; /// Phase ε.1.5 — which edge runtime an agent identifies as. /// @@ -290,7 +290,7 @@ fn apply_cold_format_from_env(edge_cfg: &mut EdgeStageConfig) { /// (`Logical`-only, unresolved `Ref`, raw Mode-3 archive). These map /// onto the raw-passthrough default pipeline in the routing emitter, /// which is correct. -pub fn extract_root_sketch_kind(expr: &PhysicalExpr) -> Option { +pub fn extract_root_sketch_kind(expr: &PhysicalExpr) -> Option { match expr { PhysicalExpr::SketchAgg { sketch_type, .. } => Some(sketch_type.clone()), PhysicalExpr::RawAtEdgeSketchAtBackend { family, .. } => Some(family.clone()), @@ -324,7 +324,7 @@ pub fn extract_root_sketch_kind(expr: &PhysicalExpr) -> Option { /// (`quantile_over_time` → DDSketch, `count`-distinct → HLL, `topk` → /// CountSketch, …). We therefore collect the UNION of every workload /// entry's committed sketch family per metric into a -/// `BTreeSet` (deterministic order). The emitter routes the +/// `BTreeSet` (deterministic order). The emitter routes the /// metric to EACH family in its set and prunes pipelines/processors to /// the union of all sets — eliminating the prior all-5 fan-out that /// shipped sketch state through every family regardless of need. @@ -344,8 +344,8 @@ pub fn extract_root_sketch_kind(expr: &PhysicalExpr) -> Option { pub fn collect_metric_to_family( registry: &WorkloadRegistry, workload_store: &WorkloadStore, -) -> std::collections::HashMap> { - let mut out: std::collections::HashMap> = +) -> std::collections::HashMap> { + let mut out: std::collections::HashMap> = std::collections::HashMap::new(); for entry in registry.entries() { // B2 (metric, role) restructure: walk EVERY role registered for @@ -733,8 +733,7 @@ mod runtime_tests { let _env = crate::test_support::env_lock(); use crate::physical::colored_dag::emitter::{EdgeSketchProcessor, ExportTarget}; use crate::physical::colored_dag::stage_id::StageId; - use crate::sketch_algebra::params::SketchKind; - use crate::sketch_algebra::params::{DDSketchParams, SketchParams}; + use asap_sketch::SummaryParams; let cfg = EdgeStageConfig { source_metric: Some("m".to_string()), @@ -742,8 +741,8 @@ mod runtime_tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "ddsketch".to_string(), - sketch_kind: SketchKind::DDSketch, - sketch_params: SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), + sketch_kind: SummaryKind::DDSketch, + sketch_params: SummaryParams::DDSketch { alpha: 0.01 }, aggregation_id: "agg0".to_string(), }], exporter_target: ExportTarget::Stage(StageId::Gateway), @@ -919,7 +918,7 @@ mod runtime_tests { #[test] fn collect_metric_to_family_binds_all_six_contract_metrics_from_live_yaml() { - use crate::sketch_algebra::params::SketchKind; + use asap_sketch::SummaryKind; // The 6 contract metrics reproduced inline (mirrors // deploy/configs/mvp-workload.yaml entries 1, 5, 6, 7, 8 plus the @@ -973,27 +972,27 @@ mod runtime_tests { // metric needs. For THIS workload every sketched metric is // queried by exactly one capability, so each set has size 1. use std::collections::BTreeSet; - let expected: Vec<(&str, Option>)> = vec![ + let expected: Vec<(&str, Option>)> = vec![ ( "http_latency_ms", - Some(BTreeSet::from([SketchKind::DDSketch])), + Some(BTreeSet::from([SummaryKind::DDSketch])), ), ("http_requests_total", None), // raw passthrough ( "request_size_bytes", - Some(BTreeSet::from([SketchKind::Kll])), + Some(BTreeSet::from([SummaryKind::Kll])), ), ( "unique_users_per_min", - Some(BTreeSet::from([SketchKind::Hll])), + Some(BTreeSet::from([SummaryKind::Hll])), ), ( "top_endpoint_qps", - Some(BTreeSet::from([SketchKind::CountSketch])), + Some(BTreeSet::from([SummaryKind::CountSketchWithHeap])), ), ( "endpoint_request_freq", - Some(BTreeSet::from([SketchKind::Cms])), + Some(BTreeSet::from([SummaryKind::Cms])), ), ]; for (metric, want) in &expected { @@ -1122,9 +1121,9 @@ mod runtime_tests { /// analyzer's query-string → AggType parsing. #[test] fn collect_metric_to_family_unions_multiple_capabilities_per_metric() { - use crate::sketch_algebra::params::SketchKind; use crate::types::{AggType, QueryWorkload, SketchType, WorkloadCharacteristics}; use crate::workload::AggRole; + use asap_sketch::SummaryKind; use std::collections::BTreeSet; use std::time::Duration; @@ -1193,7 +1192,7 @@ mod runtime_tests { .unwrap_or_else(|| panic!("http_requests must be in the map\nmap: {map:?}")); assert_eq!( got, - BTreeSet::from([SketchKind::DDSketch, SketchKind::Hll, SketchKind::Cms]), + BTreeSet::from([SummaryKind::DDSketch, SummaryKind::Hll, SummaryKind::Cms]), "a metric queried by 3 capabilities must accumulate 3 families (UNION, not first-wins)\nmap: {map:?}" ); } @@ -1260,7 +1259,7 @@ mod runtime_tests { let _env = crate::test_support::env_lock(); use crate::physical::colored_dag::emitter::{EdgeStageConfig, ExportTarget}; use crate::physical::colored_dag::stage_id::StageId; - use crate::sketch_algebra::params::SketchKind; + use asap_sketch::SummaryKind; let yaml = r#" - metric_name: http_requests_total_latency_ms @@ -1286,7 +1285,7 @@ mod runtime_tests { warm_passthrough_metrics: Vec::new(), metric_to_family: std::collections::HashMap::from([( "http_requests_total_latency_ms".to_string(), - std::collections::BTreeSet::from([SketchKind::DDSketch]), + std::collections::BTreeSet::from([SummaryKind::DDSketch]), )]), metric_to_grouping_labels: std::collections::HashMap::new(), cumulative_counter_metrics: Vec::new(), diff --git a/control_plane/src/emit/otap.rs b/control_plane/src/emit/otap.rs index 1403e607..13fb0635 100644 --- a/control_plane/src/emit/otap.rs +++ b/control_plane/src/emit/otap.rs @@ -40,7 +40,7 @@ use std::collections::BTreeMap; use crate::physical::colored_dag::emitter::{EdgeSketchProcessor, EdgeStageConfig, ExportTarget}; use crate::physical::colored_dag::stage_id::StageId; -use crate::sketch_algebra::params::{SketchKind, SketchParams}; +use asap_sketch::{SummaryKind, SummaryParams}; /// Default URL for Prometheus's native OTLP HTTP receiver. /// Matches `super::stage_config::emit_edge_yaml`'s placeholder so the @@ -296,39 +296,68 @@ fn build_asap_sketches_config(sp: &EdgeSketchProcessor, window_secs: Option Value::String(sketch_kind_tag(&sp.sketch_kind).into()), ); match &sp.sketch_params { - SketchParams::Kll(p) => { - m.insert("k".into(), Value::Number((p.k as u64).into())); + SummaryParams::Kll { k } => { + m.insert("k".into(), Value::Number((*k as u64).into())); } - SketchParams::DDSketch(p) => { - m.insert("relative_accuracy".into(), Value::Number(p.alpha.into())); + SummaryParams::DDSketch { alpha } => { + m.insert("relative_accuracy".into(), Value::Number((*alpha).into())); m.insert("delta_transmission".into(), Value::Bool(true)); } - SketchParams::Hll(_p) => { + SummaryParams::Hll { .. } => { m.insert("delta_transmission".into(), Value::Bool(true)); } - SketchParams::Cms(p) => { - m.insert("rows".into(), Value::Number((p.d as u64).into())); - m.insert("columns".into(), Value::Number((p.w as u64).into())); + // Heap-bearing width/depth extraction is identical to the bare + // kind — this path never distinguished `with_heap` even before + // `SummaryKind` split it into its own variant (heap_size wasn't + // emitted here either way). + SummaryParams::Cms { width, depth } | SummaryParams::CmsWithHeap { width, depth, .. } => { + m.insert("rows".into(), Value::Number((*depth as u64).into())); + m.insert("columns".into(), Value::Number((*width as u64).into())); m.insert("delta_transmission".into(), Value::Bool(true)); } - SketchParams::CountSketch(p) => { - let epsilon = std::f64::consts::E / (p.w as f64); - let delta = 2f64.powi(-(p.d as i32)); + SummaryParams::CountSketch { width, depth } + | SummaryParams::CountSketchWithHeap { width, depth, .. } => { + let epsilon = std::f64::consts::E / (*width as f64); + let delta = 2f64.powi(-(*depth as i32)); m.insert("epsilon".into(), Value::Number(epsilon.into())); m.insert("delta".into(), Value::Number(delta.into())); m.insert("delta_transmission".into(), Value::Bool(true)); } + SummaryParams::Sum + | SummaryParams::Count + | SummaryParams::MinMax + | SummaryParams::Increase + | SummaryParams::Rate + | SummaryParams::Kmv { .. } + | SummaryParams::Theta { .. } => { + unreachable!( + "edge sketch processor config requested for a non-sketch or unsupported \ + SummaryKind; no Bind* rule in this repo produces one" + ) + } } Value::Mapping(m) } -fn sketch_kind_tag(kind: &SketchKind) -> &'static str { +fn sketch_kind_tag(kind: &SummaryKind) -> &'static str { match kind { - SketchKind::Kll => "kll", - SketchKind::DDSketch => "ddsketch", - SketchKind::Hll => "hll", - SketchKind::Cms => "cms", - SketchKind::CountSketch => "count_sketch", + SummaryKind::Kll => "kll", + SummaryKind::DDSketch => "ddsketch", + SummaryKind::Hll => "hll", + SummaryKind::Cms | SummaryKind::CmsWithHeap => "cms", + SummaryKind::CountSketch | SummaryKind::CountSketchWithHeap => "count_sketch", + SummaryKind::Sum + | SummaryKind::Count + | SummaryKind::MinMax + | SummaryKind::Increase + | SummaryKind::Rate + | SummaryKind::Kmv + | SummaryKind::Theta => { + unreachable!( + "edge sketch processor config requested for a non-sketch or unsupported \ + SummaryKind; no Bind* rule in this repo produces one" + ) + } } } @@ -338,7 +367,7 @@ fn sketch_kind_tag(kind: &SketchKind) -> &'static str { mod tests { use super::*; use crate::physical::colored_dag::emitter::{EdgeSketchProcessor, PrometheusArchiveMetric}; - use crate::sketch_algebra::params::DDSketchParams; + use asap_sketch::{SummaryKind, SummaryParams}; /// Minimal struct-stub used to validate the emitted DAG parses as the /// otap-dataflow schema. We don't pull in the otap-df-config crate @@ -385,8 +414,8 @@ mod tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "ddsketch".to_string(), - sketch_kind: SketchKind::DDSketch, - sketch_params: SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), + sketch_kind: SummaryKind::DDSketch, + sketch_params: SummaryParams::DDSketch { alpha: 0.01 }, aggregation_id: "agg0".to_string(), }], exporter_target: ExportTarget::Stage(StageId::Gateway), diff --git a/control_plane/src/emit/stage_config.rs b/control_plane/src/emit/stage_config.rs index 2e29cc3b..931ba54c 100644 --- a/control_plane/src/emit/stage_config.rs +++ b/control_plane/src/emit/stage_config.rs @@ -63,8 +63,8 @@ use crate::physical::colored_dag::emitter::{ // unused-import warning on every non-test build, so they're scoped into the // test module's `use super::*` instead (P2-5). use crate::physical::colored_dag::stage_id::StageId; -use crate::sketch_algebra::params::{SketchKind, SketchParams}; use crate::sketch_algebra::physical_expr::EstimateOp; +use asap_sketch::{SummaryKind, SummaryParams}; // ── YAML structural types ───────────────────────────────────────────────────── // @@ -223,7 +223,7 @@ pub fn emit_edge_yaml( // ── MVP §46: 5-sketch routing-connector dispatch ─────────────────────── // // When the planner has populated `cfg.metric_to_family` (the per-metric - // → SketchKind table sourced from the workload spec), we switch to the + // → SummaryKind table sourced from the workload spec), we switch to the // canonical 5-sketch routing-connector wire shape: all referenced // sketch processors live at the top level, the OTel `routing` // *connector* (NOT the deprecated routing processor) lives under @@ -592,11 +592,11 @@ pub fn emit_gateway_yaml( // Processors — one merge processor per merge entry. Naming // convention matches the patched contrib build: - // * SketchKind::DDSketch → `ddsketchmerge` - // * SketchKind::Kll → `kllmerge` - // * SketchKind::Hll → `hllmerge` - // * SketchKind::Cms → `countminsketchmerge` - // * SketchKind::CountSketch → `countsketchmerge` + // * SummaryKind::DDSketch → `ddsketchmerge` + // * SummaryKind::Kll → `kllmerge` + // * SummaryKind::Hll → `hllmerge` + // * SummaryKind::Cms → `countminsketchmerge` + // * SummaryKind::CountSketch → `countsketchmerge` // // We honour `GatewayMergeProcessor::processor_name` if non-empty // (the typed emitter today populates it as `"sketchmergeprocessor"` @@ -874,7 +874,7 @@ pub fn emit_backend_storage_routing_with_prometheus_for_tenant( /// where each `` is either `{ "engine": , "applies_to_query_shape": [...] }` /// or `{ "engine": }` for the default slot. fn build_routing_entry(metric_name: &str, cfg: &BackendStageConfig) -> JsonValue { - let kinds: Vec = cfg + let kinds: Vec = cfg .aggregations .iter() .map(|a| a.sketch_kind.clone()) @@ -885,20 +885,31 @@ fn build_routing_entry(metric_name: &str, cfg: &BackendStageConfig) -> JsonValue let mut warm_shapes: Vec<&'static str> = Vec::new(); let has_quantile_sketch = kinds .iter() - .any(|k| matches!(k, SketchKind::DDSketch | SketchKind::Kll)); + .any(|k| matches!(k, SummaryKind::DDSketch | SummaryKind::Kll)); if has_quantile_sketch { warm_shapes.push("quantile"); warm_shapes.push("quantile_over_time"); } - let has_hll = kinds.iter().any(|k| matches!(k, SketchKind::Hll)); + let has_hll = kinds.iter().any(|k| matches!(k, SummaryKind::Hll)); if has_hll { warm_shapes.push("count"); } - let has_count_sketch = kinds.iter().any(|k| matches!(k, SketchKind::CountSketch)); + // Heap-bearing kinds count too — `SummaryKind` (unlike the retired + // `sketch_algebra::SketchKind`) promotes `with_heap` to a distinct + // identity variant, but a topk-bound Count-Sketch/CMS aggregation + // still needs to register here exactly as it did before the split. + let has_count_sketch = kinds.iter().any(|k| { + matches!( + k, + SummaryKind::CountSketch | SummaryKind::CountSketchWithHeap + ) + }); if has_count_sketch { warm_shapes.push("topk"); } - let has_cms = kinds.iter().any(|k| matches!(k, SketchKind::Cms)); + let has_cms = kinds + .iter() + .any(|k| matches!(k, SummaryKind::Cms | SummaryKind::CmsWithHeap)); if has_cms { // CMS's `Estimate` readout serves point-count / count queries. // If HLL also planned, `count` is already in the list — push @@ -1033,7 +1044,7 @@ fn emit_edge_yaml_5sketch_routing( opamp_endpoint: &str, agent_id: &str, ) -> Result { - use crate::sketch_algebra::params::SketchKind; + use asap_sketch::SummaryKind; let otlp_receiver: Value = serde_yaml::from_str( "protocols:\n grpc:\n endpoint: \"0.0.0.0:4317\"\n max_recv_msg_size_mib: 64\n http:\n endpoint: \"0.0.0.0:4318\"\n", @@ -1052,18 +1063,18 @@ fn emit_edge_yaml_5sketch_routing( // The canonical 5-family order below is the iteration order for // every emit (processors, pipelines, hints) so the YAML is stable // across controller runs regardless of HashMap iteration order. - const FAMILY_ORDER: [SketchKind; 5] = [ - SketchKind::DDSketch, - SketchKind::Kll, - SketchKind::Hll, - SketchKind::CountSketch, - SketchKind::Cms, + const FAMILY_ORDER: [SummaryKind; 5] = [ + SummaryKind::DDSketch, + SummaryKind::Kll, + SummaryKind::Hll, + SummaryKind::CountSketch, + SummaryKind::Cms, ]; - let mut needed_families: std::collections::BTreeSet = + let mut needed_families: std::collections::BTreeSet = std::collections::BTreeSet::new(); for families in cfg.metric_to_family.values() { for kind in families { - needed_families.insert(kind.clone()); + needed_families.insert(base_family(kind)); } } @@ -1081,10 +1092,14 @@ fn emit_edge_yaml_5sketch_routing( // Build per-family processor blocks. We pull from // `cfg.sketch_processors` when an entry exists for that family // (so the params flow through), otherwise we synthesise a - // default-param block. - let mut family_to_proc: HashMap = HashMap::new(); + // default-param block. Keyed by `base_family` — `FAMILY_ORDER` is a + // fixed 5-bare-family list with no heap-bearing entries, exactly + // matching pre-`SummaryKind`-split behavior (heap-bearing-ness was + // never visible to this bare-kind lookup even when it lived as a + // `with_heap` params flag). + let mut family_to_proc: HashMap = HashMap::new(); for sp in &cfg.sketch_processors { - family_to_proc.insert(sp.sketch_kind.clone(), sp); + family_to_proc.insert(base_family(&sp.sketch_kind), sp); } for kind in FAMILY_ORDER { @@ -1096,7 +1111,7 @@ fn emit_edge_yaml_5sketch_routing( .metric_to_family .iter() .filter_map(|(metric, mapped)| { - if mapped.contains(&kind) { + if mapped.iter().any(|k| base_family(k) == kind) { Some(metric.as_str()) } else { None @@ -1261,7 +1276,7 @@ fn emit_edge_yaml_5sketch_routing( // connector fans its samples into both pipelines. Family order // within each metric's pipeline list follows the canonical // `FAMILY_ORDER` so the YAML is stable. - let mut metric_family_pairs: Vec<(&String, &std::collections::BTreeSet)> = + let mut metric_family_pairs: Vec<(&String, &std::collections::BTreeSet)> = cfg.metric_to_family.iter().collect(); metric_family_pairs.sort_by(|a, b| a.0.cmp(b.0)); @@ -1291,7 +1306,7 @@ fn emit_edge_yaml_5sketch_routing( // added to EACH of its families' pipelines (the `where metric.name // == ""` guard makes it a no-op on the family's other // metrics). - let mut family_to_keep_processors: HashMap> = HashMap::new(); + let mut family_to_keep_processors: HashMap> = HashMap::new(); for (metric, families) in &metric_family_pairs { let Some(labels) = cfg.metric_to_grouping_labels.get(*metric) else { continue; @@ -1301,7 +1316,7 @@ fn emit_edge_yaml_5sketch_routing( processors.insert(proc_name.clone(), proc_block); for kind in *families { family_to_keep_processors - .entry(kind.clone()) + .entry(base_family(kind)) .or_default() .push(proc_name.clone()); } @@ -1603,19 +1618,24 @@ fn emit_edge_yaml_5sketch_routing( serde_yaml::to_string(&doc).context("serialize edge stage config (5-sketch)") } -/// Map a `SketchKind` to the `family:` token the fused `asap_edge` +/// Map a `SummaryKind` to the `family:` token the fused `asap_edge` /// processor's `metrics[]` list expects. These differ from the OTel /// component-id processor names (`KLL`, `countmin`, …) used by the /// routing-connector path — the fused processor takes a lower-case /// family discriminant per entry, matching the hand-written contract in /// `asap-otel-agent-b6-asap-single-sketch.yaml`. -fn sketch_kind_to_asap_edge_family(kind: &SketchKind) -> &'static str { +fn sketch_kind_to_asap_edge_family(kind: &SummaryKind) -> &'static str { match kind { - SketchKind::DDSketch => "ddsketch", - SketchKind::Kll => "kll", - SketchKind::Hll => "hll", - SketchKind::CountSketch => "countsketch", - SketchKind::Cms => "countminsketch", + SummaryKind::DDSketch => "ddsketch", + SummaryKind::Kll => "kll", + SummaryKind::Hll => "hll", + SummaryKind::CountSketch => "countsketch", + SummaryKind::Cms => "countminsketch", + // Every caller iterates the fixed 5-bare-family `FAMILY_ORDER` + // list (heap-bearing kinds normalize through `base_family` + // before reaching here), and no Bind* rule in this repo + // produces the exact-accumulator / Kmv / Theta kinds at all. + other => unreachable!("sketch_kind_to_asap_edge_family: unexpected kind {other:?}"), } } @@ -1681,7 +1701,7 @@ fn emit_edge_yaml_asap_edge( _opamp_endpoint: &str, _agent_id: &str, ) -> Result { - use crate::sketch_algebra::params::SketchKind; + use asap_sketch::SummaryKind; // ── Receivers ────────────────────────────────────────────────────────── // OTLP gRPC on 4317 + HTTP on 4318 — same as every other edge emit. @@ -1860,22 +1880,28 @@ fn emit_edge_yaml_asap_edge( // (keyed by family) so the per-metric param block mirrors the // routing path; fall back to catalog defaults when the planner // mapped a family with no enumerated processor. - let mut family_to_proc: HashMap = HashMap::new(); + let mut family_to_proc: HashMap = HashMap::new(); for sp in &cfg.sketch_processors { - family_to_proc.insert(sp.sketch_kind.clone(), sp); - } - const FAMILY_ORDER: [SketchKind; 5] = [ - SketchKind::DDSketch, - SketchKind::Kll, - SketchKind::Hll, - SketchKind::CountSketch, - SketchKind::Cms, + family_to_proc.insert(base_family(&sp.sketch_kind), sp); + } + const FAMILY_ORDER: [SummaryKind; 5] = [ + SummaryKind::DDSketch, + SummaryKind::Kll, + SummaryKind::Hll, + SummaryKind::CountSketch, + SummaryKind::Cms, ]; - let mut metric_family_pairs: Vec<(&String, &std::collections::BTreeSet)> = + let mut metric_family_pairs: Vec<(&String, &std::collections::BTreeSet)> = cfg.metric_to_family.iter().collect(); metric_family_pairs.sort_by(|a, b| a.0.cmp(b.0)); for (metric, families) in &metric_family_pairs { - for kind in FAMILY_ORDER.iter().filter(|k| families.contains(*k)) { + // Normalize to bare families before filtering against the fixed + // `FAMILY_ORDER` list — same reasoning as `family_to_proc` above: + // a committed heap-bearing kind (`CmsWithHeap`/`CountSketchWithHeap`) + // must still match its bare `FAMILY_ORDER` entry. + let bare_families: std::collections::BTreeSet = + families.iter().map(base_family).collect(); + for kind in FAMILY_ORDER.iter().filter(|k| bare_families.contains(*k)) { let mut e = Mapping::new(); e.insert("metric".into(), Value::String((*metric).clone())); e.insert( @@ -1968,7 +1994,7 @@ fn emit_edge_yaml_asap_edge( let whole_stream = effective_by.is_empty() && matches!( kind, - SketchKind::Hll | SketchKind::Cms | SketchKind::CountSketch + SummaryKind::Hll | SummaryKind::Cms | SummaryKind::CountSketch ); if whole_stream { e.insert("mode".into(), Value::String("whole_stream".to_string())); @@ -2015,7 +2041,7 @@ fn emit_edge_yaml_asap_edge( // lossless and serializes byte-identically to dense). We emit the // flag ONLY for the HLL family; non-HLL families carry no // `hll_sparse` key. - if matches!(kind, SketchKind::Hll) { + if matches!(kind, SummaryKind::Hll) { let hll_sparse = if whole_stream { false } else { @@ -2038,36 +2064,54 @@ fn emit_edge_yaml_asap_edge( // flag (set by `BindCountSketchOnTopK` when the family is // CountSketch picked for a `topk(...)` query). It drives the // warm-topk heap keys emitted below for the CountSketch family. - let mut countsketch_with_heap = false; + // Heap-bearing-ness now lives on `sketch_kind`, not a params + // flag — read it off the processor's kind before matching + // its params. + let mut countsketch_with_heap = family_to_proc + .get(kind) + .is_some_and(|sp| matches!(sp.sketch_kind, SummaryKind::CountSketchWithHeap)); match family_to_proc.get(kind).map(|sp| &sp.sketch_params) { - Some(SketchParams::DDSketch(p)) => { - e.insert("relative_accuracy".into(), Value::Number(p.alpha.into())); + Some(SummaryParams::DDSketch { alpha }) => { + e.insert("relative_accuracy".into(), Value::Number((*alpha).into())); } - Some(SketchParams::Kll(p)) => { - e.insert("k".into(), Value::Number((p.k as u64).into())); + Some(SummaryParams::Kll { k }) => { + e.insert("k".into(), Value::Number((*k as u64).into())); } - Some(SketchParams::Hll(_)) => { /* HLL takes no per-entry knob */ } - Some(SketchParams::CountSketch(p)) => { - e.insert("rows".into(), Value::Number((p.d as u64).into())); - e.insert("cols".into(), Value::Number((p.w as u64).into())); - countsketch_with_heap = p.with_heap; + Some(SummaryParams::Hll { .. }) => { /* HLL takes no per-entry knob */ } + Some(SummaryParams::CountSketch { width, depth }) + | Some(SummaryParams::CountSketchWithHeap { width, depth, .. }) => { + e.insert("rows".into(), Value::Number((*depth as u64).into())); + e.insert("cols".into(), Value::Number((*width as u64).into())); } - Some(SketchParams::Cms(p)) => { - e.insert("rows".into(), Value::Number((p.d as u64).into())); - e.insert("cols".into(), Value::Number((p.w as u64).into())); + Some(SummaryParams::Cms { width, depth }) + | Some(SummaryParams::CmsWithHeap { width, depth, .. }) => { + e.insert("rows".into(), Value::Number((*depth as u64).into())); + e.insert("cols".into(), Value::Number((*width as u64).into())); } + Some( + SummaryParams::Sum + | SummaryParams::Count + | SummaryParams::MinMax + | SummaryParams::Increase + | SummaryParams::Rate + | SummaryParams::Kmv { .. } + | SummaryParams::Theta { .. }, + ) => unreachable!( + "5-sketch routing: non-sketch or unsupported SummaryParams; \ + no Bind* rule in this repo produces one" + ), None => { // Family with no enumerated processor — emit catalog // defaults so the entry is still well-formed. match kind { - SketchKind::DDSketch => { + SummaryKind::DDSketch => { e.insert("relative_accuracy".into(), Value::Number(0.01.into())); } - SketchKind::Kll => { + SummaryKind::Kll => { e.insert("k".into(), Value::Number(200u64.into())); } - SketchKind::Hll => {} - SketchKind::CountSketch => { + SummaryKind::Hll => {} + SummaryKind::CountSketch => { e.insert("rows".into(), Value::Number(5u64.into())); e.insert("cols".into(), Value::Number(2048u64.into())); // P1-4: NO enumerated EdgeSketchProcessor for this @@ -2091,10 +2135,15 @@ fn emit_edge_yaml_asap_edge( // step with the backend `with_heap` registration. countsketch_with_heap = cfg.metric_to_item_label.contains_key(*metric); } - SketchKind::Cms => { + SummaryKind::Cms => { e.insert("rows".into(), Value::Number(5u64.into())); e.insert("cols".into(), Value::Number(2048u64.into())); } + // `kind` always comes from the bare 5-family + // `FAMILY_ORDER` list. + other => unreachable!( + "5-sketch routing catalog defaults: unexpected kind {other:?}" + ), } } } @@ -2102,7 +2151,7 @@ fn emit_edge_yaml_asap_edge( // families (CMS / HLL) only when `p < 1.0`. Mirrors // `build_edge_processor_block`'s guarded emit so an unset / // 1.0 probability keeps the fused entry byte-identical. - if matches!(kind, SketchKind::Cms | SketchKind::Hll) { + if matches!(kind, SummaryKind::Cms | SummaryKind::Hll) { insert_sample_p(&mut e, cfg.metric_to_sample_p.get(*metric).copied()); } @@ -2123,7 +2172,10 @@ fn emit_edge_yaml_asap_edge( // depending on that default. if matches!( kind, - SketchKind::DDSketch | SketchKind::Hll | SketchKind::CountSketch | SketchKind::Cms + SummaryKind::DDSketch + | SummaryKind::Hll + | SummaryKind::CountSketch + | SummaryKind::Cms ) { e.insert("delta_transmission".into(), Value::Bool(true)); } @@ -2154,7 +2206,7 @@ fn emit_edge_yaml_asap_edge( // that lands — `mapstructure` ignores unknown keys by default — // but the warm-topk behaviour only activates once the asapedge // build carries the fields. See the report's cross-repo note. - if matches!(kind, SketchKind::CountSketch) && countsketch_with_heap { + if matches!(kind, SummaryKind::CountSketch) && countsketch_with_heap { e.insert("emit_heap".into(), Value::Bool(true)); e.insert("heap_size".into(), Value::Number(100u64.into())); // Prefer the workload-declared inner dimension @@ -2198,7 +2250,7 @@ fn emit_edge_yaml_asap_edge( // pure YAML text here (`mapstructure` ignores unknown keys), so // emitting it is safe even before that lands; the corrected // keying only activates once the asapedge build carries it. - if matches!(kind, SketchKind::Hll | SketchKind::Cms) { + if matches!(kind, SummaryKind::Hll | SummaryKind::Cms) { if let Some(item_label) = cfg.metric_to_item_label.get(*metric) { if !item_label.is_empty() { e.insert("item_label".into(), Value::String(item_label.clone())); @@ -2405,28 +2457,32 @@ tsdb_block_duration: {window_secs}s\n", ) } -/// Map a `SketchKind` to the OTel processor name registered by the +/// Map a `SummaryKind` to the OTel processor name registered by the /// patched contrib build's factory. Keep in sync with /// `crate::physical::colored_dag::emitter::edge_processor_name`. -fn sketch_kind_to_processor_name(kind: &SketchKind) -> &'static str { +fn sketch_kind_to_processor_name(kind: &SummaryKind) -> &'static str { match kind { - SketchKind::DDSketch => "ddsketch", - SketchKind::Kll => "KLL", - SketchKind::Hll => "HLL", - SketchKind::CountSketch => "countsketch", - SketchKind::Cms => "countmin", + SummaryKind::DDSketch => "ddsketch", + SummaryKind::Kll => "KLL", + SummaryKind::Hll => "HLL", + SummaryKind::CountSketch => "countsketch", + SummaryKind::Cms => "countmin", + // Callers only ever pass a bare `FAMILY_ORDER` entry. + other => unreachable!("sketch_kind_to_processor_name: unexpected kind {other:?}"), } } -/// Map a `SketchKind` to its per-family pipeline name in the routing +/// Map a `SummaryKind` to its per-family pipeline name in the routing /// connector layout. -fn sketch_kind_to_pipeline_name(kind: &SketchKind) -> &'static str { +fn sketch_kind_to_pipeline_name(kind: &SummaryKind) -> &'static str { match kind { - SketchKind::DDSketch => "metrics/ddsketch_path", - SketchKind::Kll => "metrics/kll_path", - SketchKind::Hll => "metrics/hll_path", - SketchKind::CountSketch => "metrics/countsketch_path", - SketchKind::Cms => "metrics/countminsketch_path", + SummaryKind::DDSketch => "metrics/ddsketch_path", + SummaryKind::Kll => "metrics/kll_path", + SummaryKind::Hll => "metrics/hll_path", + SummaryKind::CountSketch => "metrics/countsketch_path", + SummaryKind::Cms => "metrics/countminsketch_path", + // Callers only ever pass a bare `FAMILY_ORDER` entry. + other => unreachable!("sketch_kind_to_pipeline_name: unexpected kind {other:?}"), } } @@ -2562,39 +2618,53 @@ fn build_transform_keep_processor_block(metric: &str, labels: &[String]) -> Valu serde_yaml::from_str(&yaml).expect("transform/keep_for_* yaml is well-formed by construction") } -/// Build a default-parameter processor block for a `SketchKind` when +/// Build a default-parameter processor block for a `SummaryKind` when /// the planner's `metric_to_family` references a family that /// `cfg.sketch_processors` didn't enumerate. Defaults match the catalog /// values used by the planner's L4 rules so the wire shape is what the /// rest of the system expects when a metric is later re-routed onto /// this family. fn build_default_edge_processor_block( - kind: &SketchKind, + kind: &SummaryKind, window_secs: Option, metric_name_hint: Option<&str>, sample_p: Option, ) -> Value { - use crate::sketch_algebra::params::{ - CmsParams, CountSketchParams, DDSketchParams, HllParams, KllParams, - }; - let params = match kind { - SketchKind::DDSketch => SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), - SketchKind::Kll => SketchParams::Kll(KllParams { k: 200 }), - SketchKind::Hll => SketchParams::Hll(HllParams { precision: 14 }), - SketchKind::CountSketch => SketchParams::CountSketch(CountSketchParams { - w: 2048, - d: 5, - with_heap: true, - }), - SketchKind::Cms => SketchParams::Cms(CmsParams { - w: 4096, - d: 4, - with_heap: false, - }), + // `kind` is always one of the 5 bare `FAMILY_ORDER` entries (every + // caller normalizes through `base_family` first) — used as-is for + // the tag/processor-name lookups below, which are keyed on the bare + // family. `stored_kind`/`params` are what actually land on the + // synthesized processor; `CountSketch`'s default stays heap-bearing + // (matching this function's pre-`SummaryKind`-split default of + // `with_heap: true` — `Cms`'s default was `with_heap: false` and + // stays bare). + let (stored_kind, params) = match kind { + SummaryKind::DDSketch => ( + SummaryKind::DDSketch, + SummaryParams::DDSketch { alpha: 0.01 }, + ), + SummaryKind::Kll => (SummaryKind::Kll, SummaryParams::Kll { k: 200 }), + SummaryKind::Hll => (SummaryKind::Hll, SummaryParams::Hll { precision: 14 }), + SummaryKind::CountSketch => ( + SummaryKind::CountSketchWithHeap, + SummaryParams::CountSketchWithHeap { + width: 2048, + depth: 5, + heap_size: 10, + }, + ), + SummaryKind::Cms => ( + SummaryKind::Cms, + SummaryParams::Cms { + width: 4096, + depth: 4, + }, + ), + other => unreachable!("build_default_edge_processor_block: unexpected kind {other:?}"), }; let synthetic = EdgeSketchProcessor { processor_name: sketch_kind_to_processor_name(kind).to_string(), - sketch_kind: kind.clone(), + sketch_kind: stored_kind, sketch_params: params, aggregation_id: format!("agg_default_{}", sketch_kind_tag(kind)), }; @@ -2728,15 +2798,15 @@ fn build_edge_processor_block( // it explicitly so the wire YAML doesn't depend on a factory // default that could regress to full-state in a future build. match &sp.sketch_params { - SketchParams::Kll(p) => { - m.insert("k".into(), Value::Number((p.k as u64).into())); + SummaryParams::Kll { k } => { + m.insert("k".into(), Value::Number((*k as u64).into())); // No delta_transmission for KLL: see comment above. } - SketchParams::DDSketch(p) => { - m.insert("relative_accuracy".into(), Value::Number(p.alpha.into())); + SummaryParams::DDSketch { alpha } => { + m.insert("relative_accuracy".into(), Value::Number((*alpha).into())); m.insert("delta_transmission".into(), Value::Bool(true)); } - SketchParams::Hll(_p) => { + SummaryParams::Hll { .. } => { // HLL takes no precision knob in its Config (the // patched build hard-codes p=14); nothing further to set. m.insert("encoding".into(), Value::String("msgpack".into())); @@ -2745,7 +2815,7 @@ fn build_edge_processor_block( // (hash-threshold element sampling in sketchlib-go). insert_sample_p(&mut m, sample_p); } - SketchParams::Cms(p) => { + SummaryParams::Cms { width, depth } | SummaryParams::CmsWithHeap { width, depth, .. } => { m.insert( "metric_name".into(), Value::String( @@ -2754,15 +2824,16 @@ fn build_edge_processor_block( .to_string(), ), ); - m.insert("rows".into(), Value::Number((p.d as u64).into())); - m.insert("columns".into(), Value::Number((p.w as u64).into())); + m.insert("rows".into(), Value::Number((*depth as u64).into())); + m.insert("columns".into(), Value::Number((*width as u64).into())); m.insert("encoding".into(), Value::String("msgpack".into())); m.insert("delta_transmission".into(), Value::Bool(true)); // Per-metric sampling: the CMS processor honours `sample_p` // (geometric admission sampling in sketchlib-go). insert_sample_p(&mut m, sample_p); } - SketchParams::CountSketch(p) => { + SummaryParams::CountSketch { width, depth } + | SummaryParams::CountSketchWithHeap { width, depth, .. } => { // P1-3: the standalone `countsketchprocessor` Config exposes ONLY // `epsilon` / `delta` (no `rows` / `cols` mapstructure keys), and // it RE-DERIVES the sketch dimensions internally via @@ -2783,12 +2854,22 @@ fn build_edge_processor_block( // epsilon = 1/sqrt(w) ⇒ ceil(1/epsilon^2) = ceil(w) = w // ⇒ nextPow2(w) = w (w is a power of 2) // delta = e^-d ⇒ ceil(ln(1/delta)) = ceil(d) = d - let (epsilon, delta) = countsketch_epsilon_delta_for(p.w, p.d); + let (epsilon, delta) = countsketch_epsilon_delta_for(*width, *depth); m.insert("epsilon".into(), Value::Number(epsilon.into())); m.insert("delta".into(), Value::Number(delta.into())); m.insert("encoding".into(), Value::String("msgpack".into())); m.insert("delta_transmission".into(), Value::Bool(true)); } + SummaryParams::Sum + | SummaryParams::Count + | SummaryParams::MinMax + | SummaryParams::Increase + | SummaryParams::Rate + | SummaryParams::Kmv { .. } + | SummaryParams::Theta { .. } => unreachable!( + "edge sketch processor config requested for a non-sketch or unsupported \ + SummaryKind; no Bind* rule in this repo produces one" + ), } Value::Mapping(m) @@ -2821,11 +2902,23 @@ fn insert_sample_p(m: &mut Mapping, sample_p: Option) { /// here so the emitted YAML round-trips through the patched build. fn gateway_merge_processor_name(mp: &GatewayMergeProcessor) -> String { match mp.sketch_kind { - SketchKind::Kll => "kllmerge".to_string(), - SketchKind::DDSketch => "ddsketchmerge".to_string(), - SketchKind::Hll => "hllmerge".to_string(), - SketchKind::Cms => "countminsketchmerge".to_string(), - SketchKind::CountSketch => "countsketchmerge".to_string(), + SummaryKind::Kll => "kllmerge".to_string(), + SummaryKind::DDSketch => "ddsketchmerge".to_string(), + SummaryKind::Hll => "hllmerge".to_string(), + SummaryKind::Cms | SummaryKind::CmsWithHeap => "countminsketchmerge".to_string(), + SummaryKind::CountSketch | SummaryKind::CountSketchWithHeap => { + "countsketchmerge".to_string() + } + SummaryKind::Sum + | SummaryKind::Count + | SummaryKind::MinMax + | SummaryKind::Increase + | SummaryKind::Rate + | SummaryKind::Kmv + | SummaryKind::Theta => unreachable!( + "gateway_merge_processor_name: non-sketch or unsupported SummaryKind; \ + no Bind* rule in this repo produces one" + ), } } @@ -2956,57 +3049,123 @@ fn build_backend_readout_json(r: &BackendReadout) -> JsonValue { } } -/// Map a `(SketchKind, SketchParams)` pair to the backend's -/// `AggregationType::Display` string — the same mapping +/// Collapse a heap-bearing `SummaryKind` to its bare counterpart. +/// Identity for every other kind. +/// +/// The 5-sketch routing-connector edge YAML path (`emit_edge_yaml`'s +/// `USE_5SKETCH_ROUTING` branch and its `metric_to_family` sibling) +/// keys its fixed `FAMILY_ORDER` list and lookup maps on the 5 bare +/// families only — matching the retired `sketch_algebra::SketchKind`, +/// which had no heap-bearing variant at all (`with_heap` was a +/// `SketchParams` field, invisible to anything keying on kind alone). +/// A committed heap-bearing kind (`CmsWithHeap`/`CountSketchWithHeap`, +/// from a topk binding) needs to normalize through this before it's +/// used as a key or set member in that path, or it silently fails to +/// match its bare `FAMILY_ORDER` entry. +fn base_family(kind: &SummaryKind) -> SummaryKind { + match kind { + SummaryKind::CmsWithHeap => SummaryKind::Cms, + SummaryKind::CountSketchWithHeap => SummaryKind::CountSketch, + other => other.clone(), + } +} + +/// Map a `SummaryKind` to the backend's `AggregationType::Display` +/// string — the same mapping /// [`crate::config::asapquery_backend::map_sketch_type_to_agg_type`] uses /// (the strings must match `AggregationType::FromStr` in the backend's /// `promql_utilities::query_logics::enums`). /// -/// The params side promotes CMS / CountSketch to their `*WithHeap` -/// variants when the planner-set `with_heap` flag is true (see -/// `bind_cms_with_heap_on_topk` and `BindCountSketchOnTopK`). This -/// is what lets the backend's `policy_capability` lookup return +/// Heap-bearing is now identity, not a params flag (`SummaryKind::CmsWithHeap` +/// / `CountSketchWithHeap`, set by `BindCountSketchOnTopK` — see +/// `sketch_algebra::rules::bind_cms_topk`), so this maps on `kind` alone; +/// `params` is unused but kept for call-site stability. This is what +/// lets the backend's `policy_capability` lookup return /// `FrequencyTopk(*WithHeap)` for heap-bearing aggregations — required /// for `topk(...)` queries to bind to the right sids. -fn sketch_kind_to_backend_type(kind: &SketchKind, params: &SketchParams) -> &'static str { - match (kind, params) { - (SketchKind::DDSketch, _) => "DDSketch", - (SketchKind::Kll, _) => "DatasketchesKLL", - (SketchKind::Hll, _) => "HLL", - (SketchKind::CountSketch, SketchParams::CountSketch(p)) if p.with_heap => { - "CountSketchWithHeap" - } - (SketchKind::CountSketch, _) => "CountSketch", - (SketchKind::Cms, SketchParams::Cms(p)) if p.with_heap => "CountMinSketchWithHeap", - (SketchKind::Cms, _) => "CountMinSketch", +fn sketch_kind_to_backend_type(kind: &SummaryKind, _params: &SummaryParams) -> &'static str { + match kind { + SummaryKind::DDSketch => "DDSketch", + SummaryKind::Kll => "DatasketchesKLL", + SummaryKind::Hll => "HLL", + SummaryKind::CountSketchWithHeap => "CountSketchWithHeap", + SummaryKind::CountSketch => "CountSketch", + SummaryKind::CmsWithHeap => "CountMinSketchWithHeap", + SummaryKind::Cms => "CountMinSketch", + SummaryKind::Sum + | SummaryKind::Count + | SummaryKind::MinMax + | SummaryKind::Increase + | SummaryKind::Rate + | SummaryKind::Kmv + | SummaryKind::Theta => unreachable!( + "sketch_kind_to_backend_type: non-sketch or unsupported SummaryKind; \ + no Bind* rule in this repo produces one" + ), } } -/// Stable lowercase tag for a `SketchKind` — used as a passthrough +/// Stable lowercase tag for a `SummaryKind` — used as a passthrough /// `sketch_kind` field in YAML so downstream consumers can dispatch -/// without round-tripping through serde. -fn sketch_kind_tag(kind: &SketchKind) -> &'static str { +/// without round-tripping through serde. Heap-bearing kinds reuse their +/// bare counterpart's tag — this field never distinguished `with_heap` +/// even before `SummaryKind` split it into its own variant. +fn sketch_kind_tag(kind: &SummaryKind) -> &'static str { match kind { - SketchKind::Kll => "kll", - SketchKind::DDSketch => "ddsketch", - SketchKind::Hll => "hll", - SketchKind::Cms => "cms", - SketchKind::CountSketch => "count_sketch", + SummaryKind::Kll => "kll", + SummaryKind::DDSketch => "ddsketch", + SummaryKind::Hll => "hll", + SummaryKind::Cms | SummaryKind::CmsWithHeap => "cms", + SummaryKind::CountSketch | SummaryKind::CountSketchWithHeap => "count_sketch", + SummaryKind::Sum + | SummaryKind::Count + | SummaryKind::MinMax + | SummaryKind::Increase + | SummaryKind::Rate + | SummaryKind::Kmv + | SummaryKind::Theta => unreachable!( + "sketch_kind_tag: non-sketch or unsupported SummaryKind; \ + no Bind* rule in this repo produces one" + ), } } /// Serialize a `SketchParams` payload to a flat JSON object the backend /// can read directly without round-tripping through the controller's /// internally-tagged enum form. -fn sketch_params_to_json(p: &SketchParams) -> JsonValue { +fn sketch_params_to_json(p: &SummaryParams) -> JsonValue { match p { - SketchParams::Kll(p) => json!({ "k": p.k }), - SketchParams::DDSketch(p) => json!({ "alpha": p.alpha }), - SketchParams::Hll(p) => json!({ "precision": p.precision }), - SketchParams::Cms(p) => json!({ "w": p.w, "d": p.d }), - SketchParams::CountSketch(p) => { - json!({ "w": p.w, "d": p.d, "with_heap": p.with_heap }) + SummaryParams::Kll { k } => json!({ "k": k }), + SummaryParams::DDSketch { alpha } => json!({ "alpha": alpha }), + SummaryParams::Hll { precision } => json!({ "precision": precision }), + // Cms/CmsWithHeap: matches the pre-split shape exactly — the old + // `SketchParams::Cms` arm never emitted `with_heap` in JSON even + // though `CmsParams.with_heap` existed as a field; heap-bearing + // and bare CMS produced identical wire JSON. `heap_size` is a + // new field with no wire representation here (nothing on the + // real backend wire path reads it — see `bind_cms_topk.rs`). + SummaryParams::Cms { width, depth } | SummaryParams::CmsWithHeap { width, depth, .. } => { + json!({ "w": width, "d": depth }) + } + // CountSketch/CountSketchWithHeap: the old arm always emitted + // `with_heap` (from `CountSketchParams.with_heap: bool`); + // that boolean is now the kind identity itself. + SummaryParams::CountSketch { width, depth } => { + json!({ "w": width, "d": depth, "with_heap": false }) } + SummaryParams::CountSketchWithHeap { width, depth, .. } => { + json!({ "w": width, "d": depth, "with_heap": true }) + } + SummaryParams::Sum + | SummaryParams::Count + | SummaryParams::MinMax + | SummaryParams::Increase + | SummaryParams::Rate + | SummaryParams::Kmv { .. } + | SummaryParams::Theta { .. } => unreachable!( + "sketch_params_to_json: non-sketch or unsupported SummaryParams; \ + no Bind* rule in this repo produces one" + ), } } @@ -3019,10 +3178,6 @@ mod tests { // module; gating them here keeps the non-test build free of the // unused-import warning they previously triggered at module scope. use crate::physical::colored_dag::emitter::{ArchiveTierMetric, PrometheusArchiveMetric}; - use crate::sketch_algebra::params::{ - CmsParams, CountSketchParams, DDSketchParams, HllParams, KllParams, - }; - fn ddsketch_edge_cfg() -> EdgeStageConfig { EdgeStageConfig { source_metric: Some("http_request_duration_seconds".to_string()), @@ -3030,8 +3185,8 @@ mod tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "ddsketch".to_string(), - sketch_kind: SketchKind::DDSketch, - sketch_params: SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), + sketch_kind: SummaryKind::DDSketch, + sketch_params: SummaryParams::DDSketch { alpha: 0.01 }, aggregation_id: "agg0".to_string(), }], exporter_target: ExportTarget::Stage(StageId::Gateway), @@ -3115,8 +3270,8 @@ mod tests { let mut cfg = ddsketch_edge_cfg(); cfg.sketch_processors[0] = EdgeSketchProcessor { processor_name: "KLL".to_string(), - sketch_kind: SketchKind::Kll, - sketch_params: SketchParams::Kll(KllParams { k: 200 }), + sketch_kind: SummaryKind::Kll, + sketch_params: SummaryParams::Kll { k: 200 }, aggregation_id: "agg7".to_string(), }; let yaml = emit_edge_yaml(&cfg, "ws://c/", "test-agent").expect("emit ok"); @@ -3150,32 +3305,31 @@ mod tests { // flag (see `edge_yaml_kll_uses_k_param`). for (kind, processor_name, params) in [ ( - SketchKind::DDSketch, + SummaryKind::DDSketch, "ddsketch", - SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), + SummaryParams::DDSketch { alpha: 0.01 }, ), ( - SketchKind::Hll, + SummaryKind::Hll, "HLL", - SketchParams::Hll(HllParams { precision: 14 }), + SummaryParams::Hll { precision: 14 }, ), ( - SketchKind::CountSketch, + SummaryKind::CountSketch, "countsketch", - SketchParams::CountSketch(CountSketchParams { - w: 2048, - d: 5, - with_heap: true, - }), + SummaryParams::CountSketchWithHeap { + width: 2048, + depth: 5, + heap_size: 10, + }, ), ( - SketchKind::Cms, + SummaryKind::Cms, "countmin", - SketchParams::Cms(CmsParams { - w: 4096, - d: 4, - with_heap: false, - }), + SummaryParams::Cms { + width: 4096, + depth: 4, + }, ), ] { let mut cfg = ddsketch_edge_cfg(); @@ -3200,12 +3354,11 @@ mod tests { cfg.source_metric = Some("endpoint_request_freq".to_string()); cfg.sketch_processors[0] = EdgeSketchProcessor { processor_name: "countmin".to_string(), - sketch_kind: SketchKind::Cms, - sketch_params: SketchParams::Cms(CmsParams { - w: 4096, - d: 4, - with_heap: false, - }), + sketch_kind: SummaryKind::Cms, + sketch_params: SummaryParams::Cms { + width: 4096, + depth: 4, + }, aggregation_id: "agg-cms".to_string(), }; let yaml = emit_edge_yaml(&cfg, "ws://c/", "test-agent").expect("emit ok"); @@ -3234,7 +3387,7 @@ mod tests { otlp_receiver_port: 4317, merge_processors: vec![GatewayMergeProcessor { processor_name: "sketchmergeprocessor".to_string(), - sketch_kind: SketchKind::DDSketch, + sketch_kind: SummaryKind::DDSketch, aggregation_id: "agg0".to_string(), }], exporter_target: ExportTarget::Stage(StageId::Backend), @@ -3278,12 +3431,12 @@ mod tests { merge_processors: vec![ GatewayMergeProcessor { processor_name: "x".into(), - sketch_kind: SketchKind::Kll, + sketch_kind: SummaryKind::Kll, aggregation_id: "agg0".into(), }, GatewayMergeProcessor { processor_name: "x".into(), - sketch_kind: SketchKind::Hll, + sketch_kind: SummaryKind::Hll, aggregation_id: "agg1".into(), }, ], @@ -3341,8 +3494,8 @@ mod tests { item_label: None, aggregation_id: "agg0".into(), metric_name: "http_latency_ms".into(), - sketch_kind: SketchKind::DDSketch, - sketch_params: SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), + sketch_kind: SummaryKind::DDSketch, + sketch_params: SummaryParams::DDSketch { alpha: 0.01 }, window_secs: 60, spatial_filter: String::new(), grouping: Vec::new(), @@ -3353,8 +3506,8 @@ mod tests { item_label: None, aggregation_id: "agg1".into(), metric_name: "http_requests_total".into(), - sketch_kind: SketchKind::Hll, - sketch_params: SketchParams::Hll(HllParams { precision: 14 }), + sketch_kind: SummaryKind::Hll, + sketch_params: SummaryParams::Hll { precision: 14 }, window_secs: 60, spatial_filter: String::new(), grouping: Vec::new(), @@ -3407,12 +3560,12 @@ mod tests { item_label: None, aggregation_id: "agg0".into(), metric_name: "endpoint_count".into(), - sketch_kind: SketchKind::CountSketch, - sketch_params: SketchParams::CountSketch(CountSketchParams { - w: 2048, - d: 5, - with_heap: true, - }), + sketch_kind: SummaryKind::CountSketchWithHeap, + sketch_params: SummaryParams::CountSketchWithHeap { + width: 2048, + depth: 5, + heap_size: 10, + }, window_secs: 60, spatial_filter: String::new(), grouping: Vec::new(), @@ -3423,12 +3576,11 @@ mod tests { item_label: None, aggregation_id: "agg1".into(), metric_name: "endpoint_hits".into(), - sketch_kind: SketchKind::Cms, - sketch_params: SketchParams::Cms(CmsParams { - w: 4096, - d: 4, - with_heap: false, - }), + sketch_kind: SummaryKind::Cms, + sketch_params: SummaryParams::Cms { + width: 4096, + depth: 4, + }, window_secs: 60, spatial_filter: String::new(), grouping: Vec::new(), @@ -3481,29 +3633,42 @@ mod tests { /// Helper: build a single-aggregation BackendStageConfig of the /// requested kind. `aggregation_id` is hard-coded — the routing - /// emitter doesn't care about it. - fn backend_cfg_with_kind(kind: SketchKind) -> BackendStageConfig { - let params = match kind { - SketchKind::DDSketch => SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), - SketchKind::Kll => SketchParams::Kll(KllParams { k: 200 }), - SketchKind::Hll => SketchParams::Hll(HllParams { precision: 14 }), - SketchKind::Cms => SketchParams::Cms(CmsParams { - w: 4096, - d: 4, - with_heap: false, - }), - SketchKind::CountSketch => SketchParams::CountSketch(CountSketchParams { - w: 2048, - d: 5, - with_heap: true, - }), + /// emitter doesn't care about it. Accepts the 5 canonical bare + /// families callers actually pass; `CountSketch` stores as the + /// heap-bearing variant (matching this fixture's pre-`SummaryKind`-split + /// behavior, when `with_heap: true` was a `CountSketchParams` field + /// rather than a distinct kind). + fn backend_cfg_with_kind(kind: SummaryKind) -> BackendStageConfig { + let (stored_kind, params) = match kind { + SummaryKind::DDSketch => ( + SummaryKind::DDSketch, + SummaryParams::DDSketch { alpha: 0.01 }, + ), + SummaryKind::Kll => (SummaryKind::Kll, SummaryParams::Kll { k: 200 }), + SummaryKind::Hll => (SummaryKind::Hll, SummaryParams::Hll { precision: 14 }), + SummaryKind::Cms => ( + SummaryKind::Cms, + SummaryParams::Cms { + width: 4096, + depth: 4, + }, + ), + SummaryKind::CountSketch => ( + SummaryKind::CountSketchWithHeap, + SummaryParams::CountSketchWithHeap { + width: 2048, + depth: 5, + heap_size: 10, + }, + ), + other => unreachable!("backend_cfg_with_kind: unsupported test fixture kind {other:?}"), }; BackendStageConfig { aggregations: vec![BackendAggregation { item_label: None, aggregation_id: "agg0".into(), metric_name: "test_metric".into(), - sketch_kind: kind.clone(), + sketch_kind: stored_kind, sketch_params: params, window_secs: 60, spatial_filter: String::new(), @@ -3514,12 +3679,15 @@ mod tests { readouts: vec![BackendReadout { aggregation_id: "agg0".into(), op: match kind { - SketchKind::DDSketch | SketchKind::Kll => EstimateOp::Quantile { q: 0.99 }, - SketchKind::Hll => EstimateOp::Cardinality, - SketchKind::CountSketch => EstimateOp::TopK { k: 10 }, - SketchKind::Cms => EstimateOp::PointCount { + SummaryKind::DDSketch | SummaryKind::Kll => EstimateOp::Quantile { q: 0.99 }, + SummaryKind::Hll => EstimateOp::Cardinality, + SummaryKind::CountSketch => EstimateOp::TopK { k: 10 }, + SummaryKind::Cms => EstimateOp::PointCount { key: "user_42".into(), }, + other => unreachable!( + "backend_cfg_with_kind: unsupported test fixture kind {other:?}" + ), }, }], } @@ -3527,7 +3695,7 @@ mod tests { #[test] fn storage_routing_emits_default_engine_and_metrics_array() { - let ddsketch = backend_cfg_with_kind(SketchKind::DDSketch); + let ddsketch = backend_cfg_with_kind(SummaryKind::DDSketch); let plans: Vec<(String, &BackendStageConfig)> = vec![("http_request_duration_seconds".to_string(), &ddsketch)]; let v = emit_backend_storage_routing(&plans).expect("emit ok"); @@ -3545,7 +3713,7 @@ mod tests { /// byte-compatible (modulo the new `tenant` field appearing). #[test] fn storage_routing_default_tenant_for_single_tenant_emit() { - let ddsketch = backend_cfg_with_kind(SketchKind::DDSketch); + let ddsketch = backend_cfg_with_kind(SummaryKind::DDSketch); let v = emit_backend_storage_routing(&[("latency".into(), &ddsketch)]).expect("emit ok"); assert_eq!(v["tenant"], DEFAULT_TENANT); } @@ -3557,7 +3725,7 @@ mod tests { /// body-tenant precedence rule. #[test] fn storage_routing_for_tenant_emits_explicit_tenant_field() { - let ddsketch = backend_cfg_with_kind(SketchKind::DDSketch); + let ddsketch = backend_cfg_with_kind(SummaryKind::DDSketch); let v = emit_backend_storage_routing_for_tenant("tenant-a", &[("latency".into(), &ddsketch)]) .expect("emit ok"); @@ -3584,7 +3752,7 @@ mod tests { #[test] fn storage_routing_ddasap_query_serves_quantile_archive_serves_others() { - let ddsketch = backend_cfg_with_kind(SketchKind::DDSketch); + let ddsketch = backend_cfg_with_kind(SummaryKind::DDSketch); let v = emit_backend_storage_routing(&[("latency".into(), &ddsketch)]).expect("emit ok"); let metric = &v["metrics"][0]; let targets = metric["targets"].as_array().expect("targets array"); @@ -3626,7 +3794,7 @@ mod tests { #[test] fn storage_routing_count_sketch_pulls_topk_off_archive() { - let cs = backend_cfg_with_kind(SketchKind::CountSketch); + let cs = backend_cfg_with_kind(SummaryKind::CountSketch); let v = emit_backend_storage_routing(&[("requests".into(), &cs)]).expect("emit ok"); let archive_shapes: Vec = v["metrics"][0]["targets"][1]["applies_to_query_shape"] .as_array() @@ -3646,7 +3814,7 @@ mod tests { #[test] fn storage_routing_hll_pulls_count_off_archive() { - let hll = backend_cfg_with_kind(SketchKind::Hll); + let hll = backend_cfg_with_kind(SummaryKind::Hll); let v = emit_backend_storage_routing(&[("active_users".into(), &hll)]).expect("emit ok"); let archive_shapes: Vec = v["metrics"][0]["targets"][1]["applies_to_query_shape"] .as_array() @@ -3671,9 +3839,9 @@ mod tests { // runs (HashMap iteration order can drift, but our impl // stages everything through a Vec so order matches input // order). - let ddsketch = backend_cfg_with_kind(SketchKind::DDSketch); - let hll = backend_cfg_with_kind(SketchKind::Hll); - let cs = backend_cfg_with_kind(SketchKind::CountSketch); + let ddsketch = backend_cfg_with_kind(SummaryKind::DDSketch); + let hll = backend_cfg_with_kind(SummaryKind::Hll); + let cs = backend_cfg_with_kind(SummaryKind::CountSketch); let plans: Vec<(String, &BackendStageConfig)> = vec![ ("http_requests_total".into(), &cs), ("active_users".into(), &hll), @@ -3838,56 +4006,54 @@ mod tests { /// silently break the backend. #[test] fn phase_b_backend_agg_type_strings_for_every_sketch_kind() { - let cases: Vec<(SketchKind, SketchParams, &str)> = vec![ + let cases: Vec<(SummaryKind, SummaryParams, &str)> = vec![ ( - SketchKind::Kll, - SketchParams::Kll(KllParams { k: 200 }), + SummaryKind::Kll, + SummaryParams::Kll { k: 200 }, "DatasketchesKLL", ), ( - SketchKind::DDSketch, - SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), + SummaryKind::DDSketch, + SummaryParams::DDSketch { alpha: 0.01 }, "DDSketch", ), ( - SketchKind::Hll, - SketchParams::Hll(HllParams { precision: 14 }), + SummaryKind::Hll, + SummaryParams::Hll { precision: 14 }, "HLL", ), ( - SketchKind::Cms, - SketchParams::Cms(CmsParams { - w: 4096, - d: 4, - with_heap: false, - }), + SummaryKind::Cms, + SummaryParams::Cms { + width: 4096, + depth: 4, + }, "CountMinSketch", ), ( - SketchKind::Cms, - SketchParams::Cms(CmsParams { - w: 4096, - d: 4, - with_heap: true, - }), + SummaryKind::CmsWithHeap, + SummaryParams::CmsWithHeap { + width: 4096, + depth: 4, + heap_size: 10, + }, "CountMinSketchWithHeap", ), ( - SketchKind::CountSketch, - SketchParams::CountSketch(CountSketchParams { - w: 2048, - d: 5, - with_heap: false, - }), + SummaryKind::CountSketch, + SummaryParams::CountSketch { + width: 2048, + depth: 5, + }, "CountSketch", ), ( - SketchKind::CountSketch, - SketchParams::CountSketch(CountSketchParams { - w: 2048, - d: 5, - with_heap: true, - }), + SummaryKind::CountSketchWithHeap, + SummaryParams::CountSketchWithHeap { + width: 2048, + depth: 5, + heap_size: 10, + }, "CountSketchWithHeap", ), ]; @@ -3912,8 +4078,8 @@ mod tests { item_label: None, aggregation_id: "agg0".into(), metric_name: "http_latency_ms".into(), - sketch_kind: SketchKind::DDSketch, - sketch_params: SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), + sketch_kind: SummaryKind::DDSketch, + sketch_params: SummaryParams::DDSketch { alpha: 0.01 }, window_secs: 30, spatial_filter: String::new(), grouping: vec!["zone".into(), "service".into()], @@ -3949,7 +4115,7 @@ mod tests { /// producing aggregation. Pins the sort order + key names. Phase β /// uses this as the wire-format anchor for the wider intent set — /// the JSON shape is intent-orthogonal, so adding new intents to L3 - /// can't drift this off so long as they bind through SketchKind / + /// can't drift this off so long as they bind through SummaryKind / /// SketchParams. #[test] fn phase_b_backend_json_aggregation_readout_alias_snapshot() { @@ -3958,8 +4124,8 @@ mod tests { item_label: None, aggregation_id: "phase_b_agg0".into(), metric_name: "phase_b_metric".into(), - sketch_kind: SketchKind::Kll, - sketch_params: SketchParams::Kll(KllParams { k: 200 }), + sketch_kind: SummaryKind::Kll, + sketch_params: SummaryParams::Kll { k: 200 }, window_secs: 60, spatial_filter: String::new(), grouping: Vec::new(), @@ -4006,8 +4172,8 @@ mod tests { item_label: None, aggregation_id: "agg0".into(), metric_name: "test_metric".into(), - sketch_kind: SketchKind::DDSketch, - sketch_params: SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), + sketch_kind: SummaryKind::DDSketch, + sketch_params: SummaryParams::DDSketch { alpha: 0.01 }, window_secs: 60, spatial_filter: String::new(), grouping: Vec::new(), @@ -4031,8 +4197,8 @@ mod tests { item_label: None, aggregation_id: "agg0".into(), metric_name: "test_metric".into(), - sketch_kind: SketchKind::DDSketch, - sketch_params: SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), + sketch_kind: SummaryKind::DDSketch, + sketch_params: SummaryParams::DDSketch { alpha: 0.01 }, window_secs: 60, spatial_filter: String::new(), grouping: Vec::new(), @@ -4072,7 +4238,7 @@ mod tests { /// Prometheus-archive metric coexist in one routing JSON. #[test] fn phase_eps1_mixed_mode1_and_mode3_share_one_routing_table() { - let ddsketch = backend_cfg_with_kind(SketchKind::DDSketch); + let ddsketch = backend_cfg_with_kind(SummaryKind::DDSketch); let plans: Vec<(String, &BackendStageConfig)> = vec![("latency_seconds".into(), &ddsketch)]; let mode3 = vec!["http_requests_total".to_string()]; let v = emit_backend_storage_routing_with_prometheus(&plans, &mode3).expect("emit ok"); @@ -4438,7 +4604,7 @@ mod tests { /// (ASAPCollector#400). Most fixtures map each metric to exactly one /// family — this keeps them concise while exercising the SET-shaped /// `metric_to_family`. - fn one(kind: SketchKind) -> std::collections::BTreeSet { + fn one(kind: SummaryKind) -> std::collections::BTreeSet { std::collections::BTreeSet::from([kind]) } @@ -4447,16 +4613,16 @@ mod tests { /// metric maps to a single-family set (this workload's per-metric set /// size is 1; see `mvp46_multi_family_metric_*` for the size>1 case). fn five_sketch_edge_cfg() -> EdgeStageConfig { - let mut metric_to_family: HashMap> = + let mut metric_to_family: HashMap> = HashMap::new(); metric_to_family.insert( "http_requests_total_latency_ms".into(), - one(SketchKind::DDSketch), + one(SummaryKind::DDSketch), ); - metric_to_family.insert("request_size_bytes".into(), one(SketchKind::Kll)); - metric_to_family.insert("unique_users_per_min".into(), one(SketchKind::Hll)); - metric_to_family.insert("top_endpoint_qps".into(), one(SketchKind::CountSketch)); - metric_to_family.insert("endpoint_request_freq".into(), one(SketchKind::Cms)); + metric_to_family.insert("request_size_bytes".into(), one(SummaryKind::Kll)); + metric_to_family.insert("unique_users_per_min".into(), one(SummaryKind::Hll)); + metric_to_family.insert("top_endpoint_qps".into(), one(SummaryKind::CountSketch)); + metric_to_family.insert("endpoint_request_freq".into(), one(SummaryKind::Cms)); // `http_requests_total` is intentionally NOT in this map — it // falls through to the `metrics/raw_passthrough` default. EdgeStageConfig { @@ -4811,7 +4977,7 @@ mod tests { /// Helper: build an `EdgeStageConfig` whose `metric_to_family` is the /// given metric→set map, with sensible defaults for the other fields. fn edge_cfg_with_families( - metric_to_family: HashMap>, + metric_to_family: HashMap>, ) -> EdgeStageConfig { EdgeStageConfig { source_metric: None, @@ -4843,7 +5009,7 @@ mod tests { // families — this is the core bandwidth fix. let cfg = edge_cfg_with_families(HashMap::from([( "latency_ms".to_string(), - one(SketchKind::DDSketch), + one(SummaryKind::DDSketch), )])); let yaml = emit_edge_yaml(&cfg, "ws://c/", "test-agent").expect("emit ok"); @@ -4884,8 +5050,8 @@ mod tests { // HLL). Exactly those two pipelines/processors must be emitted; // KLL/CountSketch/CMS pruned. let cfg = edge_cfg_with_families(HashMap::from([ - ("latency_ms".to_string(), one(SketchKind::DDSketch)), - ("uniques".to_string(), one(SketchKind::Hll)), + ("latency_ms".to_string(), one(SummaryKind::DDSketch)), + ("uniques".to_string(), one(SummaryKind::Hll)), ])); let yaml = emit_edge_yaml(&cfg, "ws://c/", "test-agent").expect("emit ok"); @@ -4923,7 +5089,7 @@ mod tests { // them). let cfg = edge_cfg_with_families(HashMap::from([( "http_requests".to_string(), - std::collections::BTreeSet::from([SketchKind::DDSketch, SketchKind::Hll]), + std::collections::BTreeSet::from([SummaryKind::DDSketch, SummaryKind::Hll]), )])); let yaml = emit_edge_yaml(&cfg, "ws://c/", "test-agent").expect("emit ok"); @@ -5735,7 +5901,6 @@ mod tests { use crate::physical::colored_dag::emitter::{ AggregationInput, BackendAggregation, BackendReadout, BackendStageConfig, }; - use crate::sketch_algebra::params::DDSketchParams; use crate::sketch_algebra::physical_expr::EstimateOp; let cfg = BackendStageConfig { @@ -5743,8 +5908,8 @@ mod tests { item_label: None, aggregation_id: "agg0".to_string(), metric_name: "http_requests_total_latency_ms".to_string(), - sketch_kind: SketchKind::DDSketch, - sketch_params: SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), + sketch_kind: SummaryKind::DDSketch, + sketch_params: SummaryParams::DDSketch { alpha: 0.01 }, window_secs: 300, // pre-clamp 5m spatial_filter: String::new(), grouping: vec!["zone".to_string()], @@ -5928,58 +6093,54 @@ mod tests { /// tier (cold), and the counter-shaped sketch inputs in the /// cumulativetodelta list. fn fused_asap_edge_cfg() -> EdgeStageConfig { - use crate::sketch_algebra::params::{ - CmsParams, CountSketchParams, DDSketchParams, HllParams, KllParams, - }; - let mut metric_to_family: HashMap> = + let mut metric_to_family: HashMap> = HashMap::new(); metric_to_family.insert( "http_requests_total_latency_ms".into(), - one(SketchKind::DDSketch), + one(SummaryKind::DDSketch), ); - metric_to_family.insert("request_size_bytes".into(), one(SketchKind::Kll)); - metric_to_family.insert("unique_users_per_min".into(), one(SketchKind::Hll)); - metric_to_family.insert("top_endpoint_qps".into(), one(SketchKind::CountSketch)); - metric_to_family.insert("endpoint_request_freq".into(), one(SketchKind::Cms)); + metric_to_family.insert("request_size_bytes".into(), one(SummaryKind::Kll)); + metric_to_family.insert("unique_users_per_min".into(), one(SummaryKind::Hll)); + metric_to_family.insert("top_endpoint_qps".into(), one(SummaryKind::CountSketch)); + metric_to_family.insert("endpoint_request_freq".into(), one(SummaryKind::Cms)); // Per-family params, mirroring the target config's per-entry knobs. let sketch_processors = vec![ EdgeSketchProcessor { processor_name: "ddsketch".into(), - sketch_kind: SketchKind::DDSketch, - sketch_params: SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), + sketch_kind: SummaryKind::DDSketch, + sketch_params: SummaryParams::DDSketch { alpha: 0.01 }, aggregation_id: "agg0".into(), }, EdgeSketchProcessor { processor_name: "KLL".into(), - sketch_kind: SketchKind::Kll, - sketch_params: SketchParams::Kll(KllParams { k: 200 }), + sketch_kind: SummaryKind::Kll, + sketch_params: SummaryParams::Kll { k: 200 }, aggregation_id: "agg1".into(), }, EdgeSketchProcessor { processor_name: "HLL".into(), - sketch_kind: SketchKind::Hll, - sketch_params: SketchParams::Hll(HllParams { precision: 14 }), + sketch_kind: SummaryKind::Hll, + sketch_params: SummaryParams::Hll { precision: 14 }, aggregation_id: "agg2".into(), }, EdgeSketchProcessor { processor_name: "countsketch".into(), - sketch_kind: SketchKind::CountSketch, - sketch_params: SketchParams::CountSketch(CountSketchParams { - w: 2048, - d: 5, - with_heap: true, - }), + sketch_kind: SummaryKind::CountSketchWithHeap, + sketch_params: SummaryParams::CountSketchWithHeap { + width: 2048, + depth: 5, + heap_size: 10, + }, aggregation_id: "agg3".into(), }, EdgeSketchProcessor { processor_name: "countmin".into(), - sketch_kind: SketchKind::Cms, - sketch_params: SketchParams::Cms(CmsParams { - w: 2048, - d: 5, - with_heap: false, - }), + sketch_kind: SummaryKind::Cms, + sketch_params: SummaryParams::Cms { + width: 2048, + depth: 5, + }, aggregation_id: "agg4".into(), }, ]; @@ -6404,12 +6565,11 @@ mod tests { /// low-cardinality so the sparse base is a memory win that auto-promotes. #[test] fn fused_asap_edge_per_group_hll_is_per_series_and_sparse() { - use crate::sketch_algebra::params::HllParams; let _env = crate::test_support::EnvVarGuard::set("ASAP_EDGE_FUSED", "1"); - let mut metric_to_family: HashMap> = + let mut metric_to_family: HashMap> = HashMap::new(); - metric_to_family.insert("distinct_users_by_region".into(), one(SketchKind::Hll)); + metric_to_family.insert("distinct_users_by_region".into(), one(SummaryKind::Hll)); let mut metric_to_grouping_labels: HashMap> = HashMap::new(); metric_to_grouping_labels.insert("distinct_users_by_region".into(), vec!["region".into()]); @@ -6428,8 +6588,8 @@ mod tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "HLL".into(), - sketch_kind: SketchKind::Hll, - sketch_params: SketchParams::Hll(HllParams { precision: 14 }), + sketch_kind: SummaryKind::Hll, + sketch_params: SummaryParams::Hll { precision: 14 }, aggregation_id: "agg0".into(), }], exporter_target: ExportTarget::Endpoint("data-plane:4317".into()), @@ -6503,12 +6663,11 @@ mod tests { /// by [`fused_asap_edge_per_group_hll_is_per_series_and_sparse`]. #[test] fn fused_asap_edge_per_series_hll_high_cardinality_hint_is_dense() { - use crate::sketch_algebra::params::HllParams; let _env = crate::test_support::EnvVarGuard::set("ASAP_EDGE_FUSED", "1"); - let mut metric_to_family: HashMap> = + let mut metric_to_family: HashMap> = HashMap::new(); - metric_to_family.insert("distinct_users_by_region".into(), one(SketchKind::Hll)); + metric_to_family.insert("distinct_users_by_region".into(), one(SummaryKind::Hll)); let mut metric_to_grouping_labels: HashMap> = HashMap::new(); metric_to_grouping_labels.insert("distinct_users_by_region".into(), vec!["region".into()]); @@ -6526,8 +6685,8 @@ mod tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "HLL".into(), - sketch_kind: SketchKind::Hll, - sketch_params: SketchParams::Hll(HllParams { precision: 14 }), + sketch_kind: SummaryKind::Hll, + sketch_params: SummaryParams::Hll { precision: 14 }, aggregation_id: "agg0".into(), }], exporter_target: ExportTarget::Endpoint("data-plane:4317".into()), @@ -6579,12 +6738,11 @@ mod tests { /// single-instance global aggregate to sparse (the scope rule wins). #[test] fn fused_asap_edge_whole_stream_hll_is_dense_regardless_of_hint() { - use crate::sketch_algebra::params::HllParams; let _env = crate::test_support::EnvVarGuard::set("ASAP_EDGE_FUSED", "1"); - let mut metric_to_family: HashMap> = + let mut metric_to_family: HashMap> = HashMap::new(); - metric_to_family.insert("distinct_users_global".into(), one(SketchKind::Hll)); + metric_to_family.insert("distinct_users_global".into(), one(SummaryKind::Hll)); // No grouping label + an item_label ⇒ effective aggregate_by empty ⇒ // whole-stream HLL. @@ -6601,8 +6759,8 @@ mod tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "HLL".into(), - sketch_kind: SketchKind::Hll, - sketch_params: SketchParams::Hll(HllParams { precision: 14 }), + sketch_kind: SummaryKind::Hll, + sketch_params: SummaryParams::Hll { precision: 14 }, aggregation_id: "agg0".into(), }], exporter_target: ExportTarget::Endpoint("data-plane:4317".into()), @@ -6654,13 +6812,12 @@ mod tests { /// pre-#471/#472 plans byte-identical (per_series is the edge default). #[test] fn fused_asap_edge_quantile_only_omits_mode_and_sparse() { - use crate::sketch_algebra::params::{DDSketchParams, KllParams}; let _env = crate::test_support::EnvVarGuard::set("ASAP_EDGE_FUSED", "1"); - let mut metric_to_family: HashMap> = + let mut metric_to_family: HashMap> = HashMap::new(); - metric_to_family.insert("latency_ms".into(), one(SketchKind::DDSketch)); - metric_to_family.insert("payload_bytes".into(), one(SketchKind::Kll)); + metric_to_family.insert("latency_ms".into(), one(SummaryKind::DDSketch)); + metric_to_family.insert("payload_bytes".into(), one(SummaryKind::Kll)); let cfg = EdgeStageConfig { source_metric: None, @@ -6669,14 +6826,14 @@ mod tests { sketch_processors: vec![ EdgeSketchProcessor { processor_name: "ddsketch".into(), - sketch_kind: SketchKind::DDSketch, - sketch_params: SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), + sketch_kind: SummaryKind::DDSketch, + sketch_params: SummaryParams::DDSketch { alpha: 0.01 }, aggregation_id: "agg0".into(), }, EdgeSketchProcessor { processor_name: "KLL".into(), - sketch_kind: SketchKind::Kll, - sketch_params: SketchParams::Kll(KllParams { k: 200 }), + sketch_kind: SummaryKind::Kll, + sketch_params: SummaryParams::Kll { k: 200 }, aggregation_id: "agg1".into(), }, ], @@ -6830,10 +6987,10 @@ mod tests { // Two metrics: a sketch-only one (warm) and one that is BOTH // sketched AND archived (both). The archive set is the precise // plan signal — only `archived_metric` is in it. - let mut metric_to_family: HashMap> = + let mut metric_to_family: HashMap> = HashMap::new(); - metric_to_family.insert("sketch_only_metric".into(), [SketchKind::Kll].into()); - metric_to_family.insert("archived_metric".into(), [SketchKind::DDSketch].into()); + metric_to_family.insert("sketch_only_metric".into(), [SummaryKind::Kll].into()); + metric_to_family.insert("archived_metric".into(), [SummaryKind::DDSketch].into()); let cfg = EdgeStageConfig { source_metric: None, @@ -6909,7 +7066,7 @@ mod tests { let mut cfg = fused_asap_edge_cfg(); cfg.metric_to_family.insert( "http_requests_total_latency_ms".into(), - one(SketchKind::Kll), + one(SummaryKind::Kll), ); let yaml = emit_edge_yaml(&cfg, "ws://c/", "agent-1").expect("emit ok"); @@ -7157,12 +7314,8 @@ mod tests { for &(w, d) in &[(2048u32, 5u32), (1024, 4), (4096, 6), (2, 1), (256, 3)] { let sp = EdgeSketchProcessor { processor_name: "countsketch".into(), - sketch_kind: SketchKind::CountSketch, - sketch_params: SketchParams::CountSketch(CountSketchParams { - w, - d, - with_heap: false, - }), + sketch_kind: SummaryKind::CountSketch, + sketch_params: SummaryParams::CountSketch { width: w, depth: d }, aggregation_id: "agg-cs".into(), }; let block = @@ -7186,11 +7339,7 @@ mod tests { // params as `{ "w", "d", "with_heap" }`. The fingerprint keys off // `parameters["w"]`, which must equal the agent-derived width. let backend_json = - sketch_params_to_json(&SketchParams::CountSketch(CountSketchParams { - w, - d, - with_heap: false, - })); + sketch_params_to_json(&SummaryParams::CountSketch { width: w, depth: d }); let backend_w = backend_json["w"].as_u64().expect("backend w present"); let backend_d = backend_json["d"].as_u64().expect("backend d present"); @@ -7211,11 +7360,14 @@ mod tests { /// `metric_to_family` but provides NO matching `EdgeSketchProcessor`, /// driving the fused emit into the catalog-default (`None`) arm. fn fused_cfg_countsketch_no_processor() -> EdgeStageConfig { - let mut metric_to_family: HashMap> = + let mut metric_to_family: HashMap> = HashMap::new(); // CountSketch family declared, but `sketch_processors` is EMPTY for // it — the `family_to_proc.get(kind)` lookup returns None. - metric_to_family.insert("endpoint_request_freq".into(), one(SketchKind::CountSketch)); + metric_to_family.insert( + "endpoint_request_freq".into(), + one(SummaryKind::CountSketch), + ); EdgeStageConfig { source_metric: None, @@ -7273,15 +7425,15 @@ mod tests { let mut cfg = fused_cfg_countsketch_no_processor(); cfg.metric_to_family.clear(); cfg.metric_to_family - .insert("top_endpoint_qps".into(), one(SketchKind::CountSketch)); + .insert("top_endpoint_qps".into(), one(SummaryKind::CountSketch)); cfg.sketch_processors = vec![EdgeSketchProcessor { processor_name: "countsketch".into(), - sketch_kind: SketchKind::CountSketch, - sketch_params: SketchParams::CountSketch(CountSketchParams { - w: 2048, - d: 5, - with_heap: true, - }), + sketch_kind: SummaryKind::CountSketchWithHeap, + sketch_params: SummaryParams::CountSketchWithHeap { + width: 2048, + depth: 5, + heap_size: 10, + }, aggregation_id: "agg-cs".into(), }]; let yaml = emit_edge_yaml_asap_edge(&cfg, "ws://c/", "agent-1").expect("fused emit ok"); diff --git a/control_plane/src/emit/telegraf.rs b/control_plane/src/emit/telegraf.rs index c5c99c6a..b332dbb9 100644 --- a/control_plane/src/emit/telegraf.rs +++ b/control_plane/src/emit/telegraf.rs @@ -39,7 +39,7 @@ use anyhow::{Context, Result}; use crate::physical::colored_dag::emitter::{EdgeSketchProcessor, EdgeStageConfig, ExportTarget}; use crate::physical::colored_dag::stage_id::StageId; -use crate::sketch_algebra::params::{SketchKind, SketchParams}; +use asap_sketch::{SummaryKind, SummaryParams}; /// Default Prometheus remote-write URL for Mode 3 — Telegraf doesn't /// support OTLP-HTTP egress, so we land in the same Prometheus archive @@ -166,39 +166,67 @@ fn emit_processors_allsketches( sketch_kind_tag(&sp.sketch_kind) )); match &sp.sketch_params { - SketchParams::Kll(p) => { - out.push_str(&format!(" k = {}\n", p.k)); + SummaryParams::Kll { k } => { + out.push_str(&format!(" k = {k}\n")); } - SketchParams::DDSketch(p) => { - out.push_str(&format!(" relative_accuracy = {}\n", p.alpha)); + SummaryParams::DDSketch { alpha } => { + out.push_str(&format!(" relative_accuracy = {alpha}\n")); out.push_str(" delta_transmission = true\n"); } - SketchParams::Hll(_p) => { + SummaryParams::Hll { .. } => { out.push_str(" delta_transmission = true\n"); } - SketchParams::Cms(p) => { - out.push_str(&format!(" rows = {}\n", p.d)); - out.push_str(&format!(" columns = {}\n", p.w)); + // Heap-bearing width/depth extraction is identical to the bare + // kind — this path never distinguished `with_heap` even before + // `SummaryKind` split it into its own variant. + SummaryParams::Cms { width, depth } | SummaryParams::CmsWithHeap { width, depth, .. } => { + out.push_str(&format!(" rows = {depth}\n")); + out.push_str(&format!(" columns = {width}\n")); out.push_str(" delta_transmission = true\n"); } - SketchParams::CountSketch(p) => { - let epsilon = std::f64::consts::E / (p.w as f64); - let delta = 2f64.powi(-(p.d as i32)); + SummaryParams::CountSketch { width, depth } + | SummaryParams::CountSketchWithHeap { width, depth, .. } => { + let epsilon = std::f64::consts::E / (*width as f64); + let delta = 2f64.powi(-(*depth as i32)); out.push_str(&format!(" epsilon = {epsilon}\n")); out.push_str(&format!(" delta = {delta}\n")); out.push_str(" delta_transmission = true\n"); } + SummaryParams::Sum + | SummaryParams::Count + | SummaryParams::MinMax + | SummaryParams::Increase + | SummaryParams::Rate + | SummaryParams::Kmv { .. } + | SummaryParams::Theta { .. } => { + unreachable!( + "edge sketch processor config requested for a non-sketch or unsupported \ + SummaryKind; no Bind* rule in this repo produces one" + ) + } } out.push('\n'); } -fn sketch_kind_tag(kind: &SketchKind) -> &'static str { +fn sketch_kind_tag(kind: &SummaryKind) -> &'static str { match kind { - SketchKind::Kll => "kll", - SketchKind::DDSketch => "ddsketch", - SketchKind::Hll => "hll", - SketchKind::Cms => "cms", - SketchKind::CountSketch => "count_sketch", + SummaryKind::Kll => "kll", + SummaryKind::DDSketch => "ddsketch", + SummaryKind::Hll => "hll", + SummaryKind::Cms | SummaryKind::CmsWithHeap => "cms", + SummaryKind::CountSketch | SummaryKind::CountSketchWithHeap => "count_sketch", + SummaryKind::Sum + | SummaryKind::Count + | SummaryKind::MinMax + | SummaryKind::Increase + | SummaryKind::Rate + | SummaryKind::Kmv + | SummaryKind::Theta => { + unreachable!( + "edge sketch processor config requested for a non-sketch or unsupported \ + SummaryKind; no Bind* rule in this repo produces one" + ) + } } } @@ -297,7 +325,7 @@ mod toml_minimal { mod tests { use super::*; use crate::physical::colored_dag::emitter::{EdgeSketchProcessor, PrometheusArchiveMetric}; - use crate::sketch_algebra::params::{DDSketchParams, KllParams}; + use asap_sketch::{SummaryKind, SummaryParams}; fn ddsketch_edge_cfg_mode1() -> EdgeStageConfig { EdgeStageConfig { @@ -306,8 +334,8 @@ mod tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "ddsketch".to_string(), - sketch_kind: SketchKind::DDSketch, - sketch_params: SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), + sketch_kind: SummaryKind::DDSketch, + sketch_params: SummaryParams::DDSketch { alpha: 0.01 }, aggregation_id: "agg0".to_string(), }], exporter_target: ExportTarget::Stage(StageId::Gateway), @@ -515,8 +543,8 @@ mod tests { window_secs: Some(60), sketch_processors: vec![EdgeSketchProcessor { processor_name: "KLL".to_string(), - sketch_kind: SketchKind::Kll, - sketch_params: SketchParams::Kll(KllParams { k: 200 }), + sketch_kind: SummaryKind::Kll, + sketch_params: SummaryParams::Kll { k: 200 }, aggregation_id: "agg0".to_string(), }], exporter_target: ExportTarget::Stage(StageId::Gateway), diff --git a/control_plane/src/emit/trait_def.rs b/control_plane/src/emit/trait_def.rs index e2862b2e..e2591e73 100644 --- a/control_plane/src/emit/trait_def.rs +++ b/control_plane/src/emit/trait_def.rs @@ -200,8 +200,8 @@ mod tests { BackendAggregation, BackendReadout, ExportTarget, GatewayMergeProcessor, }; use crate::physical::colored_dag::stage_id::StageId; - use crate::sketch_algebra::params::{DDSketchParams, SketchKind, SketchParams}; use crate::sketch_algebra::physical_expr::EstimateOp; + use asap_sketch::{SummaryKind, SummaryParams}; use std::collections::HashMap; fn empty_edge_cfg() -> EdgeStageConfig { @@ -232,7 +232,7 @@ mod tests { otlp_receiver_port: 4317, merge_processors: vec![GatewayMergeProcessor { processor_name: "sketchmergeprocessor".to_string(), - sketch_kind: SketchKind::DDSketch, + sketch_kind: SummaryKind::DDSketch, aggregation_id: "agg0".to_string(), }], exporter_target: ExportTarget::Stage(StageId::Backend), @@ -245,8 +245,8 @@ mod tests { item_label: None, aggregation_id: "agg0".to_string(), metric_name: "test_metric".to_string(), - sketch_kind: SketchKind::DDSketch, - sketch_params: SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), + sketch_kind: SummaryKind::DDSketch, + sketch_params: SummaryParams::DDSketch { alpha: 0.01 }, window_secs: 60, spatial_filter: String::new(), grouping: Vec::new(), diff --git a/control_plane/src/optimizer/cost/sketch_capability.rs b/control_plane/src/optimizer/cost/sketch_capability.rs index 30f8bbaa..370fba82 100644 --- a/control_plane/src/optimizer/cost/sketch_capability.rs +++ b/control_plane/src/optimizer/cost/sketch_capability.rs @@ -7,7 +7,7 @@ //! size, which intents each sketch family serves), read by the optimizer //! for cost-based plan rewriting and by the physical planner to check //! whether a sketch fits within a stage's budget — it was never L4 IR, just -//! filed alongside it because both modules touched `SketchKind`. +//! filed alongside it because both modules touched `SummaryKind`. //! //! Distinct from [`crate::sketch_algebra::schema::SketchStateMetadata`] — //! that struct carries the **L4 type-system flags** (`mergeable` / @@ -22,7 +22,7 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; -use crate::sketch_algebra::params::SketchKind; +use asap_sketch::SummaryKind; /// Performance and capability profile for a single sketch family. /// @@ -118,15 +118,15 @@ struct SketchCapabilitiesFile { count_min_sketch: SketchCapabilityYaml, } -/// Compiled-in capability defaults — one entry per [`SketchKind`]. +/// Compiled-in capability defaults — one entry per [`SummaryKind`]. /// Replaces the per-variant `sketch_capability(SketchType)` function /// that previously lived in `algebra/optimizer.rs`. Numerical values /// are mirrored from the YAML so the in-process defaults match the /// reference deployment file. -pub fn default_capability_table() -> HashMap { +pub fn default_capability_table() -> HashMap { let mut map = HashMap::new(); map.insert( - SketchKind::DDSketch, + SummaryKind::DDSketch, SketchCapability { insert_throughput: 10_000_000.0, query_throughput: 50_000_000.0, @@ -140,7 +140,7 @@ pub fn default_capability_table() -> HashMap { }, ); map.insert( - SketchKind::Kll, + SummaryKind::Kll, SketchCapability { insert_throughput: 5_000_000.0, query_throughput: 20_000_000.0, @@ -154,7 +154,7 @@ pub fn default_capability_table() -> HashMap { }, ); map.insert( - SketchKind::Hll, + SummaryKind::Hll, SketchCapability { insert_throughput: 20_000_000.0, query_throughput: 100_000_000.0, @@ -168,7 +168,7 @@ pub fn default_capability_table() -> HashMap { }, ); map.insert( - SketchKind::CountSketch, + SummaryKind::CountSketch, SketchCapability { insert_throughput: 8_000_000.0, query_throughput: 10_000_000.0, @@ -182,7 +182,7 @@ pub fn default_capability_table() -> HashMap { }, ); map.insert( - SketchKind::Cms, + SummaryKind::Cms, SketchCapability { insert_throughput: 8_000_000.0, query_throughput: 10_000_000.0, @@ -203,15 +203,15 @@ pub fn default_capability_table() -> HashMap { /// Replaces `algebra::optimizer::load_sketch_capabilities`. /// /// Env var: `CONTROLLER_SKETCH_CAPABILITIES=path/to/this/file.yml`. -pub fn load_capability_overrides(path: &str) -> HashMap { +pub fn load_capability_overrides(path: &str) -> HashMap { if let Ok(contents) = std::fs::read_to_string(path) { if let Ok(file) = serde_yaml::from_str::(&contents) { let mut map = HashMap::new(); - map.insert(SketchKind::DDSketch, file.ddsketch.to_capability()); - map.insert(SketchKind::Kll, file.kll.to_capability()); - map.insert(SketchKind::Hll, file.hll.to_capability()); - map.insert(SketchKind::CountSketch, file.count_sketch.to_capability()); - map.insert(SketchKind::Cms, file.count_min_sketch.to_capability()); + map.insert(SummaryKind::DDSketch, file.ddsketch.to_capability()); + map.insert(SummaryKind::Kll, file.kll.to_capability()); + map.insert(SummaryKind::Hll, file.hll.to_capability()); + map.insert(SummaryKind::CountSketch, file.count_sketch.to_capability()); + map.insert(SummaryKind::Cms, file.count_min_sketch.to_capability()); return map; } } @@ -225,17 +225,17 @@ mod tests { #[test] fn default_table_carries_all_five_sketch_kinds() { let t = default_capability_table(); - assert!(t.contains_key(&SketchKind::DDSketch)); - assert!(t.contains_key(&SketchKind::Kll)); - assert!(t.contains_key(&SketchKind::Hll)); - assert!(t.contains_key(&SketchKind::Cms)); - assert!(t.contains_key(&SketchKind::CountSketch)); + assert!(t.contains_key(&SummaryKind::DDSketch)); + assert!(t.contains_key(&SummaryKind::Kll)); + assert!(t.contains_key(&SummaryKind::Hll)); + assert!(t.contains_key(&SummaryKind::Cms)); + assert!(t.contains_key(&SummaryKind::CountSketch)); } #[test] fn default_table_ddsketch_serves_quantile_intent() { let t = default_capability_table(); - let cap = t.get(&SketchKind::DDSketch).unwrap(); + let cap = t.get(&SummaryKind::DDSketch).unwrap(); assert!(cap.supported_intents.contains(&SupportedIntent::Quantile)); assert!(cap.mergeable); } @@ -243,7 +243,7 @@ mod tests { #[test] fn default_table_hll_serves_cardinality_intent() { let t = default_capability_table(); - let cap = t.get(&SketchKind::Hll).unwrap(); + let cap = t.get(&SummaryKind::Hll).unwrap(); assert!(cap .supported_intents .contains(&SupportedIntent::Cardinality)); diff --git a/control_plane/src/optimizer/cost/wire.rs b/control_plane/src/optimizer/cost/wire.rs index 27a80f47..f6fbdd42 100644 --- a/control_plane/src/optimizer/cost/wire.rs +++ b/control_plane/src/optimizer/cost/wire.rs @@ -29,7 +29,8 @@ #![allow(dead_code)] -use crate::sketch_algebra::params::SketchKind; +use asap_sketch::SummaryKind; + use crate::types::WorkloadCharacteristics; // ── Wire-cost table ────────────────────────────────────────────────────────── @@ -103,13 +104,36 @@ impl WireCostTable { } /// Lookup the per-flush cost for a sketch family. - pub const fn for_kind(&self, kind: &SketchKind) -> SketchWireCost { + /// + /// `SummaryKind` (unlike the retired `sketch_algebra::SketchKind`) + /// distinguishes heap-bearing from bare frequency sketches at the + /// kind level rather than via a `with_heap` param flag. This table + /// never modeled the heap's extra bytes separately (the old + /// `for_kind` took a bare `SketchKind` with no visibility into + /// `with_heap` at all) — `CmsWithHeap`/`CountSketchWithHeap` reuse + /// their bare counterpart's cost to preserve that exact behavior. + /// `Kmv`/`Theta` have no established cost number (nothing in this + /// repo binds a cardinality intent to either today — the candidate + /// list stays `Hll`-only, see `capability.rs`); they reuse `hll_delta` + /// as a same-order-of-magnitude placeholder pending real numbers if + /// this repo ever adopts them. + pub const fn for_kind(&self, kind: &SummaryKind) -> SketchWireCost { match kind { - SketchKind::DDSketch => self.ddsketch_delta, - SketchKind::Kll => self.kll_full, - SketchKind::Hll => self.hll_delta, - SketchKind::Cms => self.count_min_delta, - SketchKind::CountSketch => self.count_sketch_delta, + SummaryKind::DDSketch => self.ddsketch_delta, + SummaryKind::Kll => self.kll_full, + SummaryKind::Hll => self.hll_delta, + SummaryKind::Kmv | SummaryKind::Theta => self.hll_delta, + SummaryKind::Cms => self.count_min_delta, + SummaryKind::CmsWithHeap => self.count_min_delta, + SummaryKind::CountSketch => self.count_sketch_delta, + SummaryKind::CountSketchWithHeap => self.count_sketch_delta, + SummaryKind::Sum + | SummaryKind::Count + | SummaryKind::MinMax + | SummaryKind::Increase + | SummaryKind::Rate => { + panic!("WireCostTable::for_kind: exact accumulators have no sketch wire-state cost") + } } } } @@ -222,13 +246,13 @@ pub enum BindMode { /// Mode 1: sketch processor at the edge ships sketch state via OTLP /// to the backend's ASAP tier. The default for high-sample-per-window /// workloads where the sketch's per-flush wire cost beats raw OTLP. - SketchAtEdge { family: SketchKind }, + SketchAtEdge { family: SummaryKind }, /// Mode 2: no sketch processor at edge; raw OTLP forwards to the /// gateway/backend, which builds sketches at ingest. Picked when the /// edge is resource-constrained (CPU / RAM) but the sketch still /// wins on backend-side bandwidth + accuracy. - RawAtEdgeSketchAtBackend { family: SketchKind }, + RawAtEdgeSketchAtBackend { family: SummaryKind }, /// Mode 3: no sketch processor at edge; raw OTLP ships directly to /// Prometheus's native OTLP receiver. Backend HTTP-forwards queries @@ -261,7 +285,7 @@ pub enum BindMode { /// future revision will consult observed processor CPU from the /// OnlineMetricsStore. pub fn select_bind_mode( - families: &[SketchKind], + families: &[SummaryKind], workload: &WireWorkload, table: &WireCostTable, ) -> BindMode { @@ -387,7 +411,7 @@ mod tests { // DDSketch state — 600 + 200 = 800 B. Sketch wins. let ddsketch = est_wire_bytes_per_window_per_series( &BindMode::SketchAtEdge { - family: SketchKind::DDSketch, + family: SummaryKind::DDSketch, }, &w, &table, @@ -396,7 +420,7 @@ mod tests { // HLL state — 10 000 + 200 = 10 200 B. HLL loses at 60 samples. let hll = est_wire_bytes_per_window_per_series( &BindMode::SketchAtEdge { - family: SketchKind::Hll, + family: SummaryKind::Hll, }, &w, &table, @@ -417,11 +441,11 @@ mod tests { edge_cpu_budget: None, edge_ram_budget: None, }; - let mode = select_bind_mode(&[SketchKind::DDSketch, SketchKind::Kll], &w, &table); + let mode = select_bind_mode(&[SummaryKind::DDSketch, SummaryKind::Kll], &w, &table); assert_eq!( mode, BindMode::SketchAtEdge { - family: SketchKind::DDSketch + family: SummaryKind::DDSketch } ); } @@ -439,11 +463,11 @@ mod tests { edge_cpu_budget: Some(0.1), edge_ram_budget: None, }; - let mode = select_bind_mode(&[SketchKind::DDSketch, SketchKind::Kll], &w, &table); + let mode = select_bind_mode(&[SummaryKind::DDSketch, SummaryKind::Kll], &w, &table); assert_eq!( mode, BindMode::RawAtEdgeSketchAtBackend { - family: SketchKind::DDSketch + family: SummaryKind::DDSketch } ); } @@ -465,7 +489,7 @@ mod tests { edge_cpu_budget: None, edge_ram_budget: None, }; - let mode = select_bind_mode(&[SketchKind::Hll, SketchKind::CountSketch], &w, &table); + let mode = select_bind_mode(&[SummaryKind::Hll, SummaryKind::CountSketch], &w, &table); assert_eq!(mode, BindMode::RawAtEdgePrometheusArchive); } @@ -509,7 +533,7 @@ mod tests { edge_cpu_budget: None, edge_ram_budget: None, }; - let mode = select_bind_mode(&[SketchKind::Hll], &w, &table); + let mode = select_bind_mode(&[SummaryKind::Hll], &w, &table); assert_eq!(mode, BindMode::RawAtEdgePrometheusArchive); } @@ -526,11 +550,11 @@ mod tests { edge_cpu_budget: None, edge_ram_budget: None, }; - let mode = select_bind_mode(&[SketchKind::Hll], &w, &table); + let mode = select_bind_mode(&[SummaryKind::Hll], &w, &table); assert_eq!( mode, BindMode::SketchAtEdge { - family: SketchKind::Hll + family: SummaryKind::Hll } ); } @@ -553,11 +577,11 @@ mod tests { edge_cpu_budget: Some(0.1), edge_ram_budget: None, }; - let mode = select_bind_mode(&[SketchKind::DDSketch, SketchKind::Kll], &w, &table); + let mode = select_bind_mode(&[SummaryKind::DDSketch, SummaryKind::Kll], &w, &table); assert_eq!( mode, BindMode::RawAtEdgeSketchAtBackend { - family: SketchKind::DDSketch + family: SummaryKind::DDSketch } ); } @@ -582,20 +606,20 @@ mod tests { // HLL: 10 200 B → loses // CountMin: 4 200 B → loses // CountSketch: 250 200 B → loses - let dd = select_bind_mode(&[SketchKind::DDSketch], &w, &table); + let dd = select_bind_mode(&[SummaryKind::DDSketch], &w, &table); assert_eq!( dd, BindMode::SketchAtEdge { - family: SketchKind::DDSketch + family: SummaryKind::DDSketch } ); - let kll = select_bind_mode(&[SketchKind::Kll], &w, &table); + let kll = select_bind_mode(&[SummaryKind::Kll], &w, &table); assert_eq!(kll, BindMode::RawAtEdgePrometheusArchive); - let hll = select_bind_mode(&[SketchKind::Hll], &w, &table); + let hll = select_bind_mode(&[SummaryKind::Hll], &w, &table); assert_eq!(hll, BindMode::RawAtEdgePrometheusArchive); - let cms = select_bind_mode(&[SketchKind::Cms], &w, &table); + let cms = select_bind_mode(&[SummaryKind::Cms], &w, &table); assert_eq!(cms, BindMode::RawAtEdgePrometheusArchive); - let cs = select_bind_mode(&[SketchKind::CountSketch], &w, &table); + let cs = select_bind_mode(&[SummaryKind::CountSketch], &w, &table); assert_eq!(cs, BindMode::RawAtEdgePrometheusArchive); } } diff --git a/control_plane/src/optimizer/engine.rs b/control_plane/src/optimizer/engine.rs index e24ddaf1..603f651a 100644 --- a/control_plane/src/optimizer/engine.rs +++ b/control_plane/src/optimizer/engine.rs @@ -92,18 +92,11 @@ pub fn load_sketch_capabilities( /// Built-in capability profile for a known sketch type. Thin shim — /// the real defaults live in `optimizer::cost::sketch_capability::default_capability_table`. pub fn sketch_capability(st: &crate::types::SketchType) -> SketchCapability { - use crate::sketch_algebra::params::SketchKind; - use crate::types::SketchType; - let kind: SketchKind = match st { - SketchType::DDSketch => SketchKind::DDSketch, - SketchType::KLL => SketchKind::Kll, - SketchType::HLL => SketchKind::Hll, - SketchType::CountSketch => SketchKind::CountSketch, - SketchType::CountMinSketch => SketchKind::Cms, - }; + use asap_sketch::SummaryKind; + let kind: SummaryKind = st.clone().into(); default_capability_table() .remove(&kind) - .expect("default_capability_table covers every SketchKind variant") + .expect("default_capability_table covers every SummaryKind variant") } // ── Stage budgets ─────────────────────────────────────────────────────────── diff --git a/control_plane/src/optimizer/rules/mod.rs b/control_plane/src/optimizer/rules/mod.rs index 08fe6dba..9235a2f6 100644 --- a/control_plane/src/optimizer/rules/mod.rs +++ b/control_plane/src/optimizer/rules/mod.rs @@ -68,13 +68,13 @@ pub fn bind_workload_typed(w: &QueryWorkload) -> Option Option { + SummaryKind::DDSketch | SummaryKind::Kll => { (StatisticClass::Quantile, AccuracyPreference::RelativeError) } - SketchKind::Hll => (StatisticClass::Cardinality, AccuracyPreference::default()), - SketchKind::Cms => (StatisticClass::Frequency, AccuracyPreference::default()), - SketchKind::CountSketch => (StatisticClass::TopK, AccuracyPreference::default()), + SummaryKind::Hll => (StatisticClass::Cardinality, AccuracyPreference::default()), + SummaryKind::Cms => (StatisticClass::Frequency, AccuracyPreference::default()), + SummaryKind::CountSketch => (StatisticClass::TopK, AccuracyPreference::default()), + // `ov` always comes from `SummaryKind::from(SketchType)` + // (`w.sketch_type_override` is the legacy 5-family enum), + // so only these 5 canonical families are ever reachable. + other => unreachable!( + "sketch_type_override resolved to an unsupported SummaryKind {other:?}" + ), }; statistic = s; accuracy_pref = ap; @@ -142,7 +148,7 @@ pub fn bind_workload_typed(w: &QueryWorkload) -> Option capability-matched default) ─ + // ── Resolve the SummaryKind (override > capability-matched default) ─ // // The workload-spec's `sketch_type_override` (= the spec's // `sketch_family_override` per orchestrator contract) wins over the @@ -151,10 +157,10 @@ pub fn bind_workload_typed(w: &QueryWorkload) -> Option = w + let override_kind: Option = w .sketch_type_override .as_ref() - .map(|st| SketchKind::from(st.clone())); + .map(|st| SummaryKind::from(st.clone())); let kind = match override_kind { Some(k) if is_valid_pair(k.clone(), statistic) => k, _ => pick_family(statistic, accuracy_pref)?, @@ -245,7 +251,7 @@ pub fn bind_workload_typed(w: &QueryWorkload) -> Option Option BindDDSketchOnQuantile.apply(&aggregate, &accuracy), - (SketchKind::Kll, _) => BindKllOnQuantile.apply(&aggregate, &accuracy), - (SketchKind::Hll, _) => BindHllOnCardinality.apply(&aggregate, &accuracy), + (SummaryKind::DDSketch, _) => BindDDSketchOnQuantile.apply(&aggregate, &accuracy), + (SummaryKind::Kll, _) => BindKllOnQuantile.apply(&aggregate, &accuracy), + (SummaryKind::Hll, _) => BindHllOnCardinality.apply(&aggregate, &accuracy), // The capability matrix already pinned the family here, so force // the matching recall tier rather than re-inferring it: a // CountSketch pick is the unbiased canonical top-k (Tight); a CMS // pick on a top-k is the cheap CMS-with-heap (Loose). This keeps // `bind_workload_typed`'s contract-row mapping deterministic — the // recall-aware default lives in `dispatch()` / `Rule::apply`. - (SketchKind::CountSketch, _) => BindCountSketchOnTopK.apply_with_tier( + (SummaryKind::CountSketch, _) => BindCountSketchOnTopK.apply_with_tier( &aggregate, &accuracy, crate::sketch_algebra::rules::bind_cms_topk::TopkRecallTier::Tight, ), - (SketchKind::Cms, StatisticClass::TopK) => BindCountSketchOnTopK.apply_with_tier( + (SummaryKind::Cms, StatisticClass::TopK) => BindCountSketchOnTopK.apply_with_tier( &aggregate, &accuracy, crate::sketch_algebra::rules::bind_cms_topk::TopkRecallTier::Loose, ), - (SketchKind::Cms, _) => BindCmsOnCount.apply(&aggregate, &accuracy), + (SummaryKind::Cms, _) => BindCmsOnCount.apply(&aggregate, &accuracy), + // `kind` always comes from `pick_family`/`sketch_type_override`, + // exhaustively the 5 canonical families. + (other, _) => unreachable!("dispatch: unsupported SummaryKind {other:?}"), } } @@ -464,13 +473,19 @@ mod tests { #[test] fn override_pins_nonquantile_family_over_misclassified_query() { use crate::emit::extract_root_sketch_kind; - use crate::sketch_algebra::params::SketchKind; + use asap_sketch::SummaryKind; for (ov, expect) in [ - (SketchType::DDSketch, SketchKind::DDSketch), - (SketchType::KLL, SketchKind::Kll), - (SketchType::HLL, SketchKind::Hll), - (SketchType::CountMinSketch, SketchKind::Cms), - (SketchType::CountSketch, SketchKind::CountSketch), + (SketchType::DDSketch, SummaryKind::DDSketch), + (SketchType::KLL, SummaryKind::Kll), + (SketchType::HLL, SummaryKind::Hll), + (SketchType::CountMinSketch, SummaryKind::Cms), + // CountSketch override re-derives statistic as TopK (see the + // override arm below), and `bind_cms_topk` always binds the + // heap-bearing kind for a top-k intent — matches this + // fixture's pre-`SummaryKind`-split expectation, when + // `with_heap: true` was a params flag rather than kind + // identity. + (SketchType::CountSketch, SummaryKind::CountSketchWithHeap), ] { // Query classifies as Quantile (the mis-derived case observed // live for count()/topk()/count_over_time()); the override must win. @@ -605,13 +620,13 @@ mod tests { // | `top_endpoint_qps` | CountSketch | // | `endpoint_request_freq` | CMS | - use crate::sketch_algebra::params::SketchKind; use crate::sketch_algebra::physical_expr::PhysicalExpr; + use asap_sketch::SummaryKind; /// Walk the L4 binding output and pull out the `SketchAgg`'s family. /// Returns `None` if no `SketchAgg` node is present (raw / pure /// logical pass-through). - fn extract_family(expr: &PhysicalExpr) -> Option { + fn extract_family(expr: &PhysicalExpr) -> Option { match expr { PhysicalExpr::SketchAgg { sketch_type, .. } => Some(sketch_type.clone()), PhysicalExpr::SketchEstimate { child, .. } => extract_family(child), @@ -670,7 +685,7 @@ mod tests { let bound = bind_workload_typed(&w).expect("http_latency_ms must bind"); assert_eq!( extract_family(&bound), - Some(SketchKind::DDSketch), + Some(SummaryKind::DDSketch), "http_latency_ms should bind to DDSketch (Quantile, rel-err)", ); } @@ -685,7 +700,7 @@ mod tests { let bound = bind_workload_typed(&w).expect("request_size_bytes must bind"); assert_eq!( extract_family(&bound), - Some(SketchKind::Kll), + Some(SummaryKind::Kll), "request_size_bytes should bind to KLL (Quantile, rank-err)", ); } @@ -697,7 +712,7 @@ mod tests { let bound = bind_workload_typed(&w).expect("unique_users_per_min must bind"); assert_eq!( extract_family(&bound), - Some(SketchKind::Hll), + Some(SummaryKind::Hll), "unique_users_per_min should bind to HLL (Cardinality)", ); } @@ -712,8 +727,8 @@ mod tests { let bound = bind_workload_typed(&w).expect("top_endpoint_qps must bind"); assert_eq!( extract_family(&bound), - Some(SketchKind::CountSketch), - "top_endpoint_qps should bind to CountSketch (TopK)", + Some(SummaryKind::CountSketchWithHeap), + "top_endpoint_qps should bind to CountSketch-with-heap (TopK)", ); } @@ -728,7 +743,7 @@ mod tests { let bound = bind_workload_typed(&w).expect("endpoint_request_freq must bind"); assert_eq!( extract_family(&bound), - Some(SketchKind::Cms), + Some(SummaryKind::Cms), "endpoint_request_freq should bind to CMS (Frequency)", ); } @@ -746,7 +761,7 @@ mod tests { let bound = bind_workload_typed(&w).expect("override should still bind"); assert_eq!( extract_family(&bound), - Some(SketchKind::Kll), + Some(SummaryKind::Kll), "sketch_type_override=KLL should pin KLL despite the contract's DDSketch default", ); } @@ -760,7 +775,7 @@ mod tests { let bound = bind_workload_typed(&w).expect("override should still bind"); assert_eq!( extract_family(&bound), - Some(SketchKind::DDSketch), + Some(SummaryKind::DDSketch), "sketch_type_override=DDSketch should pin DDSketch despite the contract's KLL default", ); } @@ -778,21 +793,22 @@ mod tests { bind_workload_typed(&w).expect("CountMin override on a TopK metric should still bind"); assert_eq!( extract_family(&bound), - Some(SketchKind::Cms), - "sketch_type_override=CountMinSketch on a TopK metric should pin CMS", + Some(SummaryKind::CmsWithHeap), + "sketch_type_override=CountMinSketch on a TopK metric should pin CMS-with-heap", ); } #[test] fn planner_default_for_topk_remains_countsketch() { // Without any override, the canonical pick for a TopK metric - // stays CountSketch — CMS-Heap is opt-in via override only. + // stays CountSketch(-with-heap) — CMS-Heap is opt-in via + // override only. let w = workload_for("top_endpoint_qps", AggType::Frequency); let bound = bind_workload_typed(&w).expect("top_endpoint_qps must bind"); assert_eq!( extract_family(&bound), - Some(SketchKind::CountSketch), - "default TopK pick must remain CountSketch (unbiased estimator)", + Some(SummaryKind::CountSketchWithHeap), + "default TopK pick must remain CountSketch-with-heap (unbiased estimator)", ); } @@ -806,7 +822,7 @@ mod tests { let bound = bind_workload_typed(&w).expect("fallback should bind"); assert_eq!( extract_family(&bound), - Some(SketchKind::DDSketch), + Some(SummaryKind::DDSketch), "invalid (HLL, Quantile) override should be rejected; planner falls back to DDSketch", ); } @@ -818,32 +834,32 @@ mod tests { // Single test that drives the full contract row set through // `bind_workload_typed` — this is the per-task acceptance test // ("verify each produces the expected `PhysicalExpr` family"). - let cases: Vec<(&str, AggType, Option)> = vec![ + let cases: Vec<(&str, AggType, Option)> = vec![ ("http_requests_total", AggType::Frequency, None), ( "http_latency_ms", AggType::Quantile, - Some(SketchKind::DDSketch), + Some(SummaryKind::DDSketch), ), ( "request_size_bytes", AggType::Quantile, - Some(SketchKind::Kll), + Some(SummaryKind::Kll), ), ( "unique_users_per_min", AggType::Cardinality, - Some(SketchKind::Hll), + Some(SummaryKind::Hll), ), ( "top_endpoint_qps", AggType::Frequency, - Some(SketchKind::CountSketch), + Some(SummaryKind::CountSketchWithHeap), ), ( "endpoint_request_freq", AggType::Frequency, - Some(SketchKind::Cms), + Some(SummaryKind::Cms), ), ]; for (metric, agg, expected) in cases { diff --git a/control_plane/src/physical/colored_dag/allocator.rs b/control_plane/src/physical/colored_dag/allocator.rs index ac7bafdb..6e18ea33 100644 --- a/control_plane/src/physical/colored_dag/allocator.rs +++ b/control_plane/src/physical/colored_dag/allocator.rs @@ -273,8 +273,8 @@ mod tests { use super::*; use crate::intent_algebra::schema::{Column, DataType}; use crate::intent_algebra::{QueryExpr, Schema, Source, WindowKind}; - use crate::sketch_algebra::params::{KllParams, SketchKind, SketchParams}; use crate::sketch_algebra::physical_expr::EstimateOp; + use asap_sketch::{SummaryKind, SummaryParams}; use std::time::Duration; fn ts_scan() -> QueryExpr { @@ -327,8 +327,8 @@ mod tests { fn three_stage_quantile_dag_basic() { let expr = PhysicalExpr::estimate_over_agg( EstimateOp::Quantile { q: 0.99 }, - SketchKind::Kll, - SketchParams::Kll(KllParams { k: 200 }), + SummaryKind::Kll, + SummaryParams::Kll { k: 200 }, windowed_scan(), ); let dag = StageAllocator diff --git a/control_plane/src/physical/colored_dag/dag.rs b/control_plane/src/physical/colored_dag/dag.rs index c2b4110c..2b946502 100644 --- a/control_plane/src/physical/colored_dag/dag.rs +++ b/control_plane/src/physical/colored_dag/dag.rs @@ -145,9 +145,9 @@ impl ColoredDag { mod tests { use super::*; use crate::intent_algebra::QueryExpr; - use crate::sketch_algebra::params::{KllParams, SketchKind, SketchParams}; use crate::sketch_algebra::physical_expr::EstimateOp; use crate::sketch_algebra::PhysicalExpr; + use asap_sketch::{SummaryKind, SummaryParams}; fn dummy_logical() -> PhysicalExpr { PhysicalExpr::Logical(QueryExpr::Ref { @@ -157,8 +157,8 @@ mod tests { fn dummy_agg() -> PhysicalExpr { PhysicalExpr::SketchAgg { - sketch_type: SketchKind::Kll, - params: SketchParams::Kll(KllParams { k: 200 }), + sketch_type: SummaryKind::Kll, + params: SummaryParams::Kll { k: 200 }, child: Box::new(dummy_logical()), } } diff --git a/control_plane/src/physical/colored_dag/emitter.rs b/control_plane/src/physical/colored_dag/emitter.rs index 0f8253d5..12147e21 100644 --- a/control_plane/src/physical/colored_dag/emitter.rs +++ b/control_plane/src/physical/colored_dag/emitter.rs @@ -33,8 +33,8 @@ use serde::{Deserialize, Serialize}; use crate::physical::colored_dag::dag::ColoredDag; use crate::physical::colored_dag::stage_id::{StageId, Topology}; -use crate::sketch_algebra::params::{SketchKind, SketchParams}; use crate::sketch_algebra::physical_expr::{EstimateOp, PhysicalExpr}; +use asap_sketch::{SummaryKind, SummaryParams}; /// Errors surfaced by [`Emitter::emit_per_stage`]. #[derive(Debug, thiserror::Error, PartialEq)] @@ -45,10 +45,10 @@ pub enum EmitError { #[error("unsupported topology for this emitter: {0:?} (expected {1:?})")] UnsupportedTopology(Topology, Topology), /// A sketch processor name could not be derived for the supplied - /// `SketchKind`. Should not occur with the catalog ranges shipped + /// `SummaryKind`. Should not occur with the catalog ranges shipped /// in Phase C — kept as a defensive error for future kinds. #[error("no edge processor known for sketch kind {0:?}")] - NoEdgeProcessor(SketchKind), + NoEdgeProcessor(SummaryKind), /// Backend would emit an empty StreamingConfig because no sketch /// state ever reaches it (e.g. a colouring with only `Logical` /// nodes). Surfaced as a clean error so callers can fall back to @@ -73,8 +73,7 @@ pub trait Emitter { /// The variants are deliberately struct-shaped (named fields) so future /// downstream consumers can pattern-match without relying on tuple-index /// stability. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(tag = "stage", rename_all = "snake_case")] +#[derive(Debug, Clone, PartialEq)] pub enum StageConfig { /// Edge agent's logical config — what the OpAMP push for this /// agent will need to materialise into OTel collector YAML. @@ -104,7 +103,7 @@ impl StageConfig { /// Mirrors the surface of `crate::types::AgentCollectorConfig` minus the /// wire-format details (delta encoding, series-id TTL, sink addressing) /// — those are emitter-side decisions Phase G+ owns. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq)] pub struct EdgeStageConfig { /// Source metric name (from the L3 `Scan{Source::TimeSeries}` /// node). `None` only for synthetic colourings used in tests. @@ -131,7 +130,6 @@ pub struct EdgeStageConfig { /// /// Empty list = no Mode 3 metrics → no `otlphttp/prometheus` /// exporter is emitted (the YAML is identical to Phase β). - #[serde(default, skip_serializing_if = "Vec::is_empty")] pub prometheus_archive_metrics: Vec, /// Phase 3.2.5 — archive-tier metrics that should flow through the /// `gorillas3` processor at the edge agent (write a Gorilla-S3 @@ -150,7 +148,6 @@ pub struct EdgeStageConfig { /// drop a `PhysicalExpr` node, but the agent still has to land its /// counter samples in MinIO so the Gorilla-S3 / Thanos archive /// can answer `last_over_time(...)`). - #[serde(default, skip_serializing_if = "Vec::is_empty")] pub archive_tier_metrics: Vec, /// Phase 3.2.5 — metrics that must be carried through the /// ASAP-tier pipeline WITHOUT the family-specific sketch processor @@ -166,7 +163,6 @@ pub struct EdgeStageConfig { /// `metrics/warm_passthrough` pipeline (gorillas3 if archive is /// declared, then exporter — NO sketch processor); everything /// else takes the existing `metrics/asap_tier` pipeline. - #[serde(default, skip_serializing_if = "Vec::is_empty")] pub warm_passthrough_metrics: Vec, /// MVP §46 / ASAPCollector#400 — per-metric → **set of** sketch /// families populated by the planner from the workload spec. When @@ -182,7 +178,7 @@ pub struct EdgeStageConfig { /// because different planned queries on the same metric require /// different capabilities (e.g. `quantile_over_time` → DDSketch, /// `count`-distinct → HLL, `topk` → CountSketch all on one metric). - /// The value type is therefore a `BTreeSet` (the UNION + /// The value type is therefore a `BTreeSet` (the UNION /// of capabilities across all of that metric's workload entries), /// not a single family. A metric in two families produces two /// routing-connector OTTL conditions → its samples fan into both @@ -195,12 +191,11 @@ pub struct EdgeStageConfig { /// This eliminates the prior multi-family fan-out (every metric /// shipped sketch state through all 5 families regardless of need). /// - /// `SketchFamily` is a control-plane-side alias for `SketchKind` per - /// `sketch_algebra::params`. Empty map ⇒ legacy single-pipeline / + /// `SketchFamily` is a control-plane-side alias for `asap_sketch::SummaryKind`. + /// Empty map ⇒ legacy single-pipeline / /// Mode-3 / warm-passthrough wire shapes are emitted unchanged /// (backward-compat). - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub metric_to_family: HashMap>, + pub metric_to_family: HashMap>, /// MVP blocker B3 — per-metric attribute allowlist the agent must /// reduce wire attrs to BEFORE the sketch processor sees them. /// Maps each metric to its grouping-label list; the 5-sketch routing @@ -212,7 +207,6 @@ pub struct EdgeStageConfig { /// Without this the agent sketches with the full wire-attr tuple, /// minting one sid per unique tuple — defeating the streaming-config's /// `grouping_labels` contract. - #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub metric_to_grouping_labels: HashMap>, /// Issue #298 — metrics whose OTel datapoints arrive with /// **cumulative** temporality (OTel SDK's default for `Counter` @@ -239,7 +233,6 @@ pub struct EdgeStageConfig { /// Empty list (default) ⇒ no `cumulativetodelta` processor is /// emitted; backward-compat for plans that never declare a counter /// metric (e.g. quantile-only workloads). - #[serde(default, skip_serializing_if = "Vec::is_empty")] pub cumulative_counter_metrics: Vec, /// PR #311 follow-up — the cold-tier (Gorilla archive) ingest URL the /// fused `asap_edge` processor ships per-emit Gorilla blocks to. This @@ -256,7 +249,6 @@ pub struct EdgeStageConfig { /// (it is deployment-independent — no `DeploymentConstraints` is /// plumbed in), so it populates the named default; a future layer that /// holds deploy info can set a concrete value. - #[serde(default, skip_serializing_if = "Option::is_none")] pub cold_ship_endpoint: Option, /// PR #311 follow-up — external labels stamped on every cold-tier /// Gorilla block the fused `asap_edge` processor ships (the merger @@ -266,7 +258,6 @@ pub struct EdgeStageConfig { /// field threads it explicitly. Empty ⇒ the emitter falls back to /// [`default_cold_external_labels`] (a single named default that reads /// `ASAP_CLUSTER`, defaulting to `asap-mvp`). - #[serde(default, skip_serializing_if = "Vec::is_empty")] pub cold_external_labels: Vec<(String, String)>, /// Per-metric sketch **sampling probability** `p` in `(0, 1]`, /// populated by the planner from each workload entry's @@ -288,7 +279,6 @@ pub struct EdgeStageConfig { /// optimizer-driven dynamic `p` is a follow-up (out of scope here). /// /// Empty map (default) ⇒ no metric carries sampling — backward-compat. - #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub metric_to_sample_p: HashMap, /// Per-metric **known distinct-key count per window** (cardinality hint), /// populated by the planner from each workload entry's @@ -307,7 +297,6 @@ pub struct EdgeStageConfig { /// (per-series ⇒ sparse, whole-stream ⇒ dense), so the emitted config stays /// byte-identical when no cardinality hint is declared. Empty map (default) /// ⇒ no metric carries a hint — backward-compat. - #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub metric_to_distinct_keys: HashMap, /// Per-metric **inner item dimension** for the item-counting sketch /// families (HLL / CountSketch / CountMinSketch): the data-point @@ -331,7 +320,6 @@ pub struct EdgeStageConfig { /// convention (`countsketch_item_label_for`) when a metric is absent /// from this map, preserving the prior behaviour. Empty map (default) ⇒ /// no metric carries an explicit item dimension — backward-compat. - #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub metric_to_item_label: HashMap, /// Cold-archive **wire format** the agent's `asapedgeprocessor` ships /// its cold tier in. Two formats are merged in the agent: @@ -348,7 +336,6 @@ pub struct EdgeStageConfig { /// emits NEITHER key, so the agent's cold block stays byte-identical to /// the pre-format emit (`ship_endpoint` only) — no behavior change when /// unset. - #[serde(default, skip_serializing_if = "ColdFormat::is_default")] pub cold_format: ColdFormat, /// Cold-archive intchunk ingest URL — the gorilla-merger's coldpart /// HTTP ingest endpoint (`http://gorilla-merger:10908/ingest/coldpart`). @@ -360,7 +347,6 @@ pub struct EdgeStageConfig { /// to `/ingest/coldpart` (same merger host:port as the fragment /// endpoint), falling back to [`default_cold_coldpart_endpoint`] when /// neither is set. Ignored entirely for [`ColdFormat::Fragment`]. - #[serde(default, skip_serializing_if = "Option::is_none")] pub cold_coldpart_endpoint: Option, } @@ -474,15 +460,20 @@ pub struct PrometheusArchiveMetric { } /// One sketch processor configured at an edge agent. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +/// +/// Not `Serialize`/`Deserialize` (see `PhysicalExpr`'s doc for why — +/// `SummaryKind`/`SummaryParams` have no serde impl, and nothing on the +/// real emit path ever whole-struct-serialized this type; every actual +/// YAML/JSON payload goes through a hand-written builder). +#[derive(Debug, Clone, PartialEq)] pub struct EdgeSketchProcessor { /// OTel processor component id — `KLL`, `ddsketch`, `HLL`, - /// `countmin`, etc. Maps 1:1 from `SketchKind`. + /// `countmin`, etc. Maps 1:1 from `SummaryKind`. pub processor_name: String, /// Sketch family (mirror of the `SketchAgg::sketch_type` field). - pub sketch_kind: SketchKind, + pub sketch_kind: SummaryKind, /// Sketch parameters (mirror of the `SketchAgg::params` field). - pub sketch_params: SketchParams, + pub sketch_params: SummaryParams, /// Internal emitter plumbing — threads `EdgeSketchProcessor` → /// `GatewayMergeProcessor` (which DOES surface it on the wire to /// route merged streams) during the DAG walk. Phase E derives a @@ -503,7 +494,7 @@ pub struct EdgeSketchProcessor { } /// Logical content of a gateway aggregator's per-stage config. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq)] pub struct GatewayStageConfig { /// OTLP receiver port — Phase E surfaces the abstract `Default` /// (`4317`); deployment-specific overrides happen at Phase G. @@ -515,13 +506,15 @@ pub struct GatewayStageConfig { } /// One sketch-merge processor configured at the gateway. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +/// +/// Not `Serialize`/`Deserialize` — same reason as `EdgeSketchProcessor`. +#[derive(Debug, Clone, PartialEq)] pub struct GatewayMergeProcessor { /// OTel processor name — `sketchmergeprocessor`. pub processor_name: String, /// Sketch family being merged. All inputs to the merge agree on /// this (L4 type checker enforces it; design.md §6.4). - pub sketch_kind: SketchKind, + pub sketch_kind: SummaryKind, /// Aggregation id — matches the upstream edge's /// `EdgeSketchProcessor::aggregation_id` so the gateway routes /// streams correctly. @@ -536,7 +529,7 @@ pub struct GatewayMergeProcessor { } /// Logical content of the backend `StreamingConfig`. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq)] pub struct BackendStageConfig { /// One entry per readout query the backend must serve. The /// `aggregation_id` in each routing entry is the backend's @@ -555,7 +548,13 @@ pub struct BackendStageConfig { /// the controller-allocated id; the backend content-addresses identity /// via `PolicyFingerprint(u64)` derived from `metric_name`, /// `sketch_kind`, `sketch_params`, grouping labels, and `spatial_filter`). -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +/// +/// Not `Serialize`/`Deserialize` — same reason as `EdgeSketchProcessor`: +/// `SummaryKind`/`SummaryParams` have no serde impl, and the real wire +/// payload is built by `emit::stage_config::build_backend_aggregation_json` +/// (a hand-written JSON builder reading these fields), never a whole-struct +/// serialize. +#[derive(Debug, Clone, PartialEq)] pub struct BackendAggregation { /// Internal-only id (see struct doc). Not on the wire. pub aggregation_id: String, @@ -564,17 +563,16 @@ pub struct BackendAggregation { /// `AggregationConfig` parser. pub metric_name: String, /// Sketch family. - pub sketch_kind: SketchKind, + pub sketch_kind: SummaryKind, /// Sketch parameters — the backend uses these to build its /// per-aggregation `Sketch` instance (KLL with the right `k`, /// DDSketch with the right `alpha`, etc.). - pub sketch_params: SketchParams, + pub sketch_params: SummaryParams, /// Tumbling window size in seconds. Required by the backend; the /// parser rejects zero-window aggregations. pub window_secs: u64, /// Spatial filter (comma-joined `k=v` pairs from the edge's /// `label_filters`). Empty string when no filter applies. - #[serde(default)] pub spatial_filter: String, /// Group-by label names — keys in `labels.grouping` on the backend /// side, where the precompute engine's accumulator pipeline keys @@ -588,7 +586,6 @@ pub struct BackendAggregation { /// columns (open-set label naming is a Step γ TODO in /// `intent_algebra::column_resolution`), so the workload-spec /// strings are the only reliable source of the names today. - #[serde(default)] pub grouping: Vec, /// Per-item dimension (the data-point attribute NAME, e.g. "endpoint" /// or "service") for an item_label-mode frequency sketch. Like @@ -596,7 +593,6 @@ pub struct BackendAggregation { /// it from the workload's `item_label`. Emitted into the aggregation's /// `parameters["item_label"]` so the data-plane ingest records it on the /// CMS sid and can answer per-item `estimate(key)` (FrequencyEstimate). - #[serde(default)] pub item_label: Option, /// Phase ε.1 — what shape the backend ingests for this /// aggregation. Mode 1 (sketch at edge) / sketch_envelope is the @@ -605,7 +601,6 @@ pub struct BackendAggregation { /// backend builds the sketch from raw OTLP samples at ingest. The /// backend's `StreamingConfig` consumer interprets the field — /// Phase ε.2 implements the raw-input ingest path. - #[serde(default)] pub aggregation_input: AggregationInput, /// Option B (post-PR-#287) — when `Some(s)`, the wire-side @@ -627,7 +622,6 @@ pub struct BackendAggregation { /// resolve. `sketch_kind` / `sketch_params` carry sentinel /// values when the override is in effect (their emitted form is /// suppressed in `build_backend_aggregation_json`). - #[serde(default)] pub agg_type_override: Option, } @@ -948,17 +942,34 @@ impl Emitter for ThreeStageEmitter { // ── Helpers ─────────────────────────────────────────────────────────────────── -/// Map a `SketchKind` to the OTel collector processor name. Mirrors the +/// Map a `SummaryKind` to the OTel collector processor name. Mirrors the /// names the existing OpAMP YAML emitter (and the per-sketch processor /// crates in `opentelemetry-collector-contrib`) already use. -pub(crate) fn edge_processor_name(kind: &SketchKind) -> Result { - Ok(match kind { - SketchKind::Kll => "KLL".into(), - SketchKind::DDSketch => "ddsketch".into(), - SketchKind::Hll => "HLL".into(), - SketchKind::Cms => "countmin".into(), - SketchKind::CountSketch => "countsketch".into(), - }) +/// +/// Heap-bearing kinds (`CmsWithHeap`/`CountSketchWithHeap`) reuse their +/// bare counterpart's processor name — the retired `sketch_algebra::SketchKind` +/// this replaces had no heap-bearing variant at all (`with_heap` was a +/// `SketchParams` field this function never received), so heap-bearing +/// and bare CMS/CountSketch already mapped to the identical processor +/// name; this preserves that exactly. Exact accumulators and `Kmv`/`Theta` +/// have no OTel edge processor — nothing in this repo's binding rules +/// constructs a `SketchAgg`/`RawAtEdgeSketchAtBackend` with one of these +/// kinds today, but the match must stay exhaustive. +pub(crate) fn edge_processor_name(kind: &SummaryKind) -> Result { + match kind { + SummaryKind::Kll => Ok("KLL".into()), + SummaryKind::DDSketch => Ok("ddsketch".into()), + SummaryKind::Hll => Ok("HLL".into()), + SummaryKind::Cms | SummaryKind::CmsWithHeap => Ok("countmin".into()), + SummaryKind::CountSketch | SummaryKind::CountSketchWithHeap => Ok("countsketch".into()), + SummaryKind::Sum + | SummaryKind::Count + | SummaryKind::MinMax + | SummaryKind::Increase + | SummaryKind::Rate + | SummaryKind::Kmv + | SummaryKind::Theta => Err(EmitError::NoEdgeProcessor(kind.clone())), + } } /// Recursively descend an L3 [`crate::intent_algebra::QueryExpr`] @@ -1058,7 +1069,7 @@ fn first_sketch_child_via_edges( dag: &ColoredDag, parent: crate::physical::colored_dag::dag::NodeId, sketch_agg_ids: &HashMap, -) -> Option<(SketchKind, String)> { +) -> Option<(SummaryKind, String)> { for cid in children_of(dag, parent) { let cnode = dag.nodes.get(cid.0)?; match &cnode.expr { diff --git a/control_plane/src/physical/colored_dag/tests.rs b/control_plane/src/physical/colored_dag/tests.rs index bf05285c..00ae83eb 100644 --- a/control_plane/src/physical/colored_dag/tests.rs +++ b/control_plane/src/physical/colored_dag/tests.rs @@ -14,11 +14,9 @@ use crate::intent_algebra::{LabelFilter, QueryExpr, Schema, Source, WindowKind}; use crate::physical::colored_dag::allocator::StageAllocator; use crate::physical::colored_dag::emitter::{EmitError, Emitter, StageConfig, ThreeStageEmitter}; use crate::physical::colored_dag::stage_id::{StageId, Topology}; -use crate::sketch_algebra::params::{ - DDSketchParams, HllParams, KllParams, SketchKind, SketchParams, -}; use crate::sketch_algebra::physical_expr::{EstimateOp, MergeAlgebra, PhysicalExpr}; use crate::types_v2::{AccuracyTarget, BindingName}; +use asap_sketch::{SummaryKind, SummaryParams}; // ── Test fixtures ───────────────────────────────────────────────────────────── @@ -76,8 +74,8 @@ fn windowed_scan() -> QueryExpr { fn quantile_kll_dag() -> PhysicalExpr { PhysicalExpr::estimate_over_agg( EstimateOp::Quantile { q: 0.99 }, - SketchKind::Kll, - SketchParams::Kll(KllParams { k: 200 }), + SummaryKind::Kll, + SummaryParams::Kll { k: 200 }, windowed_scan(), ) } @@ -122,8 +120,8 @@ fn allocator_sketch_agg_under_scan_pinned_edge() { // Exact design.md §6 invariant: a SketchAgg whose child is a Scan // (wrapped in Logical) MUST land on Edge. let expr = PhysicalExpr::SketchAgg { - sketch_type: SketchKind::Hll, - params: SketchParams::Hll(HllParams { precision: 14 }), + sketch_type: SummaryKind::Hll, + params: SummaryParams::Hll { precision: 14 }, child: Box::new(PhysicalExpr::Logical(ts_scan("events", None))), }; let dag = StageAllocator @@ -152,8 +150,8 @@ fn allocator_let_binding_color_propagates() { // LetBinding takes the bound expression's stage. Bind a // SketchAgg{KLL} (edge) and verify the LetBinding node colors edge. let inner_agg = PhysicalExpr::SketchAgg { - sketch_type: SketchKind::Kll, - params: SketchParams::Kll(KllParams { k: 200 }), + sketch_type: SummaryKind::Kll, + params: SummaryParams::Kll { k: 200 }, child: Box::new(PhysicalExpr::Logical(windowed_scan())), }; let bind = PhysicalExpr::LetBinding { @@ -183,8 +181,8 @@ fn allocator_ref_resolves_to_binding_stage() { // Ref takes the stage of its binding. Same fixture as above; Ref // child of SketchEstimate must color Edge (the binding's stage). let inner_agg = PhysicalExpr::SketchAgg { - sketch_type: SketchKind::Kll, - params: SketchParams::Kll(KllParams { k: 200 }), + sketch_type: SummaryKind::Kll, + params: SummaryParams::Kll { k: 200 }, child: Box::new(PhysicalExpr::Logical(windowed_scan())), }; let bind = PhysicalExpr::LetBinding { @@ -212,8 +210,8 @@ fn allocator_ref_resolves_to_binding_stage() { fn allocator_sketch_merge_lands_gateway() { // SketchMerge over edge-built KLL sketches → Gateway. let one_agg = || PhysicalExpr::SketchAgg { - sketch_type: SketchKind::Kll, - params: SketchParams::Kll(KllParams { k: 200 }), + sketch_type: SummaryKind::Kll, + params: SummaryParams::Kll { k: 200 }, child: Box::new(PhysicalExpr::Logical(windowed_scan())), }; let merge = PhysicalExpr::SketchMerge { @@ -243,8 +241,8 @@ fn emitter_three_stage_emits_three_configs() { // Build a DAG with all three stages occupied: SketchEstimate over // SketchMerge over two SketchAggs. let one_agg = || PhysicalExpr::SketchAgg { - sketch_type: SketchKind::Kll, - params: SketchParams::Kll(KllParams { k: 200 }), + sketch_type: SummaryKind::Kll, + params: SummaryParams::Kll { k: 200 }, child: Box::new(PhysicalExpr::Logical(windowed_scan())), }; let merge = PhysicalExpr::SketchMerge { @@ -275,7 +273,7 @@ fn emitter_edge_config_has_correct_processor_kll() { StageConfig::Edge(e) => { assert_eq!(e.sketch_processors.len(), 1); assert_eq!(e.sketch_processors[0].processor_name, "KLL"); - assert_eq!(e.sketch_processors[0].sketch_kind, SketchKind::Kll); + assert_eq!(e.sketch_processors[0].sketch_kind, SummaryKind::Kll); assert_eq!( e.source_metric.as_deref(), Some("http_request_duration_seconds") @@ -290,8 +288,8 @@ fn emitter_edge_config_has_correct_processor_kll() { fn emitter_edge_config_has_correct_processor_ddsketch() { let expr = PhysicalExpr::estimate_over_agg( EstimateOp::Quantile { q: 0.99 }, - SketchKind::DDSketch, - SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), + SummaryKind::DDSketch, + SummaryParams::DDSketch { alpha: 0.01 }, windowed_scan(), ); let dag = StageAllocator @@ -320,7 +318,7 @@ fn emitter_backend_config_routes_aggregation_id() { StageConfig::Backend(b) => { assert_eq!(b.aggregations.len(), 1); assert_eq!(b.aggregations[0].aggregation_id, edge_aid); - assert_eq!(b.aggregations[0].sketch_kind, SketchKind::Kll); + assert_eq!(b.aggregations[0].sketch_kind, SummaryKind::Kll); assert_eq!(b.readouts.len(), 1); assert_eq!(b.readouts[0].aggregation_id, edge_aid); assert_eq!(b.readouts[0].op, EstimateOp::Quantile { q: 0.99 }); @@ -368,8 +366,8 @@ fn end_to_end_quantile_workload() { // Gateway: SketchMerge + Merge; Backend: SketchEstimate + final // root). let agg = || PhysicalExpr::SketchAgg { - sketch_type: SketchKind::Kll, - params: SketchParams::Kll(KllParams { k: 200 }), + sketch_type: SummaryKind::Kll, + params: SummaryParams::Kll { k: 200 }, child: Box::new(PhysicalExpr::Logical(windowed_scan())), }; let merge_kll = PhysicalExpr::SketchMerge { @@ -403,7 +401,7 @@ fn end_to_end_quantile_workload() { StageConfig::Gateway(g) => { assert!(!g.merge_processors.is_empty()); assert_eq!(g.merge_processors[0].processor_name, "sketchmergeprocessor"); - assert_eq!(g.merge_processors[0].sketch_kind, SketchKind::Kll); + assert_eq!(g.merge_processors[0].sketch_kind, SummaryKind::Kll); } _ => unreachable!(), } diff --git a/control_plane/src/replan.rs b/control_plane/src/replan.rs index 4b1aa750..55ca78a1 100644 --- a/control_plane/src/replan.rs +++ b/control_plane/src/replan.rs @@ -655,7 +655,7 @@ impl Replanner { use crate::physical::colored_dag::emitter::{ AggregationInput, BackendAggregation, BackendStageConfig, }; - use crate::sketch_algebra::params::{DDSketchParams, SketchKind, SketchParams}; + use asap_sketch::{SummaryKind, SummaryParams}; let window_secs = workload.time_window.as_secs().max(1); Some(BackendStageConfig { aggregations: vec![BackendAggregation { @@ -667,9 +667,9 @@ impl Replanner { // these are not emitted on the wire. DDSketch is the // chosen sentinel because every backend that recognises // `AggregationType::FromStr` also accepts DDSketch (and - // we don't have a `SketchKind::None` variant today). - sketch_kind: SketchKind::DDSketch, - sketch_params: SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), + // we don't have a `SummaryKind::None` variant today). + sketch_kind: SummaryKind::DDSketch, + sketch_params: SummaryParams::DDSketch { alpha: 0.01 }, window_secs, spatial_filter: String::new(), grouping: workload.group_by_labels.clone(), @@ -1254,7 +1254,7 @@ mod tests { use crate::physical::colored_dag::emitter::{ AggregationInput, BackendAggregation, BackendStageConfig, }; - use crate::sketch_algebra::params::{DDSketchParams, SketchKind, SketchParams}; + use asap_sketch::{SummaryKind, SummaryParams}; let (url, hits) = start_repost_mock().await; let client = StdArc::new(BackendClient::new(url)); @@ -1304,8 +1304,8 @@ mod tests { item_label: None, aggregation_id: "exact-http_requests_total-sum".to_string(), metric_name: "http_requests_total".to_string(), - sketch_kind: SketchKind::DDSketch, - sketch_params: SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), + sketch_kind: SummaryKind::DDSketch, + sketch_params: SummaryParams::DDSketch { alpha: 0.01 }, grouping: vec!["zone".to_string()], spatial_filter: String::new(), window_secs: 60, diff --git a/control_plane/src/sketch_algebra/capability.rs b/control_plane/src/sketch_algebra/capability.rs index 9f8042d7..849aa36f 100644 --- a/control_plane/src/sketch_algebra/capability.rs +++ b/control_plane/src/sketch_algebra/capability.rs @@ -21,7 +21,6 @@ #![allow(dead_code)] use crate::intent_algebra::agg_intent::AggIntent; -use crate::sketch_algebra::params::SketchKind; use crate::types_v2::AccuracyTarget; use promql_utilities::query_logics::enums::AggregationType; @@ -287,8 +286,8 @@ impl OuterAgg { } /// Compact, hashable handle for sketch implementation choice. Mirrors -/// [`SketchKind`] but adds the `CmsWithHeap` and `Any` query-side -/// concepts (which aren't sketch families, they're dispatch hints). +/// `asap_sketch::SummaryKind` but adds the `Any` query-side wildcard +/// (not a sketch family — a dispatch hint). #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum SketchKindHandle { DDSketch, diff --git a/control_plane/src/sketch_algebra/capability_matching.rs b/control_plane/src/sketch_algebra/capability_matching.rs index 6461e8fb..53c5608f 100644 --- a/control_plane/src/sketch_algebra/capability_matching.rs +++ b/control_plane/src/sketch_algebra/capability_matching.rs @@ -50,7 +50,7 @@ #![allow(dead_code)] -use crate::sketch_algebra::params::SketchKind; +use asap_sketch::SummaryKind; /// Query intent the user is expressing — abstracted away from the L1 /// language (PromQL `quantile_over_time`, SQL `PERCENTILE_CONT`, etc.) and @@ -137,9 +137,9 @@ pub enum AccuracyPreference { /// Muthukrishnan 2005). NB: this is the planner-side capability /// declaration; the backend's "top-K from CountMin state" readout path /// is a separate workstream — see the module-level docs. -pub fn is_valid_pair(sketch: SketchKind, statistic: StatisticClass) -> bool { - use SketchKind::*; +pub fn is_valid_pair(sketch: SummaryKind, statistic: StatisticClass) -> bool { use StatisticClass::*; + use SummaryKind::*; match (sketch, statistic) { (DDSketch, Quantile) | (Kll, Quantile) @@ -167,10 +167,10 @@ pub fn is_valid_pair(sketch: SketchKind, statistic: StatisticClass) -> bool { /// `sketch_family_override` (treated as `QueryWorkload::sketch_type_override` /// at the planner-rules layer) wins over the capability-matched default — /// see `planner::rules::bind_workload_typed`. -pub fn pick_family(statistic: StatisticClass, accuracy: AccuracyPreference) -> Option { +pub fn pick_family(statistic: StatisticClass, accuracy: AccuracyPreference) -> Option { use AccuracyPreference::*; - use SketchKind::*; use StatisticClass::*; + use SummaryKind::*; let kind = match (statistic, accuracy) { (Quantile, RelativeError) => DDSketch, (Quantile, RankError) => Kll, @@ -220,65 +220,71 @@ mod tests { #[test] fn ddsketch_is_quantile_only() { assert!(is_valid_pair( - SketchKind::DDSketch, + SummaryKind::DDSketch, StatisticClass::Quantile )); assert!(!is_valid_pair( - SketchKind::DDSketch, + SummaryKind::DDSketch, StatisticClass::Cardinality )); - assert!(!is_valid_pair(SketchKind::DDSketch, StatisticClass::TopK)); + assert!(!is_valid_pair(SummaryKind::DDSketch, StatisticClass::TopK)); assert!(!is_valid_pair( - SketchKind::DDSketch, + SummaryKind::DDSketch, StatisticClass::Frequency )); assert!(!is_valid_pair( - SketchKind::DDSketch, + SummaryKind::DDSketch, StatisticClass::SumRateCount )); } #[test] fn kll_is_quantile_only() { - assert!(is_valid_pair(SketchKind::Kll, StatisticClass::Quantile)); - assert!(!is_valid_pair(SketchKind::Kll, StatisticClass::Cardinality)); - assert!(!is_valid_pair(SketchKind::Kll, StatisticClass::TopK)); - assert!(!is_valid_pair(SketchKind::Kll, StatisticClass::Frequency)); + assert!(is_valid_pair(SummaryKind::Kll, StatisticClass::Quantile)); assert!(!is_valid_pair( - SketchKind::Kll, + SummaryKind::Kll, + StatisticClass::Cardinality + )); + assert!(!is_valid_pair(SummaryKind::Kll, StatisticClass::TopK)); + assert!(!is_valid_pair(SummaryKind::Kll, StatisticClass::Frequency)); + assert!(!is_valid_pair( + SummaryKind::Kll, StatisticClass::SumRateCount )); } #[test] fn hll_is_cardinality_only() { - assert!(is_valid_pair(SketchKind::Hll, StatisticClass::Cardinality)); - assert!(!is_valid_pair(SketchKind::Hll, StatisticClass::Quantile)); - assert!(!is_valid_pair(SketchKind::Hll, StatisticClass::TopK)); - assert!(!is_valid_pair(SketchKind::Hll, StatisticClass::Frequency)); + assert!(is_valid_pair(SummaryKind::Hll, StatisticClass::Cardinality)); + assert!(!is_valid_pair(SummaryKind::Hll, StatisticClass::Quantile)); + assert!(!is_valid_pair(SummaryKind::Hll, StatisticClass::TopK)); + assert!(!is_valid_pair(SummaryKind::Hll, StatisticClass::Frequency)); assert!(!is_valid_pair( - SketchKind::Hll, + SummaryKind::Hll, StatisticClass::SumRateCount )); } #[test] fn countsketch_is_topk_only() { - assert!(is_valid_pair(SketchKind::CountSketch, StatisticClass::TopK)); + assert!(is_valid_pair( + SummaryKind::CountSketch, + StatisticClass::TopK + )); assert!(!is_valid_pair( - SketchKind::CountSketch, + SummaryKind::CountSketch, StatisticClass::Quantile )); assert!(!is_valid_pair( - SketchKind::CountSketch, + SummaryKind::CountSketch, StatisticClass::Cardinality )); assert!(!is_valid_pair( - SketchKind::CountSketch, + SummaryKind::CountSketch, StatisticClass::Frequency )); assert!(!is_valid_pair( - SketchKind::CountSketch, + SummaryKind::CountSketch, StatisticClass::SumRateCount )); } @@ -287,12 +293,15 @@ mod tests { fn cms_supports_frequency_and_topk() { // CMS validly answers Frequency (point-frequency, additive bound) // AND TopK via the CMS-Heap pattern (Cormode & Muthukrishnan 2005). - assert!(is_valid_pair(SketchKind::Cms, StatisticClass::Frequency)); - assert!(is_valid_pair(SketchKind::Cms, StatisticClass::TopK)); - assert!(!is_valid_pair(SketchKind::Cms, StatisticClass::Quantile)); - assert!(!is_valid_pair(SketchKind::Cms, StatisticClass::Cardinality)); + assert!(is_valid_pair(SummaryKind::Cms, StatisticClass::Frequency)); + assert!(is_valid_pair(SummaryKind::Cms, StatisticClass::TopK)); + assert!(!is_valid_pair(SummaryKind::Cms, StatisticClass::Quantile)); + assert!(!is_valid_pair( + SummaryKind::Cms, + StatisticClass::Cardinality + )); assert!(!is_valid_pair( - SketchKind::Cms, + SummaryKind::Cms, StatisticClass::SumRateCount )); } @@ -306,7 +315,7 @@ mod tests { // TopK-shaped workload (CMS-Heap pattern, Cormode & // Muthukrishnan 2005). assert!( - is_valid_pair(SketchKind::Cms, StatisticClass::TopK), + is_valid_pair(SummaryKind::Cms, StatisticClass::TopK), "CMS should support TopK via the CMS-Heap pattern", ); } @@ -317,7 +326,7 @@ mod tests { fn pick_family_quantile_relative_picks_ddsketch() { assert_eq!( pick_family(StatisticClass::Quantile, AccuracyPreference::RelativeError), - Some(SketchKind::DDSketch), + Some(SummaryKind::DDSketch), ); } @@ -325,7 +334,7 @@ mod tests { fn pick_family_quantile_rank_picks_kll() { assert_eq!( pick_family(StatisticClass::Quantile, AccuracyPreference::RankError), - Some(SketchKind::Kll), + Some(SummaryKind::Kll), ); } @@ -337,7 +346,7 @@ mod tests { ] { assert_eq!( pick_family(StatisticClass::Cardinality, pref), - Some(SketchKind::Hll), + Some(SummaryKind::Hll), ); } } @@ -346,7 +355,7 @@ mod tests { fn pick_family_topk_picks_countsketch() { assert_eq!( pick_family(StatisticClass::TopK, AccuracyPreference::default()), - Some(SketchKind::CountSketch), + Some(SummaryKind::CountSketch), ); } @@ -354,7 +363,7 @@ mod tests { fn pick_family_frequency_picks_cms() { assert_eq!( pick_family(StatisticClass::Frequency, AccuracyPreference::default()), - Some(SketchKind::Cms), + Some(SummaryKind::Cms), ); } @@ -420,17 +429,17 @@ mod tests { assert_eq!(classify_demo_metric(""), None); } - // ── End-to-end: every contract row maps to its expected SketchKind ──────── + // ── End-to-end: every contract row maps to its expected SummaryKind ──────── #[test] fn every_contract_metric_picks_its_contract_family() { let cases = [ ("http_requests_total", None), - ("http_latency_ms", Some(SketchKind::DDSketch)), - ("request_size_bytes", Some(SketchKind::Kll)), - ("unique_users_per_min", Some(SketchKind::Hll)), - ("top_endpoint_qps", Some(SketchKind::CountSketch)), - ("endpoint_request_freq", Some(SketchKind::Cms)), + ("http_latency_ms", Some(SummaryKind::DDSketch)), + ("request_size_bytes", Some(SummaryKind::Kll)), + ("unique_users_per_min", Some(SummaryKind::Hll)), + ("top_endpoint_qps", Some(SummaryKind::CountSketch)), + ("endpoint_request_freq", Some(SummaryKind::Cms)), ]; for (metric, want_kind) in cases { let (stat, pref) = classify_demo_metric(metric) diff --git a/control_plane/src/sketch_algebra/mod.rs b/control_plane/src/sketch_algebra/mod.rs index 957c14d0..88b13db9 100644 --- a/control_plane/src/sketch_algebra/mod.rs +++ b/control_plane/src/sketch_algebra/mod.rs @@ -6,9 +6,11 @@ //! deployment-independent). Single-rooted per query; multi-root //! workload-level fan-in lives one layer up in //! `types_v2::WorkloadPlan`. -//! - [`SketchKind`] / [`SketchParams`] — typed sketch-family selector + -//! parameter payload. Convertible to the legacy `crate::types` -//! shape via [`SketchParams::to_legacy`] for the L5 emitter side. +//! - `asap_sketch::SummaryKind` / `SummaryParams` — typed sketch-family +//! selector + parameter payload (moved out of this crate — formerly +//! `sketch_params::SketchKind`/`SketchParams` — Stage 3 of the +//! sketch-identity unification; see +//! `scratchpad/artifacts/enum-unification-plan.md`). //! - [`bind_query_expr`] — the L3→L4 lowering driver: bottom-up walk //! that fires `Bind*` rules. //! - [`rules`] — the `Bind*` rule family. Each rule pattern-matches on a @@ -34,13 +36,6 @@ pub mod lower; pub mod matcher; pub mod physical_expr; pub mod rules; -pub mod sketch_params; - -// Back-compat alias. External call sites that imported -// `control_plane::sketch_algebra::params::*` (and the in-tree -// `crate::sketch_algebra::params::SketchKind` use sites that this -// touch-up didn't migrate) keep compiling. -pub use sketch_params as params; #[cfg(test)] mod tests; @@ -53,6 +48,3 @@ pub use capability_matching::{ pub use lower::{bind_query_expr, BindingError}; pub use matcher::SummaryFamilyMatcher; pub use physical_expr::{EstimateOp, MergeAlgebra, PhysicalExpr}; -pub use sketch_params::{ - CmsParams, CountSketchParams, DDSketchParams, HllParams, KllParams, SketchKind, SketchParams, -}; diff --git a/control_plane/src/sketch_algebra/physical_expr.rs b/control_plane/src/sketch_algebra/physical_expr.rs index 90a24b03..c80eadb9 100644 --- a/control_plane/src/sketch_algebra/physical_expr.rs +++ b/control_plane/src/sketch_algebra/physical_expr.rs @@ -17,10 +17,10 @@ #![allow(dead_code)] +use asap_sketch::{SummaryKind, SummaryParams}; use serde::{Deserialize, Serialize}; use crate::intent_algebra::QueryExpr; -use crate::sketch_algebra::params::{SketchKind, SketchParams}; use crate::types_v2::BindingName; use promql_utilities::query_logics::enums::AggregationType; @@ -59,13 +59,18 @@ pub enum MergeAlgebra { /// L4 algebra node. See module doc for the variant subset rationale. /// -/// Serde tag is `"sketch_node"` (not `"node"`) so it doesn't collide with -/// the L3 `QueryExpr`'s `"node"` tag — `PhysicalExpr::Logical(QueryExpr)` -/// nests a JSON-tagged enum inside an internally-tagged outer enum, and -/// reusing the same tag would surface as a `duplicate field "node"` -/// deserialization error. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(tag = "sketch_node", rename_all = "snake_case")] +/// Not `Serialize`/`Deserialize` — `asap_sketch::SummaryKind`/`SummaryParams` +/// (carried by `SketchAgg`/`RawAtEdgeSketchAtBackend`) have no serde impl +/// (nothing in ASAPController needs one; see +/// `scratchpad/artifacts/enum-unification-plan.md` §4 — the old +/// `sketch_algebra::SketchKind`/`SketchParams` this replaces derived +/// `Serialize`/`Deserialize` too, but nothing on the real emit path ever +/// exercised it — every actual wire payload goes through a hand-written +/// JSON/YAML builder, never a whole-struct serialize). Previously tagged +/// `"sketch_node"` (not `"node"`, to avoid colliding with the L3 +/// `QueryExpr`'s `"node"` tag when nested via `PhysicalExpr::Logical`); +/// that tag is dropped along with the derive. +#[derive(Debug, Clone, PartialEq)] pub enum PhysicalExpr { /// Logical pass-through: an L3 node that no L4 rule rewrote. A /// `Filter`, a row-shaped `Aggregate{Sum}`, or any other operator @@ -78,9 +83,9 @@ pub enum PhysicalExpr { /// committed `(sketch_type, params)` pair. SketchAgg { /// Sketch family (KLL / DDSketch / HLL / CMS / CountSketch). - sketch_type: SketchKind, + sketch_type: SummaryKind, /// Sketch parameters (validated by the catalog at bind time). - params: SketchParams, + params: SummaryParams, /// Input sub-tree — typically `Logical(Window{...})` or /// `Logical(Scan{...})`. child: Box, @@ -142,9 +147,9 @@ pub enum PhysicalExpr { /// this metric (Phase ε.2 implements the raw-input ingest path). RawAtEdgeSketchAtBackend { /// Sketch family the backend will build at ingest. - family: SketchKind, + family: SummaryKind, /// Sketch parameters (validated by the catalog at bind time). - params: SketchParams, + params: SummaryParams, /// Input sub-tree — typically `Logical(Window{...})` or /// `Logical(Scan{...})`. Mirrors `SketchAgg`'s child field so the /// L5 emitter's walk uniform. @@ -187,7 +192,7 @@ pub enum PhysicalExpr { /// /// Distinct from `SketchAgg` + `SketchEstimate` in two ways: /// 1. No `EstimateOp` wrapper. ExactAgg is its own answer. - /// 2. No `SketchParams`. The `AggregationType` enum captures the + /// 2. No `SummaryParams`. The `AggregationType` enum captures the /// parameterization (DDSketch's α etc. don't apply — exact /// aggregations are parameter-free up to the accumulator /// family choice). @@ -215,8 +220,8 @@ impl PhysicalExpr { /// `Bind*` rule. Keeps rule call sites short. pub fn estimate_over_agg( op: EstimateOp, - sketch_type: SketchKind, - params: SketchParams, + sketch_type: SummaryKind, + params: SummaryParams, logical: QueryExpr, ) -> Self { PhysicalExpr::SketchEstimate { @@ -246,9 +251,7 @@ impl PhysicalExpr { mod tests { use super::*; use crate::intent_algebra::schema::{Column, DataType}; - use crate::intent_algebra::{AggIntent, LabelFilter, QueryExpr, Schema, Source, WindowKind}; - use crate::sketch_algebra::params::{CountSketchParams, DDSketchParams, HllParams, KllParams}; - use crate::types_v2::AccuracyTarget; + use crate::intent_algebra::{LabelFilter, QueryExpr, Schema, Source, WindowKind}; use std::time::Duration; fn ts_scan() -> QueryExpr { @@ -297,8 +300,8 @@ mod tests { fn estimate_over_agg_ctor_shape() { let e = PhysicalExpr::estimate_over_agg( EstimateOp::Quantile { q: 0.99 }, - SketchKind::Kll, - SketchParams::Kll(KllParams { k: 200 }), + SummaryKind::Kll, + SummaryParams::Kll { k: 200 }, windowed_scan(), ); match e { @@ -310,8 +313,8 @@ mod tests { params, child, } => { - assert_eq!(sketch_type, SketchKind::Kll); - assert_eq!(params, SketchParams::Kll(KllParams { k: 200 })); + assert_eq!(sketch_type, SummaryKind::Kll); + assert_eq!(params, SummaryParams::Kll { k: 200 }); assert!(matches!(*child, PhysicalExpr::Logical(_))); } other => panic!("expected SketchAgg, got {other:?}"), @@ -320,92 +323,4 @@ mod tests { other => panic!("expected SketchEstimate, got {other:?}"), } } - - fn agg_quantile() -> QueryExpr { - QueryExpr::Aggregate { - by: vec![], - aggs: vec![AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::Epsilon(0.01), - }], - having: None, - child: Box::new(windowed_scan()), - } - } - - #[test] - fn physical_expr_serde_roundtrip_logical() { - let e = PhysicalExpr::Logical(agg_quantile()); - let json = serde_json::to_string(&e).unwrap(); - let back: PhysicalExpr = serde_json::from_str(&json).unwrap(); - assert_eq!(e, back); - } - - #[test] - fn physical_expr_serde_roundtrip_sketch_agg() { - let e = PhysicalExpr::SketchAgg { - sketch_type: SketchKind::Kll, - params: SketchParams::Kll(KllParams { k: 200 }), - child: Box::new(PhysicalExpr::Logical(windowed_scan())), - }; - let json = serde_json::to_string(&e).unwrap(); - let back: PhysicalExpr = serde_json::from_str(&json).unwrap(); - assert_eq!(e, back); - } - - #[test] - fn physical_expr_serde_roundtrip_estimate() { - let e = PhysicalExpr::estimate_over_agg( - EstimateOp::Quantile { q: 0.95 }, - SketchKind::DDSketch, - SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), - windowed_scan(), - ); - let json = serde_json::to_string(&e).unwrap(); - let back: PhysicalExpr = serde_json::from_str(&json).unwrap(); - assert_eq!(e, back); - } - - #[test] - fn physical_expr_serde_roundtrip_merge() { - let leaf = PhysicalExpr::SketchAgg { - sketch_type: SketchKind::Hll, - params: SketchParams::Hll(HllParams { precision: 14 }), - child: Box::new(PhysicalExpr::Logical(windowed_scan())), - }; - let e = PhysicalExpr::SketchMerge { - algebra: MergeAlgebra::Union, - children: vec![leaf.clone(), leaf], - }; - let json = serde_json::to_string(&e).unwrap(); - let back: PhysicalExpr = serde_json::from_str(&json).unwrap(); - assert_eq!(e, back); - } - - #[test] - fn physical_expr_serde_roundtrip_let_ref() { - let inner_agg = PhysicalExpr::SketchAgg { - sketch_type: SketchKind::CountSketch, - params: SketchParams::CountSketch(CountSketchParams { - w: 2048, - d: 5, - with_heap: true, - }), - child: Box::new(PhysicalExpr::Logical(windowed_scan())), - }; - let e = PhysicalExpr::LetBinding { - name: BindingName::new("kll_state"), - expr: Box::new(inner_agg), - child: Box::new(PhysicalExpr::SketchEstimate { - op: EstimateOp::TopK { k: 10 }, - child: Box::new(PhysicalExpr::Ref { - name: BindingName::new("kll_state"), - }), - }), - }; - let json = serde_json::to_string(&e).unwrap(); - let back: PhysicalExpr = serde_json::from_str(&json).unwrap(); - assert_eq!(e, back); - } } diff --git a/control_plane/src/sketch_algebra/rules/bind_cms_count.rs b/control_plane/src/sketch_algebra/rules/bind_cms_count.rs index 9cac62e7..030574bd 100644 --- a/control_plane/src/sketch_algebra/rules/bind_cms_count.rs +++ b/control_plane/src/sketch_algebra/rules/bind_cms_count.rs @@ -21,10 +21,10 @@ #![allow(dead_code)] use crate::intent_algebra::{AggIntent, QueryExpr}; -use crate::sketch_algebra::params::{CmsParams, SketchKind, SketchParams}; use crate::sketch_algebra::physical_expr::{EstimateOp, PhysicalExpr}; use crate::sketch_algebra::rules::Rule; use crate::types_v2::AccuracyTarget; +use asap_sketch::{SummaryKind, SummaryParams}; /// Bind `Aggregate{Count}` / `Aggregate{Frequency}` to CMS. pub struct BindCmsOnCount; @@ -108,16 +108,14 @@ impl Rule for BindCmsOnCount { Some(PhysicalExpr::estimate_over_agg( readout, - SketchKind::Cms, + SummaryKind::Cms, // Heap-LESS CMS — `BindCmsOnCount` is the // BindCmsOnCount path (frequency / count without TopK). // The CMS-with-heap binding fires from - // `bind_cms_with_heap_on_topk` and sets `with_heap: true`. - SketchParams::Cms(CmsParams { - w, - d, - with_heap: false, - }), + // `bind_cms_with_heap_on_topk` and binds `SummaryKind::CmsWithHeap` + // instead (heap-bearing is a distinct kind in `SummaryKind`, + // not a param flag). + SummaryParams::Cms { width: w, depth: d }, (**child).clone(), )) } diff --git a/control_plane/src/sketch_algebra/rules/bind_cms_topk.rs b/control_plane/src/sketch_algebra/rules/bind_cms_topk.rs index bdef52c6..59935ded 100644 --- a/control_plane/src/sketch_algebra/rules/bind_cms_topk.rs +++ b/control_plane/src/sketch_algebra/rules/bind_cms_topk.rs @@ -66,10 +66,10 @@ use crate::intent_algebra::{AggIntent, QueryExpr}; use crate::optimizer::cost::wire::WireCostTable; -use crate::sketch_algebra::params::{CmsParams, CountSketchParams, SketchKind, SketchParams}; use crate::sketch_algebra::physical_expr::{EstimateOp, PhysicalExpr}; use crate::sketch_algebra::rules::Rule; use crate::types_v2::AccuracyTarget; +use asap_sketch::{SummaryKind, SummaryParams}; /// Recall tier for a top-k binding — drives the family pick. /// @@ -117,10 +117,10 @@ impl BindCountSketchOnTopK { /// recall bar, so the cost model is free to pick the cheaper one. /// * `Tight`: only CountSketch-heap (unbiased / signed / exact-rank) /// clears the bar. - fn candidate_families(tier: TopkRecallTier) -> &'static [SketchKind] { + fn candidate_families(tier: TopkRecallTier) -> &'static [SummaryKind] { match tier { - TopkRecallTier::Loose => &[SketchKind::Cms, SketchKind::CountSketch], - TopkRecallTier::Tight => &[SketchKind::CountSketch], + TopkRecallTier::Loose => &[SummaryKind::Cms, SummaryKind::CountSketch], + TopkRecallTier::Tight => &[SummaryKind::CountSketch], } } @@ -128,14 +128,14 @@ impl BindCountSketchOnTopK { /// `candidates` is already filtered to the families that meet the /// recall SLA (see [`Self::candidate_families`]); this is the /// "min cost s.t. SLA" tie-break the oracle uses. - fn cheapest_family(candidates: &[SketchKind], table: &WireCostTable) -> SketchKind { + fn cheapest_family(candidates: &[SummaryKind], table: &WireCostTable) -> SummaryKind { candidates .iter() .min_by_key(|k| table.for_kind(k).per_flush()) .cloned() - // (above: k is &&SketchKind; for_kind autoderefs to &SketchKind) + // (above: k is &&SummaryKind; for_kind autoderefs to &SummaryKind) // candidate_families never returns empty. - .unwrap_or(SketchKind::CountSketch) + .unwrap_or(SummaryKind::CountSketch) } /// Bind a top-k under an explicit recall tier — bypasses the @@ -252,27 +252,38 @@ impl BindCountSketchOnTopK { let table = WireCostTable::default(); let family = Self::cheapest_family(Self::candidate_families(tier), &table); + // `heap_size` — `SummaryKind::CmsWithHeap`/`CountSketchWithHeap` + // promotes the heap from a `SketchParams::{Cms,CountSketch}.with_heap` + // bool flag to a first-class identity variant carrying the real + // heap size. `k_topk` is the correct value (this is the requested + // top-k `k`); note the backend wire builder + // (`emit::stage_config::sketch_params_to_json`) doesn't read this + // field today regardless (a pre-existing gap, out of scope for + // this rename), so this is strictly more correct in-memory data + // with no observable wire-behavior change. + let heap_size = k_topk as u32; + let (kind, params) = match family { - SketchKind::Cms => ( - SketchKind::Cms, + SummaryKind::Cms => ( + SummaryKind::CmsWithHeap, // CMS-Heap pattern: pair the CMS matrix with a size-k // heavy-hitter heap. The streaming-config emit promotes // this to `CountMinSketchWithHeap` (servable as // FrequencyTopk per `asap_tier_analysis`). - SketchParams::Cms(CmsParams { - w, - d, - with_heap: true, - }), + SummaryParams::CmsWithHeap { + width: w, + depth: d, + heap_size, + }, ), // Tight tier (and any future family) → CountSketch-with-heap. _ => ( - SketchKind::CountSketch, - SketchParams::CountSketch(CountSketchParams { - w, - d, - with_heap: true, - }), + SummaryKind::CountSketchWithHeap, + SummaryParams::CountSketchWithHeap { + width: w, + depth: d, + heap_size, + }, ), }; diff --git a/control_plane/src/sketch_algebra/rules/bind_ddsketch_quantile.rs b/control_plane/src/sketch_algebra/rules/bind_ddsketch_quantile.rs index 8773de9d..17b687cf 100644 --- a/control_plane/src/sketch_algebra/rules/bind_ddsketch_quantile.rs +++ b/control_plane/src/sketch_algebra/rules/bind_ddsketch_quantile.rs @@ -21,10 +21,10 @@ #![allow(dead_code)] use crate::intent_algebra::{AggIntent, QueryExpr}; -use crate::sketch_algebra::params::{DDSketchParams, SketchKind, SketchParams}; use crate::sketch_algebra::physical_expr::{EstimateOp, PhysicalExpr}; use crate::sketch_algebra::rules::Rule; use crate::types_v2::AccuracyTarget; +use asap_sketch::{SummaryKind, SummaryParams}; /// Bind a single-intent `Aggregate{Quantile{q, accuracy}}` to DDSketch. pub struct BindDDSketchOnQuantile; @@ -78,8 +78,8 @@ impl Rule for BindDDSketchOnQuantile { Some(PhysicalExpr::estimate_over_agg( EstimateOp::Quantile { q }, - SketchKind::DDSketch, - SketchParams::DDSketch(DDSketchParams { alpha }), + SummaryKind::DDSketch, + SummaryParams::DDSketch { alpha }, (**child).clone(), )) } diff --git a/control_plane/src/sketch_algebra/rules/bind_hll_cardinality.rs b/control_plane/src/sketch_algebra/rules/bind_hll_cardinality.rs index 6fe4b700..530827c1 100644 --- a/control_plane/src/sketch_algebra/rules/bind_hll_cardinality.rs +++ b/control_plane/src/sketch_algebra/rules/bind_hll_cardinality.rs @@ -14,8 +14,9 @@ #![allow(dead_code)] +use asap_sketch::{SummaryKind, SummaryParams}; + use crate::intent_algebra::{AggIntent, QueryExpr}; -use crate::sketch_algebra::params::{HllParams, SketchKind, SketchParams}; use crate::sketch_algebra::physical_expr::{EstimateOp, PhysicalExpr}; use crate::sketch_algebra::rules::Rule; use crate::types_v2::AccuracyTarget; @@ -64,8 +65,8 @@ impl Rule for BindHllOnCardinality { Some(PhysicalExpr::estimate_over_agg( EstimateOp::Cardinality, - SketchKind::Hll, - SketchParams::Hll(HllParams { precision }), + SummaryKind::Hll, + SummaryParams::Hll { precision }, (**child).clone(), )) } @@ -74,7 +75,9 @@ impl Rule for BindHllOnCardinality { /// Map an ε standard-error budget to the HLL `precision` (log2 register /// count). Mirrors the in-tree default rungs in `algebra::directory` — /// precision 10 (ε≈3.25%) / 12 (ε≈1.6%) / 14 (ε≈0.81%) / 16 (ε≈0.41%). -fn hll_precision_for_eps(eps: f64) -> u32 { +/// `u8` to match `asap_sketch::SummaryParams::Hll`'s field type — every +/// rung here is well within range. +fn hll_precision_for_eps(eps: f64) -> u8 { if eps <= 0.0 { return 16; } diff --git a/control_plane/src/sketch_algebra/rules/bind_kll_quantile.rs b/control_plane/src/sketch_algebra/rules/bind_kll_quantile.rs index 209275c9..664b9d78 100644 --- a/control_plane/src/sketch_algebra/rules/bind_kll_quantile.rs +++ b/control_plane/src/sketch_algebra/rules/bind_kll_quantile.rs @@ -16,10 +16,10 @@ #![allow(dead_code)] use crate::intent_algebra::{AggIntent, QueryExpr}; -use crate::sketch_algebra::params::{KllParams, SketchKind, SketchParams}; use crate::sketch_algebra::physical_expr::{EstimateOp, PhysicalExpr}; use crate::sketch_algebra::rules::Rule; use crate::types_v2::AccuracyTarget; +use asap_sketch::{SummaryKind, SummaryParams}; /// Bind a single-intent `Aggregate{Quantile{q, accuracy}}` to KLL. pub struct BindKllOnQuantile; @@ -81,8 +81,8 @@ impl Rule for BindKllOnQuantile { Some(PhysicalExpr::estimate_over_agg( EstimateOp::Quantile { q }, - SketchKind::Kll, - SketchParams::Kll(KllParams { k }), + SummaryKind::Kll, + SummaryParams::Kll { k }, (**child).clone(), )) } diff --git a/control_plane/src/sketch_algebra/sketch_params.rs b/control_plane/src/sketch_algebra/sketch_params.rs deleted file mode 100644 index c8b17992..00000000 --- a/control_plane/src/sketch_algebra/sketch_params.rs +++ /dev/null @@ -1,317 +0,0 @@ -//! Layer 4 sketch parameters — typed per family. -//! -//! Mirrors `control_plane/docs/design.md` §6 `core::sketch_algebra` (around -//! line ~580). The L3 IR carries an `AggIntent` + `AccuracyTarget`; L4 -//! `Bind*` rules read those, consult the catalog, and emit a typed -//! [`SketchParams`] payload alongside the chosen [`SketchType`]. -//! -//! Why a parallel typed enum rather than reusing `crate::types::SketchParams`? -//! The legacy `SketchParams` (in `crate::types`) is shaped for the wire -//! format the OTel-collector consumes: it carries pre-baked `quantiles` -//! grids, a `metric_name` field for the CMS partition operator, etc. — -//! all of which are L5 emitter concerns. L4 needs only the parameters -//! that affect cost / accuracy: the KLL sketch's `k`, DDSketch's `alpha`, -//! HLL's `precision`, CMS's `(w, d)`. Keeping the L4 parameter shape -//! minimal makes the `Bind*` rule signatures narrow and the typed -//! `PhysicalExpr` IR independent of wire-format drift. -//! -//! Convertibility — see [`SketchParams::to_legacy`]. The legacy form is -//! what the existing `algebra::directory::build_sketch_params` produces; -//! this `to_legacy` adapter is what lets the typed-path rewrite of -//! `planner::rules` (Phase E) land additively. - -#![allow(dead_code)] - -use serde::{Deserialize, Serialize}; - -use crate::types::{SketchDefaults, SketchParams as LegacySketchParams, SketchType}; - -/// L4 sketch family selector. Mirrors `crate::types::SketchType` but with -/// the variants L4 sketch-binding rules emit. Naming kept distinct from -/// the legacy enum so the typed path is unambiguous in error messages / -/// diagnostics. -/// -/// The values are 1:1 convertible to the legacy `SketchType` via -/// [`SketchType::from`]; all rule outputs are round-trippable. -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum SketchKind { - /// Karnin-Lang-Liberty quantile sketch. Mergeable, exact rank-error - /// bound. Picked for `Quantile` intents when relative-error budget is - /// not specified or when the quantile target is in the tail. - Kll, - /// DDSketch — log-bucketed quantile sketch with a relative-error - /// guarantee. Picked for `Quantile` intents when an explicit ε is - /// supplied and tail-relative error matters more than rank-error. - DDSketch, - /// HyperLogLog — approximate cardinality. Picked for `Cardinality`. - Hll, - /// Count-Min sketch. Picked for `Count` and `Frequency` intents when - /// approximation is allowed. - Cms, - /// Count-Sketch (with optional heavy-hitter heap). Picked for `TopK` - /// when paired with a Misra-Gries / heap-of-counters extractor; also - /// the substrate for general `Frequency` sketching when balanced - /// (zero-mean) error is preferable to CMS's one-sided bias. - CountSketch, -} - -impl From for SketchType { - fn from(k: SketchKind) -> Self { - match k { - SketchKind::Kll => SketchType::KLL, - SketchKind::DDSketch => SketchType::DDSketch, - SketchKind::Hll => SketchType::HLL, - SketchKind::Cms => SketchType::CountMinSketch, - SketchKind::CountSketch => SketchType::CountSketch, - } - } -} - -/// Inverse of `From for SketchType`. Round-trippable: -/// `SketchKind::from(SketchType::from(k)) == k` for every variant. Used -/// by `planner::rules::bind_workload_typed` to translate the legacy -/// `QueryWorkload::sketch_type_override` field into the typed `SketchKind` -/// the capability matrix consumes. -impl From for SketchKind { - fn from(t: SketchType) -> Self { - match t { - SketchType::KLL => SketchKind::Kll, - SketchType::DDSketch => SketchKind::DDSketch, - SketchType::HLL => SketchKind::Hll, - SketchType::CountMinSketch => SketchKind::Cms, - SketchType::CountSketch => SketchKind::CountSketch, - } - } -} - -/// L4 sketch parameters per family. Parameter names match the canonical -/// sketch literature: `k` for KLL stream size, `alpha` for DDSketch -/// relative-accuracy bound, `precision` for HLL register width, `(w, d)` -/// for CMS / CountSketch dimensions. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(tag = "family", rename_all = "snake_case")] -pub enum SketchParams { - /// KLL stream size. `k=200` ≈ ε≈0.01 rank error, `k=2048` ≈ ε≈0.0035 - /// — see `accuracy_profile.rs` (ASAPQuery-backend) for the formal - /// rank-error bound. - Kll(KllParams), - /// DDSketch relative-accuracy. `alpha=0.01` is the catalog default - /// for `Epsilon(0.01)` quantile budgets. - DDSketch(DDSketchParams), - /// HLL register width. `precision=14` ≈ ε≈0.81%/√m; `precision=10` is - /// the coarse default. - Hll(HllParams), - /// Count-Min: `w` columns × `d` rows. Error bound: ε ≤ e/w with prob. - /// 1 − 2^(−d) (Cormode-Muthukrishnan). - Cms(CmsParams), - /// Count-Sketch: `w` columns × `d` rows; balanced (zero-mean) error. - /// When `with_heap` is set, pairs with a heavy-hitter heap to extract - /// `TopK`. - CountSketch(CountSketchParams), -} - -/// KLL parameters. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub struct KllParams { - /// Stream-size parameter. Higher k → tighter rank-error / more memory. - pub k: u32, -} - -/// DDSketch parameters. -#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] -pub struct DDSketchParams { - /// Relative-error bound — DDSketch guarantees |estimate − true| ≤ - /// alpha · true on every quantile. - pub alpha: f64, -} - -/// HLL parameters. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub struct HllParams { - /// log2 of the register count. `precision=p` → 2^p registers, error - /// ≈ 1.04 / √(2^p). - pub precision: u32, -} - -/// Count-Min parameters. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub struct CmsParams { - /// Number of columns (width). Drives the additive ε bound (≤ e/w). - pub w: u32, - /// Number of rows (depth). Drives the failure probability (≤ 2^−d). - pub d: u32, - /// Whether to pair the CMS matrix with a heavy-hitter heap (CMS-Heap - /// pattern from Cormode & Muthukrishnan 2005). Set by - /// `bind_cms_with_heap_on_topk` when the planner picks CMS for a - /// TopK statistic. The streaming-config emit consults this flag to - /// pick `CountMinSketchWithHeap` vs `CountMinSketch` for the - /// `aggregationType` string the backend's `policy_capability` keys - /// on. - #[serde(default)] - pub with_heap: bool, -} - -/// Count-Sketch parameters. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub struct CountSketchParams { - /// Number of columns. - pub w: u32, - /// Number of rows. - pub d: u32, - /// Whether to pair with a heavy-hitter heap (for TopK extraction). - pub with_heap: bool, -} - -impl SketchParams { - /// The matching [`SketchKind`] for this parameter payload. - pub fn kind(&self) -> SketchKind { - match self { - SketchParams::Kll(_) => SketchKind::Kll, - SketchParams::DDSketch(_) => SketchKind::DDSketch, - SketchParams::Hll(_) => SketchKind::Hll, - SketchParams::Cms(_) => SketchKind::Cms, - SketchParams::CountSketch(_) => SketchKind::CountSketch, - } - } - - /// Convert to the legacy wire-shaped `crate::types::SketchParams`. - /// `quantile_grid` is read from defaults — Phase C does not yet plumb - /// query-specific quantile grids into the typed path; that's an L5 - /// emitter concern picked up in Phase E (stage_split + emitter - /// migration). - pub fn to_legacy(&self, defaults: &SketchDefaults) -> LegacySketchParams { - match self { - SketchParams::Kll(p) => LegacySketchParams::KLL { - k: p.k, - quantiles: defaults.quantile_grid.clone(), - }, - SketchParams::DDSketch(p) => LegacySketchParams::DDSketch { - relative_accuracy: p.alpha, - quantiles: defaults.quantile_grid.clone(), - }, - SketchParams::Hll(p) => LegacySketchParams::HLL { - precision: p.precision, - }, - SketchParams::Cms(p) => LegacySketchParams::CountMinSketch { - rows: p.d, - cols: p.w, - metric_name: defaults.count_min_sketch.metric_name.clone(), - }, - SketchParams::CountSketch(p) => LegacySketchParams::CountSketch { - // Translate (w, d) back into the legacy (epsilon, delta) - // surface — `epsilon ≈ e/w`, `delta ≈ 2^−d`. The legacy - // CountSketch processor expects these directly. - epsilon: std::f64::consts::E / (p.w as f64), - delta: 2f64.powi(-(p.d as i32)), - }, - } - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn sketch_kind_to_legacy_roundtrip() { - let cases = [ - (SketchKind::Kll, SketchType::KLL), - (SketchKind::DDSketch, SketchType::DDSketch), - (SketchKind::Hll, SketchType::HLL), - (SketchKind::Cms, SketchType::CountMinSketch), - (SketchKind::CountSketch, SketchType::CountSketch), - ]; - for (k, expected) in cases { - let legacy: SketchType = k.into(); - assert_eq!(legacy, expected); - } - } - - #[test] - fn sketch_params_kind_matches() { - assert_eq!( - SketchParams::Kll(KllParams { k: 200 }).kind(), - SketchKind::Kll - ); - assert_eq!( - SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }).kind(), - SketchKind::DDSketch - ); - assert_eq!( - SketchParams::Hll(HllParams { precision: 14 }).kind(), - SketchKind::Hll - ); - assert_eq!( - SketchParams::Cms(CmsParams { - w: 2048, - d: 5, - with_heap: false - }) - .kind(), - SketchKind::Cms - ); - assert_eq!( - SketchParams::CountSketch(CountSketchParams { - w: 2048, - d: 5, - with_heap: true, - }) - .kind(), - SketchKind::CountSketch - ); - } - - #[test] - fn to_legacy_kll_carries_k_and_grid() { - let defaults = SketchDefaults::default(); - let p = SketchParams::Kll(KllParams { k: 200 }).to_legacy(&defaults); - match p { - LegacySketchParams::KLL { k, quantiles } => { - assert_eq!(k, 200); - assert_eq!(quantiles, defaults.quantile_grid); - } - other => panic!("expected KLL legacy, got {other:?}"), - } - } - - #[test] - fn to_legacy_ddsketch_carries_alpha() { - let defaults = SketchDefaults::default(); - let p = SketchParams::DDSketch(DDSketchParams { alpha: 0.005 }).to_legacy(&defaults); - match p { - LegacySketchParams::DDSketch { - relative_accuracy, .. - } => { - assert!((relative_accuracy - 0.005).abs() < 1e-12); - } - other => panic!("expected DDSketch legacy, got {other:?}"), - } - } - - #[test] - fn params_serde_roundtrip() { - let cases = [ - SketchParams::Kll(KllParams { k: 200 }), - SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), - SketchParams::Hll(HllParams { precision: 14 }), - SketchParams::Cms(CmsParams { - w: 2048, - d: 5, - with_heap: false, - }), - SketchParams::CountSketch(CountSketchParams { - w: 2048, - d: 5, - with_heap: true, - }), - ]; - for c in cases { - let json = serde_json::to_string(&c).unwrap(); - let back: SketchParams = serde_json::from_str(&json).unwrap(); - assert_eq!(c, back); - } - } -} diff --git a/control_plane/src/sketch_algebra/tests.rs b/control_plane/src/sketch_algebra/tests.rs index 8a50a65f..94856661 100644 --- a/control_plane/src/sketch_algebra/tests.rs +++ b/control_plane/src/sketch_algebra/tests.rs @@ -7,12 +7,12 @@ use std::time::Duration; use crate::intent_algebra::schema::{Column, DataType}; use crate::intent_algebra::{AggIntent, LabelFilter, QueryExpr, Schema, Source, WindowKind}; use crate::sketch_algebra::lower::bind_query_expr; -use crate::sketch_algebra::params::{KllParams, SketchKind, SketchParams}; use crate::sketch_algebra::physical_expr::{EstimateOp, MergeAlgebra, PhysicalExpr}; use crate::sketch_algebra::rules::{ bind_ddsketch_quantile::BindDDSketchOnQuantile, bind_kll_quantile::BindKllOnQuantile, Rule, }; use crate::types_v2::{AccuracyTarget, BindingName}; +use asap_sketch::{SummaryKind, SummaryParams}; // ── Test fixtures ───────────────────────────────────────────────────────────── @@ -67,76 +67,6 @@ fn agg_quantile(q: f64, accuracy: AccuracyTarget) -> QueryExpr { } } -// ── Serde round-trip across all variants ────────────────────────────────────── - -#[test] -fn physical_expr_serde_roundtrip() { - use crate::sketch_algebra::params::{CmsParams, CountSketchParams, DDSketchParams, HllParams}; - let cases = vec![ - PhysicalExpr::Logical(windowed_scan()), - PhysicalExpr::SketchAgg { - sketch_type: SketchKind::Kll, - params: SketchParams::Kll(KllParams { k: 200 }), - child: Box::new(PhysicalExpr::Logical(windowed_scan())), - }, - PhysicalExpr::SketchEstimate { - op: EstimateOp::Quantile { q: 0.5 }, - child: Box::new(PhysicalExpr::SketchAgg { - sketch_type: SketchKind::DDSketch, - params: SketchParams::DDSketch(DDSketchParams { alpha: 0.005 }), - child: Box::new(PhysicalExpr::Logical(windowed_scan())), - }), - }, - PhysicalExpr::SketchMerge { - algebra: MergeAlgebra::Union, - children: vec![ - PhysicalExpr::SketchAgg { - sketch_type: SketchKind::Hll, - params: SketchParams::Hll(HllParams { precision: 14 }), - child: Box::new(PhysicalExpr::Logical(windowed_scan())), - }, - PhysicalExpr::SketchAgg { - sketch_type: SketchKind::Hll, - params: SketchParams::Hll(HllParams { precision: 14 }), - child: Box::new(PhysicalExpr::Logical(windowed_scan())), - }, - ], - }, - PhysicalExpr::LetBinding { - name: BindingName::new("kll_state"), - expr: Box::new(PhysicalExpr::SketchAgg { - sketch_type: SketchKind::CountSketch, - params: SketchParams::CountSketch(CountSketchParams { - w: 2048, - d: 5, - with_heap: false, - }), - child: Box::new(PhysicalExpr::Logical(windowed_scan())), - }), - child: Box::new(PhysicalExpr::Ref { - name: BindingName::new("kll_state"), - }), - }, - PhysicalExpr::Ref { - name: BindingName::new("alone"), - }, - PhysicalExpr::SketchAgg { - sketch_type: SketchKind::Cms, - params: SketchParams::Cms(CmsParams { - w: 2048, - d: 5, - with_heap: false, - }), - child: Box::new(PhysicalExpr::Logical(windowed_scan())), - }, - ]; - for c in cases { - let json = serde_json::to_string(&c).unwrap(); - let back: PhysicalExpr = serde_json::from_str(&json).unwrap(); - assert_eq!(c, back); - } -} - // ── Bind rule tests ─────────────────────────────────────────────────────────── #[test] @@ -156,8 +86,8 @@ fn bind_kll_quantile_basic() { params, child, } => { - assert_eq!(sketch_type, SketchKind::Kll); - assert_eq!(params, SketchParams::Kll(KllParams { k: 200 })); + assert_eq!(sketch_type, SummaryKind::Kll); + assert_eq!(params, SummaryParams::Kll { k: 200 }); assert!(matches!( *child, PhysicalExpr::Logical(QueryExpr::Window { .. }) @@ -185,10 +115,12 @@ fn bind_ddsketch_quantile_basic() { params, .. } => { - assert_eq!(sketch_type, SketchKind::DDSketch); + assert_eq!(sketch_type, SummaryKind::DDSketch); match params { - SketchParams::DDSketch(p) => assert!((p.alpha - 0.01).abs() < 1e-12), - other => panic!("expected DDSketchParams, got {other:?}"), + SummaryParams::DDSketch { alpha } => { + assert!((alpha - 0.01).abs() < 1e-12) + } + other => panic!("expected DDSketch params, got {other:?}"), } } other => panic!("expected SketchAgg, got {other:?}"), @@ -212,7 +144,7 @@ fn bind_picks_ddsketch_over_kll_when_eps_explicit() { PhysicalExpr::SketchAgg { sketch_type, .. } => { assert_eq!( sketch_type, - SketchKind::DDSketch, + SummaryKind::DDSketch, "dispatcher should pick DDSketch (priority 6) over KLL (priority 5) on ε-driven Quantile" ); } @@ -232,9 +164,12 @@ fn agg_topk(k: usize, accuracy: AccuracyTarget) -> QueryExpr { } } -/// Pull the bound `(SketchKind, with_heap, w, d)` out of a top-k binding. -fn topk_binding_family(bound: &PhysicalExpr) -> (SketchKind, bool, u32, u32) { - use crate::sketch_algebra::params::{CmsParams, CountSketchParams}; +/// Pull the bound `(SummaryKind, w, d)` out of a top-k binding. +/// `SummaryKind` (unlike the retired `sketch_algebra::SketchKind`) +/// promotes `with_heap` to kind identity — `bind_cms_topk` always binds +/// `CmsWithHeap`/`CountSketchWithHeap` for a top-k intent, never the +/// bare kind, so there's no separate heap flag to return anymore. +fn topk_binding_family(bound: &PhysicalExpr) -> (SummaryKind, u32, u32) { match bound { PhysicalExpr::SketchEstimate { op, child } => { assert_eq!(*op, EstimateOp::TopK { k: 10 }); @@ -244,13 +179,15 @@ fn topk_binding_family(bound: &PhysicalExpr) -> (SketchKind, bool, u32, u32) { params, .. } => match params { - SketchParams::Cms(CmsParams { w, d, with_heap }) => { - (sketch_type.clone(), *with_heap, *w, *d) + SummaryParams::CmsWithHeap { width, depth, .. } => { + (sketch_type.clone(), *width, *depth) } - SketchParams::CountSketch(CountSketchParams { w, d, with_heap }) => { - (sketch_type.clone(), *with_heap, *w, *d) + SummaryParams::CountSketchWithHeap { width, depth, .. } => { + (sketch_type.clone(), *width, *depth) + } + other => { + panic!("expected CmsWithHeap/CountSketchWithHeap params, got {other:?}") } - other => panic!("expected CMS/CountSketch params, got {other:?}"), }, other => panic!("expected SketchAgg, got {other:?}"), } @@ -270,13 +207,12 @@ fn bind_cms_topk_loose_recall_picks_cms_heap() { }; let expr = agg_topk(10, acc.clone()); let bound = bind_query_expr(&expr, acc).expect("bind_query_expr should not error"); - let (kind, with_heap, w, d) = topk_binding_family(&bound); + let (kind, w, d) = topk_binding_family(&bound); assert_eq!( kind, - SketchKind::Cms, + SummaryKind::CmsWithHeap, "loose-recall top-k must bind the cheap CMS-with-heap, not CountSketch" ); - assert!(with_heap, "top-k binding must enable the heavy-hitter heap"); assert!(w >= 2); assert!(d >= 1); } @@ -290,13 +226,12 @@ fn bind_cms_topk_tight_recall_picks_countsketch() { let expr = agg_topk(10, AccuracyTarget::Exact); let bound = bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)) .expect("bind_query_expr should not error"); - let (kind, with_heap, w, d) = topk_binding_family(&bound); + let (kind, w, d) = topk_binding_family(&bound); assert_eq!( kind, - SketchKind::CountSketch, + SummaryKind::CountSketchWithHeap, "exact-rank top-k must bind the unbiased CountSketch-with-heap" ); - assert!(with_heap, "top-k binding must enable the heavy-hitter heap"); assert!(w >= 2); assert!(d >= 1); } @@ -310,8 +245,8 @@ fn bind_cms_topk_tight_recall_picks_countsketch() { fn bind_cms_topk_picks_cost_min_meeting_sla() { use crate::optimizer::cost::wire::WireCostTable; let table = WireCostTable::default(); - let cms = table.for_kind(&SketchKind::Cms).per_flush(); - let cs = table.for_kind(&SketchKind::CountSketch).per_flush(); + let cms = table.for_kind(&SummaryKind::Cms).per_flush(); + let cs = table.for_kind(&SummaryKind::CountSketch).per_flush(); assert!( cms < cs, "CMS-heap ({cms} B) must be cheaper than CountSketch ({cs} B) on the wire" @@ -333,7 +268,7 @@ fn bind_cms_topk_picks_cost_min_meeting_sla() { cms.min(cs), "must pick the cost-min family that meets the SLA" ); - assert_eq!(kind, SketchKind::Cms); + assert_eq!(kind, SummaryKind::CmsWithHeap); } #[test] @@ -357,11 +292,11 @@ fn bind_hll_cardinality_basic() { params, .. } => { - assert_eq!(sketch_type, SketchKind::Hll); + assert_eq!(sketch_type, SummaryKind::Hll); match params { - SketchParams::Hll(p) => { + SummaryParams::Hll { precision } => { assert!( - p.precision >= 12, + precision >= 12, "ε=0.01 should land on at least precision 12 (~1.6%) per the rung table" ); } @@ -412,49 +347,6 @@ fn bind_exact_accuracy_disables_quantile_binding() { ); } -/// Two `SketchEstimate` parents reading different quantiles can share -/// one underlying `SketchAgg{KLL}` via `LetBinding` / `Ref`. Mirrors the -/// design.md §6 batched-queries example (line ~1326) — within the L4 -/// IR, fan-in is expressible as a `LetBinding` whose bound expression -/// is the shared `SketchAgg`. -#[test] -fn let_binding_ref_through_sketch_dag() { - let shared_agg = PhysicalExpr::SketchAgg { - sketch_type: SketchKind::Kll, - params: SketchParams::Kll(KllParams { k: 200 }), - child: Box::new(PhysicalExpr::Logical(windowed_scan())), - }; - let expr = PhysicalExpr::LetBinding { - name: BindingName::new("kll_state"), - expr: Box::new(shared_agg), - child: Box::new(PhysicalExpr::SketchMerge { - algebra: MergeAlgebra::Union, - // Two `SketchEstimate` parents reading the shared sketch via - // `Ref` — the design.md §6 line ~1339 two-tier fan-in shape. - children: vec![ - PhysicalExpr::SketchEstimate { - op: EstimateOp::Quantile { q: 0.99 }, - child: Box::new(PhysicalExpr::Ref { - name: BindingName::new("kll_state"), - }), - }, - PhysicalExpr::SketchEstimate { - op: EstimateOp::Quantile { q: 0.95 }, - child: Box::new(PhysicalExpr::Ref { - name: BindingName::new("kll_state"), - }), - }, - ], - }), - }; - // Round-trip the DAG through serde to verify the multi-parent fan-in - // shape survives wire encoding (the L4 type checker, when it lands, - // will assert the matching sketch-state schema on each `Ref` reader). - let json = serde_json::to_string(&expr).unwrap(); - let back: PhysicalExpr = serde_json::from_str(&json).unwrap(); - assert_eq!(expr, back); -} - // ── Phase β: pattern-migration coverage ─────────────────────────────────────── // // The five PromQL pattern shapes defined in `asap-planner-rs/src/planner/ @@ -486,7 +378,7 @@ fn phase_b_pattern_only_temporal_quantile_binds_to_sketch() { PhysicalExpr::SketchAgg { sketch_type, .. } => { assert!(matches!( sketch_type, - SketchKind::Kll | SketchKind::DDSketch + SummaryKind::Kll | SummaryKind::DDSketch )); } other => panic!("expected SketchAgg under SketchEstimate, got {other:?}"), @@ -630,9 +522,9 @@ fn pipeline_l1_to_l4(query: &str, accuracy: AccuracyTarget) -> PhysicalExpr { /// Walk a `PhysicalExpr` and collect every `SketchAgg`'s sketch_kind. The /// number of entries + the kind set is the wire-equivalent of /// asap-planner-rs's "aggregation_id rows in StreamingConfig output". -fn collect_sketch_kinds(expr: &PhysicalExpr) -> Vec { +fn collect_sketch_kinds(expr: &PhysicalExpr) -> Vec { let mut out = Vec::new(); - fn walk(e: &PhysicalExpr, out: &mut Vec) { + fn walk(e: &PhysicalExpr, out: &mut Vec) { match e { PhysicalExpr::SketchAgg { sketch_type, child, .. @@ -659,7 +551,7 @@ fn collect_sketch_kinds(expr: &PhysicalExpr) -> Vec { walk(child, out); } PhysicalExpr::RawAtEdgePrometheusArchive { .. } => {} - // ExactAgg has no SketchKind to collect; its child may + // ExactAgg has no SummaryKind to collect; its child may // carry one transitively (rare but possible if nested). PhysicalExpr::ExactAgg { child, .. } => walk(child, out), } @@ -713,7 +605,7 @@ fn phase_b_e2e_quantile_over_time_binds_to_quantile_sketch() { let kinds = collect_sketch_kinds(&bound); assert_eq!(kinds.len(), 1, "expected 1 sketch agg, got {kinds:?}"); assert!( - matches!(kinds[0], SketchKind::Kll | SketchKind::DDSketch), + matches!(kinds[0], SummaryKind::Kll | SummaryKind::DDSketch), "expected quantile sketch family, got {:?}", kinds[0] ); @@ -764,14 +656,17 @@ fn phase_b_e2e_sum_by_preserves_grouping_label() { // ParsedQuery only carries `aggregations: Vec` not the // by-axis directly). In either case the metric name + label survive // somewhere in the L3 sub-tree — assert that. - let json = serde_json::to_string(&bound).unwrap(); + // `PhysicalExpr` is no longer `Serialize` (see its doc) — `Debug` + // output still contains every string literal in the tree, so it + // works just as well for this substring search. + let dbg = format!("{bound:?}"); assert!( - json.contains("http_requests_total"), - "metric name lost through pipeline: {json}" + dbg.contains("http_requests_total"), + "metric name lost through pipeline: {dbg}" ); assert!( - json.contains("instance"), - "by-label `instance` lost through pipeline: {json}" + dbg.contains("instance"), + "by-label `instance` lost through pipeline: {dbg}" ); } diff --git a/control_plane/src/types.rs b/control_plane/src/types.rs index 0f703c63..b7cdc325 100644 --- a/control_plane/src/types.rs +++ b/control_plane/src/types.rs @@ -194,6 +194,44 @@ impl std::fmt::Display for SketchType { } } +// Moved from the retired `sketch_algebra::sketch_params` (Stage 3 of the +// sketch-identity unification — see +// scratchpad/artifacts/enum-unification-plan.md) when `sketch_algebra::SketchKind` +// was replaced by `asap_sketch::SummaryKind`. `SketchType::from(SummaryKind)` +// only covers the 5 canonical families this legacy type has room for — +// callers (`optimizer::engine::load_sketch_capabilities`) only ever feed it +// kinds sourced from `default_capability_table`/`load_capability_overrides`, +// which are exhaustively those 5. +impl From for SketchType { + fn from(k: asap_sketch::SummaryKind) -> Self { + match k { + asap_sketch::SummaryKind::Kll => SketchType::KLL, + asap_sketch::SummaryKind::DDSketch => SketchType::DDSketch, + asap_sketch::SummaryKind::Hll => SketchType::HLL, + asap_sketch::SummaryKind::Cms => SketchType::CountMinSketch, + asap_sketch::SummaryKind::CountSketch => SketchType::CountSketch, + other => unreachable!( + "SketchType::from(SummaryKind): no legacy equivalent for {other:?} \ + (callers only pass the 5 canonical families)" + ), + } + } +} + +/// Inverse of `From for SketchType`. Round-trippable for the 5 +/// canonical families: `SummaryKind::from(SketchType::from(k)) == k`. +impl From for asap_sketch::SummaryKind { + fn from(t: SketchType) -> Self { + match t { + SketchType::KLL => asap_sketch::SummaryKind::Kll, + SketchType::DDSketch => asap_sketch::SummaryKind::DDSketch, + SketchType::HLL => asap_sketch::SummaryKind::Hll, + SketchType::CountMinSketch => asap_sketch::SummaryKind::Cms, + SketchType::CountSketch => asap_sketch::SummaryKind::CountSketch, + } + } +} + #[derive(Debug, Clone, PartialEq)] pub enum OutputMode { Raw,