From df9233fec3fb6e58676d442efc7237c0fb11b5d8 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 11 May 2026 17:09:57 -0600 Subject: [PATCH 1/4] refactor(query-engine): split engine and storage naming --- asap-query-engine/src/data_model/mod.rs | 4 +- .../src/drivers/query/controller_client.rs | 1 - .../src/drivers/query/fallback/metrics.rs | 2 +- .../src/drivers/query/servers/http.rs | 605 ++++++++---------- .../engines/{simple => asap_query}/engine.rs | 189 +++--- .../src/engines/asap_query/mod.rs | 24 + .../engines/{simple => asap_query}/tests.rs | 4 +- asap-query-engine/src/engines/mod.rs | 39 +- .../src/engines/no_data_archive.rs | 19 +- .../src/engines/prometheus/forward.rs | 47 +- .../src/engines/prometheus/mod.rs | 2 +- asap-query-engine/src/engines/simple/mod.rs | 29 - .../forward.rs} | 223 +++---- .../src/engines/thanos_query/mod.rs | 13 + .../src/engines/warm_tier/decoders.rs | 39 +- .../src/engines/warm_tier/delta_apply.rs | 38 +- .../src/engines/warm_tier/mod.rs | 4 +- .../src/engines/warm_tier/sketch_reducer.rs | 114 ++-- .../src/engines/warm_tier/tests.rs | 36 +- asap-query-engine/src/lib.rs | 2 +- asap-query-engine/src/main.rs | 138 ++-- .../src/precompute_engine/worker.rs | 7 +- .../count_min_sketch_accumulator.rs | 18 +- .../edge_runtime_adapter.rs | 52 +- .../src/precompute_operators/mod.rs | 2 +- .../multiple_increase_accumulator.rs | 14 +- .../src/routing/backend_storage_routing.rs | 362 +++++------ .../src/routing/engine_router.rs | 150 +---- .../src/routing/freshness_probe_cache.rs | 16 +- asap-query-engine/src/routing/mod.rs | 8 +- .../gorilla_object_store}/mod.rs | 84 +-- .../gorilla_object_store}/postings.rs | 2 +- .../gorilla_object_store/query_engine.rs} | 29 +- .../gorilla_object_store}/s3_cost.rs | 29 +- .../gorilla_object_store}/store.rs | 65 +- .../gorilla_object_store}/tests.rs | 81 +-- asap-query-engine/src/stores/mod.rs | 6 + .../src/stores/sketch_db/epoch_columnar.rs | 13 +- .../src/stores/sketch_db/sketch_index.rs | 31 +- .../src/tests/capability_matching_tests.rs | 2 +- .../tests/capability_miss_http_e2e_tests.rs | 3 +- .../src/tests/test_utilities/comparison.rs | 2 +- .../tests/test_utilities/engine_factories.rs | 3 +- .../edge_runtime_consumes_precompute_rs.rs | 11 +- .../tests/inference_yaml_pattern_coverage.rs | 36 +- controller/src/accuracy.rs | 5 +- controller/src/backend_client.rs | 21 +- controller/src/emit/agent.rs | 139 ++-- controller/src/emit/backend.rs | 56 +- controller/src/emit/mod.rs | 128 ++-- controller/src/emit/otap.rs | 77 ++- controller/src/emit/stage_config.rs | 85 +-- controller/src/emit/telegraf.rs | 68 +- controller/src/intent_algebra/cse.rs | 5 +- controller/src/intent_algebra/lower.rs | 18 +- controller/src/intent_algebra/schema.rs | 16 +- controller/src/language_logical_plan/lower.rs | 4 +- controller/src/language_logical_plan/tests.rs | 25 +- controller/src/main.rs | 16 +- controller/src/metrics_exposer.rs | 22 +- controller/src/monitor/mod.rs | 167 ++--- controller/src/opamp/mod.rs | 194 +++--- controller/src/optimizer/baseline.rs | 37 +- controller/src/optimizer/cost/delta.rs | 48 +- controller/src/optimizer/cost/mod.rs | 107 ++-- controller/src/optimizer/cost/online.rs | 49 +- controller/src/optimizer/cost/pareto.rs | 214 ++++--- controller/src/optimizer/cost/tco.rs | 30 +- controller/src/optimizer/cost/wire.rs | 28 +- controller/src/optimizer/mod.rs | 4 +- controller/src/optimizer/rules/mod.rs | 94 ++- .../src/physical/colored_dag/allocator.rs | 11 +- controller/src/physical/colored_dag/dag.rs | 7 +- .../src/physical/colored_dag/emitter.rs | 4 +- controller/src/physical/colored_dag/tests.rs | 6 +- controller/src/physical/mod.rs | 2 +- controller/src/physical/plan.rs | 164 +++-- controller/src/pipeline.rs | 313 +++++---- .../query_parser/language/elastic_dsl/mod.rs | 2 +- controller/src/query_parser/language/mod.rs | 9 +- .../src/query_parser/language/promql/ast.rs | 14 +- .../src/query_parser/language/promql/mod.rs | 16 +- .../src/query_parser/language/sql/mod.rs | 2 +- controller/src/query_parser/language/tests.rs | 4 +- controller/src/query_parser/sql.rs | 456 ++++++++----- controller/src/replan.rs | 147 +++-- controller/src/sketch_algebra/capability.rs | 10 +- .../src/sketch_algebra/capability_matching.rs | 69 +- .../sketch_algebra/rules/bind_archive_only.rs | 2 +- .../sketch_algebra/rules/bind_cms_count.rs | 10 +- .../src/sketch_algebra/rules/bind_cms_topk.rs | 10 +- controller/src/sketch_algebra/schema.rs | 30 +- controller/src/sketch_algebra/sketch_expr.rs | 8 +- controller/src/sketch_algebra/tests.rs | 35 +- controller/src/store/mod.rs | 40 +- controller/src/store/workload.rs | 26 +- controller/src/types.rs | 39 +- controller/src/warm_tier_analysis.rs | 5 +- controller/src/workload.rs | 71 +- crates/asap_types/src/capability_matching.rs | 147 +++-- crates/asap_types/src/lib.rs | 4 +- crates/asap_types/src/streaming_config.rs | 10 +- 102 files changed, 3172 insertions(+), 2756 deletions(-) rename asap-query-engine/src/engines/{simple => asap_query}/engine.rs (97%) create mode 100644 asap-query-engine/src/engines/asap_query/mod.rs rename asap-query-engine/src/engines/{simple => asap_query}/tests.rs (77%) delete mode 100644 asap-query-engine/src/engines/simple/mod.rs rename asap-query-engine/src/engines/{gorilla/thanos_forward.rs => thanos_query/forward.rs} (82%) create mode 100644 asap-query-engine/src/engines/thanos_query/mod.rs rename asap-query-engine/src/{engines/gorilla => stores/gorilla_object_store}/mod.rs (83%) rename asap-query-engine/src/{engines/gorilla => stores/gorilla_object_store}/postings.rs (98%) rename asap-query-engine/src/{engines/gorilla/engine.rs => stores/gorilla_object_store/query_engine.rs} (98%) rename asap-query-engine/src/{engines/gorilla => stores/gorilla_object_store}/s3_cost.rs (92%) rename asap-query-engine/src/{engines/gorilla => stores/gorilla_object_store}/store.rs (96%) rename asap-query-engine/src/{engines/gorilla => stores/gorilla_object_store}/tests.rs (94%) diff --git a/asap-query-engine/src/data_model/mod.rs b/asap-query-engine/src/data_model/mod.rs index 3b1e26e4..ebdacf8f 100644 --- a/asap-query-engine/src/data_model/mod.rs +++ b/asap-query-engine/src/data_model/mod.rs @@ -30,6 +30,6 @@ pub use traits::*; // `crate::data_model::BackendStorageRouting` compiling for any // transitive caller that hasn't been migrated yet. pub use crate::routing::{ - classify_query_shape, BackendStorageRouting, HotReloadBackendStorageRouting, - QueryShape, RoutingTarget, + classify_query_shape, BackendStorageRouting, HotReloadBackendStorageRouting, QueryShape, + RoutingTarget, }; diff --git a/asap-query-engine/src/drivers/query/controller_client.rs b/asap-query-engine/src/drivers/query/controller_client.rs index 60368dc4..3d9dcfa9 100644 --- a/asap-query-engine/src/drivers/query/controller_client.rs +++ b/asap-query-engine/src/drivers/query/controller_client.rs @@ -334,5 +334,4 @@ mod tests { assert_eq!(payload["statistics"], serde_json::json!(["Sum"])); assert_eq!(payload["data_range_ms"], 60_000); } - } diff --git a/asap-query-engine/src/drivers/query/fallback/metrics.rs b/asap-query-engine/src/drivers/query/fallback/metrics.rs index 7237601c..27e25aa1 100644 --- a/asap-query-engine/src/drivers/query/fallback/metrics.rs +++ b/asap-query-engine/src/drivers/query/fallback/metrics.rs @@ -9,7 +9,7 @@ //! * **Hot** = the `SimpleEngine` handled the query from live //! sketch-backed state. //! * **Cold** = the query was answered from the Gorilla archive -//! tier ([`crate::engines::gorilla::GorillaQueryEngine`]). +//! tier ([`crate::stores::gorilla_object_store::GorillaQueryEngine`]). //! //! The "shape" label is the parsed query's root op (`sum`, //! `count`, `avg`, `selector`, ...) — low-cardinality by design, diff --git a/asap-query-engine/src/drivers/query/servers/http.rs b/asap-query-engine/src/drivers/query/servers/http.rs index 2b33dcc9..a3519c3d 100644 --- a/asap-query-engine/src/drivers/query/servers/http.rs +++ b/asap-query-engine/src/drivers/query/servers/http.rs @@ -16,10 +16,8 @@ use tracing::{debug, info, warn}; use crate::drivers::query::adapters::{create_http_adapter, AdapterConfig, HttpProtocolAdapter}; use crate::drivers::query::servers::metrics as srv_metrics; -use crate::engines::SimpleEngine; -use crate::routing::{ - EngineRouter, EngineRouterError, FreshnessProbeCache, QueryEngine, -}; +use crate::engines::ASAPQueryEngine; +use crate::routing::{EngineRouter, EngineRouterError, FreshnessProbeCache, QueryEngine}; use crate::stores::Store; use asap_types::{AccuracyTarget, StorageBackend}; use promql_utilities::query_logics::enums::Statistic; @@ -104,7 +102,7 @@ impl std::fmt::Debug for PrecomputeJobRegistry { /// (Fix 1) for the design rationale. /// /// Recognised values match `StorageBackend::data_source_id()` — -/// `sketch_warm`, `gorilla_archive`, `double_write`. (Step-1 of +/// `asap_query`, `thanos_query`, `double_write`. (Step-1 of /// the JSONL deprecation removed the `cold_jsonl` value.) An /// unknown value returns 400. pub const ENGINE_OVERRIDE_HEADER: &str = "X-ASAP-Engine"; @@ -150,14 +148,14 @@ pub struct HttpServerConfig { #[derive(Clone)] pub struct HttpServer { config: HttpServerConfig, - query_engine: Arc, + query_engine: Arc, /// Phase-5/6 capability router. Built from `query_engine` at /// construction time (`SimpleEngine` registered as the warm-tier /// `QueryEngine`) and extended via [`Self::with_query_engine`] — /// e.g. to plug in a `GorillaQueryEngine` for the cold archive /// tier. Instant-query dispatch consults this for metrics whose /// `StreamingConfig::storage_backend()` is anything other than - /// `SketchWarmTier`; warm-tier queries still take the direct + /// `SketchStore`; warm-tier queries still take the direct /// `SimpleEngine::handle_query` path so they keep the /// `KeyByLabelNames` Prometheus needs to populate the `metric` /// map. See `docs/design-gorilla-s3-cold-engine.md` §8. @@ -173,7 +171,7 @@ pub struct HttpServer { /// `EngineRouter` for any per-metric override. When `None` the /// handler falls back to the pre-Phase-5 behaviour of consulting /// the streaming-config's single `storage_backend()` axis (which - /// itself defaults to `SketchWarmTier`). Wired by the binary via + /// itself defaults to `SketchStore`). Wired by the binary via /// [`Self::with_backend_storage_routing`]; production deploys /// bootstrap from `deploy/configs/backend-storage-routing.yaml` /// (legacy form) or the controller's first @@ -224,7 +222,7 @@ pub struct HttpServer { #[derive(Clone)] struct AppState { config: HttpServerConfig, - query_engine: Arc, + query_engine: Arc, /// See [`HttpServer::query_router`]. query_router: Arc, store: Arc, @@ -253,13 +251,11 @@ struct AppState { impl HttpServer { pub fn new( config: HttpServerConfig, - query_engine: Arc, + query_engine: Arc, store: Arc, ) -> Self { - // Bootstrap the capability router with `SimpleEngine` registered - // for the warm-tier (`sketch_warm`) `data_source_id`. Callers - // wiring up additional engines (e.g. `GorillaQueryEngine` for - // the cold archive) extend the router via `with_query_engine`. + // Bootstrap the capability router with `ASAPQueryEngine` + // registered under its canonical query-engine id. let mut router = EngineRouter::new(); router.register(query_engine.clone() as Arc); let query_router = Arc::new(router); @@ -296,23 +292,12 @@ impl HttpServer { self } - /// Like [`Self::with_query_engine`] but registers the engine - /// under an explicit `data_source_id` instead of the one its - /// `capabilities()` reports. Used by Step-2.3's Path A2 wiring: - /// the same `ThanosForwardEngine` instance is registered under - /// both `thanos_archive` (its native id, for explicit overrides) - /// and `gorilla_archive` (the legacy archive slot that the - /// existing `compatible_storage_backends` failover sequence - /// walks). The legacy in-process `GorillaQueryEngine` is only - /// registered when `ASAP_THANOS_QUERY_URL` is unset, so the two - /// registrations never collide on the same id. - pub fn with_query_engine_aliased( - mut self, - id: &'static str, - engine: Arc, - ) -> Self { + /// Register the archive engine. The only public archive query + /// engine id is `thanos_query`; Gorilla is only an + /// encoding/storage detail. + pub fn with_archive_query_engine(mut self, engine: Arc) -> Self { let mut router: EngineRouter = (*self.query_router).clone(); - router.register_aliased(id, engine); + router.register(engine); self.query_router = Arc::new(router); self } @@ -669,8 +654,8 @@ async fn process_query_request( // If the caller explicitly named an engine (via `X-ASAP-Engine` // header or `?engine=` query param) bypass `BackendStorageRouting` // and dispatch directly. The accuracy reducer relies on this to - // ask the same PromQL against the warm sketch (`sketch_warm`) and - // the Gorilla archive (`gorilla_archive`) so it can compute + // ask the same PromQL against the warm sketch (`asap_query`) and + // the Gorilla archive (`thanos_query`) so it can compute // apples-to-apples relative error per replay row. if let Some(override_id) = engine_override.as_deref() { return process_via_named_engine(state, parsed_request, start_time, override_id).await; @@ -694,9 +679,7 @@ async fn process_query_request( // dispatch falls through to the normal path — preserving the // long-window queries (≥1m) that the cold archive still answers // correctly. - if let Some(response) = - try_answer_freshness_probe(state, parsed_request, start_time).await - { + if let Some(response) = try_answer_freshness_probe(state, parsed_request, start_time).await { return response; } @@ -709,18 +692,18 @@ async fn process_query_request( // query is parsed; the metric name is extracted from the // AST and looked up in the table. This is the production // path the issue-46 MVP relies on so cold-archive metrics - // (e.g. `http_requests_total` → `gorilla_archive`) actually + // (e.g. `http_requests_total` → `thanos_query`) actually // route through the `EngineRouter`. // (b) Single-axis `StreamingConfig::storage_backend()` from the // hot-reload config (the pre-Phase-5 fallback). Pre-controller - // deploys ride this path; it always lands on `SketchWarmTier` + // deploys ride this path; it always lands on `SketchStore` // unless the YAML was hand-patched. - // (c) Default — `SketchWarmTier`. Keeps the direct + // (c) Default — `SketchStore`. Keeps the direct // `SimpleEngine::handle_query` path so the response carries // the `KeyByLabelNames` the Prometheus adapter needs to // populate the `metric` map. // - // For non-`SketchWarmTier` axes the dispatch goes through the + // For non-`SketchStore` axes the dispatch goes through the // `EngineRouter`. Phase-6 (Gorilla MVP) returns a scalar with // empty labels, so dropping `KeyByLabelNames` is acceptable; the // response carries `accuracy` + `data_source` via the @@ -734,7 +717,7 @@ async fn process_query_request( state.hot_reload_config.is_some(), ); - if matches!(metric_storage, StorageBackend::SketchWarmTier) { + if matches!(metric_storage, StorageBackend::SketchStore) { process_via_simple_engine(state, parsed_request, start_time, headers).await } else { process_via_router(state, parsed_request, start_time, metric_storage).await @@ -749,7 +732,7 @@ async fn process_query_request( /// This is the path issue #46's MVP demo relies on. /// 2. Streaming-config single-axis fallback — preserves pre-Phase-5 /// behaviour for deploys that haven't loaded a routing table. -/// 3. Default `SketchWarmTier`. +/// 3. Default `SketchStore`. /// /// Parsing failures fall through to (2)/(3) so a malformed PromQL /// doesn't surface as a routing 5xx (the engines themselves will @@ -836,7 +819,7 @@ fn first_metric_name(expr: &promql_parser::parser::Expr) -> Option { /// /// Response shape mirrors `process_via_router`'s success path: a /// Prometheus instant vector with one element (empty labels, scalar -/// value), a `data_source: sketch_warm` info-line so the wire format +/// value), a `data_source: asap_query` info-line so the wire format /// is consistent with the warm-tier path the routing table comment /// describes as the right home for the `_warm` probe. async fn try_answer_freshness_probe( @@ -874,10 +857,8 @@ async fn try_answer_freshness_probe( "freshness-probe cache hit; answering last_over_time from RAM" ); - let element = InstantVectorElement::new( - crate::data_model::KeyByLabelValues::new(), - sample.value, - ); + let element = + InstantVectorElement::new(crate::data_model::KeyByLabelValues::new(), sample.value); // The instant-vector timestamp is unix milliseconds — match the // adapter's expectations downstream (the Prometheus adapter // divides by 1000 to render the wire `value: [, ...]`). @@ -899,11 +880,10 @@ async fn try_answer_freshness_probe( .format_success_response(&execution_result) .await { - Ok(response) => annotate_data_source( - response, - StorageBackend::SketchWarmTier.data_source_id(), - ) - .await, + Ok(response) => { + annotate_data_source(response, StorageBackend::SketchStore.data_source_id()) + .await + } Err(status) => status.into_response(), }, ) @@ -950,7 +930,7 @@ fn parse_last_over_time_probe(query: &str) -> Option<(String, i64)> { /// `KeyByLabelNames` the Prometheus adapter needs to fill in the /// `metric` map. Used for warm-tier metrics (the default) so the /// response surface is byte-identical to the pre-router path. Adds a -/// `data_source: sketch_warm` info-line at the JSON layer so Phase-6 +/// `data_source: asap_query` info-line at the JSON layer so Phase-6 /// callers can byte-compare regardless of the dispatch path. async fn process_via_simple_engine( state: &AppState, @@ -997,11 +977,10 @@ async fn process_via_simple_engine( .format_success_response(&execution_result) .await { - Ok(response) => annotate_data_source( - response, - StorageBackend::SketchWarmTier.data_source_id(), - ) - .await, + Ok(response) => { + annotate_data_source(response, StorageBackend::SketchStore.data_source_id()) + .await + } Err(status) => status.into_response(), } } @@ -1027,18 +1006,20 @@ async fn process_via_simple_engine( } else { debug!("Query not supported and forwarding disabled, returning error"); // Adapter formats the unsupported query error for its protocol. - // We still annotate `data_source: sketch_warm` so callers + // We still annotate `data_source: asap_query` so callers // see which tier the request was dispatched against — // the routing decision happened, the metric just had no // compatible aggregation. Mirrors the // SimpleEngine-as-router-engine path where a // `EngineError::CapabilityMiss` response is still tagged. match state.adapter.format_unsupported_query_response().await { - Ok(response) => annotate_data_source( - response, - StorageBackend::SketchWarmTier.data_source_id(), - ) - .await, + Ok(response) => { + annotate_data_source( + response, + StorageBackend::SketchStore.data_source_id(), + ) + .await + } Err(status) => status.into_response(), } } @@ -1164,7 +1145,7 @@ async fn process_via_named_engine( } /// Dispatch through the [`EngineRouter`] — used for any metric whose -/// pinned `StorageBackend` is something other than `SketchWarmTier`. +/// pinned `StorageBackend` is something other than `SketchStore`. /// /// The router's `execute` API is `(&str, Statistic, AccuracyTarget, /// StorageBackend) -> QueryResult`. Three of those four axes are @@ -1180,7 +1161,7 @@ async fn process_via_named_engine( /// [`compatible_storage_backends`] consults them only for the /// `DoubleWrite` head-selection heuristic (other deploy shapes /// degenerate to a fixed list keyed only by `metric_storage`), so -/// defaulting to `(Sum, Approximate)` is safe for `GorillaS3Archive`- +/// defaulting to `(Sum, Approximate)` is safe for `GorillaObjectStore`- /// only deploys. A follow-up will thread /// the real values through once the Phase-6 query-tracker exposes /// them per request. @@ -1209,7 +1190,7 @@ async fn process_via_router( // Default `(Sum, Approximate)` — see fn doc above. The router's // capability table only consults these axes for `DoubleWrite` - // metrics; for `GorillaS3Archive`-only deploys the dispatch + // metrics; for `GorillaObjectStore`-only deploys the dispatch // is a function of `metric_storage` alone. let stat = Statistic::Sum; let accuracy = AccuracyTarget::Approximate; @@ -1252,11 +1233,9 @@ async fn process_via_router( .format_success_response(&execution_result) .await { - Ok(response) => annotate_data_source( - response, - metric_storage.data_source_id(), - ) - .await, + Ok(response) => { + annotate_data_source(response, metric_storage.data_source_id()).await + } Err(status) => status.into_response(), } } @@ -1281,12 +1260,8 @@ async fn process_via_router( Err(EngineRouterError::AllFailed { last }) => { warn!(error = %last, "EngineRouter: all compatible engines failed"); let (status, error_type) = match &last { - EngineError::CapabilityMiss { .. } => { - (StatusCode::NOT_FOUND, "bad_data") - } - EngineError::Backend { .. } => { - (StatusCode::INTERNAL_SERVER_ERROR, "internal") - } + EngineError::CapabilityMiss { .. } => (StatusCode::NOT_FOUND, "bad_data"), + EngineError::Backend { .. } => (StatusCode::INTERNAL_SERVER_ERROR, "internal"), }; ( status, @@ -1392,9 +1367,15 @@ async fn handle_instant_query( } }; - let response = - process_query_request(&state, &parsed_request, start_time, HashMap::new(), engine_override, &tenant) - .await; + let response = process_query_request( + &state, + &parsed_request, + start_time, + HashMap::new(), + engine_override, + &tenant, + ) + .await; srv_metrics::record_query_outcome( srv_metrics::QUERY_TYPE_INSTANT, query_status_label(&response), @@ -1626,8 +1607,7 @@ async fn handle_metrics() -> impl IntoResponse { // mvp/v5: append the S3 cost counters in Prometheus text // exposition. Mirrors `/internal/s3_cost.csv` — the CSV is for // the demo, this is for live dashboards. - let counters = - crate::engines::gorilla::global_s3_cost_counters(); + let counters = crate::stores::gorilla_object_store::global_s3_cost_counters(); buffer.extend_from_slice(counters.render_prometheus().as_bytes()); ( [( @@ -1644,13 +1624,9 @@ async fn handle_metrics() -> impl IntoResponse { /// operations have been issued (the counters default to zero, so /// the CSV is still well-formed). async fn handle_s3_cost_csv() -> impl IntoResponse { - let counters = - crate::engines::gorilla::global_s3_cost_counters(); + let counters = crate::stores::gorilla_object_store::global_s3_cost_counters(); ( - [( - axum::http::header::CONTENT_TYPE, - "text/csv; charset=utf-8", - )], + [(axum::http::header::CONTENT_TYPE, "text/csv; charset=utf-8")], counters.render_csv(), ) } @@ -2981,10 +2957,8 @@ aggregations: metric_storage_backend: StorageBackend, extra_engines: Vec>, ) -> u16 { - let adapter_config = AdapterConfig::prometheus_promql( - "http://127.0.0.1:9999".to_string(), - false, - ); + let adapter_config = + AdapterConfig::prometheus_promql("http://127.0.0.1:9999".to_string(), false); let config = HttpServerConfig { port: 0, handle_http_requests: true, @@ -3011,8 +2985,8 @@ aggregations: 15000, crate::data_model::QueryLanguage::promql, )); - let mut server = HttpServer::new(config, query_engine, store) - .with_hot_reload_config(hot_reload); + let mut server = + HttpServer::new(config, query_engine, store).with_hot_reload_config(hot_reload); for engine in extra_engines { server = server.with_query_engine(engine); } @@ -3025,7 +2999,7 @@ aggregations: /// Build an `HttpServer` wired with a per-metric /// `BackendStorageRouting` table — the **production path** the /// issue-46 MVP relies on. The streaming-config single axis stays - /// at `SketchWarmTier` (the realistic deploy state); the routing + /// at `SketchStore` (the realistic deploy state); the routing /// table is what flips per-metric dispatch over to the /// `EngineRouter`. This proves the production code path /// (`process_query_request → resolve_metric_storage → routing @@ -3036,10 +3010,8 @@ aggregations: routing: crate::data_model::BackendStorageRouting, extra_engines: Vec>, ) -> u16 { - let adapter_config = AdapterConfig::prometheus_promql( - "http://127.0.0.1:9999".to_string(), - false, - ); + let adapter_config = + AdapterConfig::prometheus_promql("http://127.0.0.1:9999".to_string(), false); let config = HttpServerConfig { port: 0, handle_http_requests: true, @@ -3049,7 +3021,7 @@ aggregations: crate::data_model::QueryLanguage::promql, crate::data_model::CleanupPolicy::NoCleanup, ); - // Streaming-config stays on the default `SketchWarmTier` axis + // Streaming-config stays on the default `SketchStore` axis // — exactly what the production deploy looks like (the YAML // loader doesn't parse `storage_backend`). All routing // decisions must come from the per-metric routing table. @@ -3086,16 +3058,14 @@ aggregations: /// surface — but registers nothing, then asks the router-path /// dispatch to route an archive metric. Used by the /// `503 NoEngineRegistered` test. - async fn setup_test_server_with_empty_router( - metric_storage_backend: StorageBackend, - ) -> u16 { + async fn setup_test_server_with_empty_router(metric_storage_backend: StorageBackend) -> u16 { // `HttpServer::new` always registers SimpleEngine for the // warm tier. To force `NoEngineRegistered` we point the // metric at a backend whose data_source_id doesn't match // any registered engine — since `HttpServer::new` only - // registers SimpleEngine (sketch_warm), routing a - // `GorillaS3Archive`-only metric trips the empty path - // (compatible_storage_backends = [GorillaS3Archive], no + // registers SimpleEngine (asap_query), routing a + // `GorillaObjectStore`-only metric trips the empty path + // (compatible_storage_backends = [GorillaObjectStore], no // engine registered for that id). setup_test_server_with_router(metric_storage_backend, Vec::new()).await } @@ -3107,11 +3077,7 @@ aggregations: let infos = body .get("infos") .and_then(|v| v.as_array()) - .unwrap_or_else(|| { - panic!( - "expected `infos` array in response body, got {body}", - ) - }); + .unwrap_or_else(|| panic!("expected `infos` array in response body, got {body}",)); let want = format!("data_source: {expected}"); assert!( infos.iter().any(|v| v.as_str() == Some(&want)), @@ -3121,12 +3087,12 @@ aggregations: #[tokio::test] async fn http_routes_warm_tier_metric_to_simple_engine() { - // Default (no hot-reload) → `SketchWarmTier`. The handler + // Default (no hot-reload) → `SketchStore`. The handler // takes the direct `SimpleEngine::handle_query` path; the - // response's `infos` array carries `data_source: sketch_warm` + // response's `infos` array carries `data_source: asap_query` // so callers can byte-compare which engine answered. let server_port = - setup_test_server_with_router(StorageBackend::SketchWarmTier, Vec::new()).await; + setup_test_server_with_router(StorageBackend::SketchStore, Vec::new()).await; let client = Client::new(); let resp = client .get(format!("http://127.0.0.1:{server_port}/api/v1/query")) @@ -3140,19 +3106,19 @@ aggregations: resp.status() ); let body: serde_json::Value = resp.json().await.unwrap(); - assert_data_source(&body, "sketch_warm"); + assert_data_source(&body, "asap_query"); } #[tokio::test] async fn http_routes_archive_metric_to_gorilla_engine() { - // Pin `storage_backend = GorillaS3Archive` and register a + // Pin `storage_backend = GorillaObjectStore` and register a // `MockQueryEngine` under that id. The handler must dispatch // through the router (not SimpleEngine) and the response's - // `infos` array must carry `data_source: gorilla_archive`. + // `infos` array must carry `data_source: thanos_query`. let (gorilla, gorilla_calls) = - MockQueryEngine::new(StorageBackend::GorillaS3Archive, MockOutcome::OkEmpty); + MockQueryEngine::new(StorageBackend::GorillaObjectStore, MockOutcome::OkEmpty); let server_port = setup_test_server_with_router( - StorageBackend::GorillaS3Archive, + StorageBackend::GorillaObjectStore, vec![gorilla as Arc], ) .await; @@ -3172,7 +3138,7 @@ aggregations: resp.status() ); let body: serde_json::Value = resp.json().await.unwrap(); - assert_data_source(&body, "gorilla_archive"); + assert_data_source(&body, "thanos_query"); assert_eq!( gorilla_calls.load(Ordering::SeqCst), 1, @@ -3183,7 +3149,7 @@ aggregations: #[tokio::test] async fn http_query_with_no_storage_config_defaults_to_warm_tier() { // `StreamingConfig::default()` has `storage_backend = - // SketchWarmTier` (per the `#[serde(default)]` on the + // SketchStore` (per the `#[serde(default)]` on the // field — see `streaming_config.rs`). A server set up // without a hot-reload handle still infers warm-tier and // takes the SimpleEngine direct path. Back-compat for @@ -3203,21 +3169,21 @@ aggregations: resp.status() ); let body: serde_json::Value = resp.json().await.unwrap(); - assert_data_source(&body, "sketch_warm"); + assert_data_source(&body, "asap_query"); } #[tokio::test] async fn http_returns_503_when_no_engines_registered() { - // Pin `storage_backend = GorillaS3Archive` but register no + // Pin `storage_backend = GorillaObjectStore` but register no // archive engine (only `SimpleEngine` is registered under - // `sketch_warm`). The router walks - // `compatible_storage_backends = [GorillaS3Archive]` and + // `asap_query`). The router walks + // `compatible_storage_backends = [GorillaObjectStore]` and // bails out with `NoEngineRegistered`, which the HTTP layer // surfaces as 503. Step-1 of the JSONL deprecation removed // the `ColdJsonlFallback` failover slot, so this is the // canonical "engine missing" path now. let server_port = - setup_test_server_with_empty_router(StorageBackend::GorillaS3Archive).await; + setup_test_server_with_empty_router(StorageBackend::GorillaObjectStore).await; let client = Client::new(); let resp = client .get(format!("http://127.0.0.1:{server_port}/api/v1/query")) @@ -3257,14 +3223,14 @@ aggregations: } fn capabilities(&self) -> EngineCapabilities { EngineCapabilities { - data_source_id: StorageBackend::GorillaS3Archive.data_source_id(), - storage_backend: StorageBackend::GorillaS3Archive, + data_source_id: StorageBackend::GorillaObjectStore.data_source_id(), + storage_backend: StorageBackend::GorillaObjectStore, supports_streams_above_bytes: 1024 * 1024, } } } let server_port = setup_test_server_with_router( - StorageBackend::GorillaS3Archive, + StorageBackend::GorillaObjectStore, vec![Arc::new(ExactStub) as Arc], ) .await; @@ -3281,7 +3247,7 @@ aggregations: // `accuracy: ε=..., δ=..., kind=exact` summary must land on // the response — proving the router-path dispatch preserves // the engine's wire annotations. - assert_data_source(&body, "gorilla_archive"); + assert_data_source(&body, "thanos_query"); let infos = body["infos"].as_array().expect("infos array"); assert!( infos @@ -3305,13 +3271,13 @@ aggregations: // The HTTP handler dispatches with default // `(Statistic::Sum, AccuracyTarget::Approximate)`, so for a // `DoubleWrite` metric the compatibility list is - // `[SketchWarmTier, GorillaS3Archive]` and the warm-tier + // `[SketchStore, GorillaObjectStore]` and the warm-tier // mock answers first. The archive must NOT be hit (no // failover needed when the head succeeds). let (warm_ok, warm_calls) = - MockQueryEngine::new(StorageBackend::SketchWarmTier, MockOutcome::OkEmpty); + MockQueryEngine::new(StorageBackend::SketchStore, MockOutcome::OkEmpty); let (archive, archive_calls) = - MockQueryEngine::new(StorageBackend::GorillaS3Archive, MockOutcome::Backend); + MockQueryEngine::new(StorageBackend::GorillaObjectStore, MockOutcome::Backend); let server_port = setup_test_server_with_router( StorageBackend::DoubleWrite, vec![ @@ -3344,10 +3310,10 @@ aggregations: // // The tests above (e.g. `http_routes_archive_metric_to_gorilla_engine`) // mock the routing decision by pinning `streaming_cfg.storage_backend - // = GorillaS3Archive` directly. That proves the dispatch BRANCH is + // = GorillaObjectStore` directly. That proves the dispatch BRANCH is // wired, but not the production code path — in real deploys the // streaming-config YAML loader drops `storage_backend` (it always - // defaults to `SketchWarmTier`), so the issue-46 v2 demo's queries + // defaults to `SketchStore`), so the issue-46 v2 demo's queries // never reached the EngineRouter. The tests below exercise the // **production path** end-to-end: streaming config stays default, // a per-metric `BackendStorageRouting` table is loaded at startup @@ -3357,28 +3323,26 @@ aggregations: #[tokio::test] async fn http_production_path_routes_archive_metric_via_routing_table() { // Production path: streaming-config single axis stays on - // `SketchWarmTier` (the YAML loader's default), but the + // `SketchStore` (the YAML loader's default), but the // per-metric routing table flips `http_requests_total` to - // `gorilla_archive`. The handler must extract the metric name + // `thanos_query`. The handler must extract the metric name // from the PromQL AST, look it up, and dispatch through the - // EngineRouter — landing the `data_source: gorilla_archive` + // EngineRouter — landing the `data_source: thanos_query` // info-line on the response. let mut metrics = std::collections::HashMap::new(); metrics.insert( "http_requests_total".to_string(), - StorageBackend::GorillaS3Archive, + StorageBackend::GorillaObjectStore, ); let routing = crate::data_model::BackendStorageRouting::new_from_single_targets( - StorageBackend::SketchWarmTier, + StorageBackend::SketchStore, metrics, ); let (gorilla, gorilla_calls) = - MockQueryEngine::new(StorageBackend::GorillaS3Archive, MockOutcome::OkEmpty); - let server_port = setup_test_server_with_routing_table( - routing, - vec![gorilla as Arc], - ) - .await; + MockQueryEngine::new(StorageBackend::GorillaObjectStore, MockOutcome::OkEmpty); + let server_port = + setup_test_server_with_routing_table(routing, vec![gorilla as Arc]) + .await; let client = Client::new(); let resp = client .get(format!("http://127.0.0.1:{server_port}/api/v1/query")) @@ -3395,7 +3359,7 @@ aggregations: resp.status(), ); let body: serde_json::Value = resp.json().await.unwrap(); - assert_data_source(&body, "gorilla_archive"); + assert_data_source(&body, "thanos_query"); assert_eq!( gorilla_calls.load(Ordering::SeqCst), 1, @@ -3411,14 +3375,13 @@ aggregations: let mut metrics = std::collections::HashMap::new(); metrics.insert( "http_requests_total".to_string(), - StorageBackend::GorillaS3Archive, + StorageBackend::GorillaObjectStore, ); let routing = crate::data_model::BackendStorageRouting::new_from_single_targets( - StorageBackend::SketchWarmTier, + StorageBackend::SketchStore, metrics, ); - let server_port = - setup_test_server_with_routing_table(routing, Vec::new()).await; + let server_port = setup_test_server_with_routing_table(routing, Vec::new()).await; let client = Client::new(); let resp = client .get(format!("http://127.0.0.1:{server_port}/api/v1/query")) @@ -3435,26 +3398,24 @@ aggregations: resp.status(), ); let body: serde_json::Value = resp.json().await.unwrap(); - assert_data_source(&body, "sketch_warm"); + assert_data_source(&body, "asap_query"); } #[tokio::test] async fn http_production_path_default_axis_routes_all_metrics() { // Routing table with no per-metric overrides but a non-default - // top-level `default: gorilla_s3_archive` — every metric must + // top-level `default: thanos_query` — every metric must // route through the router. Pins the §8 "all-metrics-archive" // deploy mode. let routing = crate::data_model::BackendStorageRouting::new_from_single_targets( - StorageBackend::GorillaS3Archive, + StorageBackend::GorillaObjectStore, std::collections::HashMap::new(), ); let (gorilla, gorilla_calls) = - MockQueryEngine::new(StorageBackend::GorillaS3Archive, MockOutcome::OkEmpty); - let server_port = setup_test_server_with_routing_table( - routing, - vec![gorilla as Arc], - ) - .await; + MockQueryEngine::new(StorageBackend::GorillaObjectStore, MockOutcome::OkEmpty); + let server_port = + setup_test_server_with_routing_table(routing, vec![gorilla as Arc]) + .await; let client = Client::new(); let resp = client .get(format!("http://127.0.0.1:{server_port}/api/v1/query")) @@ -3467,7 +3428,7 @@ aggregations: .expect("Failed to send request"); assert!(resp.status().is_success()); let body: serde_json::Value = resp.json().await.unwrap(); - assert_data_source(&body, "gorilla_archive"); + assert_data_source(&body, "thanos_query"); assert_eq!(gorilla_calls.load(Ordering::SeqCst), 1); } @@ -3484,29 +3445,25 @@ aggregations: #[tokio::test] async fn http_v7_dual_routing_count_lands_on_archive() { - use crate::data_model::{ - BackendStorageRouting, QueryShape, RoutingTarget, - }; + use crate::data_model::{BackendStorageRouting, QueryShape, RoutingTarget}; let mut metrics = std::collections::HashMap::new(); metrics.insert( "http_requests_total".to_string(), vec![ - RoutingTarget::always(StorageBackend::SketchWarmTier), + RoutingTarget::always(StorageBackend::SketchStore), RoutingTarget::for_shapes( - StorageBackend::GorillaS3Archive, + StorageBackend::GorillaObjectStore, vec![QueryShape::Count, QueryShape::Topk, QueryShape::RatePostHoc], ), ], ); - let routing = BackendStorageRouting::new(StorageBackend::SketchWarmTier, metrics); + let routing = BackendStorageRouting::new(StorageBackend::SketchStore, metrics); let (gorilla, gorilla_calls) = - MockQueryEngine::new(StorageBackend::GorillaS3Archive, MockOutcome::OkEmpty); - let server_port = setup_test_server_with_routing_table( - routing, - vec![gorilla as Arc], - ) - .await; + MockQueryEngine::new(StorageBackend::GorillaObjectStore, MockOutcome::OkEmpty); + let server_port = + setup_test_server_with_routing_table(routing, vec![gorilla as Arc]) + .await; let client = Client::new(); let resp = client @@ -3524,7 +3481,7 @@ aggregations: resp.status() ); let body: serde_json::Value = resp.json().await.unwrap(); - assert_data_source(&body, "gorilla_archive"); + assert_data_source(&body, "thanos_query"); assert_eq!( gorilla_calls.load(Ordering::SeqCst), 1, @@ -3534,47 +3491,40 @@ aggregations: #[tokio::test] async fn http_v7_dual_routing_quantile_stays_on_warm_tier() { - use crate::data_model::{ - BackendStorageRouting, QueryShape, RoutingTarget, - }; + use crate::data_model::{BackendStorageRouting, QueryShape, RoutingTarget}; let mut metrics = std::collections::HashMap::new(); metrics.insert( "http_requests_total".to_string(), vec![ - RoutingTarget::always(StorageBackend::SketchWarmTier), + RoutingTarget::always(StorageBackend::SketchStore), RoutingTarget::for_shapes( - StorageBackend::GorillaS3Archive, + StorageBackend::GorillaObjectStore, vec![QueryShape::Count, QueryShape::Topk, QueryShape::RatePostHoc], ), ], ); - let routing = BackendStorageRouting::new(StorageBackend::SketchWarmTier, metrics); + let routing = BackendStorageRouting::new(StorageBackend::SketchStore, metrics); // Register a Gorilla mock so a misroute would surface as a // failed assertion rather than a silent fall-through. The // mock starts with 0 calls; a quantile must NOT touch it. let (gorilla, gorilla_calls) = - MockQueryEngine::new(StorageBackend::GorillaS3Archive, MockOutcome::OkEmpty); - let server_port = setup_test_server_with_routing_table( - routing, - vec![gorilla as Arc], - ) - .await; + MockQueryEngine::new(StorageBackend::GorillaObjectStore, MockOutcome::OkEmpty); + let server_port = + setup_test_server_with_routing_table(routing, vec![gorilla as Arc]) + .await; let client = Client::new(); let resp = client .get(format!("http://127.0.0.1:{server_port}/api/v1/query")) .query(&[ - ( - "query", - "quantile_over_time(0.99, http_requests_total[1m])", - ), + ("query", "quantile_over_time(0.99, http_requests_total[1m])"), ("time", "1700000000"), ]) .send() .await .expect("Failed to send request"); - // Warm tier path returns 2xx with `data_source: sketch_warm` + // Warm tier path returns 2xx with `data_source: asap_query` // (the SimpleEngine returns None for this unconfigured // metric, but the handler still annotates the wire response // with the warm-tier source). @@ -3584,7 +3534,7 @@ aggregations: resp.status() ); let body: serde_json::Value = resp.json().await.unwrap(); - assert_data_source(&body, "sketch_warm"); + assert_data_source(&body, "asap_query"); assert_eq!( gorilla_calls.load(Ordering::SeqCst), 0, @@ -3614,7 +3564,7 @@ aggregations: use crate::data_model::{BackendStorageRouting, QueryShape, RoutingTarget}; let (gorilla, gorilla_calls) = - MockQueryEngine::new(StorageBackend::GorillaS3Archive, MockOutcome::OkEmpty); + MockQueryEngine::new(StorageBackend::GorillaObjectStore, MockOutcome::OkEmpty); // Dual-routing for `metric_warm`: quantiles → warm, count → // archive. Without the header the test query routes to warm. @@ -3623,28 +3573,26 @@ aggregations: "metric_warm".to_string(), vec![ RoutingTarget::for_shapes( - StorageBackend::SketchWarmTier, + StorageBackend::SketchStore, vec![QueryShape::Quantile], ), RoutingTarget::for_shapes( - StorageBackend::GorillaS3Archive, + StorageBackend::GorillaObjectStore, vec![QueryShape::Count], ), ], ); - let routing = BackendStorageRouting::new(StorageBackend::SketchWarmTier, metrics); - let server_port = setup_test_server_with_routing_table( - routing, - vec![gorilla as Arc], - ) - .await; + let routing = BackendStorageRouting::new(StorageBackend::SketchStore, metrics); + let server_port = + setup_test_server_with_routing_table(routing, vec![gorilla as Arc]) + .await; let client = Client::new(); // Ask a quantile query (default routing → warm) but override // to archive via the header. let resp = client .get(format!("http://127.0.0.1:{server_port}/api/v1/query")) - .header(ENGINE_OVERRIDE_HEADER, "gorilla_archive") + .header(ENGINE_OVERRIDE_HEADER, "thanos_query") .query(&[ ("query", "quantile_over_time(0.5, metric_warm[1m])"), ("time", "1700000000"), @@ -3658,7 +3606,7 @@ aggregations: resp.status() ); let body: serde_json::Value = resp.json().await.unwrap(); - assert_data_source(&body, "gorilla_archive"); + assert_data_source(&body, "thanos_query"); assert_eq!( gorilla_calls.load(Ordering::SeqCst), 1, @@ -3674,28 +3622,26 @@ aggregations: use crate::data_model::{BackendStorageRouting, QueryShape, RoutingTarget}; let (gorilla, gorilla_calls) = - MockQueryEngine::new(StorageBackend::GorillaS3Archive, MockOutcome::OkEmpty); + MockQueryEngine::new(StorageBackend::GorillaObjectStore, MockOutcome::OkEmpty); let mut metrics = std::collections::HashMap::new(); metrics.insert( "metric_warm".to_string(), vec![ RoutingTarget::for_shapes( - StorageBackend::SketchWarmTier, + StorageBackend::SketchStore, vec![QueryShape::Quantile], ), RoutingTarget::for_shapes( - StorageBackend::GorillaS3Archive, + StorageBackend::GorillaObjectStore, vec![QueryShape::Count], ), ], ); - let routing = BackendStorageRouting::new(StorageBackend::SketchWarmTier, metrics); - let server_port = setup_test_server_with_routing_table( - routing, - vec![gorilla as Arc], - ) - .await; + let routing = BackendStorageRouting::new(StorageBackend::SketchStore, metrics); + let server_port = + setup_test_server_with_routing_table(routing, vec![gorilla as Arc]) + .await; let client = Client::new(); // No override → quantile shape routes to warm tier. @@ -3710,7 +3656,7 @@ aggregations: .expect("Failed to send request"); assert!(resp.status().is_success(), "default routing must still 2xx"); let body: serde_json::Value = resp.json().await.unwrap(); - assert_data_source(&body, "sketch_warm"); + assert_data_source(&body, "asap_query"); assert_eq!( gorilla_calls.load(Ordering::SeqCst), 0, @@ -3718,15 +3664,15 @@ aggregations: ); } - /// Query-param fallback: `?engine=gorilla_archive` overrides the + /// Query-param fallback: `?engine=thanos_query` overrides the /// routing table when the header is absent. #[tokio::test] async fn http_engine_override_query_param_routes_to_named_engine() { let (gorilla, gorilla_calls) = - MockQueryEngine::new(StorageBackend::GorillaS3Archive, MockOutcome::OkEmpty); + MockQueryEngine::new(StorageBackend::GorillaObjectStore, MockOutcome::OkEmpty); let server_port = setup_test_server_with_router( - StorageBackend::SketchWarmTier, + StorageBackend::SketchStore, vec![gorilla as Arc], ) .await; @@ -3737,14 +3683,14 @@ aggregations: .query(&[ ("query", "sum_over_time(foo[5m])"), ("time", "1700000000"), - (ENGINE_OVERRIDE_QUERY_PARAM, "gorilla_archive"), + (ENGINE_OVERRIDE_QUERY_PARAM, "thanos_query"), ]) .send() .await .expect("Failed to send request"); assert!(resp.status().is_success(), "query-param override must 2xx"); let body: serde_json::Value = resp.json().await.unwrap(); - assert_data_source(&body, "gorilla_archive"); + assert_data_source(&body, "thanos_query"); assert_eq!( gorilla_calls.load(Ordering::SeqCst), 1, @@ -3759,31 +3705,25 @@ aggregations: #[tokio::test] async fn http_engine_override_unknown_id_returns_400() { let server_port = - setup_test_server_with_router(StorageBackend::SketchWarmTier, Vec::new()).await; + setup_test_server_with_router(StorageBackend::SketchStore, Vec::new()).await; let client = Client::new(); let resp = client .get(format!("http://127.0.0.1:{server_port}/api/v1/query")) .header(ENGINE_OVERRIDE_HEADER, "does_not_exist") - .query(&[ - ("query", "sum_over_time(foo[5m])"), - ("time", "1700000000"), - ]) + .query(&[("query", "sum_over_time(foo[5m])"), ("time", "1700000000")]) .send() .await .expect("Failed to send request"); assert_eq!(resp.status(), reqwest::StatusCode::BAD_REQUEST); let body: serde_json::Value = resp.json().await.unwrap(); - let err = body - .get("error") - .and_then(|v| v.as_str()) - .unwrap_or(""); + let err = body.get("error").and_then(|v| v.as_str()).unwrap_or(""); assert!( err.contains("does_not_exist"), "error must name the bad id; got {err}", ); assert!( - err.contains("sketch_warm"), + err.contains("asap_query"), "error must list registered engines; got {err}", ); } @@ -3793,9 +3733,9 @@ aggregations: #[tokio::test] async fn http_engine_override_post_header_routes_to_named_engine() { let (gorilla, gorilla_calls) = - MockQueryEngine::new(StorageBackend::GorillaS3Archive, MockOutcome::OkEmpty); + MockQueryEngine::new(StorageBackend::GorillaObjectStore, MockOutcome::OkEmpty); let server_port = setup_test_server_with_router( - StorageBackend::SketchWarmTier, + StorageBackend::SketchStore, vec![gorilla as Arc], ) .await; @@ -3804,7 +3744,7 @@ aggregations: let form_body = "query=sum_over_time(foo%5B5m%5D)&time=1700000000"; let resp = client .post(format!("http://127.0.0.1:{server_port}/api/v1/query")) - .header(ENGINE_OVERRIDE_HEADER, "gorilla_archive") + .header(ENGINE_OVERRIDE_HEADER, "thanos_query") .header("Content-Type", "application/x-www-form-urlencoded") .body(form_body) .send() @@ -3816,7 +3756,7 @@ aggregations: resp.status() ); let body: serde_json::Value = resp.json().await.unwrap(); - assert_data_source(&body, "gorilla_archive"); + assert_data_source(&body, "thanos_query"); assert_eq!(gorilla_calls.load(Ordering::SeqCst), 1); } @@ -3825,14 +3765,13 @@ aggregations: /// Standard test wiring for the `/api/v1/storage_routing` endpoint: /// install an empty hot-reload routing handle, hold the handle so /// the test can introspect the swap result. - async fn setup_test_server_for_storage_routing() -> (u16, crate::routing::HotReloadBackendStorageRouting) { + async fn setup_test_server_for_storage_routing( + ) -> (u16, crate::routing::HotReloadBackendStorageRouting) { use crate::data_model::{HotReloadStreamingConfig, StreamingConfig}; use crate::routing::HotReloadBackendStorageRouting; - let adapter_config = AdapterConfig::prometheus_promql( - "http://127.0.0.1:9999".to_string(), - false, - ); + let adapter_config = + AdapterConfig::prometheus_promql("http://127.0.0.1:9999".to_string(), false); let config = HttpServerConfig { port: 0, handle_http_requests: true, @@ -3866,14 +3805,14 @@ aggregations: fn fixture_routing_json() -> serde_json::Value { serde_json::json!({ - "default_engine": "sketch_warm_tier", + "default_engine": "asap_query", "metrics": [ { "name": "http_requests_total", "targets": [ - { "engine": "sketch_warm_tier" }, + { "engine": "asap_query" }, { - "engine": "thanos_archive", + "engine": "thanos_query", "applies_to_query_shape": [ "histogram_quantile", "delta", "absent", "rate_post_hoc", "count" @@ -3900,7 +3839,11 @@ aggregations: .await .expect("send ok"); - assert!(resp.status().is_success(), "swap must 2xx; got {}", resp.status()); + assert!( + resp.status().is_success(), + "swap must 2xx; got {}", + resp.status() + ); let body: serde_json::Value = resp.json().await.unwrap(); assert_eq!(body["status"], "success"); assert_eq!(body["metrics_count"], 1); @@ -3932,7 +3875,7 @@ aggregations: // Valid JSON but invalid schema (unknown engine). let bad = serde_json::json!({ - "default_engine": "sketch_warm_tier", + "default_engine": "asap_query", "metrics": [{ "name": "x", "targets": [{ "engine": "not_a_real_engine" }] @@ -3960,10 +3903,9 @@ aggregations: async fn storage_routing_get_returns_current_snapshot() { let (port, handle) = setup_test_server_for_storage_routing().await; // Pre-load the table. - let new = crate::data_model::BackendStorageRouting::from_json_payload( - &fixture_routing_json(), - ) - .expect("parse"); + let new = + crate::data_model::BackendStorageRouting::from_json_payload(&fixture_routing_json()) + .expect("parse"); handle.swap(new); let client = Client::new(); @@ -3975,7 +3917,7 @@ aggregations: assert!(resp.status().is_success()); let body: serde_json::Value = resp.json().await.unwrap(); assert_eq!(body["status"], "success"); - assert_eq!(body["default_engine"], "sketch_warm"); + assert_eq!(body["default_engine"], "asap_query"); assert_eq!(body["metrics_count"], 1); let snap_hash = body["table_hash"].as_str().unwrap(); let live_hash = crate::routing::routing_table_hash(handle.snapshot().as_ref()); @@ -4132,74 +4074,58 @@ aggregations: // the very next read. let snap = handle.snapshot(); assert_eq!( - snap.lookup_with_shape( - "http_requests_total", - crate::data_model::QueryShape::Count, - ), - StorageBackend::GorillaS3Archive, + snap.lookup_with_shape("http_requests_total", crate::data_model::QueryShape::Count,), + StorageBackend::GorillaObjectStore, ); assert_eq!( snap.lookup_with_shape( "http_requests_total", crate::data_model::QueryShape::Quantile, ), - StorageBackend::SketchWarmTier, + StorageBackend::SketchStore, ); } // ── Step 2.3: Path A2 thanos forwarder integration tests ──────────────── // // Pin the full HTTP path: backend receives PromQL → routes to - // `gorilla_archive` (via the alias) → forwards to a mock + // `thanos_query` (via the alias) → forwards to a mock // `thanos-query` sidecar → returns wrapped Prometheus response. // // The mock thanos sidecar is a tiny in-process axum server bound // to an ephemeral 127.0.0.1 port; the real wire path runs end-to- // end (reqwest serialises the form, axum parses it, the mock // returns canned JSON, the engine parses it back, the HTTP - // handler annotates `data_source: thanos_archive` on the wire + // handler annotates `data_source: thanos_query` on the wire // response). #[tokio::test] async fn http_archive_metric_forwards_to_thanos_query() { - use crate::engines::gorilla::thanos_forward::test_support::{ + use crate::engines::thanos_query::forward::test_support::{ spawn_mock_thanos, CANNED_VECTOR_BODY, }; - use crate::engines::gorilla::{ - ThanosForwardConfig, ThanosForwardEngine, DATA_SOURCE_THANOS_ARCHIVE_ID, - }; + use crate::engines::thanos_query::{ThanosQueryConfig, ThanosQueryEngine}; let (mock_url, _mock_handle) = spawn_mock_thanos(CANNED_VECTOR_BODY).await; - let cfg = ThanosForwardConfig { + let cfg = ThanosQueryConfig { base_url: mock_url, request_timeout: std::time::Duration::from_secs(5), }; - let engine = ThanosForwardEngine::new(cfg).expect("engine"); + let engine = ThanosQueryEngine::new(cfg).expect("engine"); let arc_engine: Arc = Arc::new(engine); - // Mirror the binary's Step-2.3 wiring: register under both - // ids so the failover sequence finds the engine via - // `gorilla_archive` and explicit overrides reach it via - // `thanos_archive`. - let server_port = setup_test_server_with_router_aliased( - StorageBackend::GorillaS3Archive, - vec![ - (DATA_SOURCE_THANOS_ARCHIVE_ID, arc_engine.clone()), - ( - StorageBackend::GorillaS3Archive.data_source_id(), - arc_engine, - ), - ], + // Mirror the binary's Step-2.3 wiring: register once under + // the engine's canonical `thanos_query` id. + let server_port = setup_test_server_with_named_router( + StorageBackend::GorillaObjectStore, + vec![arc_engine], ) .await; let client = Client::new(); let resp = client .get(format!("http://127.0.0.1:{server_port}/api/v1/query")) - .query(&[ - ("query", "up"), - ("time", "1700000000"), - ]) + .query(&[("query", "up"), ("time", "1700000000")]) .send() .await .expect("Failed to send request"); @@ -4211,41 +4137,33 @@ aggregations: let body: serde_json::Value = resp.json().await.unwrap(); // The failover-dispatch path annotates `data_source: // .data_source_id()`, which for an - // archive-pinned metric is `gorilla_archive`. Path A2 + // archive-pinned metric is `thanos_query`. Path A2 // re-uses the archive tier slot in the routing matrix — // the wire `data_source` reflects the *tier* (archive), // not which engine implementation answered. The explicit - // `X-ASAP-Engine: thanos_archive` override path (covered - // by `http_engine_override_can_target_thanos_archive_id`) - // is the route that pins `data_source: thanos_archive` + // `X-ASAP-Engine: thanos_query` override path (covered + // by `http_engine_override_can_target_thanos_query_id`) + // is the route that pins `data_source: thanos_query` // on the wire. - assert_data_source(&body, "gorilla_archive"); + assert_data_source(&body, "thanos_query"); } #[tokio::test] async fn http_thanos_unreachable_returns_503_with_quirk() { - use crate::engines::gorilla::thanos_forward::test_support::spawn_mock_thanos_503; - use crate::engines::gorilla::{ - ThanosForwardConfig, ThanosForwardEngine, DATA_SOURCE_THANOS_ARCHIVE_ID, - }; + use crate::engines::thanos_query::forward::test_support::spawn_mock_thanos_503; + use crate::engines::thanos_query::{ThanosQueryConfig, ThanosQueryEngine}; let (mock_url, _mock_handle) = spawn_mock_thanos_503().await; - let cfg = ThanosForwardConfig { + let cfg = ThanosQueryConfig { base_url: mock_url, request_timeout: std::time::Duration::from_secs(2), }; - let engine = ThanosForwardEngine::new(cfg).expect("engine"); + let engine = ThanosQueryEngine::new(cfg).expect("engine"); let arc_engine: Arc = Arc::new(engine); - let server_port = setup_test_server_with_router_aliased( - StorageBackend::GorillaS3Archive, - vec![ - (DATA_SOURCE_THANOS_ARCHIVE_ID, arc_engine.clone()), - ( - StorageBackend::GorillaS3Archive.data_source_id(), - arc_engine, - ), - ], + let server_port = setup_test_server_with_named_router( + StorageBackend::GorillaObjectStore, + vec![arc_engine], ) .await; @@ -4276,61 +4194,55 @@ aggregations: } #[tokio::test] - async fn http_engine_override_can_target_thanos_archive_id() { - // X-ASAP-Engine: thanos_archive must reach the forwarder + async fn http_engine_override_can_target_thanos_query_id() { + // X-ASAP-Engine: thanos_query must reach the forwarder // even when the metric's storage axis would otherwise route // to the warm tier. Path A2's accuracy reducer relies on // this for apples-to-apples comparison runs. - use crate::engines::gorilla::thanos_forward::test_support::{ + use crate::engines::thanos_query::forward::test_support::{ spawn_mock_thanos, CANNED_VECTOR_BODY, }; - use crate::engines::gorilla::{ - ThanosForwardConfig, ThanosForwardEngine, DATA_SOURCE_THANOS_ARCHIVE_ID, + use crate::engines::thanos_query::{ + ThanosQueryConfig, ThanosQueryEngine, DATA_SOURCE_THANOS_QUERY_ID, }; let (mock_url, _mock_handle) = spawn_mock_thanos(CANNED_VECTOR_BODY).await; - let cfg = ThanosForwardConfig { + let cfg = ThanosQueryConfig { base_url: mock_url, request_timeout: std::time::Duration::from_secs(5), }; - let engine = ThanosForwardEngine::new(cfg).expect("engine"); + let engine = ThanosQueryEngine::new(cfg).expect("engine"); let arc_engine: Arc = Arc::new(engine); - let server_port = setup_test_server_with_router_aliased( - StorageBackend::SketchWarmTier, // Default storage axis is warm tier. - vec![(DATA_SOURCE_THANOS_ARCHIVE_ID, arc_engine)], + let server_port = setup_test_server_with_named_router( + StorageBackend::SketchStore, // Default storage axis is warm tier. + vec![arc_engine], ) .await; let client = Client::new(); let resp = client .get(format!("http://127.0.0.1:{server_port}/api/v1/query")) .query(&[("query", "up"), ("time", "1700000000")]) - .header(ENGINE_OVERRIDE_HEADER, DATA_SOURCE_THANOS_ARCHIVE_ID) + .header(ENGINE_OVERRIDE_HEADER, DATA_SOURCE_THANOS_QUERY_ID) .send() .await .expect("Failed to send request"); assert!( resp.status().is_success(), - "X-ASAP-Engine: thanos_archive must reach the forwarder; got {}", + "X-ASAP-Engine: thanos_query must reach the forwarder; got {}", resp.status() ); let body: serde_json::Value = resp.json().await.unwrap(); - assert_data_source(&body, "thanos_archive"); + assert_data_source(&body, "thanos_query"); } - /// Build an `HttpServer` whose router holds the supplied set of - /// `(alias_id, engine)` pairs. Mirrors `setup_test_server_with_router` - /// but uses [`HttpServer::with_query_engine_aliased`] so a single - /// engine instance can register under multiple ids — the Step-2.3 - /// pattern Path A2 relies on. - async fn setup_test_server_with_router_aliased( + /// Build an `HttpServer` whose router holds the supplied engines. + async fn setup_test_server_with_named_router( metric_storage_backend: StorageBackend, - aliased_engines: Vec<(&'static str, Arc)>, + engines: Vec>, ) -> u16 { - let adapter_config = AdapterConfig::prometheus_promql( - "http://127.0.0.1:9999".to_string(), - false, - ); + let adapter_config = + AdapterConfig::prometheus_promql("http://127.0.0.1:9999".to_string(), false); let config = HttpServerConfig { port: 0, handle_http_requests: true, @@ -4355,10 +4267,10 @@ aggregations: 15000, crate::data_model::QueryLanguage::promql, )); - let mut server = HttpServer::new(config, query_engine, store) - .with_hot_reload_config(hot_reload); - for (id, engine) in aliased_engines { - server = server.with_query_engine_aliased(id, engine); + let mut server = + HttpServer::new(config, query_engine, store).with_hot_reload_config(hot_reload); + for engine in engines { + server = server.with_query_engine(engine); } server .start_test_server() @@ -4386,14 +4298,10 @@ aggregations: /// hitting `/api/v1/query`. The router holds no cold-archive /// engine; the freshness probe short-circuit must answer /// without ever consulting the cold tier. - async fn setup_test_server_with_probe_cache() -> ( - u16, - Arc, - ) { - let adapter_config = AdapterConfig::prometheus_promql( - "http://127.0.0.1:9999".to_string(), - false, - ); + async fn setup_test_server_with_probe_cache() -> (u16, Arc) + { + let adapter_config = + AdapterConfig::prometheus_promql("http://127.0.0.1:9999".to_string(), false); let config = HttpServerConfig { port: 0, handle_http_requests: true, @@ -4416,8 +4324,7 @@ aggregations: crate::data_model::QueryLanguage::promql, )); let cache = Arc::new(crate::routing::FreshnessProbeCache::new()); - let server = HttpServer::new(config, query_engine, store) - .with_probe_cache(cache.clone()); + let server = HttpServer::new(config, query_engine, store).with_probe_cache(cache.clone()); let port = server .start_test_server() .await @@ -4479,7 +4386,7 @@ aggregations: // Sample is older than the lookback window — the cache lookup // returns None and the handler falls through to the normal // routing path. The default routing landed on - // `SketchWarmTier`, which the test's empty `SimpleEngine` + // `SketchStore`, which the test's empty `SimpleEngine` // can't answer, so the response is a structured error or an // empty-result success — anything but a crash. The test // pins the no-crash contract; the exact error surface is @@ -4518,10 +4425,9 @@ aggregations: // Different range — millis are extracted from the matrix // selector, not hard-coded. - let parsed = super::parse_last_over_time_probe( - "last_over_time(http_freshness_probe_archive[5m])", - ) - .expect("5m range must parse"); + let parsed = + super::parse_last_over_time_probe("last_over_time(http_freshness_probe_archive[5m])") + .expect("5m range must parse"); assert_eq!(parsed.1, 5 * 60_000); } @@ -4539,10 +4445,7 @@ aggregations: ); // Wrong arg count for last_over_time (which takes one matrix // selector). - assert_eq!( - super::parse_last_over_time_probe("last_over_time()"), - None, - ); + assert_eq!(super::parse_last_over_time_probe("last_over_time()"), None,); // Aggregation around the call — outermost shape isn't a // bare `last_over_time` call. assert_eq!( @@ -4593,11 +4496,7 @@ aggregations: /// second call. #[tokio::test] async fn http_precompute_jobs_register_then_delete_roundtrip() { - let port = setup_test_server_with_router( - StorageBackend::SketchWarmTier, - Vec::new(), - ) - .await; + let port = setup_test_server_with_router(StorageBackend::SketchStore, Vec::new()).await; let client = Client::new(); let body = serde_json::json!({ @@ -4650,11 +4549,7 @@ aggregations: /// `DELETE /api/v1/precompute/jobs/{unknown}` returns 404. #[tokio::test] async fn http_precompute_jobs_delete_unknown_id_returns_404() { - let port = setup_test_server_with_router( - StorageBackend::SketchWarmTier, - Vec::new(), - ) - .await; + let port = setup_test_server_with_router(StorageBackend::SketchStore, Vec::new()).await; let client = Client::new(); let resp = client .delete(format!( diff --git a/asap-query-engine/src/engines/simple/engine.rs b/asap-query-engine/src/engines/asap_query/engine.rs similarity index 97% rename from asap-query-engine/src/engines/simple/engine.rs rename to asap-query-engine/src/engines/asap_query/engine.rs index cb897abe..876fd779 100644 --- a/asap-query-engine/src/engines/simple/engine.rs +++ b/asap-query-engine/src/engines/asap_query/engine.rs @@ -28,10 +28,8 @@ use promql_utilities::query_logics::parsing::{ get_metric_and_spatial_filter, get_spatial_aggregation_output_labels, get_statistics_to_compute, }; - // SQL issue: refactor simpleengine to create matchresult similar to SQLquerydata - // Type alias for merged outputs (single aggregate per key after merging) type MergedOutputsMap = HashMap, Box>; @@ -58,7 +56,8 @@ fn replace_metric_token(haystack: &str, needle: &str, replacement: &str) -> Stri let mut out = String::with_capacity(haystack.len()); let mut i = 0; while i < bytes.len() { - if i + needle_bytes.len() <= bytes.len() && &bytes[i..i + needle_bytes.len()] == needle_bytes + if i + needle_bytes.len() <= bytes.len() + && &bytes[i..i + needle_bytes.len()] == needle_bytes { let prev_ok = i == 0 || !is_ident(bytes[i - 1]); let next_idx = i + needle_bytes.len(); @@ -107,12 +106,7 @@ fn extract_metric_and_label_keys( use promql_parser::parser::Expr; let ast = promql_parser::parser::parse(query).ok()?; - fn walk( - expr: &Expr, - ) -> Option<( - String, - std::collections::BTreeSet, - )> { + fn walk(expr: &Expr) -> Option<(String, std::collections::BTreeSet)> { match expr { Expr::VectorSelector(vs) => { let mut keys = std::collections::BTreeSet::new(); @@ -309,10 +303,15 @@ pub struct SimpleEngine { /// When `None` (no archive engine wired), the engine returns the /// warm answer as-is; the existing `EngineRouter` failover handles /// the rest of the routing matrix. - archive_engine: - Option>, + archive_engine: Option>, } +/// Public production name for the warm-tier sketch query engine. +/// +/// `SimpleEngine` remains as a compatibility alias in existing tests +/// and downstream code, but new code should use `ASAPQueryEngine`. +pub type ASAPQueryEngine = SimpleEngine; + impl SimpleEngine { /// Construct a `SimpleEngine` with a static `Arc`. /// Wraps the config in a fresh `HotReloadStreamingConfig` internally @@ -741,9 +740,7 @@ impl SimpleEngine { Expr::MatrixSelector(ms) => ms.vs.name.clone(), Expr::Call(call) => call.args.args.iter().find_map(|a| first_metric(a)), Expr::Aggregate(agg) => first_metric(&agg.expr), - Expr::Binary(bin) => { - first_metric(&bin.lhs).or_else(|| first_metric(&bin.rhs)) - } + Expr::Binary(bin) => first_metric(&bin.lhs).or_else(|| first_metric(&bin.rhs)), Expr::Subquery(sq) => first_metric(&sq.expr), Expr::Paren(p) => first_metric(&p.expr), Expr::Unary(u) => first_metric(&u.expr), @@ -2206,9 +2203,7 @@ impl SimpleEngine { // downstream stage (pattern match, `QueryConfig` lookup, // capability matching, `StoreQueryParams.metric`, schema // label lookup) sees the same suffixed name. - let query = self - .resolve_sketch_metric_alias(&query) - .unwrap_or(query); + let query = self.resolve_sketch_metric_alias(&query).unwrap_or(query); // Binary arithmetic dispatch was previously handled here via a // DataFusion-based plan combiner. That path was removed alongside @@ -3555,9 +3550,9 @@ fn stitch_warm_and_archive( // For each archive series, merge into by_labels. for arch_el in archive_matrix { - let entry = by_labels.entry(arch_el.labels.labels.clone()).or_insert_with( - || RangeVectorElement::new(arch_el.labels.clone()), - ); + let entry = by_labels + .entry(arch_el.labels.labels.clone()) + .or_insert_with(|| RangeVectorElement::new(arch_el.labels.clone())); // Build a set of warm timestamps inside coverage (kept). let warm_ts: std::collections::HashSet = entry .samples @@ -3639,7 +3634,7 @@ impl crate::routing::engine_router::QueryEngine for SimpleEngine { // // 1. `WarmTierAnalysis::unsupported` is `Some(_)` — the // PromQL shape isn't warm-tier-servable. Surface as - // `EngineError::CapabilityMiss(SketchWarmTier, …)` with the + // `EngineError::CapabilityMiss(SketchStore, …)` with the // structured `UnsupportedReason` in the detail string. The // EngineRouter fails over to the archive engine. This // covers all of: @@ -3669,9 +3664,9 @@ impl crate::routing::engine_router::QueryEngine for SimpleEngine { // Branch 1 — the controller analyzer rejects the shape. if let Some(reason) = &analysis.unsupported { return Err(crate::engines::EngineError::capability_miss( - asap_types::StorageBackend::SketchWarmTier.data_source_id(), + asap_types::StorageBackend::SketchStore.data_source_id(), format!( - "SketchWarmTier analyzer rejected `{query}`: {reason:?} — \ + "SketchStore analyzer rejected `{query}`: {reason:?} — \ failing over to archive" ), )); @@ -3682,9 +3677,9 @@ impl crate::routing::engine_router::QueryEngine for SimpleEngine { // when `candidates.is_empty()` but we keep the // belt-and-braces miss-path for safety. return Err(crate::engines::EngineError::capability_miss( - asap_types::StorageBackend::SketchWarmTier.data_source_id(), + asap_types::StorageBackend::SketchStore.data_source_id(), format!( - "SketchWarmTier analyzer produced no warm-tier candidates for \ + "SketchStore analyzer produced no warm-tier candidates for \ `{query}` — failing over to archive" ), )); @@ -3717,15 +3712,12 @@ impl crate::routing::engine_router::QueryEngine for SimpleEngine { let mut combined_t0: u64 = u64::MAX; for candidate in &analysis.candidates { - let sids = idx.instances_matching( - &candidate.metric_name, - &candidate.group_by_keys, - ); + let sids = idx.instances_matching(&candidate.metric_name, &candidate.group_by_keys); if sids.is_empty() { return Err(crate::engines::EngineError::capability_miss( - asap_types::StorageBackend::SketchWarmTier.data_source_id(), + asap_types::StorageBackend::SketchStore.data_source_id(), format!( - "SketchWarmTier has no instance for metric `{}` \ + "SketchStore has no instance for metric `{}` \ with group_by_keys ⊇ {:?} (analyzer required \ {:?}) — failing over to archive", candidate.metric_name, @@ -3749,9 +3741,9 @@ impl crate::routing::engine_router::QueryEngine for SimpleEngine { crate::stores::sketch_db::sketch_index::SidLookup::Ghost | crate::stores::sketch_db::sketch_index::SidLookup::Unknown => { return Err(crate::engines::EngineError::capability_miss( - asap_types::StorageBackend::SketchWarmTier.data_source_id(), + asap_types::StorageBackend::SketchStore.data_source_id(), format!( - "SketchWarmTier ghost/unknown sid {sid} for metric \ + "SketchStore ghost/unknown sid {sid} for metric \ `{}` — failing over to archive", candidate.metric_name ), @@ -3768,9 +3760,9 @@ impl crate::routing::engine_router::QueryEngine for SimpleEngine { } if hit_sids.is_empty() { return Err(crate::engines::EngineError::capability_miss( - asap_types::StorageBackend::SketchWarmTier.data_source_id(), + asap_types::StorageBackend::SketchStore.data_source_id(), format!( - "SketchWarmTier has no sid satisfying capability \ + "SketchStore has no sid satisfying capability \ {:?} for metric `{}` — failing over to archive", candidate.required_capability, candidate.metric_name ), @@ -3795,13 +3787,11 @@ impl crate::routing::engine_router::QueryEngine for SimpleEngine { now_ms, ) { Ok(r) => r, - Err(crate::engines::warm_tier::WarmTierError::UnsupportedFunction( - name, - )) => { + Err(crate::engines::warm_tier::WarmTierError::UnsupportedFunction(name)) => { return Err(crate::engines::EngineError::capability_miss( - asap_types::StorageBackend::SketchWarmTier.data_source_id(), + asap_types::StorageBackend::SketchStore.data_source_id(), format!( - "SketchWarmTier reducer does not support function `{name}` \ + "SketchStore reducer does not support function `{name}` \ — failing over to archive" ), )); @@ -3811,9 +3801,9 @@ impl crate::routing::engine_router::QueryEngine for SimpleEngine { capability, }) => { return Err(crate::engines::EngineError::capability_miss( - asap_types::StorageBackend::SketchWarmTier.data_source_id(), + asap_types::StorageBackend::SketchStore.data_source_id(), format!( - "SketchWarmTier reducer cannot answer `{function}` against \ + "SketchStore reducer cannot answer `{function}` against \ capability {capability:?} — failing over to archive" ), )); @@ -3824,21 +3814,19 @@ impl crate::routing::engine_router::QueryEngine for SimpleEngine { reason, }) => { return Err(crate::engines::EngineError::capability_miss( - asap_types::StorageBackend::SketchWarmTier.data_source_id(), + asap_types::StorageBackend::SketchStore.data_source_id(), format!( - "SketchWarmTier reducer failed to decode sketch for sid \ + "SketchStore reducer failed to decode sketch for sid \ {sid} (encoding={encoding:?}): {reason} — failing over \ to archive" ), )); } - Err(crate::engines::warm_tier::WarmTierError::NoData { - metric_name: m, - }) => { + Err(crate::engines::warm_tier::WarmTierError::NoData { metric_name: m }) => { return Err(crate::engines::EngineError::capability_miss( - asap_types::StorageBackend::SketchWarmTier.data_source_id(), + asap_types::StorageBackend::SketchStore.data_source_id(), format!( - "SketchWarmTier reducer found no samples for metric \ + "SketchStore reducer found no samples for metric \ `{m}` in window — failing over to archive" ), )); @@ -3848,9 +3836,9 @@ impl crate::routing::engine_router::QueryEngine for SimpleEngine { sketch_kind, }) => { return Err(crate::engines::EngineError::capability_miss( - asap_types::StorageBackend::SketchWarmTier.data_source_id(), + asap_types::StorageBackend::SketchStore.data_source_id(), format!( - "SketchWarmTier reducer cannot enumerate top-k for sid \ + "SketchStore reducer cannot enumerate top-k for sid \ {sid} (sketch_kind={sketch_kind:?}, no heap) — \ failing over to archive" ), @@ -3877,10 +3865,7 @@ impl crate::routing::engine_router::QueryEngine for SimpleEngine { let archive_qr = archive.execute(query).await; if let Ok(archive_qr) = archive_qr { return Ok(stitch_warm_and_archive( - warm_qr, - archive_qr, - cov_lo, - cov_hi, + warm_qr, archive_qr, cov_lo, cov_hi, )); } // On archive error, fall back to warm-only. @@ -3900,16 +3885,16 @@ impl crate::routing::engine_router::QueryEngine for SimpleEngine { match self.handle_query(query.to_string(), now_ms) { Some((_labels, result)) => Ok(result), None => Err(crate::engines::EngineError::capability_miss( - asap_types::StorageBackend::SketchWarmTier.data_source_id(), - format!("SimpleEngine has no compatible aggregation for `{query}`"), + asap_types::StorageBackend::SketchStore.data_source_id(), + format!("ASAPQueryEngine has no compatible aggregation for `{query}`"), )), } } fn capabilities(&self) -> crate::routing::engine_router::EngineCapabilities { crate::routing::engine_router::EngineCapabilities { - data_source_id: asap_types::StorageBackend::SketchWarmTier.data_source_id(), - storage_backend: asap_types::StorageBackend::SketchWarmTier, + data_source_id: asap_types::StorageBackend::SketchStore.data_source_id(), + storage_backend: asap_types::StorageBackend::SketchStore, // Warm-tier sketches are O(sketch-size); call it 16 MiB ceiling // for buffered ops (KLL with k=200 is well below this). supports_streams_above_bytes: 16 * 1024 * 1024, @@ -4695,7 +4680,7 @@ mod range_query_tests { #[cfg(test)] mod sketch_query_tests { // use crate::data_model::{CleanupPolicy, InferenceConfig, QueryLanguage, StreamingConfig}; - // use crate::engines::simple::engine::SimpleEngine; + // use crate::engines::asap_query::engine::SimpleEngine; // use crate::stores::promsketch_store::PromSketchStore; // use crate::stores::{Store, TimestampedBucketsMap}; // use std::collections::HashMap; @@ -5770,8 +5755,8 @@ mod forced_agg_id_tests { mod sketch_alias_resolver_tests { use super::*; use crate::data_model::{ - AggregationConfig, CleanupPolicy, HotReloadStreamingConfig, InferenceConfig, - PromQLSchema, QueryLanguage, SchemaConfig, StreamingConfig, WindowType, + AggregationConfig, CleanupPolicy, HotReloadStreamingConfig, InferenceConfig, PromQLSchema, + QueryLanguage, SchemaConfig, StreamingConfig, WindowType, }; use crate::stores::sketch_db::simple_map_store::SimpleMapStore; use std::sync::Arc; @@ -5908,8 +5893,7 @@ mod sketch_alias_resolver_tests { fn rate_query_is_not_touched() { // CountMin processor doesn't rename today; `rate(metric[5m])` // must pass through unchanged. - let engine = - engine_with(&[("endpoint_request_freq", AggregationType::CountMinSketch)]); + let engine = engine_with(&[("endpoint_request_freq", AggregationType::CountMinSketch)]); let q = "rate(endpoint_request_freq[5m])"; assert!(engine.resolve_sketch_metric_alias(q).is_none()); } @@ -5963,9 +5947,14 @@ mod hll_count_query_tests { for &v in observations { let h = v.wrapping_mul(0x9E37_79B9_7F4A_7C15); let bucket = (h >> (64 - 8)) as usize; // top 8 bits - // Remaining 56 bits — count leading zeros + 1 (capped at 64). + // Remaining 56 bits — count leading zeros + 1 (capped at 64). let rem = h << 8; - let lz = if rem == 0 { 64 - 8 } else { rem.leading_zeros() } as u8 + 1; + let lz = if rem == 0 { + 64 - 8 + } else { + rem.leading_zeros() + } as u8 + + 1; if (bucket as u64) < m { let r = &mut acc.inner.registers[bucket]; if lz > *r { @@ -6001,10 +5990,7 @@ mod hll_count_query_tests { .query_statistic(Statistic::Count, &None, &HashMap::new()) .expect("empty HLL still answers Count"); // Linear-counting branch returns 0 when all registers are 0. - assert!( - v.abs() < 1e-9, - "empty HLL cardinality should be 0, got {v}" - ); + assert!(v.abs() < 1e-9, "empty HLL cardinality should be 0, got {v}"); } #[test] @@ -6031,10 +6017,7 @@ mod hll_count_query_tests { // the resolved agg is the HLL one. Regression guard for the // PR #111 honest-gap closure on HLL-Count capability. let acc = hll_with_observations(&(1..=50).collect::>()); - let data = vec![( - None, - Box::new(acc) as Box, - )]; + let data = vec![(None, Box::new(acc) as Box)]; // No `by (...)` modifier on the query → empty grouping. The // engine factory's HLL agg is registered with empty grouping // labels; this matches the warm-tier production shape. @@ -6077,10 +6060,7 @@ mod kll_quantile_query_tests { // `request_size_bytes_quantile` and verify // `quantile_over_time(0.99, ...)` resolves to it. let acc = DatasketchesKLLAccumulator::new(200); - let data = vec![( - None, - Box::new(acc) as Box, - )]; + let data = vec![(None, Box::new(acc) as Box)]; let engine = create_engine_single_pop( "request_size_bytes_quantile", AggregationType::DatasketchesKLL, @@ -6100,7 +6080,10 @@ mod kll_quantile_query_tests { ); assert_eq!(ctx.metadata.statistic_to_compute, Statistic::Quantile); assert_eq!( - ctx.metadata.query_kwargs.get("quantile").map(String::as_str), + ctx.metadata + .query_kwargs + .get("quantile") + .map(String::as_str), Some("0.99") ); } @@ -6121,10 +6104,7 @@ mod cms_rate_capability_tests { #[test] fn capability_matching_resolves_rate_to_count_min_sketch() { let acc = CountMinSketchAccumulator::new(4, 64); - let data = vec![( - None, - Box::new(acc) as Box, - )]; + let data = vec![(None, Box::new(acc) as Box)]; let engine = create_engine_single_pop( "endpoint_request_freq", AggregationType::CountMinSketch, @@ -6146,7 +6126,10 @@ mod cms_rate_capability_tests { // The engine pushes range_ms into kwargs so the CMS // accumulator can divide events by seconds at query time. assert_eq!( - ctx.metadata.query_kwargs.get("range_ms").map(String::as_str), + ctx.metadata + .query_kwargs + .get("range_ms") + .map(String::as_str), Some("60000") ); } @@ -6156,7 +6139,7 @@ mod cms_rate_capability_tests { /// Pre-Phase-5 the trait adapter unconditionally delegated to /// `handle_query`. After Phase 5 wire-in, when a `SketchIndex` is /// attached, the adapter classifies first and surfaces -/// `EngineError::CapabilityMiss(SketchWarmTier, ...)` on Ghost / Unknown +/// `EngineError::CapabilityMiss(SketchStore, ...)` on Ghost / Unknown /// / no-instance outcomes so the EngineRouter (Phase 6) can fall /// through to the archive engine. #[cfg(test)] @@ -6194,11 +6177,16 @@ mod warm_tier_classify_tests { } fn dd_meta(sid: u64, metric: &str, group_by: &[&str]) -> SketchInstanceMetadata { - let cfg = SketchConfig::DDSketch { relative_accuracy: 0.01 }; + let cfg = SketchConfig::DDSketch { + relative_accuracy: 0.01, + }; SketchInstanceMetadata { sid, metric_name: metric.to_string(), - group_by_keys: group_by.iter().map(|s| s.to_string()).collect::>(), + group_by_keys: group_by + .iter() + .map(|s| s.to_string()) + .collect::>(), capability: Capability::QuantileApprox(SketchKindHandle::DDSketch), sketch_kind: SketchKindHandle::DDSketch, sketch_config: cfg.clone(), @@ -6213,14 +6201,15 @@ mod warm_tier_classify_tests { // capability-miss rather than burn a `handle_query` round-trip. let idx = Arc::new(SketchIndex::new()); let engine = build_engine_with_index(idx); - let err = engine.execute("unknown_metric{zone=\"z0\"}").await.expect_err( - "warm-tier with no matching instance must yield CapabilityMiss", - ); + let err = engine + .execute("unknown_metric{zone=\"z0\"}") + .await + .expect_err("warm-tier with no matching instance must yield CapabilityMiss"); match err { EngineError::CapabilityMiss { engine_id, .. } => { assert_eq!( engine_id, - asap_types::StorageBackend::SketchWarmTier.data_source_id() + asap_types::StorageBackend::SketchStore.data_source_id() ); } other => panic!("expected CapabilityMiss, got {other:?}"), @@ -6247,10 +6236,12 @@ mod warm_tier_classify_tests { EngineError::CapabilityMiss { engine_id, detail } => { assert_eq!( engine_id, - asap_types::StorageBackend::SketchWarmTier.data_source_id() + asap_types::StorageBackend::SketchStore.data_source_id() ); assert!( - detail.contains("ghost") || detail.contains("Ghost") || detail.contains("unknown"), + detail.contains("ghost") + || detail.contains("Ghost") + || detail.contains("unknown"), "detail mentions ghost/unknown: {detail}" ); } @@ -6287,14 +6278,11 @@ mod warm_tier_classify_tests { match result { Err(EngineError::CapabilityMiss { detail, .. }) => { assert!( - detail.contains("NoCallNodeFound") - || detail.contains("analyzer rejected"), + detail.contains("NoCallNodeFound") || detail.contains("analyzer rejected"), "expected NoCallNodeFound analyzer rejection: {detail}" ); } - other => panic!( - "expected analyzer-rejected CapabilityMiss, got {other:?}" - ), + other => panic!("expected analyzer-rejected CapabilityMiss, got {other:?}"), } } } @@ -6323,10 +6311,7 @@ mod hybrid_stitch_tests { #[test] fn stitch_fills_archive_prefix_and_suffix() { // Warm covers [100, 200] with timestamps 100, 150, 200. - let warm = matrix_with_samples( - "host=a", - vec![(100, 10.0), (150, 11.0), (200, 12.0)], - ); + let warm = matrix_with_samples("host=a", vec![(100, 10.0), (150, 11.0), (200, 12.0)]); // Archive covers [50, 250] with timestamps every 50ms. let archive = matrix_with_samples( "host=a", diff --git a/asap-query-engine/src/engines/asap_query/mod.rs b/asap-query-engine/src/engines/asap_query/mod.rs new file mode 100644 index 00000000..4cf6d568 --- /dev/null +++ b/asap-query-engine/src/engines/asap_query/mod.rs @@ -0,0 +1,24 @@ +//! Warm-tier sketch query engine. +//! +//! `ASAPQueryEngine` is the long-standing PromQL/SQL/Elasticsearch-DSL +//! query path that answers from the in-memory sketch DB +//! ([`crate::stores::sketch_db::SimpleMapStore`]) and its +//! per-`agg_id` precomputed accumulators. It returns ε/δ-bounded +//! approximate answers for sketch-resident queries and `None` on +//! a capability miss (router falls through, which after Step-1 of +//! the JSONL deprecation means the archive tier or a hard 404 — +//! the JSONL leg has been deleted). +//! +//! `SimpleEngine` remains as a compatibility type alias in the +//! implementation, but new code should refer to this engine as +//! `ASAPQueryEngine`. + +pub mod engine; + +#[cfg(test)] +pub mod tests; + +pub use engine::{ + ASAPQueryEngine, QueryExecutionContext, QueryMetadata, QueryTimestamps, SimpleEngine, + StoreQueryParams, StoreQueryPlan, +}; diff --git a/asap-query-engine/src/engines/simple/tests.rs b/asap-query-engine/src/engines/asap_query/tests.rs similarity index 77% rename from asap-query-engine/src/engines/simple/tests.rs rename to asap-query-engine/src/engines/asap_query/tests.rs index e3765b5c..667e7c75 100644 --- a/asap-query-engine/src/engines/simple/tests.rs +++ b/asap-query-engine/src/engines/asap_query/tests.rs @@ -2,10 +2,10 @@ //! //! Step-1 of the JSONL deprecation refactor moved //! `engines/simple_engine.rs` to `engines/simple/engine.rs`. The -//! engine's tests live inline in [`super::engine`] (~6 distinct +//! engine's tests live inline in [`super::query_engine`] (~6 distinct //! `#[cfg(test)] mod tests { ... }` blocks, each pinning a //! specific dispatch axis). They are exercised under -//! `crate::engines::simple::engine::tests` rather than this file +//! `crate::engines::asap_query::engine::tests` rather than this file //! to preserve `git blame` continuity across the move. //! //! Step-2 (Prometheus-block format + Thanos store-gateway) can diff --git a/asap-query-engine/src/engines/mod.rs b/asap-query-engine/src/engines/mod.rs index 2e0462d7..43e263f1 100644 --- a/asap-query-engine/src/engines/mod.rs +++ b/asap-query-engine/src/engines/mod.rs @@ -1,21 +1,18 @@ -//! Tier-co-located query engines. +//! Query engines. //! -//! Step-1 of the JSONL deprecation refactor split this module -//! into two tier sub-directories ([`simple`] for the warm sketch -//! tier, [`gorilla`] for the archive tier) plus the shared -//! infrastructure (`logical`/`physical` plan helpers, -//! `query_result`, `timeline_dispatch`, `window_merger`) used by -//! both. The capability `EngineRouter` and per-metric -//! `BackendStorageRouting` config loader moved out into -//! [`crate::routing`]. +//! The public query-engine surface is intentionally small: +//! [`asap_query`] answers from ASAP's sketch store, and +//! [`thanos_query`] forwards exact/archive queries to `thanos-query`. +//! Gorilla object storage lives under [`crate::stores::gorilla_object_store`] +//! because it is a storage implementation detail, not a public query-engine +//! family. //! //! ## Public surface //! -//! Three engines + the shared error envelope: +//! Engines + the shared error envelope: //! -//! * [`simple::SimpleEngine`] — warm-tier sketch query engine. -//! * [`gorilla::GorillaQueryEngine`] — archive-tier exact query -//! engine over the [`gorilla::store::GorillaS3Store`]. +//! * [`asap_query::ASAPQueryEngine`] — warm-tier sketch query engine. +//! * [`thanos_query::ThanosQueryEngine`] — archive-tier query engine. //! * [`prometheus::PrometheusForwardEngine`] — HTTP-forwarder to a //! Prometheus `/api/v1/query` endpoint, registered under the //! `prometheus_remote` engine id when @@ -23,22 +20,24 @@ //! * [`EngineError`] — the trait-level error envelope every //! `crate::routing::QueryEngine` impl returns. -pub mod gorilla; +pub mod asap_query; pub mod no_data_archive; pub mod prometheus; pub mod query_result; -pub mod simple; +pub mod thanos_query; pub mod timeline_dispatch; pub mod warm_tier; pub mod window_merger; -pub use gorilla::{ - EngineError as GorillaEngineError, GorillaEngineConfig, GorillaQueryEngine, -}; +pub use asap_query::{ASAPQueryEngine, SimpleEngine}; pub use no_data_archive::{NoDataArchiveEngine, DATA_SOURCE_ID_NO_DATA_ARCHIVE}; pub use prometheus::{PrometheusForwardConfig, PrometheusForwardEngine, PrometheusForwardError}; pub use query_result::{InstantVector, QueryResult, RangeVector, RangeVectorElement, Sample}; -pub use simple::SimpleEngine; +pub use thanos_query::{ + thanos_engine_from_env, ThanosQueryConfig, ThanosQueryEngine, ThanosQueryError, + ASAP_THANOS_QUERY_URL_ENV, DATA_SOURCE_THANOS_QUERY_ID, DATA_SOURCE_THANOS_QUERY_INFO, + DEFAULT_THANOS_QUERY_URL, QUIRK_THANOS_UNREACHABLE, +}; pub use timeline_dispatch::{combine_statistic, CombinedResult}; pub use window_merger::{create_window_merger, NaiveMerger, WindowMerger}; @@ -47,7 +46,7 @@ pub use window_merger::{create_window_merger, NaiveMerger, WindowMerger}; // // The trait is engine-agnostic, so its `execute` must return an error type // that can wrap *any* concrete engine's failure mode. Today's two engines -// — `SimpleEngine` (capability-miss → `None`) and `GorillaQueryEngine` +// — `ASAPQueryEngine` (capability-miss → `None`) and archive/query forwarders // (rich `EngineError`) — fold into this common envelope. // --------------------------------------------------------------------------- diff --git a/asap-query-engine/src/engines/no_data_archive.rs b/asap-query-engine/src/engines/no_data_archive.rs index 06608201..402e9193 100644 --- a/asap-query-engine/src/engines/no_data_archive.rs +++ b/asap-query-engine/src/engines/no_data_archive.rs @@ -7,15 +7,14 @@ //! has neither `ASAP_THANOS_QUERY_URL` nor `ASAP_GORILLA_S3_*` env //! vars set, no archive engine is registered on the //! [`crate::routing::EngineRouter`]. Cold queries (queries the -//! per-metric routing table sends to `gorilla_archive` / -//! `thanos_archive`) then surface as +//! per-metric routing table sends to `thanos_query`) then surface as //! `503 NoEngineRegistered` from the HTTP handler. //! //! Treating "no archive configured" as a 503 trips up dashboards and //! freshness probes that just want a degraded but successful answer. //! This engine flips that default: when the archive env vars are //! unset, the binary registers a [`NoDataArchiveEngine`] under the -//! `gorilla_archive` slot. Cold queries return an **empty result +//! `thanos_query` slot. Cold queries return an **empty result //! set** with `data_source_id = "no_data_archive"` so the wire //! response carries enough signal for operators to notice the //! misconfig without breaking the request path. @@ -68,11 +67,11 @@ impl QueryEngine for NoDataArchiveEngine { fn capabilities(&self) -> EngineCapabilities { EngineCapabilities { - data_source_id: DATA_SOURCE_ID_NO_DATA_ARCHIVE, - // Register under the archive slot. The binary aliases this - // engine onto the `gorilla_archive` id so the routing - // table's archive entries dispatch here transparently. - storage_backend: StorageBackend::GorillaS3Archive, + data_source_id: asap_types::ENGINE_ID_THANOS_QUERY, + // Register under the canonical archive query-engine id so + // archive entries dispatch here transparently when no real + // ThanosQueryEngine is configured. + storage_backend: StorageBackend::GorillaObjectStore, supports_streams_above_bytes: 0, } } @@ -98,7 +97,7 @@ mod tests { fn capabilities_use_no_data_archive_id() { let engine = NoDataArchiveEngine::new(); let caps = engine.capabilities(); - assert_eq!(caps.data_source_id, "no_data_archive"); - assert_eq!(caps.storage_backend, StorageBackend::GorillaS3Archive); + assert_eq!(caps.data_source_id, asap_types::ENGINE_ID_THANOS_QUERY); + assert_eq!(caps.storage_backend, StorageBackend::GorillaObjectStore); } } diff --git a/asap-query-engine/src/engines/prometheus/forward.rs b/asap-query-engine/src/engines/prometheus/forward.rs index f80b5f29..87ded7a9 100644 --- a/asap-query-engine/src/engines/prometheus/forward.rs +++ b/asap-query-engine/src/engines/prometheus/forward.rs @@ -27,10 +27,10 @@ //! `NoEngineRegistered` 503 from the HTTP handler — the correct //! fail-loud behaviour for a misconfigured deploy. //! -//! This is a near-mirror of [`crate::engines::gorilla::thanos_forward`] +//! This is a near-mirror of [`crate::engines::thanos_query::forward`] //! (the Step-2.3 archive forwarder), pointed at Prometheus's standard //! `/api/v1/query` endpoint instead of a `thanos-query` sidecar. The -//! two engines coexist: `thanos_archive` answers archive-tier queries +//! two engines coexist: `thanos_query` answers archive-tier queries //! over Prometheus TSDB blocks emitted by `gorillas3processor`; //! `prometheus_remote` answers queries for metrics whose raw data is //! shipped to Prometheus's native OTLP receiver (no ASAP archive at @@ -351,7 +351,9 @@ fn build_result_from_prometheus_payload( ) -> Result { if payload.status != "success" { let detail = payload.error.unwrap_or_else(|| "unknown error".to_string()); - let kind = payload.error_type.unwrap_or_else(|| "execution".to_string()); + let kind = payload + .error_type + .unwrap_or_else(|| "execution".to_string()); return Err(format!("prometheus error ({kind}): {detail}")); } let data = payload @@ -559,7 +561,10 @@ pub mod test_support { let app: Router = Router::new() .route("/api/v1/query", post(move || async move { canned_body })) - .route("/api/v1/query_range", post(move || async move { canned_body })); + .route( + "/api/v1/query_range", + post(move || async move { canned_body }), + ); let handle = tokio::spawn(async move { axum::serve(listener, app) @@ -680,7 +685,9 @@ mod tests { let infos = PrometheusForwardEngine::success_infos(0); assert!( - infos.iter().any(|s| s == DATA_SOURCE_PROMETHEUS_REMOTE_INFO), + infos + .iter() + .any(|s| s == DATA_SOURCE_PROMETHEUS_REMOTE_INFO), "success_infos must carry the data_source marker; got {infos:?}", ); assert!( @@ -749,7 +756,9 @@ mod tests { // dashboards / e2e demos pin against. let infos = PrometheusForwardEngine::unreachable_infos("upstream returned 503", 0); assert!(infos.iter().any(|s| s == QUIRK_PROMETHEUS_UNREACHABLE)); - assert!(infos.iter().any(|s| s.contains("prometheus_unreachable_reason"))); + assert!(infos + .iter() + .any(|s| s.contains("prometheus_unreachable_reason"))); } #[tokio::test] @@ -769,10 +778,8 @@ mod tests { #[tokio::test] async fn config_from_env_strips_trailing_slash() { let _g = ENV_LOCK.lock().expect("lock"); - let _scope = test_support::EnvGuard::set( - ASAP_PROMETHEUS_QUERY_URL_ENV, - "http://prometheus:9090/", - ); + let _scope = + test_support::EnvGuard::set(ASAP_PROMETHEUS_QUERY_URL_ENV, "http://prometheus:9090/"); let cfg = PrometheusForwardConfig::from_env().expect("set"); assert_eq!(cfg.base_url, "http://prometheus:9090"); } @@ -780,10 +787,8 @@ mod tests { #[tokio::test] async fn engine_from_env_returns_some_when_set() { let _g = ENV_LOCK.lock().expect("lock"); - let _scope = test_support::EnvGuard::set( - ASAP_PROMETHEUS_QUERY_URL_ENV, - "http://127.0.0.1:1", - ); + let _scope = + test_support::EnvGuard::set(ASAP_PROMETHEUS_QUERY_URL_ENV, "http://127.0.0.1:1"); let engine = engine_from_env().expect("ok"); assert!(engine.is_some(), "env set → engine constructed"); } @@ -810,16 +815,16 @@ mod tests { // Env set → engine registered. { let _g = ENV_LOCK.lock().expect("lock"); - let _scope = test_support::EnvGuard::set( - ASAP_PROMETHEUS_QUERY_URL_ENV, - "http://127.0.0.1:1", - ); + let _scope = + test_support::EnvGuard::set(ASAP_PROMETHEUS_QUERY_URL_ENV, "http://127.0.0.1:1"); let mut router = EngineRouter::new(); if let Ok(Some(engine)) = engine_from_env() { router.register(Arc::new(engine)); } assert!( - router.engine_by_id(DATA_SOURCE_PROMETHEUS_REMOTE_ID).is_some(), + router + .engine_by_id(DATA_SOURCE_PROMETHEUS_REMOTE_ID) + .is_some(), "env set must yield prometheus_remote engine in the router", ); } @@ -833,7 +838,9 @@ mod tests { router.register(Arc::new(engine)); } assert!( - router.engine_by_id(DATA_SOURCE_PROMETHEUS_REMOTE_ID).is_none(), + router + .engine_by_id(DATA_SOURCE_PROMETHEUS_REMOTE_ID) + .is_none(), "env unset must leave prometheus_remote unregistered", ); } diff --git a/asap-query-engine/src/engines/prometheus/mod.rs b/asap-query-engine/src/engines/prometheus/mod.rs index 78734853..cd66551e 100644 --- a/asap-query-engine/src/engines/prometheus/mod.rs +++ b/asap-query-engine/src/engines/prometheus/mod.rs @@ -12,7 +12,7 @@ //! 503 from the HTTP handler (the correct fail-loud behaviour for a //! misconfigured deploy). //! -//! Sibling of [`crate::engines::gorilla::thanos_forward`] (the +//! Sibling of [`crate::engines::thanos_query::forward`] (the //! Step-2.3 archive forwarder); the two engines coexist in the //! router under different ids and answer different routing-table //! entries. diff --git a/asap-query-engine/src/engines/simple/mod.rs b/asap-query-engine/src/engines/simple/mod.rs deleted file mode 100644 index c8bffef1..00000000 --- a/asap-query-engine/src/engines/simple/mod.rs +++ /dev/null @@ -1,29 +0,0 @@ -//! Warm-tier sketch query engine. -//! -//! `SimpleEngine` is the long-standing PromQL/SQL/Elasticsearch-DSL -//! query path that answers from the in-memory sketch DB -//! ([`crate::stores::sketch_db::SimpleMapStore`]) and its -//! per-`agg_id` precomputed accumulators. It returns ε/δ-bounded -//! approximate answers for sketch-resident queries and `None` on -//! a capability miss (router falls through, which after Step-1 of -//! the JSONL deprecation means the archive tier or a hard 404 — -//! the JSONL leg has been deleted). -//! -//! Step-1 of the JSONL deprecation refactor moved this module -//! from `engines/simple_engine.rs` (single file) into -//! `engines/simple/{mod.rs, engine.rs, tests.rs}` so the -//! warm-tier query engine sits under its own tier-co-located -//! directory, mirroring [`crate::engines::gorilla`] for the -//! archive tier. The engine's data model + execution code is -//! kept verbatim in [`engine`]; this `mod.rs` is the public -//! surface re-exporting the long-standing types. - -pub mod engine; - -#[cfg(test)] -pub mod tests; - -pub use engine::{ - QueryExecutionContext, QueryMetadata, QueryTimestamps, SimpleEngine, StoreQueryParams, - StoreQueryPlan, -}; diff --git a/asap-query-engine/src/engines/gorilla/thanos_forward.rs b/asap-query-engine/src/engines/thanos_query/forward.rs similarity index 82% rename from asap-query-engine/src/engines/gorilla/thanos_forward.rs rename to asap-query-engine/src/engines/thanos_query/forward.rs index 12d9ca8a..6e28dca7 100644 --- a/asap-query-engine/src/engines/gorilla/thanos_forward.rs +++ b/asap-query-engine/src/engines/thanos_query/forward.rs @@ -1,4 +1,4 @@ -//! `ThanosForwardEngine` — HTTP forwarder to a `thanos-query` +//! `ThanosQueryEngine` — HTTP forwarder to a `thanos-query` //! sidecar for Path A2 of the Step-2 archive deprecation. //! //! Step-2.1 (PR #311) teaches `gorillas3processor` to emit @@ -11,24 +11,25 @@ //! [`ASAP_THANOS_QUERY_URL_ENV`] env var, consulted at backend //! startup: //! -//! * **Path A2 mode** (env set) — `ThanosForwardEngine` is +//! * **Path A2 mode** (env set) — `ThanosQueryEngine` is //! registered in the [`crate::routing::EngineRouter`]. Archive //! queries POST to `${ASAP_THANOS_QUERY_URL}/api/v1/query` and //! the answer is wrapped in ASAP's standard //! [`crate::engines::QueryResult`] shape. //! * **Legacy mode** (env unset) — the in-process -//! [`super::GorillaQueryEngine`] handles archive queries from -//! the per-hour Gorilla chunks the -//! [`super::store::GorillaS3Store`] streams from S3 / MinIO. +//! [`crate::stores::gorilla_object_store::GorillaQueryEngine`] +//! handles archive queries from the per-hour Gorilla chunks that +//! [`crate::stores::gorilla_object_store::GorillaS3Store`] streams +//! from S3 / MinIO. //! Phase δ deletes this leg after Path A2 is verified //! end-to-end. //! //! The two modes are mutually exclusive: when Path A2 is active, -//! both the legacy id (`gorilla_archive`) and the alias id -//! (`thanos_archive`) point at the same `ThanosForwardEngine` +//! both the legacy id (`thanos_query`) and the alias id +//! (`thanos_query`) point at the same `ThanosQueryEngine` //! instance, so the per-metric `BackendStorageRouting` config can //! target either name without surprise. See the binary's -//! `register_thanos_or_gorilla_archive` helper for the +//! `register_thanos_or_thanos_query` helper for the //! registration site. use std::time::{Duration, Instant}; @@ -52,10 +53,10 @@ use crate::stores::sketch_db::accuracy::{AccuracyEnvelope, AccuracyProfile}; // --------------------------------------------------------------------------- /// Env var consulted at backend startup. When set, the binary -/// registers a [`ThanosForwardEngine`] pointing at the URL and the +/// registers a [`ThanosQueryEngine`] pointing at the URL and the /// router dispatches archive-tier queries to it. When unset, the -/// legacy in-process [`super::GorillaQueryEngine`] handles archive -/// queries. +/// legacy in-process [`crate::stores::gorilla_object_store::GorillaQueryEngine`] +/// handles archive queries. pub const ASAP_THANOS_QUERY_URL_ENV: &str = "ASAP_THANOS_QUERY_URL"; /// Default upstream URL when `ASAP_THANOS_QUERY_URL` is set to the @@ -64,16 +65,16 @@ pub const ASAP_THANOS_QUERY_URL_ENV: &str = "ASAP_THANOS_QUERY_URL"; /// `mvp-thanos-archive.yml` pins `thanos-query:10903`). pub const DEFAULT_THANOS_QUERY_URL: &str = "http://thanos-query:10903"; -/// `data_source_id` the [`ThanosForwardEngine`] registers under +/// `data_source_id` the [`ThanosQueryEngine`] registers under /// for explicit per-query overrides via the `X-ASAP-Engine` header /// or the `?engine=` query param. Pinned so dashboards / e2e /// scripts can byte-compare without parsing. -pub const DATA_SOURCE_THANOS_ARCHIVE_ID: &str = "thanos_archive"; +pub const DATA_SOURCE_THANOS_QUERY_ID: &str = asap_types::ENGINE_ID_THANOS_QUERY; -/// Marker line every `ThanosForwardEngine` answer carries on its +/// Marker line every `ThanosQueryEngine` answer carries on its /// `infos` array. Pinned so dashboards and the upcoming Step-2.4 /// e2e demo can byte-compare without parsing. -pub const DATA_SOURCE_THANOS_ARCHIVE_INFO: &str = "data_source: thanos_archive"; +pub const DATA_SOURCE_THANOS_QUERY_INFO: &str = "data_source: thanos_query"; /// `data_source_quirk` line surfaced when the upstream /// `thanos-query` sidecar is unreachable (network error / 5xx / @@ -91,10 +92,10 @@ pub const DEFAULT_THANOS_REQUEST_TIMEOUT: Duration = Duration::from_secs(60); // Config + engine. // --------------------------------------------------------------------------- -/// Tunable runtime knobs for [`ThanosForwardEngine`]. Built from -/// env via [`ThanosForwardConfig::from_env`]. +/// Tunable runtime knobs for [`ThanosQueryEngine`]. Built from +/// env via [`ThanosQueryConfig::from_env`]. #[derive(Debug, Clone)] -pub struct ThanosForwardConfig { +pub struct ThanosQueryConfig { /// Base URL of the upstream `thanos-query` sidecar — e.g. /// `http://thanos-query:10903`. The engine appends /// `/api/v1/query` (or `/api/v1/query_range`) when forwarding. @@ -104,7 +105,7 @@ pub struct ThanosForwardConfig { pub request_timeout: Duration, } -impl Default for ThanosForwardConfig { +impl Default for ThanosQueryConfig { fn default() -> Self { Self { base_url: DEFAULT_THANOS_QUERY_URL.to_string(), @@ -113,7 +114,7 @@ impl Default for ThanosForwardConfig { } } -impl ThanosForwardConfig { +impl ThanosQueryConfig { /// Build a config from the [`ASAP_THANOS_QUERY_URL_ENV`] env /// var, returning `None` when the var is unset / empty / blank /// (the binary should then fall through to the legacy @@ -147,54 +148,40 @@ impl ThanosForwardConfig { /// Implements the [`QueryEngine`] trait so the /// [`crate::routing::EngineRouter`] can hold it as `Arc`. Reports `data_source_id = -/// "thanos_archive"` and (for the compatibility-list dispatch path) -/// `storage_backend = StorageBackend::GorillaS3Archive` — Path A2 +/// "thanos_query"` and (for the compatibility-list dispatch path) +/// `storage_backend = StorageBackend::GorillaObjectStore` — Path A2 /// re-uses the archive tier slot in the routing matrix, so any -/// metric configured for `GorillaS3Archive` keeps routing through +/// metric configured for `GorillaObjectStore` keeps routing through /// the archive tier; only the engine answering changes. -pub struct ThanosForwardEngine { - config: ThanosForwardConfig, +pub struct ThanosQueryEngine { + config: ThanosQueryConfig, client: reqwest::Client, - /// Pinned id we register under. Defaults to - /// [`DATA_SOURCE_THANOS_ARCHIVE_ID`]; `with_data_source_id` - /// lets the binary's "alias under the legacy slot" wiring use - /// the same engine instance under both `thanos_archive` and - /// `gorilla_archive`. + /// Pinned id we register under. data_source_id: &'static str, } -impl ThanosForwardEngine { +impl ThanosQueryEngine { /// Build with an explicit config. Used by tests + the binary's /// startup wiring. - pub fn new(config: ThanosForwardConfig) -> Result { + pub fn new(config: ThanosQueryConfig) -> Result { let client = reqwest::Client::builder() .timeout(config.request_timeout) .build() - .map_err(|e| ThanosForwardError::ConfigInvalid(e.to_string()))?; + .map_err(|e| ThanosQueryError::ConfigInvalid(e.to_string()))?; Ok(Self { config, client, - data_source_id: DATA_SOURCE_THANOS_ARCHIVE_ID, + data_source_id: DATA_SOURCE_THANOS_QUERY_ID, }) } /// Build the production config from /// [`ASAP_THANOS_QUERY_URL_ENV`] or return `None` when the env /// var is unset / blank. The binary calls this first; if it - /// returns `None`, the legacy in-process `GorillaQueryEngine` - /// is registered instead. - pub fn from_env() -> Option> { - ThanosForwardConfig::from_env().map(Self::new) - } - - /// Override the registered `data_source_id`. Used by the - /// binary's "alias under the legacy slot" wiring to register - /// the same engine instance under `gorilla_archive` so the - /// existing `compatible_storage_backends` failover sequence - /// finds it transparently. - pub fn with_data_source_id(mut self, id: &'static str) -> Self { - self.data_source_id = id; - self + /// returns `None`, the legacy in-process Gorilla object-store + /// executor is registered instead. + pub fn from_env() -> Option> { + ThanosQueryConfig::from_env().map(Self::new) } /// Read-only access to the configured base URL — useful for @@ -204,13 +191,13 @@ impl ThanosForwardEngine { } /// The infos a successful forwarded answer carries. The - /// `data_source: thanos_archive` line is added by the HTTP + /// `data_source: thanos_query` line is added by the HTTP /// handler's `annotate_data_source` step (driven from /// `capabilities().data_source_id`), so tests pin the strings /// here without re-implementing the wire path. pub fn success_infos(elapsed_ms: u128) -> Vec { vec![ - DATA_SOURCE_THANOS_ARCHIVE_INFO.to_string(), + DATA_SOURCE_THANOS_QUERY_INFO.to_string(), AccuracyProfile::exact().summary(), format!("query_latency_ms: {elapsed_ms}"), ] @@ -221,7 +208,7 @@ impl ThanosForwardEngine { /// fail-loud behaviour. pub fn unreachable_infos(reason: &str, elapsed_ms: u128) -> Vec { vec![ - DATA_SOURCE_THANOS_ARCHIVE_INFO.to_string(), + DATA_SOURCE_THANOS_QUERY_INFO.to_string(), QUIRK_THANOS_UNREACHABLE.to_string(), format!("thanos_unreachable_reason: {reason}"), format!("query_latency_ms: {elapsed_ms}"), @@ -231,9 +218,9 @@ impl ThanosForwardEngine { /// Forward `query` to `${base_url}/api/v1/query` and parse the /// Prometheus-format response back into a [`QueryResult`]. /// - /// Errors are folded into [`ThanosForwardError`] variants — + /// Errors are folded into [`ThanosQueryError`] variants — /// the [`QueryEngine`] impl decides how to surface each. - pub async fn query(&self, query: &str) -> Result { + pub async fn query(&self, query: &str) -> Result { let started = Instant::now(); let url = self.config.instant_endpoint(); debug!( @@ -248,11 +235,11 @@ impl ThanosForwardEngine { .form(&[("query", query)]) .send() .await - .map_err(|e| ThanosForwardError::Unreachable(e.to_string()))?; + .map_err(|e| ThanosQueryError::Unreachable(e.to_string()))?; let status = resp.status(); if status.is_server_error() { - return Err(ThanosForwardError::Unreachable(format!( + return Err(ThanosQueryError::Unreachable(format!( "upstream returned {status}", ))); } @@ -261,7 +248,7 @@ impl ThanosForwardEngine { // not an unreachable upstream, it's a query the // sidecar doesn't accept. let body = resp.text().await.unwrap_or_default(); - return Err(ThanosForwardError::BadQuery { + return Err(ThanosQueryError::BadQuery { status: status.as_u16(), body, }); @@ -270,21 +257,21 @@ impl ThanosForwardEngine { let payload: ThanosResponse = resp .json() .await - .map_err(|e| ThanosForwardError::ParseError(e.to_string()))?; + .map_err(|e| ThanosQueryError::ParseError(e.to_string()))?; let elapsed_ms = started.elapsed().as_millis(); let result = build_result_from_thanos_payload(payload, elapsed_ms) - .map_err(ThanosForwardError::ParseError)?; + .map_err(ThanosQueryError::ParseError)?; Ok(result) } } #[async_trait] -impl QueryEngine for ThanosForwardEngine { +impl QueryEngine for ThanosQueryEngine { async fn execute(&self, query: &str) -> Result { match self.query(query).await { Ok(result) => Ok(result), - Err(ThanosForwardError::Unreachable(reason)) => { + Err(ThanosQueryError::Unreachable(reason)) => { // Surface fail-loud as a backend error so the // router's failover sequence can fall through to // the warm-tier sketch on a `DoubleWrite` deploy. @@ -301,24 +288,20 @@ impl QueryEngine for ThanosForwardEngine { format!("thanos_unreachable: {reason}"), )) } - Err(ThanosForwardError::BadQuery { status, body }) => { + Err(ThanosQueryError::BadQuery { status, body }) => { Err(crate::engines::EngineError::capability_miss( self.data_source_id, format!("thanos rejected query (status {status}): {body}"), )) } - Err(ThanosForwardError::ParseError(msg)) => { - Err(crate::engines::EngineError::backend( - self.data_source_id, - format!("thanos response parse error: {msg}"), - )) - } - Err(ThanosForwardError::ConfigInvalid(msg)) => { - Err(crate::engines::EngineError::backend( - self.data_source_id, - format!("thanos client misconfigured: {msg}"), - )) - } + Err(ThanosQueryError::ParseError(msg)) => Err(crate::engines::EngineError::backend( + self.data_source_id, + format!("thanos response parse error: {msg}"), + )), + Err(ThanosQueryError::ConfigInvalid(msg)) => Err(crate::engines::EngineError::backend( + self.data_source_id, + format!("thanos client misconfigured: {msg}"), + )), } } @@ -328,7 +311,7 @@ impl QueryEngine for ThanosForwardEngine { // Re-uses the archive tier slot in the routing // matrix; Path A2 swaps the engine answering, not the // tier classification. See module docstring. - storage_backend: asap_types::StorageBackend::GorillaS3Archive, + storage_backend: asap_types::StorageBackend::GorillaObjectStore, // Forwarder doesn't materialise samples locally; // upstream thanos-query owns the memory budget. We // surface a generous ceiling so the cost-aware @@ -374,7 +357,9 @@ fn build_result_from_thanos_payload( ) -> Result { if payload.status != "success" { let detail = payload.error.unwrap_or_else(|| "unknown error".to_string()); - let kind = payload.error_type.unwrap_or_else(|| "execution".to_string()); + let kind = payload + .error_type + .unwrap_or_else(|| "execution".to_string()); return Err(format!("thanos error ({kind}): {detail}")); } let data = payload @@ -410,11 +395,11 @@ fn build_result_from_thanos_payload( // `infos: []` array is appended downstream from the // PrometheusResponse adapter (`with_accuracy` already mirrors // a one-liner there). For dashboards that pin - // `data_source: thanos_archive` byte-compares, the HTTP + // `data_source: thanos_query` byte-compares, the HTTP // handler's `annotate_data_source` step adds the marker line // to `infos` after dispatch — but only when the engine reports - // the `thanos_archive` id; aliased registrations under - // `gorilla_archive` get the gorilla marker instead, which is + // the `thanos_query` id; aliased registrations under + // `thanos_query` get the gorilla marker instead, which is // fine for Path A2 backwards compat. let _ = elapsed_ms; // surfaced via tests directly via `success_infos`. Ok(result) @@ -494,7 +479,7 @@ fn parse_matrix(values: &[Value]) -> Result { /// the values into `KeyByLabelValues` (the same shape the /// in-process engine pins on its results). Unknown / non-object /// shapes fall through to an empty label set rather than failing -/// the parse — the wrapped `data_source: thanos_archive` info is +/// the parse — the wrapped `data_source: thanos_query` info is /// the meaningful annotation. fn labels_from_metric(metric: &Value) -> KeyByLabelValues { if let Some(obj) = metric.as_object() { @@ -519,7 +504,7 @@ fn labels_from_metric(metric: &Value) -> KeyByLabelValues { /// envelope; the public `query` method returns the richer surface /// for tests and direct callers. #[derive(Debug, thiserror::Error)] -pub enum ThanosForwardError { +pub enum ThanosQueryError { /// Upstream returned a network error / timeout / 5xx — /// dashboard-level "thanos is down." #[error("thanos unreachable: {0}")] @@ -549,16 +534,16 @@ pub enum ThanosForwardError { // --------------------------------------------------------------------------- /// Convenience combinator the binary uses at startup: try -/// [`ThanosForwardEngine::from_env`]; if it returns `None`, the +/// [`ThanosQueryEngine::from_env`]; if it returns `None`, the /// caller falls through to the legacy in-process -/// [`super::GorillaQueryEngine`] path. +/// [`crate::stores::gorilla_object_store::GorillaQueryEngine`] path. /// /// Returning `Result, ...>` instead of unwrapping in /// `main.rs` keeps the construction failure (bad URL / bad TLS /// init) inspectable so the binary can emit a helpful warning /// instead of crashing on startup. -pub fn engine_from_env() -> Result, ThanosForwardError> { - match ThanosForwardEngine::from_env() { +pub fn engine_from_env() -> Result, ThanosQueryError> { + match ThanosQueryEngine::from_env() { Some(Ok(engine)) => Ok(Some(engine)), Some(Err(e)) => Err(e), None => Ok(None), @@ -596,12 +581,13 @@ pub mod test_support { let app: Router = Router::new() .route("/api/v1/query", post(move || async move { canned_body })) - .route("/api/v1/query_range", post(move || async move { canned_body })); + .route( + "/api/v1/query_range", + post(move || async move { canned_body }), + ); let handle = tokio::spawn(async move { - axum::serve(listener, app) - .await - .expect("mock_thanos serve"); + axum::serve(listener, app).await.expect("mock_thanos serve"); }); // Best-effort: yield once so the listener is definitely @@ -700,8 +686,8 @@ mod tests { use super::*; use crate::engines::query_result::QueryResult; - fn config_for(url: &str) -> ThanosForwardConfig { - ThanosForwardConfig { + fn config_for(url: &str) -> ThanosQueryConfig { + ThanosQueryConfig { base_url: url.trim_end_matches('/').to_string(), request_timeout: Duration::from_secs(5), } @@ -710,12 +696,12 @@ mod tests { #[tokio::test] async fn forwards_promql_and_wraps_response() { let (url, _handle) = spawn_mock_thanos(CANNED_VECTOR_BODY).await; - let engine = ThanosForwardEngine::new(config_for(&url)).expect("engine"); + let engine = ThanosQueryEngine::new(config_for(&url)).expect("engine"); let result = engine.query("up").await.expect("ok response"); - let infos = ThanosForwardEngine::success_infos(0); + let infos = ThanosQueryEngine::success_infos(0); assert!( - infos.iter().any(|s| s == DATA_SOURCE_THANOS_ARCHIVE_INFO), + infos.iter().any(|s| s == DATA_SOURCE_THANOS_QUERY_INFO), "success_infos must carry the data_source marker; got {infos:?}", ); assert!( @@ -739,36 +725,27 @@ mod tests { } #[tokio::test] - async fn capabilities_report_thanos_archive_id() { + async fn capabilities_report_thanos_query_id() { // No upstream needed — we only inspect capabilities. - let engine = - ThanosForwardEngine::new(config_for("http://127.0.0.1:1")).expect("engine"); + let engine = ThanosQueryEngine::new(config_for("http://127.0.0.1:1")).expect("engine"); let caps = engine.capabilities(); - assert_eq!(caps.data_source_id, DATA_SOURCE_THANOS_ARCHIVE_ID); + assert_eq!(caps.data_source_id, DATA_SOURCE_THANOS_QUERY_ID); assert_eq!( caps.storage_backend, - asap_types::StorageBackend::GorillaS3Archive, + asap_types::StorageBackend::GorillaObjectStore, "Path A2 re-uses the archive tier slot in the routing matrix", ); } - #[tokio::test] - async fn alias_registration_reports_overridden_id() { - let engine = ThanosForwardEngine::new(config_for("http://127.0.0.1:1")) - .expect("engine") - .with_data_source_id("gorilla_archive"); - assert_eq!(engine.capabilities().data_source_id, "gorilla_archive"); - } - #[tokio::test] async fn unreachable_upstream_returns_503_quirk_via_engine_trait() { let (url, _handle) = spawn_mock_thanos_503().await; - let engine = ThanosForwardEngine::new(config_for(&url)).expect("engine"); + let engine = ThanosQueryEngine::new(config_for(&url)).expect("engine"); // The richer surface returns Unreachable. let direct = engine.query("up").await; match direct { - Err(ThanosForwardError::Unreachable(_)) => {} + Err(ThanosQueryError::Unreachable(_)) => {} other => panic!("expected Unreachable error, got {other:?}"), } @@ -779,7 +756,7 @@ mod tests { let trait_path = QueryEngine::execute(&engine, "up").await; match trait_path { Err(crate::engines::EngineError::Backend { engine_id, message }) => { - assert_eq!(engine_id, DATA_SOURCE_THANOS_ARCHIVE_ID); + assert_eq!(engine_id, DATA_SOURCE_THANOS_QUERY_ID); assert!( message.contains("thanos_unreachable"), "Backend error must carry thanos_unreachable marker; got {message:?}", @@ -790,43 +767,40 @@ mod tests { // The unreachable_infos helper exposes the wire shape // dashboards / e2e demos pin against. - let infos = ThanosForwardEngine::unreachable_infos("upstream returned 503", 0); + let infos = ThanosQueryEngine::unreachable_infos("upstream returned 503", 0); assert!(infos.iter().any(|s| s == QUIRK_THANOS_UNREACHABLE)); - assert!(infos.iter().any(|s| s.contains("thanos_unreachable_reason"))); + assert!(infos + .iter() + .any(|s| s.contains("thanos_unreachable_reason"))); } #[tokio::test] async fn config_from_env_returns_none_when_unset() { let _g = ENV_LOCK.lock().expect("lock"); let _scope = test_support::EnvGuard::unset(ASAP_THANOS_QUERY_URL_ENV); - assert!(ThanosForwardConfig::from_env().is_none()); + assert!(ThanosQueryConfig::from_env().is_none()); } #[tokio::test] async fn config_from_env_returns_none_when_blank() { let _g = ENV_LOCK.lock().expect("lock"); let _scope = test_support::EnvGuard::set(ASAP_THANOS_QUERY_URL_ENV, " "); - assert!(ThanosForwardConfig::from_env().is_none()); + assert!(ThanosQueryConfig::from_env().is_none()); } #[tokio::test] async fn config_from_env_strips_trailing_slash() { let _g = ENV_LOCK.lock().expect("lock"); - let _scope = test_support::EnvGuard::set( - ASAP_THANOS_QUERY_URL_ENV, - "http://thanos-query:10903/", - ); - let cfg = ThanosForwardConfig::from_env().expect("set"); + let _scope = + test_support::EnvGuard::set(ASAP_THANOS_QUERY_URL_ENV, "http://thanos-query:10903/"); + let cfg = ThanosQueryConfig::from_env().expect("set"); assert_eq!(cfg.base_url, "http://thanos-query:10903"); } #[tokio::test] async fn engine_from_env_returns_some_when_set() { let _g = ENV_LOCK.lock().expect("lock"); - let _scope = test_support::EnvGuard::set( - ASAP_THANOS_QUERY_URL_ENV, - "http://127.0.0.1:1", - ); + let _scope = test_support::EnvGuard::set(ASAP_THANOS_QUERY_URL_ENV, "http://127.0.0.1:1"); let engine = engine_from_env().expect("ok"); assert!(engine.is_some(), "env set → engine constructed"); } @@ -836,7 +810,10 @@ mod tests { let _g = ENV_LOCK.lock().expect("lock"); let _scope = test_support::EnvGuard::unset(ASAP_THANOS_QUERY_URL_ENV); let engine = engine_from_env().expect("ok"); - assert!(engine.is_none(), "env unset → caller must use legacy engine"); + assert!( + engine.is_none(), + "env unset → caller must use legacy engine" + ); } #[test] diff --git a/asap-query-engine/src/engines/thanos_query/mod.rs b/asap-query-engine/src/engines/thanos_query/mod.rs new file mode 100644 index 00000000..14914dcb --- /dev/null +++ b/asap-query-engine/src/engines/thanos_query/mod.rs @@ -0,0 +1,13 @@ +//! Thanos query-engine wrapper. +//! +//! This module owns the public archive query engine, [`ThanosQueryEngine`]. +//! Gorilla object storage and the legacy in-process Gorilla executor live +//! under [`crate::stores::gorilla_object_store`]. + +pub mod forward; + +pub use forward::{ + engine_from_env as thanos_engine_from_env, ThanosQueryConfig, ThanosQueryEngine, + ThanosQueryError, ASAP_THANOS_QUERY_URL_ENV, DATA_SOURCE_THANOS_QUERY_ID, + DATA_SOURCE_THANOS_QUERY_INFO, DEFAULT_THANOS_QUERY_URL, QUIRK_THANOS_UNREACHABLE, +}; diff --git a/asap-query-engine/src/engines/warm_tier/decoders.rs b/asap-query-engine/src/engines/warm_tier/decoders.rs index 0cab96ab..a380bb74 100644 --- a/asap-query-engine/src/engines/warm_tier/decoders.rs +++ b/asap-query-engine/src/engines/warm_tier/decoders.rs @@ -34,16 +34,20 @@ pub fn decode_cms_from_proto(buffer: &[u8]) -> Result { Ok(env) => match env.sketch_state { Some(sketch_envelope::SketchState::CountMin(st)) => st, Some(_) => return Err("SketchEnvelope contains non-CountMin sketch".to_string()), - None => CountMinState::decode(buffer) - .map_err(|e| format!("decode CountMinState: {e}"))?, + None => { + CountMinState::decode(buffer).map_err(|e| format!("decode CountMinState: {e}"))? + } }, - Err(_) => CountMinState::decode(buffer) - .map_err(|e| format!("decode CountMinState: {e}"))?, + Err(_) => { + CountMinState::decode(buffer).map_err(|e| format!("decode CountMinState: {e}"))? + } }; let rows = state.rows as usize; let cols = state.cols as usize; if rows == 0 || cols == 0 { - return Err(format!("CountMinState has zero dims (rows={rows}, cols={cols})")); + return Err(format!( + "CountMinState has zero dims (rows={rows}, cols={cols})" + )); } let expected_len = rows * cols; let counter_type = CounterType::try_from(state.counter_type) @@ -103,23 +107,28 @@ pub fn decode_cs_from_proto(buffer: &[u8]) -> Result { let state = match SketchEnvelope::decode(buffer) { Ok(env) => match env.sketch_state { Some(sketch_envelope::SketchState::CountSketch(st)) => st, - Some(_) => { - return Err("SketchEnvelope contains non-CountSketch sketch".to_string()) - } + Some(_) => return Err("SketchEnvelope contains non-CountSketch sketch".to_string()), None => CountSketchState::decode(buffer) .map_err(|e| format!("decode CountSketchState: {e}"))?, }, - Err(_) => CountSketchState::decode(buffer) - .map_err(|e| format!("decode CountSketchState: {e}"))?, + Err(_) => { + CountSketchState::decode(buffer).map_err(|e| format!("decode CountSketchState: {e}"))? + } }; let rows = state.rows as usize; let cols = state.cols as usize; if rows == 0 || cols == 0 { - return Err(format!("CountSketchState has zero dims (rows={rows}, cols={cols})")); + return Err(format!( + "CountSketchState has zero dims (rows={rows}, cols={cols})" + )); } let expected_len = rows * cols; - let counter_type = CounterType::try_from(state.counter_type) - .map_err(|_| format!("CountSketchState unknown counter_type {}", state.counter_type))?; + let counter_type = CounterType::try_from(state.counter_type).map_err(|_| { + format!( + "CountSketchState unknown counter_type {}", + state.counter_type + ) + })?; let flat: Vec = match counter_type { CounterType::Int32 | CounterType::Int64 => { if state.counts_int.len() != expected_len { @@ -166,9 +175,7 @@ pub fn decode_cs_from_msgpack(buffer: &[u8]) -> Result { /// marked the sid as CmsWithHeap (heap embedded in the /// `CountMinSketchWithHeapSerialized` outer wrapper). Mirrors /// `precompute_operators::count_min_sketch_with_heap_accumulator::deserialize_from_bytes_arroyo`. -pub fn decode_cms_with_heap_from_msgpack( - buffer: &[u8], -) -> Result { +pub fn decode_cms_with_heap_from_msgpack(buffer: &[u8]) -> Result { CountMinSketchWithHeap::deserialize_msgpack(buffer) .map_err(|e| format!("deserialize CountMinSketchWithHeap msgpack: {e}")) } diff --git a/asap-query-engine/src/engines/warm_tier/delta_apply.rs b/asap-query-engine/src/engines/warm_tier/delta_apply.rs index abf25847..ed185278 100644 --- a/asap-query-engine/src/engines/warm_tier/delta_apply.rs +++ b/asap-query-engine/src/engines/warm_tier/delta_apply.rs @@ -82,9 +82,7 @@ fn decode_full( .map_err(|e| format!("deserialize KllSketch msgpack: {e}"))?; Ok(RollingState::Kll(sk)) } - (_, e) => Err(format!( - "decode_full called with non-Full encoding {e:?}" - )), + (_, e) => Err(format!("decode_full called with non-Full encoding {e:?}")), } } @@ -113,7 +111,9 @@ impl RollingState { encoding, SketchEncoding::ProtoDelta | SketchEncoding::MsgpackDelta ) { - return Err(format!("apply_delta_bytes called with non-Delta encoding {encoding:?}")); + return Err(format!( + "apply_delta_bytes called with non-Delta encoding {encoding:?}" + )); } match self { RollingState::Dd(sk) => { @@ -129,9 +129,7 @@ impl RollingState { let other = match decode_full(&DeltaSketchKind::DDSketch, bytes, full_enc) { Ok(RollingState::Dd(s)) => s, Ok(_) => { - return Err( - "decode_full(DDSketch) returned non-DDSketch state".to_string() - ) + return Err("decode_full(DDSketch) returned non-DDSketch state".to_string()) } Err(e) => return Err(e), }; @@ -164,9 +162,7 @@ impl RollingState { }; let other = match decode_full(&DeltaSketchKind::Kll, bytes, full_enc) { Ok(RollingState::Kll(s)) => s, - Ok(_) => { - return Err("decode_full(Kll) returned non-Kll state".to_string()) - } + Ok(_) => return Err("decode_full(Kll) returned non-Kll state".to_string()), Err(e) => return Err(e), }; sk.merge(&other) @@ -278,14 +274,11 @@ where RollingState::Hll(a) } (Some(RollingState::Kll(mut a)), RollingState::Kll(b)) => { - a.merge(&b) - .map_err(|e| format!("cum merge KLL: {e}"))?; + a.merge(&b).map_err(|e| format!("cum merge KLL: {e}"))?; RollingState::Kll(a) } (Some(_), _) => { - return Err( - "cumulative merge across sketch family mismatch".to_string() - ) + return Err("cumulative merge across sketch family mismatch".to_string()) } }); } @@ -315,11 +308,13 @@ fn dd_from_proto(buffer: &[u8]) -> Result { Ok(env) => match env.sketch_state { Some(sketch_envelope::SketchState::Ddsketch(st)) => st, Some(_) => return Err("SketchEnvelope contains non-DDSketch sketch".to_string()), - None => DdSketchState::decode(buffer) - .map_err(|e| format!("decode DDSketchState: {e}"))?, + None => { + DdSketchState::decode(buffer).map_err(|e| format!("decode DDSketchState: {e}"))? + } }, - Err(_) => DdSketchState::decode(buffer) - .map_err(|e| format!("decode DDSketchState: {e}"))?, + Err(_) => { + DdSketchState::decode(buffer).map_err(|e| format!("decode DDSketchState: {e}"))? + } }; if !(state.alpha > 0.0 && state.alpha < 1.0) { return Err(format!( @@ -380,8 +375,9 @@ fn hll_from_proto(buffer: &[u8]) -> Result { None => HyperLogLogState::decode(buffer) .map_err(|e| format!("decode HyperLogLogState: {e}"))?, }, - Err(_) => HyperLogLogState::decode(buffer) - .map_err(|e| format!("decode HyperLogLogState: {e}"))?, + Err(_) => { + HyperLogLogState::decode(buffer).map_err(|e| format!("decode HyperLogLogState: {e}"))? + } }; if state.precision == 0 || state.precision > 20 { return Err(format!( diff --git a/asap-query-engine/src/engines/warm_tier/mod.rs b/asap-query-engine/src/engines/warm_tier/mod.rs index bc8e7d08..2e6087aa 100644 --- a/asap-query-engine/src/engines/warm_tier/mod.rs +++ b/asap-query-engine/src/engines/warm_tier/mod.rs @@ -1,12 +1,12 @@ //! Warm-tier sketch query evaluator (Phase 5 follow-up to PR #122). //! //! PR #122 wired the warm-tier classification hook in -//! [`crate::engines::simple::engine::SimpleEngine`]'s +//! [`crate::engines::asap_query::engine::SimpleEngine`]'s //! `QueryEngine::execute` adapter: parse the PromQL, extract //! `(metric_name, label_keys)`, look up candidate sids via //! [`crate::stores::sketch_db::sketch_index::SketchIndex::instances_matching`], //! and classify each sid. On `Ghost`/`Unknown`, return -//! `EngineError::CapabilityMiss(SketchWarmTier, …)` so the +//! `EngineError::CapabilityMiss(SketchStore, …)` so the //! `EngineRouter` fails over to the archive engine. //! //! That hook today still falls through to `handle_query` (legacy diff --git a/asap-query-engine/src/engines/warm_tier/sketch_reducer.rs b/asap-query-engine/src/engines/warm_tier/sketch_reducer.rs index 8c3a17ad..b45002b4 100644 --- a/asap-query-engine/src/engines/warm_tier/sketch_reducer.rs +++ b/asap-query-engine/src/engines/warm_tier/sketch_reducer.rs @@ -51,11 +51,11 @@ use std::collections::BTreeMap; +use asap_sketchlib::sketches::countminsketch::CountMinSketch; +use asap_sketchlib::sketches::countsketch::CountSketch; use asap_sketchlib::sketches::ddsketch::DdSketch; use asap_sketchlib::sketches::hll::HllSketch; use asap_sketchlib::sketches::kll::KllSketch; -use asap_sketchlib::sketches::countminsketch::CountMinSketch; -use asap_sketchlib::sketches::countsketch::CountSketch; use crate::engines::warm_tier::decoders::{ decode_cms_from_msgpack, decode_cms_from_proto, decode_cms_with_heap_from_msgpack, @@ -379,7 +379,11 @@ impl<'a> SketchReducer<'a> { continue; }; any_window = true; - let w_end_u64 = if *window_end >= 0 { *window_end as u64 } else { 0 }; + let w_end_u64 = if *window_end >= 0 { + *window_end as u64 + } else { + 0 + }; if w_end_u64 < cov_lo { cov_lo = w_end_u64; } @@ -478,25 +482,25 @@ impl<'a> SketchReducer<'a> { } let samples_out: Vec<(i64, f64)> = if is_cumulative { - let (one, _skipped) = - cumulative_evaluate(&samples_vec, delta_kind, &evaluator) - .map_err(|e| WarmTierError::DeserializeFailure { - sid, - encoding: SketchEncoding::ProtoFull, - reason: e, - })?; + let (one, _skipped) = cumulative_evaluate(&samples_vec, delta_kind, &evaluator) + .map_err(|e| WarmTierError::DeserializeFailure { + sid, + encoding: SketchEncoding::ProtoFull, + reason: e, + })?; match one { Some(s) => vec![s], None => Vec::new(), } } else { let (per_win, _skipped) = - per_window_evaluate(&samples_vec, delta_kind, &evaluator) - .map_err(|e| WarmTierError::DeserializeFailure { + per_window_evaluate(&samples_vec, delta_kind, &evaluator).map_err(|e| { + WarmTierError::DeserializeFailure { sid, encoding: SketchEncoding::ProtoFull, reason: e, - })?; + } + })?; per_win }; out_series.push((ts.series_label_values, samples_out)); @@ -626,18 +630,17 @@ impl<'a> SketchReducer<'a> { // --------------------------------------------------------------------------- #[allow(dead_code)] -fn decode_ddsketch( - sid: u64, - state: &SketchSampleState, -) -> Result { +fn decode_ddsketch(sid: u64, state: &SketchSampleState) -> Result { match state.encoding { - SketchEncoding::ProtoFull => DdSketch_from_sketchlib_proto_bytes(&state.bytes).map_err(|e| { - WarmTierError::DeserializeFailure { - sid, - encoding: state.encoding, - reason: e.to_string(), - } - }), + SketchEncoding::ProtoFull => { + DdSketch_from_sketchlib_proto_bytes(&state.bytes).map_err(|e| { + WarmTierError::DeserializeFailure { + sid, + encoding: state.encoding, + reason: e.to_string(), + } + }) + } SketchEncoding::MsgpackFull => DdSketch::deserialize_msgpack(&state.bytes).map_err(|e| { WarmTierError::DeserializeFailure { sid, @@ -659,18 +662,17 @@ fn decode_ddsketch( } #[allow(dead_code)] -fn decode_kll( - sid: u64, - state: &SketchSampleState, -) -> Result { +fn decode_kll(sid: u64, state: &SketchSampleState) -> Result { match state.encoding { - SketchEncoding::ProtoFull => KllSketch_from_sketchlib_proto_bytes(&state.bytes).map_err(|e| { - WarmTierError::DeserializeFailure { - sid, - encoding: state.encoding, - reason: e.to_string(), - } - }), + SketchEncoding::ProtoFull => { + KllSketch_from_sketchlib_proto_bytes(&state.bytes).map_err(|e| { + WarmTierError::DeserializeFailure { + sid, + encoding: state.encoding, + reason: e.to_string(), + } + }) + } SketchEncoding::MsgpackFull => KllSketch::deserialize_msgpack(&state.bytes).map_err(|e| { WarmTierError::DeserializeFailure { sid, @@ -682,26 +684,24 @@ fn decode_kll( Err(WarmTierError::DeserializeFailure { sid, encoding: state.encoding, - reason: "KLL delta encodings not implemented in warm-tier reducer" - .to_string(), + reason: "KLL delta encodings not implemented in warm-tier reducer".to_string(), }) } } } #[allow(dead_code)] -fn decode_hll( - sid: u64, - state: &SketchSampleState, -) -> Result { +fn decode_hll(sid: u64, state: &SketchSampleState) -> Result { match state.encoding { - SketchEncoding::ProtoFull => HllSketch_from_sketchlib_proto_bytes(&state.bytes).map_err(|e| { - WarmTierError::DeserializeFailure { - sid, - encoding: state.encoding, - reason: e.to_string(), - } - }), + SketchEncoding::ProtoFull => { + HllSketch_from_sketchlib_proto_bytes(&state.bytes).map_err(|e| { + WarmTierError::DeserializeFailure { + sid, + encoding: state.encoding, + reason: e.to_string(), + } + }) + } SketchEncoding::MsgpackFull => HllSketch::deserialize_msgpack(&state.bytes).map_err(|e| { WarmTierError::DeserializeFailure { sid, @@ -713,8 +713,7 @@ fn decode_hll( Err(WarmTierError::DeserializeFailure { sid, encoding: state.encoding, - reason: "HLL delta encodings not implemented in warm-tier reducer" - .to_string(), + reason: "HLL delta encodings not implemented in warm-tier reducer".to_string(), }) } } @@ -733,11 +732,13 @@ fn DdSketch_from_sketchlib_proto_bytes(buffer: &[u8]) -> Result match env.sketch_state { Some(sketch_envelope::SketchState::Ddsketch(st)) => st, Some(_) => return Err("SketchEnvelope contains non-DDSketch sketch".to_string()), - None => DdSketchState::decode(buffer) - .map_err(|e| format!("decode DDSketchState: {e}"))?, + None => { + DdSketchState::decode(buffer).map_err(|e| format!("decode DDSketchState: {e}"))? + } }, - Err(_) => DdSketchState::decode(buffer) - .map_err(|e| format!("decode DDSketchState: {e}"))?, + Err(_) => { + DdSketchState::decode(buffer).map_err(|e| format!("decode DDSketchState: {e}"))? + } }; if !(state.alpha > 0.0 && state.alpha < 1.0) { return Err(format!( @@ -800,8 +801,9 @@ fn HllSketch_from_sketchlib_proto_bytes(buffer: &[u8]) -> Result HyperLogLogState::decode(buffer) .map_err(|e| format!("decode HyperLogLogState: {e}"))?, }, - Err(_) => HyperLogLogState::decode(buffer) - .map_err(|e| format!("decode HyperLogLogState: {e}"))?, + Err(_) => { + HyperLogLogState::decode(buffer).map_err(|e| format!("decode HyperLogLogState: {e}"))? + } }; if state.precision == 0 || state.precision > 20 { return Err(format!( diff --git a/asap-query-engine/src/engines/warm_tier/tests.rs b/asap-query-engine/src/engines/warm_tier/tests.rs index 3018f506..540b6adf 100644 --- a/asap-query-engine/src/engines/warm_tier/tests.rs +++ b/asap-query-engine/src/engines/warm_tier/tests.rs @@ -17,8 +17,8 @@ use asap_sketchlib::sketches::hll::{HllSketch, HllVariant}; use crate::engines::warm_tier::{SketchReducer, WarmTierError}; use crate::stores::sketch_db::sketch_index::{ - AccuracyBound, Capability, SketchConfig, SketchEncoding, SketchIndex, - SketchInstanceMetadata, SketchKindHandle, SketchSampleState, + AccuracyBound, Capability, SketchConfig, SketchEncoding, SketchIndex, SketchInstanceMetadata, + SketchKindHandle, SketchSampleState, }; // --------------------------------------------------------------------------- @@ -448,10 +448,7 @@ fn multi_series_one_per_label_value() { use asap_sketchlib::sketches::countminsketch_topk::CountMinSketchWithHeap; fn cms_heap_meta(sid: u64) -> SketchInstanceMetadata { - let cfg = SketchConfig::CountMin { - rows: 4, - cols: 256, - }; + let cfg = SketchConfig::CountMin { rows: 4, cols: 256 }; SketchInstanceMetadata { sid, metric_name: "endpoint_hits".to_string(), @@ -465,10 +462,7 @@ fn cms_heap_meta(sid: u64) -> SketchInstanceMetadata { } fn cms_only_meta(sid: u64) -> SketchInstanceMetadata { - let cfg = SketchConfig::CountMin { - rows: 4, - cols: 256, - }; + let cfg = SketchConfig::CountMin { rows: 4, cols: 256 }; SketchInstanceMetadata { sid, metric_name: "endpoint_hits".to_string(), @@ -560,7 +554,10 @@ fn cms_without_heap_returns_missing_heap() { .evaluate(&[sid], "topk", &[5.0], 1000, 1010) .expect_err("topk against CountMin (no heap) must surface MissingHeap"); match err { - WarmTierError::MissingHeap { sid: s, sketch_kind } => { + WarmTierError::MissingHeap { + sid: s, + sketch_kind, + } => { assert_eq!(s, sid); assert_eq!(sketch_kind, SketchKindHandle::CountMin); } @@ -678,13 +675,7 @@ fn hll_cumulative_full_plus_one_delta() { let reducer = SketchReducer::new(&idx); let result = reducer - .evaluate( - &[sid], - "count_distinct_over_time", - &[], - 1000, - 1020, - ) + .evaluate(&[sid], "count_distinct_over_time", &[], 1000, 1020) .expect("cumulative HLL evaluate should succeed"); assert_eq!(result.series.len(), 1); let (_, samples) = &result.series[0]; @@ -708,7 +699,7 @@ fn hll_cumulative_full_plus_one_delta() { // We don't drive the full SimpleEngine here (that would require // constructing the whole streaming-config plumbing). Instead we exercise // the `stitch_warm_and_archive` helper directly via a small wrapper -// test in `engines::simple::tests` would be ideal — but to keep this +// test in `engines::asap_query::tests` would be ideal — but to keep this // PR additive, we verify the `coverage` field is populated correctly // on a multi-window evaluate so the downstream stitch path has the // information it needs. @@ -732,7 +723,12 @@ fn coverage_reports_observed_window_range() { let bytes = encode_ddsketch(&sk); let window_start = 100 + (i as u64) * 100; let window_end = window_start + 100; - idx.append_sample(sid, BTreeMap::new(), (window_start, window_end), proto_full(bytes)); + idx.append_sample( + sid, + BTreeMap::new(), + (window_start, window_end), + proto_full(bytes), + ); } let reducer = SketchReducer::new(&idx); diff --git a/asap-query-engine/src/lib.rs b/asap-query-engine/src/lib.rs index efb1e1b0..c7279a2b 100644 --- a/asap-query-engine/src/lib.rs +++ b/asap-query-engine/src/lib.rs @@ -24,7 +24,7 @@ pub use precompute_operators::{ pub use stores::{SimpleMapStore, Store, StoreResult}; -pub use engines::{InstantVector, QueryResult, SimpleEngine}; +pub use engines::{ASAPQueryEngine, InstantVector, QueryResult, SimpleEngine}; pub use drivers::{ HttpServer, HttpServerConfig, KafkaConsumer, KafkaConsumerConfig, OtlpReceiver, diff --git a/asap-query-engine/src/main.rs b/asap-query-engine/src/main.rs index fa7d9aab..66b32ab2 100644 --- a/asap-query-engine/src/main.rs +++ b/asap-query-engine/src/main.rs @@ -17,7 +17,9 @@ use std::sync::Arc; use tokio::signal; use tracing::{error, info, warn}; -use query_engine_rust::data_model::enums::{CleanupPolicy, InputFormat, LockStrategy, StreamingEngine}; +use query_engine_rust::data_model::enums::{ + CleanupPolicy, InputFormat, LockStrategy, StreamingEngine, +}; use query_engine_rust::data_model::InferenceConfig; use query_engine_rust::drivers::AdapterConfig; use query_engine_rust::precompute_engine::config::LateDataPolicy; @@ -282,7 +284,7 @@ struct Args { /// pick the right engine (`SimpleEngine` for warm-tier sketches, /// `GorillaQueryEngine` for the cold archive, etc.). Without /// this flag the handler falls back to the streaming-config - /// single axis (always `SketchWarmTier`) and the EngineRouter is + /// single axis (always `SketchStore`) and the EngineRouter is /// effectively bypassed — the issue-46 v2 demo's criterion ⑤ /// failure mode. Mirrors the `precompute_engine` binary's flag /// of the same name. @@ -438,10 +440,10 @@ async fn main() -> Result<()> { // are constructed so both can be wired with a single canonical // instance — even when precompute is disabled, the engine still // needs the index for the Phase 6 archive failover trigger. - let series_resolver = Arc::new( - query_engine_rust::drivers::ingest::series_resolver::SeriesIdResolver::new(), - ); - let sketch_index = Arc::new(query_engine_rust::stores::sketch_db::sketch_index::SketchIndex::new()); + let series_resolver = + Arc::new(query_engine_rust::drivers::ingest::series_resolver::SeriesIdResolver::new()); + let sketch_index = + Arc::new(query_engine_rust::stores::sketch_db::sketch_index::SketchIndex::new()); // Setup query engine. SimpleEngine shares the same // HotReloadStreamingConfig handle as the HTTP server, so a POST @@ -686,7 +688,7 @@ async fn main() -> Result<()> { // from `--backend-storage-routing` (or its env-var alias) so the // HTTP handler consults a per-metric `StorageBackend` map on // every PromQL query instead of bypassing the EngineRouter when - // the streaming-config single axis defaults to `SketchWarmTier`. + // the streaming-config single axis defaults to `SketchStore`. // // Phase α (MVP): even when no static YAML is loaded, install an // empty hot-reload handle so the controller's first @@ -721,20 +723,15 @@ async fn main() -> Result<()> { }; server = server.with_backend_storage_routing(Arc::new(bootstrap_routing)); - // Phase-5/6 + Step-2.3: register an archive-tier engine on the + // Phase-5/6 + Step-2.3: register the Thanos query engine on the // capability router. Two operating modes, selected at startup: // // * **Path A2 mode** — when `ASAP_THANOS_QUERY_URL` is set, the // backend forwards archive-tier PromQL queries to a // `thanos-query` sidecar via the - // [`ThanosForwardEngine`]. The forwarder is registered under - // both `thanos_archive` (its native id, for explicit - // `X-ASAP-Engine` overrides) and `gorilla_archive` (the legacy - // archive slot that the existing - // `compatible_storage_backends` failover sequence walks), so - // per-metric routing config can target either name without - // surprise. The legacy in-process `GorillaQueryEngine` is - // skipped in this mode. + // [`ThanosQueryEngine`], registered under the single public id + // `thanos_query`. The legacy in-process `GorillaQueryEngine` + // is skipped in this mode. // * **Legacy mode** — when `ASAP_THANOS_QUERY_URL` is unset, the // in-process `GorillaQueryEngine` answers archive queries // from per-hour Gorilla chunks landed on S3 / MinIO via the @@ -743,72 +740,63 @@ async fn main() -> Result<()> { // end-to-end. // // When neither env-var family is configured the binary registers - // a `NoDataArchiveEngine` stub under the `gorilla_archive` alias + // a `NoDataArchiveEngine` stub under `thanos_query` // so cold queries succeed with an empty result instead of // surfacing as `503 NoEngineRegistered`. Operators that want the // original fail-loud behaviour can opt back in by setting // `ASAP_REQUIRE_ARCHIVE_ENGINE=1`. let mut archive_registered = false; - match query_engine_rust::engines::gorilla::thanos_engine_from_env() { + match query_engine_rust::engines::thanos_query::thanos_engine_from_env() { Ok(Some(thanos)) => { - use query_engine_rust::engines::gorilla::DATA_SOURCE_THANOS_ARCHIVE_ID; use query_engine_rust::routing::QueryEngine; info!( upstream = thanos.base_url(), - "Path A2: registering ThanosForwardEngine for the archive tier (data_source_id=thanos_archive, alias=gorilla_archive); legacy in-process GorillaQueryEngine skipped", + "Path A2: registering ThanosQueryEngine for the archive tier (data_source_id=thanos_query); legacy in-process GorillaQueryEngine skipped", ); let thanos_arc: Arc = Arc::new(thanos); - server = server - .with_query_engine_aliased( - DATA_SOURCE_THANOS_ARCHIVE_ID, - thanos_arc.clone(), - ) - .with_query_engine_aliased( - asap_types::StorageBackend::GorillaS3Archive.data_source_id(), - thanos_arc, - ); + server = server.with_archive_query_engine(thanos_arc); archive_registered = true; } - Ok(None) => { - match query_engine_rust::engines::gorilla::GorillaS3Config::from_env() { - Ok(s3_cfg) => { - match query_engine_rust::engines::gorilla::GorillaS3Store::with_default_backend(s3_cfg) { - Ok(store) => { - use query_engine_rust::engines::{GorillaEngineConfig, GorillaQueryEngine}; - use query_engine_rust::routing::QueryEngine; - let gorilla = Arc::new(GorillaQueryEngine::with_gorilla_s3( - Arc::new(store), - GorillaEngineConfig::default(), - )); - info!( - "Registering legacy in-process GorillaQueryEngine on the capability router (data_source_id=gorilla_archive); set ASAP_THANOS_QUERY_URL to switch to Path A2 thanos forwarding", + Ok(None) => match query_engine_rust::stores::gorilla_object_store::GorillaS3Config::from_env() { + Ok(s3_cfg) => { + match query_engine_rust::stores::gorilla_object_store::GorillaS3Store::with_default_backend( + s3_cfg, + ) { + Ok(store) => { + use query_engine_rust::stores::{GorillaEngineConfig, GorillaQueryEngine}; + use query_engine_rust::routing::QueryEngine; + let gorilla = Arc::new(GorillaQueryEngine::with_gorilla_s3( + Arc::new(store), + GorillaEngineConfig::default(), + )); + info!( + "Registering legacy in-process GorillaQueryEngine on the archive slot (canonical data_source_id=thanos_query); set ASAP_THANOS_QUERY_URL to use the intended Thanos archive path", ); - server = server.with_query_engine(gorilla as Arc); - archive_registered = true; - } - Err(e) => { - warn!( + server = server.with_archive_query_engine(gorilla as Arc); + archive_registered = true; + } + Err(e) => { + warn!( "ASAP_GORILLA_S3_* env vars present but GorillaS3Store failed to build ({e}); router will not have an archive engine", ); - } } } - Err(_) => { - info!( + } + Err(_) => { + info!( "ASAP_GORILLA_S3_* env vars not configured — router serves warm-tier metrics only (set ASAP_GORILLA_S3_BUCKET + ASAP_GORILLA_S3_REGION to enable archive routing, or set ASAP_THANOS_QUERY_URL to enable Path A2 thanos forwarding)", ); - } } - } + }, Err(e) => { warn!( - "ASAP_THANOS_QUERY_URL set but ThanosForwardEngine failed to build ({e}); router will not have an archive engine", + "ASAP_THANOS_QUERY_URL set but ThanosQueryEngine failed to build ({e}); router will not have an archive engine", ); } } // No archive engine configured — register a `NoDataArchiveEngine` - // stub under the `gorilla_archive` alias so cold queries succeed + // stub under `thanos_query` so cold queries succeed // with an empty result. `ASAP_REQUIRE_ARCHIVE_ENGINE=1` opts back // into the original fail-loud (`503 NoEngineRegistered`) behaviour. if !archive_registered { @@ -823,48 +811,10 @@ async fn main() -> Result<()> { use query_engine_rust::engines::NoDataArchiveEngine; use query_engine_rust::routing::QueryEngine; info!( - "Registering NoDataArchiveEngine stub on the archive slot (data_source_id=no_data_archive, alias=gorilla_archive); set ASAP_REQUIRE_ARCHIVE_ENGINE=1 to disable", + "Registering NoDataArchiveEngine stub on the archive slot (canonical data_source_id=thanos_query); set ASAP_REQUIRE_ARCHIVE_ENGINE=1 to disable", ); let stub: Arc = Arc::new(NoDataArchiveEngine::new()); - server = server.with_query_engine_aliased( - asap_types::StorageBackend::GorillaS3Archive.data_source_id(), - stub, - ); - } - } - - // Phase ε.2: register a `PrometheusForwardEngine` under the - // `prometheus_remote` engine id when `ASAP_PROMETHEUS_QUERY_URL` - // is set. The controller's Mode 3 (`RawAtEdgePrometheusArchive`) - // emits routing-table entries with `engine: prometheus_remote` - // for metrics whose raw data is shipped to Prometheus's native - // OTLP receiver. When the env var is unset the engine is not - // registered; if a routing-table entry references - // `prometheus_remote` in that case, the dispatcher returns a - // clear `NoEngineRegistered` 503 — fail-loud is the correct - // behaviour for a misconfigured deploy. - // - // Mirrors the `ASAP_THANOS_QUERY_URL` wiring above; the two - // engines coexist on the router under different ids and answer - // different routing-table entries. - match query_engine_rust::engines::prometheus::prometheus_engine_from_env() { - Ok(Some(prom)) => { - use query_engine_rust::routing::QueryEngine; - info!( - upstream = prom.base_url(), - "Phase ε.2: registering PrometheusForwardEngine on the capability router (data_source_id=prometheus_remote); routing-table entries that reference `prometheus_remote` will dispatch here", - ); - server = server.with_query_engine(Arc::new(prom) as Arc); - } - Ok(None) => { - info!( - "ASAP_PROMETHEUS_QUERY_URL not set — PrometheusForwardEngine skipped; routing-table entries referencing `prometheus_remote` will surface NoEngineRegistered", - ); - } - Err(e) => { - warn!( - "ASAP_PROMETHEUS_QUERY_URL set but PrometheusForwardEngine failed to build ({e}); router will not have a prometheus_remote engine", - ); + server = server.with_archive_query_engine(stub); } } diff --git a/asap-query-engine/src/precompute_engine/worker.rs b/asap-query-engine/src/precompute_engine/worker.rs index 9c99ef85..df0d7672 100644 --- a/asap-query-engine/src/precompute_engine/worker.rs +++ b/asap-query-engine/src/precompute_engine/worker.rs @@ -2540,9 +2540,10 @@ aggregations: total_buckets > 0, "query returned a key entry but with zero buckets — persistence is half-broken" ); - let any_in_first_window = results.values().flat_map(|v| v.iter()).any(|(range, _)| { - range.0 == 60_000 && range.1 == 90_000 - }); + let any_in_first_window = results + .values() + .flat_map(|v| v.iter()) + .any(|(range, _)| range.0 == 60_000 && range.1 == 90_000); assert!( any_in_first_window, "no bucket landed in the closed window [60_000, 90_000) — persistence pathway misroutes" diff --git a/asap-query-engine/src/precompute_operators/count_min_sketch_accumulator.rs b/asap-query-engine/src/precompute_operators/count_min_sketch_accumulator.rs index d22184d5..a8d61440 100644 --- a/asap-query-engine/src/precompute_operators/count_min_sketch_accumulator.rs +++ b/asap-query-engine/src/precompute_operators/count_min_sketch_accumulator.rs @@ -432,9 +432,9 @@ impl AggregateCore for CountMinSketchAccumulator { let Some(s) = range_ms_str else { return Ok(total); }; - let range_ms: f64 = s.parse().map_err(|e| { - format!("CountMinSketchAccumulator: bad range_ms='{s}': {e}") - })?; + let range_ms: f64 = s + .parse() + .map_err(|e| format!("CountMinSketchAccumulator: bad range_ms='{s}': {e}"))?; if range_ms <= 0.0 { return Err("CountMinSketchAccumulator: range_ms must be positive".into()); } @@ -920,11 +920,7 @@ mod tests { // for instant rate-shape queries that bypass the matrix-selector // code path. let cms = CountMinSketchAccumulator { - inner: CountMinSketch::from_legacy_matrix( - vec![vec![42.0, 0.0], vec![42.0, 0.0]], - 2, - 2, - ), + inner: CountMinSketch::from_legacy_matrix(vec![vec![42.0, 0.0], vec![42.0, 0.0]], 2, 2), }; let trait_obj: &dyn AggregateCore = &cms; let v = trait_obj @@ -939,11 +935,7 @@ mod tests { // same min-row-sum as Sum / Count. Differs from Rate only in // that it never divides by range. let cms = CountMinSketchAccumulator { - inner: CountMinSketch::from_legacy_matrix( - vec![vec![5.0, 7.0], vec![3.0, 9.0]], - 2, - 2, - ), + inner: CountMinSketch::from_legacy_matrix(vec![vec![5.0, 7.0], vec![3.0, 9.0]], 2, 2), }; let trait_obj: &dyn AggregateCore = &cms; let v = trait_obj diff --git a/asap-query-engine/src/precompute_operators/edge_runtime_adapter.rs b/asap-query-engine/src/precompute_operators/edge_runtime_adapter.rs index f55fdee9..c86f1ae8 100644 --- a/asap-query-engine/src/precompute_operators/edge_runtime_adapter.rs +++ b/asap-query-engine/src/precompute_operators/edge_runtime_adapter.rs @@ -96,8 +96,8 @@ pub use asap_precompute_rs::{CardinalitySketch, FrequencySketch, QuantileSketch} pub fn unwrap_envelope_state( bytes: &[u8], ) -> Result, Box> { - let env = ProtoSketchEnvelope::decode(bytes) - .map_err(|e| format!("decode SketchEnvelope: {e}"))?; + let env = + ProtoSketchEnvelope::decode(bytes).map_err(|e| format!("decode SketchEnvelope: {e}"))?; Ok(env.sketch_state) } @@ -190,18 +190,16 @@ pub fn reconstruct_via_runtime( .map_err(|e| format!("KLLWrapper snapshot: {e}"))?; Ok(ReconstructedSketch::Kll { snapshot_bytes }) } - SketchType::HLLSketch | SketchType::CountSketch | SketchType::CountMinSketch => Err( - format!( + SketchType::HLLSketch | SketchType::CountSketch | SketchType::CountMinSketch => { + Err(format!( "reconstruct_via_runtime({sketch_type:?}): byte parity for \ HLL / CountSketch / CountMinSketch not yet in upstream \ asap_sketchlib — tracked at ProjectASAP/ASAPCollector#243. \ Caller must fall back to backend's per-accumulator decoder." ) - .into(), - ), - SketchType::Unspecified => { - Err("reconstruct_via_runtime: SketchType::Unspecified".into()) + .into()) } + SketchType::Unspecified => Err("reconstruct_via_runtime: SketchType::Unspecified".into()), } } @@ -253,11 +251,7 @@ pub fn encode_ddsketch_envelope(sk: &asap_sketchlib::sketches::ddsketch::DdSketc store_offset: sk.store_offset, count: sk.count, sum: sk.sum, - min: if sk.count == 0 { - f64::INFINITY - } else { - sk.min - }, + min: if sk.count == 0 { f64::INFINITY } else { sk.min }, max: if sk.count == 0 { f64::NEG_INFINITY } else { @@ -351,8 +345,8 @@ mod tests { w.update(i as f64); } let original_bytes = w.snapshot().expect("snapshot ok"); - let reconstructed = reconstruct_via_runtime(SketchType::DDSketch, &original_bytes) - .expect("reconstruct ok"); + let reconstructed = + reconstruct_via_runtime(SketchType::DDSketch, &original_bytes).expect("reconstruct ok"); let dd = match reconstructed { ReconstructedSketch::DdSketch(d) => d, ReconstructedSketch::Kll { .. } => panic!("got KLL, expected DDSketch"), @@ -379,24 +373,16 @@ mod tests { b.update(i as f64); } // Reach into the wrapper's inner via snapshot/decode. - let a_inner = match reconstruct_via_runtime( - SketchType::DDSketch, - &a.snapshot().unwrap(), - ) - .unwrap() - { - ReconstructedSketch::DdSketch(d) => d, - _ => panic!(), - }; - let b_inner = match reconstruct_via_runtime( - SketchType::DDSketch, - &b.snapshot().unwrap(), - ) - .unwrap() - { - ReconstructedSketch::DdSketch(d) => d, - _ => panic!(), - }; + let a_inner = + match reconstruct_via_runtime(SketchType::DDSketch, &a.snapshot().unwrap()).unwrap() { + ReconstructedSketch::DdSketch(d) => d, + _ => panic!(), + }; + let b_inner = + match reconstruct_via_runtime(SketchType::DDSketch, &b.snapshot().unwrap()).unwrap() { + ReconstructedSketch::DdSketch(d) => d, + _ => panic!(), + }; let merged = merge_ddsketches_via_runtime(&a_inner, &b_inner).expect("merge ok"); assert_eq!(merged.count, 20); } diff --git a/asap-query-engine/src/precompute_operators/mod.rs b/asap-query-engine/src/precompute_operators/mod.rs index 00d69fa5..08f7cb10 100644 --- a/asap-query-engine/src/precompute_operators/mod.rs +++ b/asap-query-engine/src/precompute_operators/mod.rs @@ -1,10 +1,10 @@ pub mod count_min_sketch_accumulator; pub mod count_min_sketch_with_heap_accumulator; -pub mod edge_runtime_adapter; pub mod count_sketch_accumulator; pub mod datasketches_kll_accumulator; pub mod dd_sketch_accumulator; pub mod delta_set_aggregator_accumulator; +pub mod edge_runtime_adapter; pub mod hll_sketch_accumulator; pub mod hydra_kll_accumulator; pub mod increase_accumulator; diff --git a/asap-query-engine/src/precompute_operators/multiple_increase_accumulator.rs b/asap-query-engine/src/precompute_operators/multiple_increase_accumulator.rs index adabf4ae..ee417d01 100644 --- a/asap-query-engine/src/precompute_operators/multiple_increase_accumulator.rs +++ b/asap-query-engine/src/precompute_operators/multiple_increase_accumulator.rs @@ -459,21 +459,11 @@ mod tests { acc.update( east.clone(), - IncreaseAccumulator::new( - Measurement::new(10.0), - 1000, - Measurement::new(100.0), - 2000, - ), + IncreaseAccumulator::new(Measurement::new(10.0), 1000, Measurement::new(100.0), 2000), ); acc.update( west.clone(), - IncreaseAccumulator::new( - Measurement::new(5.0), - 1000, - Measurement::new(50.0), - 2000, - ), + IncreaseAccumulator::new(Measurement::new(5.0), 1000, Measurement::new(50.0), 2000), ); assert_eq!(acc.query(Statistic::Sum, &east, None).unwrap(), 100.0); diff --git a/asap-query-engine/src/routing/backend_storage_routing.rs b/asap-query-engine/src/routing/backend_storage_routing.rs index e1542a07..b3382027 100644 --- a/asap-query-engine/src/routing/backend_storage_routing.rs +++ b/asap-query-engine/src/routing/backend_storage_routing.rs @@ -9,10 +9,10 @@ //! the entire streaming config. In production deploys (the //! `precompute_engine` binary loading `backend-streaming.yaml`) the //! field decodes via `Self::new(...)` which always defaults to -//! `SketchWarmTier`, so the handler always took the +//! `SketchStore`, so the handler always took the //! `SimpleEngine`-direct-dispatch branch and the `EngineRouter` was //! effectively bypassed for every query — the `data_source: -//! gorilla_archive` info-line never landed on cold-archive responses +//! thanos_query` info-line never landed on cold-archive responses //! even when the chunks were on disk in MinIO. //! //! The fix lives **outside** the streaming pipeline: the streaming @@ -30,7 +30,7 @@ //! accuracy) and criterion ⑤ (cold-fallback). Every quantile/sum-by //! query on `http_requests_total` had to go to either the warm tier //! (so the accuracy reducer could compute relative error) or the -//! archive (so the `data_source: gorilla_archive` info-line landed on +//! archive (so the `data_source: thanos_query` info-line landed on //! the cold-fallback probe). v7 closes this by letting one metric have //! multiple targets, each with an optional query-shape filter; the //! HTTP handler inspects the parsed PromQL and picks the matching @@ -46,27 +46,27 @@ //! //! ```yaml //! # v6.1 form (single-target): -//! default: sketch_warm_tier +//! default: asap_query //! metrics: -//! audit_events: gorilla_s3_archive +//! audit_events: thanos_query //! ``` //! //! ```yaml //! # v7 form (multi-target with query-shape selection): -//! default: sketch_warm_tier +//! default: asap_query //! routes: //! - metric: http_requests_total //! targets: -//! - backend: sketch_warm_tier +//! - backend: asap_query //! # default — predictable / planned queries land here -//! - backend: gorilla_s3_archive +//! - backend: thanos_query //! applies_to_query_shape: [count, topk, rate_post_hoc] //! - metric: http_freshness_probe_warm //! targets: -//! - backend: sketch_warm_tier +//! - backend: asap_query //! - metric: http_freshness_probe_archive //! targets: -//! - backend: gorilla_s3_archive +//! - backend: thanos_query //! ``` //! //! The two shapes can be mixed in the same YAML — metrics under @@ -75,14 +75,14 @@ //! in BOTH wins from `routes:` (multi-target overrides single-target). //! //! Valid `StorageBackend` values mirror the snake-cased serde tags on -//! `asap_types::StorageBackend`: `sketch_warm_tier`, -//! `gorilla_s3_archive`, `double_write`. (Step-1 of the JSONL +//! `asap_types::StorageBackend`: `asap_query`, +//! `thanos_query`, `double_write`. (Step-1 of the JSONL //! deprecation refactor removed the `cold_jsonl_fallback` tag.) //! //! Loaded once at backend startup (CLI flag `--backend-storage-routing` //! on `precompute_engine`) and stored in `AppState`. Lookup is //! O(metric-name-hash); a query that doesn't match any entry falls back -//! to `default` (which itself falls back to `SketchWarmTier`). +//! to `default` (which itself falls back to `SketchStore`). //! //! ## Out of scope //! @@ -97,7 +97,7 @@ use std::collections::HashMap; use std::path::Path; use anyhow::{Context, Result}; -use asap_types::StorageBackend; +use asap_types::{parse_storage_backend_engine_id, StorageBackend, CANONICAL_QUERY_ENGINE_IDS}; use serde::{Deserialize, Serialize}; use serde_json::Value as JsonValue; use tracing::{debug, info}; @@ -313,7 +313,7 @@ struct BackendStorageRoutingYaml { #[serde(default, skip_serializing_if = "Option::is_none")] tenant: Option, /// Fallback storage backend for any metric not explicitly listed. - /// Optional; defaults to `SketchWarmTier`. + /// Optional; defaults to `SketchStore`. #[serde(default)] default: StorageBackend, /// v6.1 form — per-metric overrides keyed by the bare metric name @@ -375,7 +375,7 @@ pub const DEFAULT_TENANT: &str = "default"; /// In-memory routing table consulted by the HTTP handler at request /// time. Build via [`Self::from_yaml_file`] / [`Self::from_yaml_str`] -/// or [`Self::empty`] (everything routes to `SketchWarmTier`). +/// or [`Self::empty`] (everything routes to `SketchStore`). /// /// ## Tenant scope (per-tenant routing, follow-up to PR #333) /// @@ -408,7 +408,7 @@ pub struct BackendStorageRouting { impl BackendStorageRouting { /// Build an empty router — every metric resolves to - /// `SketchWarmTier`. Equivalent to "no routing config at all" and + /// `SketchStore`. Equivalent to "no routing config at all" and /// preserves pre-Phase-5 dispatch (`SimpleEngine` direct path). /// Scoped to the [`DEFAULT_TENANT`] tenant. pub fn empty() -> Self { @@ -536,12 +536,12 @@ impl BackendStorageRouting { /// /// ```json /// { - /// "default_engine": "sketch_warm_tier", + /// "default_engine": "asap_query", /// "metrics": [ /// { "name": "http_requests_total", /// "targets": [ - /// { "engine": "sketch_warm_tier" }, - /// { "engine": "thanos_archive", + /// { "engine": "asap_query" }, + /// { "engine": "thanos_query", /// "applies_to_query_shape": ["count", "topk", "rate_post_hoc", /// "histogram_quantile", "delta", "absent"] } /// ] @@ -552,11 +552,9 @@ impl BackendStorageRouting { /// /// Engine-name compatibility (controller → backend `StorageBackend`): /// - /// * `sketch_warm_tier` → `SketchWarmTier` - /// * `thanos_archive` → `GorillaS3Archive` (Phase α uses the existing - /// archive engine; future phases may register a real Thanos engine). - /// * `gorilla_s3_archive` → `GorillaS3Archive` (back-compat alias). - /// * `double_write` → `DoubleWrite`. + /// * `asap_query` → `SketchStore` + /// * `thanos_query` → `GorillaObjectStore` storage, served by + /// `ThanosQueryEngine`. /// /// Unknown query-shape strings are mapped to [`QueryShape::Other`] /// rather than failing the parse — the controller's vocabulary may @@ -575,7 +573,7 @@ impl BackendStorageRouting { let default_engine = value .get("default_engine") .and_then(|v| v.as_str()) - .unwrap_or("sketch_warm_tier"); + .unwrap_or("asap_query"); let default = parse_engine_string(default_engine).with_context(|| { format!( "backend-storage-routing JSON: invalid default_engine '{}'", @@ -619,23 +617,23 @@ impl BackendStorageRouting { } let mut targets: Vec = Vec::with_capacity(targets_arr.len()); for (j, t) in targets_arr.iter().enumerate() { - let engine_str = t - .get("engine") - .and_then(|v| v.as_str()) - .ok_or_else(|| { - anyhow::anyhow!( - "backend-storage-routing JSON: metric '{}' targets[{}] missing 'engine'", - name, j, - ) - })?; + let engine_str = t.get("engine").and_then(|v| v.as_str()).ok_or_else(|| { + anyhow::anyhow!( + "backend-storage-routing JSON: metric '{}' targets[{}] missing 'engine'", + name, + j, + ) + })?; let backend = parse_engine_string(engine_str).with_context(|| { format!( "backend-storage-routing JSON: metric '{}' targets[{}] invalid engine '{}'", name, j, engine_str, ) })?; - let applies_to_query_shape = - t.get("applies_to_query_shape").and_then(|v| v.as_array()).map(|arr| { + let applies_to_query_shape = t + .get("applies_to_query_shape") + .and_then(|v| v.as_array()) + .map(|arr| { arr.iter() .filter_map(|s| s.as_str()) .map(parse_query_shape_string) @@ -687,10 +685,7 @@ impl BackendStorageRouting { pub fn lookup(&self, metric_name: &str) -> StorageBackend { match self.metrics.get(metric_name) { Some(targets) => { - let backend = targets - .first() - .map(|t| t.backend) - .unwrap_or(self.default); + let backend = targets.first().map(|t| t.backend).unwrap_or(self.default); debug!( metric = metric_name, backend = ?backend, @@ -766,10 +761,7 @@ impl BackendStorageRouting { // Pass 3: the first target, regardless of filter // (only reachable when the metric only has // shape-specific targets and none matched). - let backend = targets - .first() - .map(|t| t.backend) - .unwrap_or(self.default); + let backend = targets.first().map(|t| t.backend).unwrap_or(self.default); debug!( metric = metric_name, shape = ?shape, @@ -823,38 +815,18 @@ impl Default for BackendStorageRouting { } /// Map a JSON `engine` string into a backend `StorageBackend` variant. -/// Phase α accepts both the controller's vocabulary (`thanos_archive`) -/// and the existing YAML's vocabulary (`gorilla_s3_archive`) — both -/// resolve to `StorageBackend::GorillaS3Archive` because the cold-archive -/// engine registered today serves both via `GorillaQueryEngine`. +/// Only the two public query engine ids are accepted: +/// `asap_query` and `thanos_query`. /// `unknown_engine` returns an error so a typo doesn't silently turn /// into a default-routing footgun. fn parse_engine_string(s: &str) -> Result { - match s { - "sketch_warm_tier" | "sketch_warm" => Ok(StorageBackend::SketchWarmTier), - // `thanos_archive` is the controller-emitted name; the backend - // currently registers the Gorilla-S3 cold archive under - // `gorilla_archive` / `gorilla_s3_archive`. They map to the - // same `StorageBackend` variant for Phase α — when a real - // Thanos engine lands the parser can split the two. - "thanos_archive" | "gorilla_s3_archive" | "gorilla_archive" => { - Ok(StorageBackend::GorillaS3Archive) - } - "double_write" => Ok(StorageBackend::DoubleWrite), - // Phase ε.2: the controller's Mode 3 - // (`RawAtEdgePrometheusArchive`) emits this when a metric's - // raw data is shipped to Prometheus's native OTLP receiver. - // The backend's `PrometheusForwardEngine` (in - // `engines::prometheus::forward`) registers under this id - // when `ASAP_PROMETHEUS_QUERY_URL` is set. - "prometheus_remote" => Ok(StorageBackend::PrometheusRemote), - other => Err(anyhow::anyhow!( - "unknown engine '{}': expected one of \ - [sketch_warm_tier, thanos_archive, gorilla_s3_archive, double_write, \ - prometheus_remote]", - other, - )), - } + parse_storage_backend_engine_id(s).ok_or_else(|| { + anyhow::anyhow!( + "unknown engine '{}': expected one of [{}]", + s, + CANONICAL_QUERY_ENGINE_IDS.join(", "), + ) + }) } /// Map a JSON `applies_to_query_shape` string into a backend @@ -967,9 +939,8 @@ pub struct HotReloadBackendStorageRouting { /// Map keyed by tenant id. Wrapped in `Arc` so swaps can /// publish a fresh map atomically; readers snapshot the whole map /// once and pick the tenant's `Arc`. - inner: std::sync::Arc< - arc_swap::ArcSwap>>, - >, + inner: + std::sync::Arc>>>, } impl HotReloadBackendStorageRouting { @@ -1047,10 +1018,7 @@ impl HotReloadBackendStorageRouting { /// the single-tenant convenience accessor. Returns the `Arc` /// that was just replaced (or `None` when no prior entry /// existed) for callers that want to log the diff. - pub fn swap( - &self, - new: BackendStorageRouting, - ) -> std::sync::Arc { + pub fn swap(&self, new: BackendStorageRouting) -> std::sync::Arc { // Preserve the original return type (always returns the // previous Arc, fabricating an empty one when none existed) // so callers depending on the old contract don't break. @@ -1078,8 +1046,7 @@ impl HotReloadBackendStorageRouting { // for *different* tenants don't lose updates. loop { let cur = self.inner.load_full(); - let mut next: HashMap> = - (*cur).clone(); + let mut next: HashMap> = (*cur).clone(); let prev = next.insert(tenant.to_string(), new_arc.clone()); let next_arc = std::sync::Arc::new(next); // `compare_and_swap` returns the value that was actually @@ -1122,11 +1089,14 @@ mod tests { #[test] fn empty_router_routes_everything_to_warm_tier() { let r = BackendStorageRouting::empty(); - assert_eq!(r.lookup("anything"), StorageBackend::SketchWarmTier); - assert_eq!(r.lookup("http_requests_total"), StorageBackend::SketchWarmTier); + assert_eq!(r.lookup("anything"), StorageBackend::SketchStore); + assert_eq!( + r.lookup("http_requests_total"), + StorageBackend::SketchStore + ); assert_eq!( r.lookup_with_shape("anything", QueryShape::Count), - StorageBackend::SketchWarmTier + StorageBackend::SketchStore ); } @@ -1134,45 +1104,42 @@ mod tests { fn yaml_with_per_metric_override_routes_correctly_v6_1_form() { // v6.1 form: `metrics:` map. Each value is a single backend. let yaml = r#" -default: sketch_warm_tier +default: asap_query metrics: - http_requests_total: gorilla_s3_archive - audit_events: gorilla_s3_archive + http_requests_total: thanos_query + audit_events: thanos_query "#; let r = BackendStorageRouting::from_yaml_str(yaml).expect("parse"); assert_eq!( r.lookup("http_requests_total"), - StorageBackend::GorillaS3Archive - ); - assert_eq!( - r.lookup("audit_events"), - StorageBackend::GorillaS3Archive + StorageBackend::GorillaObjectStore ); - assert_eq!(r.lookup("unlisted"), StorageBackend::SketchWarmTier); + assert_eq!(r.lookup("audit_events"), StorageBackend::GorillaObjectStore); + assert_eq!(r.lookup("unlisted"), StorageBackend::SketchStore); assert_eq!(r.len(), 2); } #[test] fn yaml_default_only_routes_all_metrics_to_default() { - let yaml = "default: gorilla_s3_archive\n"; + let yaml = "default: thanos_query\n"; let r = BackendStorageRouting::from_yaml_str(yaml).expect("parse"); - assert_eq!(r.lookup("anything"), StorageBackend::GorillaS3Archive); + assert_eq!(r.lookup("anything"), StorageBackend::GorillaObjectStore); assert!(r.is_empty()); - assert_eq!(r.default_backend(), StorageBackend::GorillaS3Archive); + assert_eq!(r.default_backend(), StorageBackend::GorillaObjectStore); } #[test] - fn yaml_omitted_default_falls_back_to_sketch_warm() { - let yaml = "metrics:\n foo: gorilla_s3_archive\n"; + fn yaml_omitted_default_falls_back_to_asap_query() { + let yaml = "metrics:\n foo: thanos_query\n"; let r = BackendStorageRouting::from_yaml_str(yaml).expect("parse"); - assert_eq!(r.lookup("foo"), StorageBackend::GorillaS3Archive); - assert_eq!(r.lookup("bar"), StorageBackend::SketchWarmTier); + assert_eq!(r.lookup("foo"), StorageBackend::GorillaObjectStore); + assert_eq!(r.lookup("bar"), StorageBackend::SketchStore); } #[test] fn empty_yaml_is_valid_and_empty() { let r = BackendStorageRouting::from_yaml_str("").expect("empty parse"); - assert_eq!(r.lookup("foo"), StorageBackend::SketchWarmTier); + assert_eq!(r.lookup("foo"), StorageBackend::SketchStore); assert!(r.is_empty()); } @@ -1190,58 +1157,58 @@ metrics: // targets — the default warm-tier slot and a cold-archive // slot scoped to count/topk/rate_post_hoc. let yaml = r#" -default: sketch_warm_tier +default: asap_query routes: - metric: http_requests_total targets: - - backend: sketch_warm_tier - - backend: gorilla_s3_archive + - backend: asap_query + - backend: thanos_query applies_to_query_shape: [count, topk, rate_post_hoc] - metric: http_freshness_probe_warm targets: - - backend: sketch_warm_tier + - backend: asap_query - metric: http_freshness_probe_archive targets: - - backend: gorilla_s3_archive + - backend: thanos_query "#; let r = BackendStorageRouting::from_yaml_str(yaml).expect("parse"); // Count + topk + rate_post_hoc → archive. assert_eq!( r.lookup_with_shape("http_requests_total", QueryShape::Count), - StorageBackend::GorillaS3Archive, + StorageBackend::GorillaObjectStore, ); assert_eq!( r.lookup_with_shape("http_requests_total", QueryShape::Topk), - StorageBackend::GorillaS3Archive, + StorageBackend::GorillaObjectStore, ); assert_eq!( r.lookup_with_shape("http_requests_total", QueryShape::RatePostHoc), - StorageBackend::GorillaS3Archive, + StorageBackend::GorillaObjectStore, ); // Quantile + sum_over_time + everything else → warm. assert_eq!( r.lookup_with_shape("http_requests_total", QueryShape::Quantile), - StorageBackend::SketchWarmTier, + StorageBackend::SketchStore, ); assert_eq!( r.lookup_with_shape("http_requests_total", QueryShape::Sum), - StorageBackend::SketchWarmTier, + StorageBackend::SketchStore, ); assert_eq!( r.lookup_with_shape("http_requests_total", QueryShape::Other), - StorageBackend::SketchWarmTier, + StorageBackend::SketchStore, ); // Single-target metrics keep v6.1 semantics regardless of shape. assert_eq!( r.lookup_with_shape("http_freshness_probe_warm", QueryShape::LastOverTime), - StorageBackend::SketchWarmTier, + StorageBackend::SketchStore, ); assert_eq!( r.lookup_with_shape("http_freshness_probe_archive", QueryShape::LastOverTime), - StorageBackend::GorillaS3Archive, + StorageBackend::GorillaObjectStore, ); } @@ -1251,7 +1218,7 @@ routes: // to the same backend (no dual-routing). let yaml = r#" metrics: - audit_events: gorilla_s3_archive + audit_events: thanos_query "#; let r = BackendStorageRouting::from_yaml_str(yaml).expect("parse"); for shape in [ @@ -1263,8 +1230,8 @@ metrics: ] { assert_eq!( r.lookup_with_shape("audit_events", shape), - StorageBackend::GorillaS3Archive, - "shape={shape:?} must resolve to gorilla_s3_archive (single-target)", + StorageBackend::GorillaObjectStore, + "shape={shape:?} must resolve to thanos_query (single-target)", ); } } @@ -1274,26 +1241,26 @@ metrics: // Both `metrics:` and `routes:` populated; a metric in BOTH // wins from `routes:` (multi-target overrides single-target). let yaml = r#" -default: sketch_warm_tier +default: asap_query metrics: - http_requests_total: gorilla_s3_archive + http_requests_total: thanos_query routes: - metric: http_requests_total targets: - - backend: sketch_warm_tier - - backend: gorilla_s3_archive + - backend: asap_query + - backend: thanos_query applies_to_query_shape: [count] "#; let r = BackendStorageRouting::from_yaml_str(yaml).expect("parse"); // Default slot (warm) wins for non-count shapes. assert_eq!( r.lookup_with_shape("http_requests_total", QueryShape::Quantile), - StorageBackend::SketchWarmTier, + StorageBackend::SketchStore, ); // Count → archive. assert_eq!( r.lookup_with_shape("http_requests_total", QueryShape::Count), - StorageBackend::GorillaS3Archive, + StorageBackend::GorillaObjectStore, ); // The `metrics:` entry was overridden by the multi-target // `routes:` entry (the single-target archive vanished). @@ -1320,19 +1287,19 @@ routes: metrics.insert( "x".to_string(), vec![ - RoutingTarget::for_shapes(StorageBackend::GorillaS3Archive, vec![QueryShape::Count]), RoutingTarget::for_shapes( - StorageBackend::SketchWarmTier, - vec![QueryShape::Topk], + StorageBackend::GorillaObjectStore, + vec![QueryShape::Count], ), + RoutingTarget::for_shapes(StorageBackend::SketchStore, vec![QueryShape::Topk]), ], ); - let r = BackendStorageRouting::new(StorageBackend::SketchWarmTier, metrics); + let r = BackendStorageRouting::new(StorageBackend::SketchStore, metrics); // No filter matches `Quantile`; must return the first target's // backend. assert_eq!( r.lookup_with_shape("x", QueryShape::Quantile), - StorageBackend::GorillaS3Archive, + StorageBackend::GorillaObjectStore, ); } @@ -1402,14 +1369,14 @@ routes: fn fixture_json() -> serde_json::Value { serde_json::json!({ - "default_engine": "sketch_warm_tier", + "default_engine": "asap_query", "metrics": [ { "name": "http_requests_total", "targets": [ - { "engine": "sketch_warm_tier" }, + { "engine": "asap_query" }, { - "engine": "thanos_archive", + "engine": "thanos_query", "applies_to_query_shape": [ "histogram_quantile", "delta", "deriv", "absent", "rate_post_hoc", "count" @@ -1421,9 +1388,9 @@ routes: { "name": "request_latency_seconds", "targets": [ - { "engine": "sketch_warm_tier" }, + { "engine": "asap_query" }, { - "engine": "thanos_archive", + "engine": "thanos_query", "applies_to_query_shape": [ "histogram_quantile", "delta", "absent", "rate_post_hoc", "topk", "count" @@ -1438,50 +1405,50 @@ routes: #[test] fn json_payload_parses_controller_fixture() { let r = BackendStorageRouting::from_json_payload(&fixture_json()).expect("parse"); - assert_eq!(r.default_backend(), StorageBackend::SketchWarmTier); + assert_eq!(r.default_backend(), StorageBackend::SketchStore); assert_eq!(r.len(), 2); // http_requests_total: histogram_quantile / delta / etc → archive, // quantile / sum / topk → warm. assert_eq!( r.lookup_with_shape("http_requests_total", QueryShape::HistogramQuantile), - StorageBackend::GorillaS3Archive, + StorageBackend::GorillaObjectStore, ); assert_eq!( r.lookup_with_shape("http_requests_total", QueryShape::Delta), - StorageBackend::GorillaS3Archive, + StorageBackend::GorillaObjectStore, ); assert_eq!( r.lookup_with_shape("http_requests_total", QueryShape::Count), - StorageBackend::GorillaS3Archive, + StorageBackend::GorillaObjectStore, ); assert_eq!( r.lookup_with_shape("http_requests_total", QueryShape::Quantile), - StorageBackend::SketchWarmTier, + StorageBackend::SketchStore, ); assert_eq!( r.lookup_with_shape("http_requests_total", QueryShape::Topk), - StorageBackend::SketchWarmTier, + StorageBackend::SketchStore, ); // LastOverTime not in the archive's filter list → falls // through to the default (warm) slot. assert_eq!( r.lookup_with_shape("http_requests_total", QueryShape::LastOverTime), - StorageBackend::SketchWarmTier, + StorageBackend::SketchStore, ); } #[test] fn json_payload_unknown_shape_defaults_to_other() { let value = serde_json::json!({ - "default_engine": "sketch_warm_tier", + "default_engine": "asap_query", "metrics": [ { "name": "x", "targets": [ - { "engine": "sketch_warm_tier" }, + { "engine": "asap_query" }, { - "engine": "thanos_archive", + "engine": "thanos_query", "applies_to_query_shape": ["some_future_shape", "count"] } ] @@ -1493,18 +1460,18 @@ routes: // to QueryShape::Other (silently — forward-compat). assert_eq!( r.lookup_with_shape("x", QueryShape::Count), - StorageBackend::GorillaS3Archive, + StorageBackend::GorillaObjectStore, ); assert_eq!( r.lookup_with_shape("x", QueryShape::Other), - StorageBackend::GorillaS3Archive, + StorageBackend::GorillaObjectStore, ); } #[test] fn json_payload_invalid_engine_errors() { let value = serde_json::json!({ - "default_engine": "sketch_warm_tier", + "default_engine": "asap_query", "metrics": [{ "name": "x", "targets": [{ "engine": "not_a_real_engine" }] @@ -1524,7 +1491,7 @@ routes: #[test] fn json_payload_empty_targets_errors() { let value = serde_json::json!({ - "default_engine": "sketch_warm_tier", + "default_engine": "asap_query", "metrics": [{ "name": "x", "targets": [] }] }); let err = BackendStorageRouting::from_json_payload(&value).expect_err("must reject"); @@ -1533,7 +1500,7 @@ routes: #[test] fn json_payload_missing_metrics_errors() { - let value = serde_json::json!({ "default_engine": "sketch_warm_tier" }); + let value = serde_json::json!({ "default_engine": "asap_query" }); let err = BackendStorageRouting::from_json_payload(&value).expect_err("must reject"); assert!(err.to_string().contains("metrics")); } @@ -1542,28 +1509,28 @@ routes: fn json_payload_default_engine_optional_falls_back_to_warm() { let value = serde_json::json!({ "metrics": [ - { "name": "x", "targets": [{ "engine": "sketch_warm_tier" }] } + { "name": "x", "targets": [{ "engine": "asap_query" }] } ] }); let r = BackendStorageRouting::from_json_payload(&value).expect("parse"); - assert_eq!(r.default_backend(), StorageBackend::SketchWarmTier); + assert_eq!(r.default_backend(), StorageBackend::SketchStore); } #[test] - fn json_payload_back_compat_gorilla_s3_archive_alias() { + fn json_payload_back_compat_thanos_query_alias() { // An older deploy might emit the YAML's vocabulary instead of - // `thanos_archive`; both must parse and resolve the same. + // `thanos_query`; both must parse and resolve the same. let value = serde_json::json!({ - "default_engine": "sketch_warm_tier", + "default_engine": "asap_query", "metrics": [{ "name": "audit_events", "targets": [ - { "engine": "gorilla_s3_archive" } + { "engine": "thanos_query" } ] }] }); let r = BackendStorageRouting::from_json_payload(&value).expect("parse"); - assert_eq!(r.lookup("audit_events"), StorageBackend::GorillaS3Archive); + assert_eq!(r.lookup("audit_events"), StorageBackend::GorillaObjectStore); } /// Phase ε.2: the controller's Mode 3 @@ -1576,7 +1543,7 @@ routes: #[test] fn json_payload_prometheus_remote_parses_to_prometheus_remote_backend() { let value = serde_json::json!({ - "default_engine": "sketch_warm_tier", + "default_engine": "asap_query", "metrics": [{ "name": "node_cpu_seconds_total", "targets": [ @@ -1600,19 +1567,16 @@ routes: #[test] fn replace_swaps_table_in_place() { let mut r = BackendStorageRouting::new_from_single_targets( - StorageBackend::SketchWarmTier, - HashMap::from([( - "old_metric".to_string(), - StorageBackend::GorillaS3Archive, - )]), + StorageBackend::SketchStore, + HashMap::from([("old_metric".to_string(), StorageBackend::GorillaObjectStore)]), ); let new = BackendStorageRouting::from_json_payload(&fixture_json()).expect("parse"); r.replace(new); // Old metric is gone; new metrics are visible. - assert_eq!(r.lookup("old_metric"), StorageBackend::SketchWarmTier); + assert_eq!(r.lookup("old_metric"), StorageBackend::SketchStore); assert_eq!( r.lookup_with_shape("http_requests_total", QueryShape::HistogramQuantile), - StorageBackend::GorillaS3Archive, + StorageBackend::GorillaObjectStore, ); } @@ -1628,7 +1592,7 @@ routes: assert_eq!(snap.len(), 2); assert_eq!( snap.lookup_with_shape("http_requests_total", QueryShape::Delta), - StorageBackend::GorillaS3Archive, + StorageBackend::GorillaObjectStore, ); } @@ -1642,9 +1606,9 @@ routes: let mut metrics = HashMap::new(); metrics.insert( format!("metric_{i}"), - vec![RoutingTarget::always(StorageBackend::GorillaS3Archive)], + vec![RoutingTarget::always(StorageBackend::GorillaObjectStore)], ); - let new = BackendStorageRouting::new(StorageBackend::SketchWarmTier, metrics); + let new = BackendStorageRouting::new(StorageBackend::SketchStore, metrics); writer_hr.swap(new); } }); @@ -1678,7 +1642,8 @@ routes: #[test] fn classifies_histogram_quantile_correctly() { - let e = parse("histogram_quantile(0.99, sum by (le) (rate(http_request_duration_bucket[5m])))"); + let e = + parse("histogram_quantile(0.99, sum by (le) (rate(http_request_duration_bucket[5m])))"); assert_eq!(classify_query_shape(&e), QueryShape::HistogramQuantile); } @@ -1715,10 +1680,10 @@ routes: fn json_payload_tenant_field_is_picked_up_when_present() { let value = serde_json::json!({ "tenant": "tenant-a", - "default_engine": "sketch_warm_tier", + "default_engine": "asap_query", "metrics": [ { "name": "http_requests_total", - "targets": [{ "engine": "sketch_warm_tier" }] } + "targets": [{ "engine": "asap_query" }] } ] }); let r = BackendStorageRouting::from_json_payload(&value).expect("parse"); @@ -1730,9 +1695,9 @@ routes: // Existing YAMLs in the wild don't have `tenant:` — they // must keep parsing and resolve to [`DEFAULT_TENANT`]. let yaml = r#" -default: sketch_warm_tier +default: asap_query metrics: - http_requests_total: gorilla_s3_archive + http_requests_total: thanos_query "#; let r = BackendStorageRouting::from_yaml_str(yaml).expect("parse"); assert_eq!(r.tenant(), DEFAULT_TENANT); @@ -1742,9 +1707,9 @@ metrics: fn yaml_tenant_field_is_picked_up_when_present() { let yaml = r#" tenant: tenant-b -default: sketch_warm_tier +default: asap_query metrics: - http_requests_total: gorilla_s3_archive + http_requests_total: thanos_query "#; let r = BackendStorageRouting::from_yaml_str(yaml).expect("parse"); assert_eq!(r.tenant(), "tenant-b"); @@ -1757,40 +1722,34 @@ metrics: let hr = HotReloadBackendStorageRouting::empty(); // Push tenant-a's table. let table_a = BackendStorageRouting::new_from_single_targets( - StorageBackend::SketchWarmTier, - HashMap::from([( - "metric_a".to_string(), - StorageBackend::GorillaS3Archive, - )]), + StorageBackend::SketchStore, + HashMap::from([("metric_a".to_string(), StorageBackend::GorillaObjectStore)]), ); hr.swap_tenant("tenant-a", table_a); // Push tenant-b's table. let table_b = BackendStorageRouting::new_from_single_targets( - StorageBackend::SketchWarmTier, - HashMap::from([( - "metric_b".to_string(), - StorageBackend::GorillaS3Archive, - )]), + StorageBackend::SketchStore, + HashMap::from([("metric_b".to_string(), StorageBackend::GorillaObjectStore)]), ); hr.swap_tenant("tenant-b", table_b); // Replace tenant-a only. let table_a_v2 = BackendStorageRouting::new_from_single_targets( - StorageBackend::SketchWarmTier, - HashMap::from([( - "metric_a_v2".to_string(), - StorageBackend::GorillaS3Archive, - )]), + StorageBackend::SketchStore, + HashMap::from([("metric_a_v2".to_string(), StorageBackend::GorillaObjectStore)]), ); hr.swap_tenant("tenant-a", table_a_v2); // tenant-a now reflects v2; tenant-b is unchanged. let snap_a = hr.snapshot_for_tenant("tenant-a"); - assert_eq!(snap_a.lookup("metric_a"), StorageBackend::SketchWarmTier); - assert_eq!(snap_a.lookup("metric_a_v2"), StorageBackend::GorillaS3Archive); + assert_eq!(snap_a.lookup("metric_a"), StorageBackend::SketchStore); + assert_eq!( + snap_a.lookup("metric_a_v2"), + StorageBackend::GorillaObjectStore + ); let snap_b = hr.snapshot_for_tenant("tenant-b"); - assert_eq!(snap_b.lookup("metric_b"), StorageBackend::GorillaS3Archive); - assert_eq!(snap_b.lookup("metric_a_v2"), StorageBackend::SketchWarmTier); + assert_eq!(snap_b.lookup("metric_b"), StorageBackend::GorillaObjectStore); + assert_eq!(snap_b.lookup("metric_a_v2"), StorageBackend::SketchStore); } #[test] @@ -1800,10 +1759,10 @@ metrics: let hr = HotReloadBackendStorageRouting::empty(); // Default tenant has an explicit override. let default_table = BackendStorageRouting::new_from_single_targets( - StorageBackend::SketchWarmTier, + StorageBackend::SketchStore, HashMap::from([( "shared_metric".to_string(), - StorageBackend::GorillaS3Archive, + StorageBackend::GorillaObjectStore, )]), ); hr.swap_tenant(DEFAULT_TENANT, default_table); @@ -1812,13 +1771,13 @@ metrics: let snap = hr.snapshot_for_tenant("nonexistent-tenant"); assert_eq!( snap.lookup("shared_metric"), - StorageBackend::GorillaS3Archive, + StorageBackend::GorillaObjectStore, ); // Default tenant: same answer. let snap_default = hr.snapshot_for_tenant(DEFAULT_TENANT); assert_eq!( snap_default.lookup("shared_metric"), - StorageBackend::GorillaS3Archive, + StorageBackend::GorillaObjectStore, ); } @@ -1829,11 +1788,8 @@ metrics: // the map key is the source of truth. let hr = HotReloadBackendStorageRouting::empty(); let mut table = BackendStorageRouting::new_from_single_targets( - StorageBackend::SketchWarmTier, - HashMap::from([( - "m".to_string(), - StorageBackend::GorillaS3Archive, - )]), + StorageBackend::SketchStore, + HashMap::from([("m".to_string(), StorageBackend::GorillaObjectStore)]), ); table = table.with_tenant("WRONG-TENANT"); hr.swap_tenant("right-tenant", table); diff --git a/asap-query-engine/src/routing/engine_router.rs b/asap-query-engine/src/routing/engine_router.rs index 53e8c4a3..a8cab7ad 100644 --- a/asap-query-engine/src/routing/engine_router.rs +++ b/asap-query-engine/src/routing/engine_router.rs @@ -133,29 +133,6 @@ impl EngineRouter { self.engines.insert(caps.data_source_id, engine); } - /// Register an engine under an alias `data_source_id`, ignoring the - /// id reported by `engine.capabilities()`. Used by Step-2.3's - /// Path A2 wiring: the same `ThanosForwardEngine` instance is - /// registered under both its native id (`thanos_archive`, for - /// explicit overrides) and under the legacy archive id - /// (`gorilla_archive`, so the existing - /// `compatible_storage_backends` failover sequence finds it - /// transparently). The legacy in-process `GorillaQueryEngine` is - /// only registered when `ASAP_THANOS_QUERY_URL` is unset; the - /// alias mechanism guarantees the two registrations never - /// collide on the same id. - /// - /// If two engines claim the same `id` the later registration wins - /// (matches [`Self::register`]'s hot-swap contract). - pub fn register_aliased(&mut self, id: &'static str, engine: Arc) { - debug!( - data_source_id = id, - engine_native_id = engine.capabilities().data_source_id, - "router: registering engine under alias", - ); - self.engines.insert(id, engine); - } - /// Number of engines registered. Test-only convenience. pub fn len(&self) -> usize { self.engines.len() @@ -326,9 +303,9 @@ mod tests { #[tokio::test] async fn router_dispatches_to_warm_tier_for_sketch_metrics() { let mut router = EngineRouter::new(); - let (warm, warm_calls) = StubEngine::new(StorageBackend::SketchWarmTier, Outcome::Ok); + let (warm, warm_calls) = StubEngine::new(StorageBackend::SketchStore, Outcome::Ok); let (archive, archive_calls) = - StubEngine::new(StorageBackend::GorillaS3Archive, Outcome::Ok); + StubEngine::new(StorageBackend::GorillaObjectStore, Outcome::Ok); router.register(warm); router.register(archive); @@ -337,7 +314,7 @@ mod tests { "sum_over_time(foo[5m])", Statistic::Sum, AccuracyTarget::Approximate, - StorageBackend::SketchWarmTier, + StorageBackend::SketchStore, ) .await; assert!(result.is_ok()); @@ -352,9 +329,9 @@ mod tests { #[tokio::test] async fn router_dispatches_to_gorilla_for_archive_metrics() { let mut router = EngineRouter::new(); - let (warm, warm_calls) = StubEngine::new(StorageBackend::SketchWarmTier, Outcome::Ok); + let (warm, warm_calls) = StubEngine::new(StorageBackend::SketchStore, Outcome::Ok); let (gorilla, gorilla_calls) = - StubEngine::new(StorageBackend::GorillaS3Archive, Outcome::Ok); + StubEngine::new(StorageBackend::GorillaObjectStore, Outcome::Ok); router.register(warm); router.register(gorilla); @@ -363,7 +340,7 @@ mod tests { "sum_over_time(audit_events[1h])", Statistic::Sum, AccuracyTarget::Exact, - StorageBackend::GorillaS3Archive, + StorageBackend::GorillaObjectStore, ) .await; assert!(result.is_ok()); @@ -382,9 +359,8 @@ mod tests { // remaining failover after Step-1 deleted JSONL). let mut router = EngineRouter::new(); let (gorilla, gorilla_calls) = - StubEngine::new(StorageBackend::GorillaS3Archive, Outcome::Backend); - let (warm, warm_calls) = - StubEngine::new(StorageBackend::SketchWarmTier, Outcome::Ok); + StubEngine::new(StorageBackend::GorillaObjectStore, Outcome::Backend); + let (warm, warm_calls) = StubEngine::new(StorageBackend::SketchStore, Outcome::Ok); router.register(gorilla); router.register(warm); @@ -412,14 +388,14 @@ mod tests { "sum_over_time(foo[5m])", Statistic::Sum, AccuracyTarget::Approximate, - StorageBackend::SketchWarmTier, + StorageBackend::SketchStore, ) .await; match result { Err(EngineRouterError::NoEngineRegistered { tried, registered }) => { // Step-1 deleted the JSONL failover slot, so the - // SketchWarmTier failover sequence is just itself. - assert_eq!(tried, vec![StorageBackend::SketchWarmTier]); + // SketchStore failover sequence is just itself. + assert_eq!(tried, vec![StorageBackend::SketchStore]); assert!(registered.is_empty()); } other => panic!("expected NoEngineRegistered, got {other:?}"), @@ -428,11 +404,11 @@ mod tests { #[tokio::test] async fn router_returns_all_failed_when_every_engine_errors() { - // After Step-1 deleted JSONL, the SketchWarmTier failover - // sequence is just `[SketchWarmTier]`. A failing warm-tier + // After Step-1 deleted JSONL, the SketchStore failover + // sequence is just `[SketchStore]`. A failing warm-tier // engine is the only error path on this metric. let mut router = EngineRouter::new(); - let (warm, _) = StubEngine::new(StorageBackend::SketchWarmTier, Outcome::Backend); + let (warm, _) = StubEngine::new(StorageBackend::SketchStore, Outcome::Backend); router.register(warm); let result = router @@ -440,7 +416,7 @@ mod tests { "sum_over_time(foo[5m])", Statistic::Sum, AccuracyTarget::Approximate, - StorageBackend::SketchWarmTier, + StorageBackend::SketchStore, ) .await; match result { @@ -454,16 +430,16 @@ mod tests { #[tokio::test] async fn engine_by_id_returns_registered_engines_or_none() { let mut router = EngineRouter::new(); - let (warm, _) = StubEngine::new(StorageBackend::SketchWarmTier, Outcome::Ok); + let (warm, _) = StubEngine::new(StorageBackend::SketchStore, Outcome::Ok); let (gorilla, gorilla_calls) = - StubEngine::new(StorageBackend::GorillaS3Archive, Outcome::Ok); + StubEngine::new(StorageBackend::GorillaObjectStore, Outcome::Ok); router.register(warm); router.register(gorilla); // Hit by id — must return the engine for that backend. let archive = router - .engine_by_id("gorilla_archive") - .expect("gorilla_archive engine registered"); + .engine_by_id("thanos_query") + .expect("thanos_query engine registered"); let _ = archive.execute("count(foo)").await; assert_eq!( gorilla_calls.load(Ordering::SeqCst), @@ -477,91 +453,36 @@ mod tests { // Iter exposes every registered id. let mut ids: Vec<&str> = router.registered_ids().collect(); ids.sort(); - assert_eq!(ids, vec!["gorilla_archive", "sketch_warm"]); - } - - #[tokio::test] - async fn register_aliased_inserts_under_explicit_id() { - // Step-2.3 wiring: a single ThanosForwardEngine instance is - // registered under both `thanos_archive` (its native id) and - // `gorilla_archive` (the legacy archive slot the failover - // sequence walks). Both lookups must hit the same engine. - let mut router = EngineRouter::new(); - let (engine, calls) = StubEngine::new(StorageBackend::SketchWarmTier, Outcome::Ok); - // First, register under the engine's native id (`sketch_warm`). - router.register(engine.clone()); - // Then alias it under a totally different id. - router.register_aliased("custom_alias", engine); - // Both lookups must return the engine — we exercise both and - // verify the call counter ticked twice. - let native = router - .engine_by_id("sketch_warm") - .expect("native id registered"); - let aliased = router - .engine_by_id("custom_alias") - .expect("alias registered"); - let _ = native.execute("foo").await; - let _ = aliased.execute("foo").await; - assert_eq!( - calls.load(Ordering::SeqCst), - 2, - "both lookups must reach the same engine instance", - ); - - let mut ids: Vec<&str> = router.registered_ids().collect(); - ids.sort(); - assert_eq!(ids, vec!["custom_alias", "sketch_warm"]); - } - - #[tokio::test] - async fn register_aliased_overrides_capability_dispatch_target() { - // Step-2.3 wiring continued: when `ThanosForwardEngine` is - // aliased onto `gorilla_archive`, the failover sequence walks - // the alias instead of the legacy in-process engine. We - // simulate this with a stub registered under - // `GorillaS3Archive`'s native id via the alias path. - let mut router = EngineRouter::new(); - let (legacy, legacy_calls) = - StubEngine::new(StorageBackend::GorillaS3Archive, Outcome::Ok); - // Use alias to register under the gorilla_archive id - // explicitly (matches Step-2.3's "thanos under legacy slot" - // wiring). - router.register_aliased( - StorageBackend::GorillaS3Archive.data_source_id(), - legacy, - ); - - let result = router - .execute( - "sum_over_time(audit_events[1h])", - Statistic::Sum, - AccuracyTarget::Exact, - StorageBackend::GorillaS3Archive, - ) - .await; - assert!(result.is_ok()); - assert_eq!(legacy_calls.load(Ordering::SeqCst), 1); + assert_eq!(ids, vec!["thanos_query", "asap_query"]); } #[tokio::test] async fn register_overwrites_same_data_source_id() { let mut router = EngineRouter::new(); - let (first, first_calls) = StubEngine::new(StorageBackend::SketchWarmTier, Outcome::Ok); - let (second, second_calls) = StubEngine::new(StorageBackend::SketchWarmTier, Outcome::Ok); + let (first, first_calls) = StubEngine::new(StorageBackend::SketchStore, Outcome::Ok); + let (second, second_calls) = StubEngine::new(StorageBackend::SketchStore, Outcome::Ok); router.register(first); router.register(second); - assert_eq!(router.len(), 1, "two registrations under same id collapse to one"); + assert_eq!( + router.len(), + 1, + "two registrations under same id collapse to one" + ); let _ = router .execute( "sum_over_time(foo[5m])", Statistic::Sum, AccuracyTarget::Approximate, - StorageBackend::SketchWarmTier, + StorageBackend::SketchStore, ) .await; assert_eq!(first_calls.load(Ordering::SeqCst), 0); - assert_eq!(second_calls.load(Ordering::SeqCst), 1, "later registration wins"); + assert_eq!( + second_calls.load(Ordering::SeqCst), + 1, + "later registration wins" + ); } /// Phase ε.2: a routing-table entry with @@ -574,9 +495,8 @@ mod tests { #[tokio::test] async fn router_dispatches_to_prometheus_remote_for_mode3_metrics() { let mut router = EngineRouter::new(); - let (prom, prom_calls) = - StubEngine::new(StorageBackend::PrometheusRemote, Outcome::Ok); - let (warm, warm_calls) = StubEngine::new(StorageBackend::SketchWarmTier, Outcome::Ok); + let (prom, prom_calls) = StubEngine::new(StorageBackend::PrometheusRemote, Outcome::Ok); + let (warm, warm_calls) = StubEngine::new(StorageBackend::SketchStore, Outcome::Ok); router.register(prom); router.register(warm); diff --git a/asap-query-engine/src/routing/freshness_probe_cache.rs b/asap-query-engine/src/routing/freshness_probe_cache.rs index 6932e330..3bd6e7e5 100644 --- a/asap-query-engine/src/routing/freshness_probe_cache.rs +++ b/asap-query-engine/src/routing/freshness_probe_cache.rs @@ -218,7 +218,13 @@ mod tests { let got = cache .lookup("http_freshness_probe_warm", 1_005, 10_000) .expect("sample should be inside window"); - assert_eq!(got, ProbeSample { ts_ms: 1_000, value: 1_000.0 }); + assert_eq!( + got, + ProbeSample { + ts_ms: 1_000, + value: 1_000.0 + } + ); } #[test] @@ -285,11 +291,9 @@ mod tests { "upper bound must be inclusive (ts_ms == now)", ); // sample.ts == now − range_ms − 1 — outside lower bound. - assert!( - cache - .lookup("http_freshness_probe_warm", 11_001, 10_000) - .is_none(), - ); + assert!(cache + .lookup("http_freshness_probe_warm", 11_001, 10_000) + .is_none(),); } #[test] diff --git a/asap-query-engine/src/routing/mod.rs b/asap-query-engine/src/routing/mod.rs index a310315b..6cc06f57 100644 --- a/asap-query-engine/src/routing/mod.rs +++ b/asap-query-engine/src/routing/mod.rs @@ -2,8 +2,8 @@ //! //! This module is the dispatch boundary between the HTTP query //! handler and the tier-co-located engines (warm sketch tier in -//! [`crate::engines::simple`], archive tier in -//! [`crate::engines::gorilla`]). Two cooperating pieces: +//! [`crate::engines::asap_query`], archive tier in +//! [`crate::stores::gorilla_object_store`]). Two cooperating pieces: //! //! * [`backend_storage_routing`] — config loader + multi-target //! per-metric lookup (`metric → [(backend, query-shape filter), ...]`). @@ -30,9 +30,7 @@ pub use backend_storage_routing::{ classify_query_shape, routing_table_hash, BackendStorageRouting, HotReloadBackendStorageRouting, QueryShape, RoutingTarget, DEFAULT_TENANT, }; -pub use engine_router::{ - EngineCapabilities, EngineRouter, EngineRouterError, QueryEngine, -}; +pub use engine_router::{EngineCapabilities, EngineRouter, EngineRouterError, QueryEngine}; pub use freshness_probe_cache::{ is_freshness_probe, now_ms as freshness_probe_now_ms, FreshnessProbeCache, ProbeSample, }; diff --git a/asap-query-engine/src/engines/gorilla/mod.rs b/asap-query-engine/src/stores/gorilla_object_store/mod.rs similarity index 83% rename from asap-query-engine/src/engines/gorilla/mod.rs rename to asap-query-engine/src/stores/gorilla_object_store/mod.rs index aded5e22..a1be91f7 100644 --- a/asap-query-engine/src/engines/gorilla/mod.rs +++ b/asap-query-engine/src/stores/gorilla_object_store/mod.rs @@ -1,14 +1,13 @@ -//! `GorillaQueryEngine` — exact PromQL execution over the Gorilla -//! archive tier. +//! Gorilla object store and legacy in-process archive executor. //! -//! Sibling of [`crate::engines::simple`] (warm-tier sketches). Both -//! consume the same PromQL surface; `GorillaQueryEngine` answers -//! exactly from per-hour Gorilla chunks landed on S3 / MinIO via -//! [`store::GorillaS3Store`]. +//! This module owns the Gorilla chunk/object-store implementation. The +//! public archive query engine is [`crate::engines::thanos_query`]; the +//! in-process [`GorillaQueryEngine`] remains here as a legacy fallback for +//! development deployments that still read per-hour Gorilla chunks directly. //! //! ## Module layout (post Step-1 refactor) //! -//! * [`engine`] — query planner + per-statistic exact executor (the +//! * [`query_engine`] — query planner + per-statistic exact executor (the //! merged form of the previous `query_planner.rs` + //! `exact_executor.rs`). //! * [`store`] — `GorillaS3Store` (the only `Store` impl after @@ -24,14 +23,14 @@ //! //! 1. an [`crate::stores::sketch_db::AccuracyEnvelope`] with //! `kind = Exact`, ε = 0, δ = 0, -//! 2. a `data_source: gorilla_archive` info line, +//! 2. a `data_source: thanos_query` info line, //! 3. cheap diagnostics (`samples_scanned`, `chunks_fetched`). //! //! See `docs/design-gorilla-s3-cold-engine.md` §6. //! //! ## Two execution strategies //! -//! Per-statistic dispatch in [`engine::ExactExecutor`]: +//! Per-statistic dispatch in [`query_engine::ExactExecutor`]: //! //! * **Streaming-additive** — `Sum`, `Count`, `Min`, `Max`, `Rate`, //! `Increase` (and `Avg` derived as Sum/Count). One chunk at a @@ -42,11 +41,10 @@ //! [`GorillaEngineConfig::max_buffered_samples`]; over-budget //! queries fail fast with [`EngineError::TooManySamples`]. -pub mod engine; pub mod postings; +pub mod query_engine; pub mod s3_cost; pub mod store; -pub mod thanos_forward; #[cfg(test)] mod tests; @@ -62,7 +60,7 @@ use crate::data_model::KeyByLabelValues; use crate::engines::query_result::{InstantVectorElement, QueryResult}; use crate::stores::sketch_db::accuracy::{AccuracyEnvelope, AccuracyProfile}; -pub use engine::{ +pub use query_engine::{ plan_query, plan_query_at, AdditiveOp, ExactExecutor, LabelMatcher, QueryPlan, QueryStatistic, }; pub use postings::PostingsHits; @@ -73,16 +71,10 @@ pub use store::{ ChunkRef, GorillaS3Config, GorillaS3ConfigError, GorillaS3Store, ObjectStore, RawSample, S3ObjectStore, Store, StoreError, }; -pub use thanos_forward::{ - engine_from_env as thanos_engine_from_env, ThanosForwardConfig, ThanosForwardEngine, - ThanosForwardError, ASAP_THANOS_QUERY_URL_ENV, DATA_SOURCE_THANOS_ARCHIVE_ID, - DATA_SOURCE_THANOS_ARCHIVE_INFO, DEFAULT_THANOS_QUERY_URL, QUIRK_THANOS_UNREACHABLE, -}; - -/// Marker line that every `GorillaQueryEngine` answer carries on -/// its `infos` array. Pinned so dashboards / Phase-5 capability -/// routers can byte-compare without parsing. -pub const DATA_SOURCE_GORILLA_ARCHIVE: &str = "data_source: gorilla_archive"; +/// Marker line that every archive answer carries on its `infos` array. +/// The legacy in-process Gorilla executor is an implementation detail; +/// the public archive engine identity is Thanos. +pub const DATA_SOURCE_GORILLA_ARCHIVE: &str = "data_source: thanos_query"; /// Tunable runtime knobs for the Gorilla query engine. /// @@ -113,7 +105,7 @@ impl Default for GorillaEngineConfig { #[derive(Debug, Error)] pub enum EngineError { /// PromQL string failed to parse, or used a construct outside - /// the engine's supported surface (see [`engine`]). + /// the engine's supported surface (see [`query_engine`]). #[error("query planning failed: {0}")] Plan(String), /// Archive-store fetch / decode failed. @@ -169,10 +161,7 @@ impl GorillaQueryEngine { /// Convenience constructor for the production /// [`store::GorillaS3Store`] path. Mirrors the design.md type /// signature. - pub fn with_gorilla_s3( - store: Arc, - config: GorillaEngineConfig, - ) -> Self { + pub fn with_gorilla_s3(store: Arc, config: GorillaEngineConfig) -> Self { Self::new(store as Arc, config) } @@ -192,11 +181,11 @@ impl GorillaQueryEngine { /// Execute a parsed PromQL query against the archive tier. /// - /// The query string is parsed via [`engine::plan_query`], + /// The query string is parsed via [`query_engine::plan_query`], /// the resulting plan dispatches to either the streaming /// additive or the buffered execution path, and the answer is /// wrapped with the exact-accuracy envelope + the - /// `data_source: gorilla_archive` annotation. + /// `data_source: thanos_query` annotation. pub async fn execute(&self, query: &str) -> Result { let now_ms = std::time::SystemTime::now() .duration_since(std::time::SystemTime::UNIX_EPOCH) @@ -209,23 +198,15 @@ impl GorillaQueryEngine { /// pinning the right edge of the request window. Used by /// tests + by callers that want to back-date a query against /// historical chunks. - pub async fn execute_at( - &self, - query: &str, - now_ms: i64, - ) -> Result { + pub async fn execute_at(&self, query: &str, now_ms: i64) -> Result { let timeout = Duration::from_secs(self.config.query_timeout_secs.max(1)); tokio::time::timeout(timeout, self.execute_inner(query, now_ms)) .await .map_err(|_| EngineError::Timeout(timeout))? } - async fn execute_inner( - &self, - query: &str, - now_ms: i64, - ) -> Result { - let plan = engine::plan_query_at(query, now_ms).map_err(EngineError::Plan)?; + async fn execute_inner(&self, query: &str, now_ms: i64) -> Result { + let plan = query_engine::plan_query_at(query, now_ms).map_err(EngineError::Plan)?; debug!( metric = plan.metric.as_str(), stat = ?plan.statistic, @@ -243,7 +224,7 @@ impl GorillaQueryEngine { /// Wrap a finished `(scalar value, sample / chunk counts)` into a /// `QueryResult` with the exact-accuracy envelope + the -/// `data_source: gorilla_archive` info line. Pulled out so tests +/// `data_source: thanos_query` info line. Pulled out so tests /// can pin the wrapping shape independently of the executor. pub fn wrap_result(plan: &QueryPlan, outcome: ExecutionOutcome) -> QueryResult { // Result timestamp is the right edge of the requested range — @@ -254,7 +235,8 @@ pub fn wrap_result(plan: &QueryPlan, outcome: ExecutionOutcome) -> QueryResult { let labels = KeyByLabelValues::new_with_labels(Vec::new()); let element = InstantVectorElement::new(labels, outcome.value); let envelope = AccuracyEnvelope::single(AccuracyProfile::exact()); - QueryResult::vector(vec![element], result_ts).with_accuracy(envelope) + QueryResult::vector(vec![element], result_ts) + .with_accuracy(envelope) // Window is the requested range, expressed in u64 ms. .with_window_used(( plan.time_range_ms.0.max(0) as u64, @@ -345,18 +327,15 @@ impl ExecutionOutcome { #[async_trait::async_trait] impl crate::routing::engine_router::QueryEngine for GorillaQueryEngine { - async fn execute( - &self, - query: &str, - ) -> Result { + async fn execute(&self, query: &str) -> Result { match GorillaQueryEngine::execute(self, query).await { Ok(result) => Ok(result), Err(EngineError::Plan(msg)) => Err(crate::engines::EngineError::capability_miss( - asap_types::StorageBackend::GorillaS3Archive.data_source_id(), + asap_types::StorageBackend::GorillaObjectStore.data_source_id(), msg, )), Err(other) => Err(crate::engines::EngineError::backend( - asap_types::StorageBackend::GorillaS3Archive.data_source_id(), + asap_types::StorageBackend::GorillaObjectStore.data_source_id(), other, )), } @@ -364,15 +343,12 @@ impl crate::routing::engine_router::QueryEngine for GorillaQueryEngine { fn capabilities(&self) -> crate::routing::engine_router::EngineCapabilities { crate::routing::engine_router::EngineCapabilities { - data_source_id: asap_types::StorageBackend::GorillaS3Archive.data_source_id(), - storage_backend: asap_types::StorageBackend::GorillaS3Archive, + data_source_id: asap_types::StorageBackend::GorillaObjectStore.data_source_id(), + storage_backend: asap_types::StorageBackend::GorillaObjectStore, // The buffered-aggregate budget gives a natural ceiling: each // sample is ~16 B (i64 ts + f64 value), so the byte budget is // ~16 × max_buffered_samples. - supports_streams_above_bytes: self - .config - .max_buffered_samples - .saturating_mul(16), + supports_streams_above_bytes: self.config.max_buffered_samples.saturating_mul(16), } } } diff --git a/asap-query-engine/src/engines/gorilla/postings.rs b/asap-query-engine/src/stores/gorilla_object_store/postings.rs similarity index 98% rename from asap-query-engine/src/engines/gorilla/postings.rs rename to asap-query-engine/src/stores/gorilla_object_store/postings.rs index aaa9110a..ba9de4b7 100644 --- a/asap-query-engine/src/engines/gorilla/postings.rs +++ b/asap-query-engine/src/stores/gorilla_object_store/postings.rs @@ -5,7 +5,7 @@ //! `gorillas3processor` alongside each per-hour `index.json`: //! `//YYYY/MM/DD/HH/postings-v1.json`. It maps //! `(label_name, label_value) → [series_id, ...]` so the -//! [`super::ExactExecutor`] can prune chunks by `label_hash` +//! [`super::query_engine::ExactExecutor`] can prune chunks by `label_hash` //! without paying the chunk-body GET cost. //! //! Step-1 of the JSONL deprecation refactor pulled this code out diff --git a/asap-query-engine/src/engines/gorilla/engine.rs b/asap-query-engine/src/stores/gorilla_object_store/query_engine.rs similarity index 98% rename from asap-query-engine/src/engines/gorilla/engine.rs rename to asap-query-engine/src/stores/gorilla_object_store/query_engine.rs index 1c3b33a9..7543a0f0 100644 --- a/asap-query-engine/src/engines/gorilla/engine.rs +++ b/asap-query-engine/src/stores/gorilla_object_store/query_engine.rs @@ -174,14 +174,8 @@ fn plan_from_ast(ast: &Expr, now_ms: i64) -> Result { fn plan_from_call(call: &Call, now_ms: i64) -> Result { let name = call.func.name.to_lowercase(); match name.as_str() { - "sum_over_time" - | "count_over_time" - | "avg_over_time" - | "min_over_time" - | "max_over_time" - | "last_over_time" - | "rate" - | "increase" => { + "sum_over_time" | "count_over_time" | "avg_over_time" | "min_over_time" + | "max_over_time" | "last_over_time" | "rate" | "increase" => { let ms = expect_single_matrix_arg(&call.args, &name)?; let (metric, range_ms) = matrix_metric_and_range_ms(ms); let stat = match name.as_str() { @@ -373,8 +367,6 @@ pub(crate) fn extract_label_matchers(vs: &VectorSelector) -> (Vec, (supported, has_unsupported) } - - // ===================================================================== // Executor // ===================================================================== @@ -435,13 +427,17 @@ impl ExactExecutor { QueryStatistic::MaxOverTime => { self.execute_streaming_additive(plan, AdditiveOp::Max).await } - QueryStatistic::Rate => self.execute_streaming_additive(plan, AdditiveOp::Rate).await, + QueryStatistic::Rate => { + self.execute_streaming_additive(plan, AdditiveOp::Rate) + .await + } QueryStatistic::Increase => { self.execute_streaming_additive(plan, AdditiveOp::Increase) .await } QueryStatistic::LastOverTime => { - self.execute_streaming_additive(plan, AdditiveOp::Last).await + self.execute_streaming_additive(plan, AdditiveOp::Last) + .await } QueryStatistic::QuantileOverTime { phi } => self.execute_quantile(plan, *phi).await, QueryStatistic::TopK { k } => self.execute_topk(plan, *k).await, @@ -477,8 +473,7 @@ impl ExactExecutor { "gorilla-engine: streaming additive over chunks" ); - let (filtered_chunks, postings_outcome) = - self.apply_postings_filter(plan, &chunks).await; + let (filtered_chunks, postings_outcome) = self.apply_postings_filter(plan, &chunks).await; let mut acc = AdditiveAccumulator::new(op); let mut samples_scanned: usize = 0; @@ -585,8 +580,7 @@ impl ExactExecutor { // are pre-mvp/v5 multi-series chunks that don't pin a // single series — keep them (they may carry matching // series; correctness > pruning). - let series_set: std::collections::BTreeSet = - hits.series_ids.iter().copied().collect(); + let series_set: std::collections::BTreeSet = hits.series_ids.iter().copied().collect(); let filtered: Vec = chunks .iter() .filter(|c| c.label_hash == 0 || series_set.contains(&c.label_hash)) @@ -711,8 +705,7 @@ impl ExactExecutor { .list_chunks(&plan.metric, start_ms, end_ms) .await?; let total_chunks = chunks.len(); - let (filtered_chunks, postings_outcome) = - self.apply_postings_filter(plan, &chunks).await; + let (filtered_chunks, postings_outcome) = self.apply_postings_filter(plan, &chunks).await; let chunks_fetched = filtered_chunks.len(); let mut buffer: Vec = Vec::new(); let limit = self.config.max_buffered_samples; diff --git a/asap-query-engine/src/engines/gorilla/s3_cost.rs b/asap-query-engine/src/stores/gorilla_object_store/s3_cost.rs similarity index 92% rename from asap-query-engine/src/engines/gorilla/s3_cost.rs rename to asap-query-engine/src/stores/gorilla_object_store/s3_cost.rs index 3162df56..54afc50c 100644 --- a/asap-query-engine/src/engines/gorilla/s3_cost.rs +++ b/asap-query-engine/src/stores/gorilla_object_store/s3_cost.rs @@ -93,8 +93,13 @@ impl S3CostCounters { "asap_backend_s3_bytes_put {}\n", "asap_backend_s3_bytes_got {}\n", ), - s.put_count, s.get_count, s.head_count, s.list_count, s.delete_count, - s.bytes_put, s.bytes_got, + s.put_count, + s.get_count, + s.head_count, + s.list_count, + s.delete_count, + s.bytes_put, + s.bytes_got, ) } @@ -105,8 +110,13 @@ impl S3CostCounters { format!( "put_count,get_count,head_count,list_count,delete_count,bytes_put,bytes_got\n\ {},{},{},{},{},{},{}\n", - s.put_count, s.get_count, s.head_count, s.list_count, s.delete_count, - s.bytes_put, s.bytes_got, + s.put_count, + s.get_count, + s.head_count, + s.list_count, + s.delete_count, + s.bytes_put, + s.bytes_got, ) } } @@ -143,10 +153,7 @@ pub struct S3CostTrackingObjectStore { impl S3CostTrackingObjectStore { /// Wrap `inner` and a counter-set for the wrapper to update. - pub fn new( - inner: Arc, - counters: Arc, - ) -> Self { + pub fn new(inner: Arc, counters: Arc) -> Self { Self { inner, counters } } @@ -175,7 +182,7 @@ impl ObjectStore for S3CostTrackingObjectStore { #[cfg(test)] mod tests { use super::*; - use crate::engines::gorilla::store::ObjectStore as _; + use crate::stores::gorilla_object_store::store::ObjectStore as _; use std::collections::HashMap; use tokio::sync::Mutex; @@ -236,7 +243,9 @@ mod tests { c.get_count.store(7, Ordering::Relaxed); c.bytes_got.store(1024, Ordering::Relaxed); let csv = c.render_csv(); - assert!(csv.starts_with("put_count,get_count,head_count,list_count,delete_count,bytes_put,bytes_got\n")); + assert!(csv.starts_with( + "put_count,get_count,head_count,list_count,delete_count,bytes_put,bytes_got\n" + )); assert!(csv.contains("3,7,0,0,0,0,1024")); } diff --git a/asap-query-engine/src/engines/gorilla/store.rs b/asap-query-engine/src/stores/gorilla_object_store/store.rs similarity index 96% rename from asap-query-engine/src/engines/gorilla/store.rs rename to asap-query-engine/src/stores/gorilla_object_store/store.rs index d90d5960..e97bf9dc 100644 --- a/asap-query-engine/src/engines/gorilla/store.rs +++ b/asap-query-engine/src/stores/gorilla_object_store/store.rs @@ -367,14 +367,10 @@ mod rust_s3_backend { .map_err(|e| StoreError::Backend(format!("region parse: {e}")))?, }; let creds = match (&cfg.access_key_id, &cfg.secret_access_key) { - (Some(ak), Some(sk)) => { - Credentials::new(Some(ak), Some(sk), None, None, None).map_err(|e| { - StoreError::Backend(format!("credentials: {e}")) - })? - } - _ => Credentials::default().map_err(|e| { - StoreError::Backend(format!("default credentials: {e}")) - })?, + (Some(ak), Some(sk)) => Credentials::new(Some(ak), Some(sk), None, None, None) + .map_err(|e| StoreError::Backend(format!("credentials: {e}")))?, + _ => Credentials::default() + .map_err(|e| StoreError::Backend(format!("default credentials: {e}")))?, }; let bucket = Bucket::new(&cfg.bucket, region, creds) .map_err(|e| StoreError::Backend(format!("bucket: {e}")))?; @@ -400,9 +396,7 @@ mod rust_s3_backend { .await .map_err(|e| StoreError::Backend(format!("s3 get {key}: {e}")))?; if resp.status_code() == 404 { - return Err(StoreError::Backend(format!( - "s3 get {key}: not found" - ))); + return Err(StoreError::Backend(format!("s3 get {key}: not found"))); } if !(200..300).contains(&resp.status_code()) { return Err(StoreError::Backend(format!( @@ -463,8 +457,7 @@ impl GorillaS3Store { // mvp/v5: postings + index caches scale with the chunk // cache (one entry per hour-bucket, mirrors typical query // cardinality). - let pc_cap = NonZeroUsize::new(cap.get().max(64)) - .unwrap_or(NonZeroUsize::new(64).unwrap()); + let pc_cap = NonZeroUsize::new(cap.get().max(64)).unwrap_or(NonZeroUsize::new(64).unwrap()); Self { object_store, config, @@ -570,9 +563,8 @@ impl GorillaS3Store { async fn fetch_index(&self, metric: &str, hour_ms: i64) -> Result { let key = self.index_key(metric, hour_ms); match self.object_store.get_object(&key).await { - Ok(bytes) => IndexFile::read(bytes.as_slice()).map_err(|e| { - StoreError::Malformed(format!("index.json at {key}: {e}")) - }), + Ok(bytes) => IndexFile::read(bytes.as_slice()) + .map_err(|e| StoreError::Malformed(format!("index.json at {key}: {e}"))), Err(e) if self.object_store.object_missing(&e) => { debug!(key = %key, "gorilla-s3: index.json missing for hour bucket; skipping"); Ok(IndexFile::new(0)) @@ -867,7 +859,10 @@ mod tests { }, IndexEntry { key: key_b.clone(), - time_range: ((h0 + 5_000) as u64 * 1_000_000, (h0 + 6_000) as u64 * 1_000_000), + time_range: ( + (h0 + 5_000) as u64 * 1_000_000, + (h0 + 6_000) as u64 * 1_000_000, + ), sample_count: 11, label_hash: 0xBBBB, size_bytes: 110, @@ -877,7 +872,10 @@ mod tests { }, IndexEntry { key: key_c.clone(), - time_range: ((h0 + 10_000) as u64 * 1_000_000, (h0 + 11_000) as u64 * 1_000_000), + time_range: ( + (h0 + 10_000) as u64 * 1_000_000, + (h0 + 11_000) as u64 * 1_000_000, + ), sample_count: 12, label_hash: 0xCCCC, size_bytes: 120, @@ -954,8 +952,14 @@ mod tests { assert_eq!(samples[1].value, 0.7); assert_eq!(samples[2].ts_ms, h0 + 3_000); assert_eq!(samples[2].value, 0.7); - assert_eq!(samples[0].labels.get("instance").map(String::as_str), Some("i-1")); - assert_eq!(samples[0].labels.get("mode").map(String::as_str), Some("user")); + assert_eq!( + samples[0].labels.get("instance").map(String::as_str), + Some("i-1") + ); + assert_eq!( + samples[0].labels.get("mode").map(String::as_str), + Some("user") + ); } #[tokio::test] @@ -1013,7 +1017,10 @@ mod tests { let block = make_block( "m", &[("i", &i.to_string())], - &[(h0 + i * 1_000, i as f64), (h0 + i * 1_000 + 100, i as f64 + 0.5)], + &[ + (h0 + i * 1_000, i as f64), + (h0 + i * 1_000 + 100, i as f64 + 0.5), + ], ); let key = format!("tenant1/m/2026/05/06/12/part-{i}.gor"); store.put(key.clone(), block.clone()).await; @@ -1100,7 +1107,10 @@ mod tests { let store = InMemoryObjectStore::new(); let cs = GorillaS3Store::new(Arc::new(store), cfg()); let h0 = ms(2026, 5, 6, 12, 0, 0); - let chunks = cs.list_chunks("never_written", h0, h0 + 60_000).await.unwrap(); + let chunks = cs + .list_chunks("never_written", h0, h0 + 60_000) + .await + .unwrap(); assert!(chunks.is_empty()); let samples = cs.scan("never_written", h0, h0 + 60_000).await.unwrap(); assert!(samples.is_empty()); @@ -1110,11 +1120,7 @@ mod tests { async fn scan_filters_to_requested_range() { let store = Arc::new(InMemoryObjectStore::new()); let h0 = ms(2026, 5, 6, 12, 0, 0); - let block = make_block( - "m", - &[], - &[(h0 + 1_000, 1.0), (h0 + 10_000, 2.0)], - ); + let block = make_block("m", &[], &[(h0 + 1_000, 1.0), (h0 + 10_000, 2.0)]); let key = "tenant1/m/2026/05/06/12/part-Z.gor".to_string(); store.put(key.clone(), block.clone()).await; store @@ -1214,10 +1220,7 @@ mod tests { let store = InMemoryObjectStore::new(); let cs = GorillaS3Store::new(Arc::new(store), config); let key = cs.bucket_prefix("http_freshness_probe_archive", ms(2026, 5, 7, 4, 0, 0)); - assert_eq!( - key, - "tenant1/http_freshness_probe_archive/2026/05/07/04/", - ); + assert_eq!(key, "tenant1/http_freshness_probe_archive/2026/05/07/04/",); } #[test] diff --git a/asap-query-engine/src/engines/gorilla/tests.rs b/asap-query-engine/src/stores/gorilla_object_store/tests.rs similarity index 94% rename from asap-query-engine/src/engines/gorilla/tests.rs rename to asap-query-engine/src/stores/gorilla_object_store/tests.rs index 63207cb0..6fbaa431 100644 --- a/asap-query-engine/src/engines/gorilla/tests.rs +++ b/asap-query-engine/src/stores/gorilla_object_store/tests.rs @@ -17,7 +17,7 @@ use tokio::time::sleep; use crate::engines::query_result::QueryResult; use crate::stores::sketch_db::accuracy::{AccuracyKind, AccuracyProfile}; -use super::engine::{plan_query_at, QueryStatistic}; +use super::query_engine::{plan_query_at, QueryStatistic}; use super::store::{ChunkRef, RawSample, Store, StoreError}; use super::{ wrap_result, EngineError, ExactExecutor, ExecutionOutcome, GorillaEngineConfig, @@ -60,10 +60,7 @@ impl MockStore { /// mvp/v5: install a postings table for the /// `list_postings_for` path. - fn with_postings( - mut self, - postings: BTreeMap<(String, String), Vec>, - ) -> Self { + fn with_postings(mut self, postings: BTreeMap<(String, String), Vec>) -> Self { self.postings = Some(postings); self } @@ -99,9 +96,7 @@ impl Store for MockStore { .chunks .iter() .filter(|(c, _)| { - c.metric == metric - && c.time_range_ms.0 < end_ms - && c.time_range_ms.1 >= start_ms + c.metric == metric && c.time_range_ms.0 < end_ms && c.time_range_ms.1 >= start_ms }) .map(|(c, _)| c.clone()) .collect()) @@ -143,18 +138,14 @@ impl Store for MockStore { }; if matchers.is_empty() { // Union of every series id in the table. - let mut set: std::collections::BTreeSet = - std::collections::BTreeSet::new(); + let mut set: std::collections::BTreeSet = std::collections::BTreeSet::new(); for ids in table.values() { set.extend(ids.iter().copied()); } hits.series_ids = set.into_iter().collect(); return Ok(hits); } - let first = table - .get(&matchers[0]) - .cloned() - .unwrap_or_default(); + let first = table.get(&matchers[0]).cloned().unwrap_or_default(); let mut acc: std::collections::BTreeSet = first.into_iter().collect(); for m in &matchers[1..] { let next = table.get(m).cloned().unwrap_or_default(); @@ -231,14 +222,7 @@ fn engine_with_config( #[tokio::test] async fn execute_sum_over_time_streaming() { // 60 samples × value 2.0 = 120.0 - let chunks = vec![linear_chunk( - "c1", - NOW_MS - 60_000, - 1_000, - 60, - 2.0, - 0.0, - )]; + let chunks = vec![linear_chunk("c1", NOW_MS - 60_000, 1_000, 60, 2.0, 0.0)]; let engine = engine_with(chunks); let plan = plan_query_at(&format!("sum_over_time({METRIC}[5m])"), NOW_MS).unwrap(); assert_eq!(plan.statistic, QueryStatistic::SumOverTime); @@ -415,8 +399,8 @@ async fn execute_last_over_time_unordered_samples_picks_largest_ts() { // claims (start, last_ts+1) but the per-sample observe() must // still pick the largest ts, not the last-arrived sample. let samples = vec![ - raw(NOW_MS - 5_000, 50.0), // largest ts is sample[2] - raw(NOW_MS - 8_000, 80.0), // smallest ts but later in vec + raw(NOW_MS - 5_000, 50.0), // largest ts is sample[2] + raw(NOW_MS - 8_000, 80.0), // smallest ts but later in vec raw(NOW_MS - 1_000, 1234.5), // largest ts raw(NOW_MS - 3_000, 30.0), ]; @@ -675,7 +659,7 @@ async fn result_carries_exact_accuracy_envelope() { } #[tokio::test] -async fn result_includes_data_source_gorilla_archive() { +async fn result_includes_data_source_thanos_query() { // The wrapping fn surfaces the data_source line on // ExecutionOutcome::info_lines — pin both the marker constant // and the assembled info strings. @@ -747,7 +731,6 @@ async fn engine_respects_config_timeout() { } } - // ───────────────────────────────────────────────────────────────────── // mvp/v5 — postings-aware path tests // ───────────────────────────────────────────────────────────────────── @@ -788,18 +771,31 @@ async fn postings_aware_path_prunes_chunks() { // Two chunks: one for zone=a (label_hash=11), one for zone=b // (label_hash=22). Postings says zone=a → [11]. The engine // must read only the zone=a chunk. - let (chunk_a, samples_a) = - labeled_chunk("k-a", 11, "a", NOW_MS - 30_000, &[(NOW_MS - 1_000, 5.0), (NOW_MS - 500, 5.0)]); - let (chunk_b, samples_b) = - labeled_chunk("k-b", 22, "b", NOW_MS - 30_000, &[(NOW_MS - 1_000, 99.0), (NOW_MS - 500, 99.0)]); + let (chunk_a, samples_a) = labeled_chunk( + "k-a", + 11, + "a", + NOW_MS - 30_000, + &[(NOW_MS - 1_000, 5.0), (NOW_MS - 500, 5.0)], + ); + let (chunk_b, samples_b) = labeled_chunk( + "k-b", + 22, + "b", + NOW_MS - 30_000, + &[(NOW_MS - 1_000, 99.0), (NOW_MS - 500, 99.0)], + ); let mut postings: BTreeMap<(String, String), Vec> = BTreeMap::new(); postings.insert(("zone".to_string(), "a".to_string()), vec![11]); postings.insert(("zone".to_string(), "b".to_string()), vec![22]); - let mock = MockStore::new(vec![(chunk_a, samples_a), (chunk_b, samples_b)]) - .with_postings(postings); + let mock = + MockStore::new(vec![(chunk_a, samples_a), (chunk_b, samples_b)]).with_postings(postings); let engine = GorillaQueryEngine::new(Arc::new(mock), cfg()); - let plan = - plan_query_at(&format!(r#"sum_over_time({METRIC}{{zone="a"}}[5m])"#), NOW_MS).unwrap(); + let plan = plan_query_at( + &format!(r#"sum_over_time({METRIC}{{zone="a"}}[5m])"#), + NOW_MS, + ) + .unwrap(); assert_eq!(plan.label_matchers.len(), 1); let exec = ExactExecutor::new(engine.store_for_tests(), cfg()); let outcome = exec.execute_plan(&plan).await.unwrap(); @@ -822,8 +818,11 @@ async fn postings_missing_falls_back_to_scan_all() { labeled_chunk("k-b", 22, "b", NOW_MS - 30_000, &[(NOW_MS - 1_000, 99.0)]); let mock = MockStore::new(vec![(chunk_a, samples_a), (chunk_b, samples_b)]); let engine = GorillaQueryEngine::new(Arc::new(mock), cfg()); - let plan = - plan_query_at(&format!(r#"sum_over_time({METRIC}{{zone="a"}}[5m])"#), NOW_MS).unwrap(); + let plan = plan_query_at( + &format!(r#"sum_over_time({METRIC}{{zone="a"}}[5m])"#), + NOW_MS, + ) + .unwrap(); let exec = ExactExecutor::new(engine.store_for_tests(), cfg()); let outcome = exec.execute_plan(&plan).await.unwrap(); // Correctness: only zone=a sample (5.0) folded in. The @@ -832,9 +831,14 @@ async fn postings_missing_falls_back_to_scan_all() { // Both chunks were fetched — postings filter no-oped. assert_eq!(outcome.chunks_fetched, 2); assert_eq!(outcome.chunks_skipped_via_postings, 0); - assert!(outcome.postings_missing, "missing-postings flag must be set"); + assert!( + outcome.postings_missing, + "missing-postings flag must be set" + ); let infos = outcome.info_lines(); - assert!(infos.iter().any(|i| i == "data_source_quirk: postings_missing")); + assert!(infos + .iter() + .any(|i| i == "data_source_quirk: postings_missing")); } #[tokio::test] @@ -856,4 +860,3 @@ async fn postings_path_no_label_predicate_skips_postings_lookup() { assert!(!outcome.postings_missing); assert_eq!(outcome.chunks_skipped_via_postings, 0); } - diff --git a/asap-query-engine/src/stores/mod.rs b/asap-query-engine/src/stores/mod.rs index 4ac8491a..80b6b7ac 100644 --- a/asap-query-engine/src/stores/mod.rs +++ b/asap-query-engine/src/stores/mod.rs @@ -17,6 +17,7 @@ //! at the `stores` top level. pub mod promsketch_store; +pub mod gorilla_object_store; pub mod sketch_db; pub mod traits; @@ -27,3 +28,8 @@ pub use sketch_db::sketch_index::{ }; pub use sketch_db::{AggSchema, AggStatus, SchemaRegistry, SimpleMapStore}; pub use traits::*; +pub use gorilla_object_store::{ + global_s3_cost_counters, ChunkRef, GorillaEngineConfig, GorillaQueryEngine, GorillaS3Config, + GorillaS3ConfigError, GorillaS3Store, ObjectStore, RawSample, S3CostCounters, S3CostSnapshot, + S3CostTrackingObjectStore, +}; diff --git a/asap-query-engine/src/stores/sketch_db/epoch_columnar.rs b/asap-query-engine/src/stores/sketch_db/epoch_columnar.rs index af0ebdf4..c4905ef8 100644 --- a/asap-query-engine/src/stores/sketch_db/epoch_columnar.rs +++ b/asap-query-engine/src/stores/sketch_db/epoch_columnar.rs @@ -303,7 +303,11 @@ impl

SealedEpoch

{ // were replaced with zeroed memory; their drop should not run). std::mem::forget(m.payloads_col); entries.sort_by(|a, b| a.0.cmp(&b.0).then(a.1.cmp(&b.1))); - Self { entries, min_start, max_end } + Self { + entries, + min_start, + max_end, + } } pub fn min_start(&self) -> Option { @@ -417,12 +421,7 @@ impl SidStoreData { // Drop oldest sealed if we exceed `max_epochs`. while self.sealed_epochs.len() + 1 > self.max_epochs { // BTreeMap::pop_first is stable in 1.66+ - if let Some((id, _)) = self - .sealed_epochs - .iter() - .next() - .map(|(k, _)| (*k, ())) - { + if let Some((id, _)) = self.sealed_epochs.iter().next().map(|(k, _)| (*k, ())) { self.sealed_epochs.remove(&id); } else { break; diff --git a/asap-query-engine/src/stores/sketch_db/sketch_index.rs b/asap-query-engine/src/stores/sketch_db/sketch_index.rs index 2cd6259f..6bc8518b 100644 --- a/asap-query-engine/src/stores/sketch_db/sketch_index.rs +++ b/asap-query-engine/src/stores/sketch_db/sketch_index.rs @@ -89,17 +89,27 @@ impl AccuracyBound { // 1/(2^(rows/2)). SketchConfig::CountSketch { rows, cols } => { let e = std::f64::consts::E; - let eps = if *cols > 0 { (e / (*cols as f64)).sqrt() } else { 1.0 }; + let eps = if *cols > 0 { + (e / (*cols as f64)).sqrt() + } else { + 1.0 + }; let half_rows = (*rows as f64) / 2.0; let delta = 2f64.powf(-half_rows); - Self { epsilon: eps, confidence: 1.0 - delta } + Self { + epsilon: eps, + confidence: 1.0 - delta, + } } // CMS: ε ≈ e/cols, δ ≈ exp(-rows). SketchConfig::CountMin { rows, cols } => { let e = std::f64::consts::E; let eps = if *cols > 0 { e / (*cols as f64) } else { 1.0 }; let delta = (-(*rows as f64)).exp(); - Self { epsilon: eps, confidence: 1.0 - delta } + Self { + epsilon: eps, + confidence: 1.0 - delta, + } } } } @@ -294,11 +304,7 @@ impl SketchIndex { by_label_id .into_iter() .map(|(label_id, samples)| { - let label_values = guard - .intern - .resolve(label_id) - .cloned() - .unwrap_or_default(); + let label_values = guard.intern.resolve(label_id).cloned().unwrap_or_default(); SketchTimeSeries { sid, series_label_values: label_values, @@ -349,7 +355,9 @@ mod tests { use super::*; fn meta(sid: u64) -> SketchInstanceMetadata { - let cfg = SketchConfig::DDSketch { relative_accuracy: 0.01 }; + let cfg = SketchConfig::DDSketch { + relative_accuracy: 0.01, + }; SketchInstanceMetadata { sid, metric_name: "m".into(), @@ -363,7 +371,10 @@ mod tests { } fn sample(b: u8) -> SketchSampleState { - SketchSampleState { bytes: vec![b], encoding: SketchEncoding::ProtoFull } + SketchSampleState { + bytes: vec![b], + encoding: SketchEncoding::ProtoFull, + } } #[test] diff --git a/asap-query-engine/src/tests/capability_matching_tests.rs b/asap-query-engine/src/tests/capability_matching_tests.rs index 52532846..fe641c01 100644 --- a/asap-query-engine/src/tests/capability_matching_tests.rs +++ b/asap-query-engine/src/tests/capability_matching_tests.rs @@ -9,7 +9,7 @@ use crate::data_model::{ PrecomputedOutput, PromQLSchema, QueryConfig, QueryLanguage, SchemaConfig, StreamingConfig, WindowType, }; -use crate::engines::simple::engine::SimpleEngine; +use crate::engines::asap_query::engine::SimpleEngine; use crate::precompute_operators::count_min_sketch_accumulator::CountMinSketchAccumulator; use crate::precompute_operators::datasketches_kll_accumulator::DatasketchesKLLAccumulator; use crate::precompute_operators::delta_set_aggregator_accumulator::DeltaSetAggregatorAccumulator; diff --git a/asap-query-engine/src/tests/capability_miss_http_e2e_tests.rs b/asap-query-engine/src/tests/capability_miss_http_e2e_tests.rs index 2122e2fc..a032e5af 100644 --- a/asap-query-engine/src/tests/capability_miss_http_e2e_tests.rs +++ b/asap-query-engine/src/tests/capability_miss_http_e2e_tests.rs @@ -182,8 +182,7 @@ async fn start_backend(controller_url: String, hot_reload: HotReloadStreamingCon handle_http_requests: true, adapter_config, }; - let server = - HttpServer::new(config, engine, store).with_hot_reload_config(hot_reload.clone()); + let server = HttpServer::new(config, engine, store).with_hot_reload_config(hot_reload.clone()); server .start_test_server() .await diff --git a/asap-query-engine/src/tests/test_utilities/comparison.rs b/asap-query-engine/src/tests/test_utilities/comparison.rs index 5382fc6b..6d781eeb 100644 --- a/asap-query-engine/src/tests/test_utilities/comparison.rs +++ b/asap-query-engine/src/tests/test_utilities/comparison.rs @@ -3,7 +3,7 @@ //! Provides assertion helpers for deep equality checking of query execution contexts. use crate::data_model::{AggregationIdInfo, AggregationType}; -use crate::engines::simple::engine::{ +use crate::engines::asap_query::engine::{ QueryExecutionContext, QueryMetadata, StoreQueryParams, StoreQueryPlan, }; use promql_utilities::data_model::KeyByLabelNames; diff --git a/asap-query-engine/src/tests/test_utilities/engine_factories.rs b/asap-query-engine/src/tests/test_utilities/engine_factories.rs index 1b360955..d10ef011 100644 --- a/asap-query-engine/src/tests/test_utilities/engine_factories.rs +++ b/asap-query-engine/src/tests/test_utilities/engine_factories.rs @@ -11,7 +11,7 @@ use crate::data_model::{ StreamingConfig, WindowType, }; use crate::engines::query_result::InstantVectorElement; -use crate::engines::simple::engine::SimpleEngine; +use crate::engines::asap_query::engine::SimpleEngine; use crate::stores::sketch_db::simple_map_store::SimpleMapStore; use crate::stores::Store; use crate::AggregateCore; @@ -636,4 +636,3 @@ pub fn create_engine_multi_timestamp_with_window( QueryLanguage::promql, ) } - diff --git a/asap-query-engine/tests/edge_runtime_consumes_precompute_rs.rs b/asap-query-engine/tests/edge_runtime_consumes_precompute_rs.rs index 2ab5dae5..32d0261b 100644 --- a/asap-query-engine/tests/edge_runtime_consumes_precompute_rs.rs +++ b/asap-query-engine/tests/edge_runtime_consumes_precompute_rs.rs @@ -76,7 +76,9 @@ fn ddsketch_envelope_structural_assertions() { w.update(2.0); w.update(3.0); let bytes = w.snapshot().expect("snapshot"); - let state = unwrap_envelope_state(&bytes).expect("unwrap").expect("state"); + let state = unwrap_envelope_state(&bytes) + .expect("unwrap") + .expect("state"); match state { SketchState::Ddsketch(s) => { assert_eq!(s.count, 3, "structural count"); @@ -106,8 +108,7 @@ fn ddsketch_envelope_ends_up_in_backend_accumulator() { } let bytes = w.snapshot().expect("snapshot"); - let reconstructed = - reconstruct_via_runtime(SketchType::DDSketch, &bytes).expect("reconstruct"); + let reconstructed = reconstruct_via_runtime(SketchType::DDSketch, &bytes).expect("reconstruct"); let dd = match reconstructed { ReconstructedSketch::DdSketch(d) => d, _ => panic!(), @@ -205,7 +206,9 @@ fn kll_envelope_structural_assertions() { w.update(i as f64); } let bytes = w.snapshot().expect("snapshot"); - let state = unwrap_envelope_state(&bytes).expect("unwrap").expect("state"); + let state = unwrap_envelope_state(&bytes) + .expect("unwrap") + .expect("state"); match state { SketchState::Kll(s) => { assert_eq!(s.k, 200, "structural k"); diff --git a/asap-query-engine/tests/inference_yaml_pattern_coverage.rs b/asap-query-engine/tests/inference_yaml_pattern_coverage.rs index 0e7c37f2..b8628474 100644 --- a/asap-query-engine/tests/inference_yaml_pattern_coverage.rs +++ b/asap-query-engine/tests/inference_yaml_pattern_coverage.rs @@ -48,11 +48,7 @@ fn promql_inference_yaml_loads_all_pattern_families() { let cfg = read_inference_config(PROMQL_YAML, QueryLanguage::promql) .expect("inference_config.yaml must parse"); - let queries: Vec<&str> = cfg - .query_configs - .iter() - .map(|q| q.query.as_str()) - .collect(); + let queries: Vec<&str> = cfg.query_configs.iter().map(|q| q.query.as_str()).collect(); // Sanity: expansion landed (pre-PR baseline was 1 entry). assert!( @@ -313,10 +309,7 @@ fn rate_routes_to_increase_accumulator_warm_tier() { ); let result = engine - .handle_query_promql( - "rate(fake_metric[1m])".to_string(), - QUERY_TIME_SEC, - ) + .handle_query_promql("rate(fake_metric[1m])".to_string(), QUERY_TIME_SEC) .expect("warm tier should answer rate(...[1m])"); let (_, qr) = result; let elements = match qr { @@ -345,10 +338,7 @@ fn increase_routes_to_increase_accumulator_warm_tier() { ); let result = engine - .handle_query_promql( - "increase(fake_metric[1m])".to_string(), - QUERY_TIME_SEC, - ) + .handle_query_promql("increase(fake_metric[1m])".to_string(), QUERY_TIME_SEC) .expect("warm tier should answer increase(...[1m])"); let (_, qr) = result; let elements = match qr { @@ -374,17 +364,17 @@ fn sum_over_time_wider_range_routes_through_warm_tier() { ); let result = engine - .handle_query_promql( - "sum_over_time(fake_metric[2m])".to_string(), - QUERY_TIME_SEC, - ) + .handle_query_promql("sum_over_time(fake_metric[2m])".to_string(), QUERY_TIME_SEC) .expect("warm tier should answer sum_over_time(...[2m])"); let (_, qr) = result; let elements = match qr { query_engine_rust::engines::QueryResult::Vector(iv) => iv.values, other => panic!("expected vector, got {other:?}"), }; - assert!(!elements.is_empty(), "sum_over_time result should not be empty"); + assert!( + !elements.is_empty(), + "sum_over_time result should not be empty" + ); } #[test] @@ -411,7 +401,10 @@ fn count_over_time_routes_through_warm_tier() { query_engine_rust::engines::QueryResult::Vector(iv) => iv.values, other => panic!("expected vector, got {other:?}"), }; - assert!(!elements.is_empty(), "count_over_time result should not be empty"); + assert!( + !elements.is_empty(), + "count_over_time result should not be empty" + ); } #[test] @@ -524,5 +517,8 @@ fn spatial_multi_quantile_routes_through_warm_tier() { query_engine_rust::engines::QueryResult::Vector(iv) => iv.values, other => panic!("expected vector, got {other:?}"), }; - assert!(!elements.is_empty(), "spatial p50 result should not be empty"); + assert!( + !elements.is_empty(), + "spatial p50 result should not be empty" + ); } diff --git a/controller/src/accuracy.rs b/controller/src/accuracy.rs index bd7aaa53..4de34785 100644 --- a/controller/src/accuracy.rs +++ b/controller/src/accuracy.rs @@ -215,7 +215,10 @@ mod tests { fn golden_parity_with_backend() { // HLL precision=14: ε = 0.008125, kind = relative_cardinality let p = AccuracyProfile::derive(&SketchParams::HLL { precision: 14 }); - assert_eq!(p.summary(), "accuracy: ε=0.008125, δ=0, kind=relative_cardinality"); + assert_eq!( + p.summary(), + "accuracy: ε=0.008125, δ=0, kind=relative_cardinality" + ); // KLL k=200: ε = 2.296 / √200, kind = rank_quantile, δ = 0.01 let p = AccuracyProfile::derive(&SketchParams::KLL { diff --git a/controller/src/backend_client.rs b/controller/src/backend_client.rs index 2815c5fb..10e1347d 100644 --- a/controller/src/backend_client.rs +++ b/controller/src/backend_client.rs @@ -312,8 +312,7 @@ mod tests { #[tokio::test] async fn json_post_non_2xx_is_error() { let sink = SharedSink(StdArc::new(Mutex::new(Vec::new()))); - let url = - start_mock_backend(sink.clone(), axum::http::StatusCode::BAD_REQUEST).await; + let url = start_mock_backend(sink.clone(), axum::http::StatusCode::BAD_REQUEST).await; let client = BackendClient::new(url); let result = client.post_streaming_config_json("{}".to_string()).await; @@ -346,10 +345,7 @@ mod tests { "http://127.0.0.1:1/api/v1/storage_routing" ); // Unrelated path passes through too — no surprise rewriting. - assert_eq!( - derive_storage_routing_url("http://x/foo"), - "http://x/foo" - ); + assert_eq!(derive_storage_routing_url("http://x/foo"), "http://x/foo"); } /// Phase α: full happy path. A mock backend hosts the storage @@ -388,9 +384,8 @@ mod tests { let url = start_mock_routing_backend(sink.clone(), axum::http::StatusCode::OK).await; let client = BackendClient::new(url); - let json = - r#"{"default_engine":"sketch_warm_tier","metrics":[{"name":"x","targets":[]}]}"# - .to_string(); + let json = r#"{"default_engine":"sketch_store","metrics":[{"name":"x","targets":[]}]}"# + .to_string(); client .post_storage_routing_json(json.clone()) .await @@ -404,11 +399,9 @@ mod tests { #[tokio::test] async fn storage_routing_post_non_2xx_is_error() { let sink = SharedSink(StdArc::new(Mutex::new(Vec::new()))); - let url = start_mock_routing_backend( - sink.clone(), - axum::http::StatusCode::INTERNAL_SERVER_ERROR, - ) - .await; + let url = + start_mock_routing_backend(sink.clone(), axum::http::StatusCode::INTERNAL_SERVER_ERROR) + .await; let client = BackendClient::new(url); let result = client.post_storage_routing_json("{}".to_string()).await; assert!(result.is_err(), "expected error on 500, got {result:?}"); diff --git a/controller/src/emit/agent.rs b/controller/src/emit/agent.rs index a2cdd988..a096fb5e 100644 --- a/controller/src/emit/agent.rs +++ b/controller/src/emit/agent.rs @@ -71,7 +71,8 @@ pub fn generate_agent_config( // OpAMP extension — allows the controller to push config updates at runtime. let opamp_ext: Value = serde_yaml::from_str(&format!( "server:\n ws:\n endpoint: \"{opamp_endpoint}\"\n" - )).unwrap(); + )) + .unwrap(); let doc = CollectorYaml { extensions: [("opamp".to_string(), opamp_ext)].into(), @@ -145,13 +146,17 @@ fn build_processor_block(cfg: &AgentCollectorConfig) -> Value { } if !cfg.label_matchers.is_empty() { // Go processors expect []LabelMatcher{Key, Value}, not flat strings. - let matchers: Vec = cfg.label_matchers.iter().filter_map(|s| { - let (k, v) = s.split_once('=')?; - let mut map = serde_yaml::Mapping::new(); - map.insert("key".into(), Value::String(k.to_string())); - map.insert("value".into(), Value::String(v.to_string())); - Some(Value::Mapping(map)) - }).collect(); + let matchers: Vec = cfg + .label_matchers + .iter() + .filter_map(|s| { + let (k, v) = s.split_once('=')?; + let mut map = serde_yaml::Mapping::new(); + map.insert("key".into(), Value::String(k.to_string())); + map.insert("value".into(), Value::String(v.to_string())); + Some(Value::Mapping(map)) + }) + .collect(); if !matchers.is_empty() { m.insert("label_matchers".into(), Value::Sequence(matchers)); } @@ -174,20 +179,38 @@ fn build_processor_block(cfg: &AgentCollectorConfig) -> Value { // Sketch-type-specific params. match &cfg.sketch_params { - SketchParams::DDSketch { relative_accuracy, quantiles } => { - m.insert("relative_accuracy".into(), Value::Number((*relative_accuracy).into())); + SketchParams::DDSketch { + relative_accuracy, + quantiles, + } => { + m.insert( + "relative_accuracy".into(), + Value::Number((*relative_accuracy).into()), + ); if !quantiles.is_empty() { - m.insert("quantiles".into(), Value::Sequence( - quantiles.iter().map(|q| Value::Number((*q).into())).collect(), - )); + m.insert( + "quantiles".into(), + Value::Sequence( + quantiles + .iter() + .map(|q| Value::Number((*q).into())) + .collect(), + ), + ); } } SketchParams::KLL { k, quantiles } => { m.insert("k".into(), Value::Number((*k as u64).into())); if !quantiles.is_empty() { - m.insert("quantiles".into(), Value::Sequence( - quantiles.iter().map(|q| Value::Number((*q).into())).collect(), - )); + m.insert( + "quantiles".into(), + Value::Sequence( + quantiles + .iter() + .map(|q| Value::Number((*q).into())) + .collect(), + ), + ); } } SketchParams::HLL { .. } => { @@ -197,7 +220,11 @@ fn build_processor_block(cfg: &AgentCollectorConfig) -> Value { m.insert("epsilon".into(), Value::Number((*epsilon).into())); m.insert("delta".into(), Value::Number((*delta).into())); } - SketchParams::CountMinSketch { rows, cols, metric_name } => { + SketchParams::CountMinSketch { + rows, + cols, + metric_name, + } => { m.insert("metric_name".into(), Value::String(metric_name.clone())); m.insert("rows".into(), Value::Number((*rows as u64).into())); m.insert("columns".into(), Value::Number((*cols as u64).into())); @@ -235,8 +262,8 @@ mod tests { drop_original: true, delta_transmission: false, delta_threshold: 0.0, - enable_series_id: true, - series_id_ttl_secs: 0, + enable_series_id: true, + series_id_ttl_secs: 0, // Pre-existing fixture tests (`contains_prometheus_exporter`, // `pipeline_has_receivers_and_exporters`) assert the legacy // prometheus exporter on :8889 — keep the test semantics by @@ -318,12 +345,15 @@ mod tests { drop_original: true, delta_transmission: false, delta_threshold: 0.0, - enable_series_id: true, - series_id_ttl_secs: 0, + enable_series_id: true, + series_id_ttl_secs: 0, data_sink: AgentDataSink::default(), }; let yaml = generate_agent_config(&cfg, "ws://ctrl:4320/v1/opamp").unwrap(); - assert!(yaml.contains("HLL:"), "YAML should contain HLL processor key\n{yaml}"); + assert!( + yaml.contains("HLL:"), + "YAML should contain HLL processor key\n{yaml}" + ); assert!( yaml.contains("- HLL"), "pipeline should reference HLL processor\n{yaml}" @@ -353,8 +383,8 @@ mod tests { drop_original: true, delta_transmission: false, delta_threshold: 0.0, - enable_series_id: true, - series_id_ttl_secs: 0, + enable_series_id: true, + series_id_ttl_secs: 0, data_sink: AgentDataSink::default(), }; let yaml = generate_agent_config(&cfg, "ws://ctrl:4320/v1/opamp").unwrap(); @@ -458,14 +488,20 @@ mod tests { drop_original: true, delta_transmission: false, delta_threshold: 0.0, - enable_series_id: true, - series_id_ttl_secs: 0, + enable_series_id: true, + series_id_ttl_secs: 0, data_sink: AgentDataSink::default(), }; let yaml = generate_agent_config(&cfg, "ws://ctrl:4320/v1/opamp").unwrap(); assert!(yaml.contains("KLL:"), "YAML should contain 'KLL:'\n{yaml}"); - assert!(yaml.contains("k:"), "YAML should contain 'k:' param\n{yaml}"); - assert!(!yaml.contains("ddsketch:"), "YAML must not contain wrong processor key\n{yaml}"); + assert!( + yaml.contains("k:"), + "YAML should contain 'k:' param\n{yaml}" + ); + assert!( + !yaml.contains("ddsketch:"), + "YAML must not contain wrong processor key\n{yaml}" + ); } #[test] @@ -486,8 +522,8 @@ mod tests { drop_original: true, delta_transmission: false, delta_threshold: 0.0, - enable_series_id: true, - series_id_ttl_secs: 0, + enable_series_id: true, + series_id_ttl_secs: 0, data_sink: AgentDataSink::default(), }; let yaml = generate_agent_config(&cfg, "ws://ctrl:4320/v1/opamp").unwrap(); @@ -508,11 +544,40 @@ mod tests { #[test] fn all_sketch_types_processor_key_matches_pipeline_ref() { let cases: &[(&str, SketchType, SketchParams)] = &[ - ("ddsketch", SketchType::DDSketch, SketchParams::DDSketch { relative_accuracy: 0.01, quantiles: vec![0.5] }), - ("KLL", SketchType::KLL, SketchParams::KLL { k: 200, quantiles: vec![0.5] }), - ("HLL", SketchType::HLL, SketchParams::HLL { precision: 14 }), - ("countsketch", SketchType::CountSketch, SketchParams::CountSketch { epsilon: CountSketchDefaults::default().epsilon, delta: CountSketchDefaults::default().delta }), - ("countmin", SketchType::CountMinSketch, SketchParams::CountMinSketch { rows: 5, cols: 2048, metric_name: "m".into() }), + ( + "ddsketch", + SketchType::DDSketch, + SketchParams::DDSketch { + relative_accuracy: 0.01, + quantiles: vec![0.5], + }, + ), + ( + "KLL", + SketchType::KLL, + SketchParams::KLL { + k: 200, + quantiles: vec![0.5], + }, + ), + ("HLL", SketchType::HLL, SketchParams::HLL { precision: 14 }), + ( + "countsketch", + SketchType::CountSketch, + SketchParams::CountSketch { + epsilon: CountSketchDefaults::default().epsilon, + delta: CountSketchDefaults::default().delta, + }, + ), + ( + "countmin", + SketchType::CountMinSketch, + SketchParams::CountMinSketch { + rows: 5, + cols: 2048, + metric_name: "m".into(), + }, + ), ]; for (expected_key, sketch_type, sketch_params) in cases { @@ -547,7 +612,9 @@ mod tests { ); // No other sketch type key should appear as a processor. for (other_key, _, _) in cases { - if other_key == expected_key { continue; } + if other_key == expected_key { + continue; + } assert!( !yaml.contains(&format!("{other_key}:")), "sketch_type={expected_key}: YAML must not contain foreign key '{other_key}:'\n{yaml}" diff --git a/controller/src/emit/backend.rs b/controller/src/emit/backend.rs index fa467675..b19e4454 100644 --- a/controller/src/emit/backend.rs +++ b/controller/src/emit/backend.rs @@ -1,13 +1,16 @@ +use crate::types::*; use anyhow::Context; use serde_json::json; -use crate::types::*; /// Generates an OTel collector YAML string for the backend merge collector. /// /// **SP-9**: when `staged.has_dedup` is true a `dedup` processor is inserted /// before the merge processor in the pipeline, honouring the `Dedup` node /// assignment from [`crate::planner::stage_split::split_expr_by_stage`]. -pub fn generate_backend_config(cfg: &BackendCollectorConfig, opamp_endpoint: &str) -> anyhow::Result { +pub fn generate_backend_config( + cfg: &BackendCollectorConfig, + opamp_endpoint: &str, +) -> anyhow::Result { generate_backend_config_staged(cfg, None, opamp_endpoint) } @@ -26,10 +29,7 @@ pub fn generate_backend_config_staged( let mut pipeline_processors: Vec = vec![]; if has_dedup { - processors.insert( - "dedup".into(), - json!({ "mode": "dedup" }), - ); + processors.insert("dedup".into(), json!({ "mode": "dedup" })); pipeline_processors.push(json!("dedup")); } @@ -69,8 +69,14 @@ mod tests { group_by: vec!["host.name".into()], }; let yaml = generate_backend_config(&cfg, "ws://ctrl:4320/v1/opamp").unwrap(); - assert!(yaml.contains("ddsketch_merge"), "YAML should contain merge key\n{yaml}"); - assert!(yaml.contains("host.name"), "YAML should contain group_by\n{yaml}"); + assert!( + yaml.contains("ddsketch_merge"), + "YAML should contain merge key\n{yaml}" + ); + assert!( + yaml.contains("host.name"), + "YAML should contain group_by\n{yaml}" + ); } #[test] @@ -86,7 +92,10 @@ mod tests { #[test] fn contains_opamp_endpoint() { let ep = "ws://custom-ctrl:9000/v1/opamp"; - let cfg = BackendCollectorConfig { merge_sketch_type: SketchType::KLL, group_by: vec![] }; + let cfg = BackendCollectorConfig { + merge_sketch_type: SketchType::KLL, + group_by: vec![], + }; let yaml = generate_backend_config(&cfg, ep).unwrap(); assert!(yaml.contains(ep), "YAML should contain endpoint\n{yaml}"); } @@ -97,9 +106,17 @@ mod tests { merge_sketch_type: SketchType::HLL, group_by: vec!["user_id".into()], }; - let staged = BackendSubPlan { has_dedup: true, has_merge: true, group_by: vec![] }; - let yaml = generate_backend_config_staged(&cfg, Some(&staged), "ws://ctrl:4320/v1/opamp").unwrap(); - assert!(yaml.contains("dedup:"), "YAML should contain dedup processor\n{yaml}"); + let staged = BackendSubPlan { + has_dedup: true, + has_merge: true, + group_by: vec![], + }; + let yaml = + generate_backend_config_staged(&cfg, Some(&staged), "ws://ctrl:4320/v1/opamp").unwrap(); + assert!( + yaml.contains("dedup:"), + "YAML should contain dedup processor\n{yaml}" + ); // dedup must appear before merge in the pipeline list let dedup_pos = yaml.find("- dedup").expect("missing dedup in pipeline"); let merge_pos = yaml.find("- HLL_merge").expect("missing merge in pipeline"); @@ -112,9 +129,16 @@ mod tests { merge_sketch_type: SketchType::DDSketch, group_by: vec![], }; - let staged = BackendSubPlan { has_dedup: false, has_merge: true, group_by: vec![] }; - let yaml = generate_backend_config_staged(&cfg, Some(&staged), "ws://ctrl:4320/v1/opamp").unwrap(); - assert!(!yaml.contains("dedup"), "YAML must not contain dedup\n{yaml}"); + let staged = BackendSubPlan { + has_dedup: false, + has_merge: true, + group_by: vec![], + }; + let yaml = + generate_backend_config_staged(&cfg, Some(&staged), "ws://ctrl:4320/v1/opamp").unwrap(); + assert!( + !yaml.contains("dedup"), + "YAML must not contain dedup\n{yaml}" + ); } } - diff --git a/controller/src/emit/mod.rs b/controller/src/emit/mod.rs index ad6df566..f5cb25b8 100644 --- a/controller/src/emit/mod.rs +++ b/controller/src/emit/mod.rs @@ -18,23 +18,23 @@ pub mod agent; pub mod asapquery_backend; pub mod backend; +pub mod otap; pub mod precompute; pub mod stage_config; -pub mod otap; pub mod telegraf; pub mod trait_def; pub use agent::generate_agent_config; pub use asapquery_backend::generate_streaming_config_yaml; pub use backend::{generate_backend_config, generate_backend_config_staged}; -pub use precompute::{should_precompute, build_precompute_jobs, PrecomputeClient}; +pub use otap::emit_otap_dag_yaml; +pub use precompute::{build_precompute_jobs, should_precompute, PrecomputeClient}; pub use stage_config::{ emit_backend_config_json, emit_backend_storage_routing, emit_backend_storage_routing_for_tenant, emit_backend_storage_routing_with_prometheus, emit_backend_storage_routing_with_prometheus_for_tenant, emit_edge_yaml, emit_gateway_yaml, DEFAULT_TENANT, }; -pub use otap::emit_otap_dag_yaml; pub use telegraf::emit_telegraf_toml; pub use trait_def::{ InferenceConfigEmitter, InferenceConfigInput, OpampEmitter, OpampGatewayEmitter, @@ -49,8 +49,8 @@ pub use trait_def::{ pub use crate::workload::WorkloadRegistry; use crate::physical::colored_dag::emitter::EdgeStageConfig; -use crate::sketch_algebra::SketchExpr; use crate::sketch_algebra::params::SketchKind; +use crate::sketch_algebra::SketchExpr; use crate::store::WorkloadStore; use anyhow::Result; @@ -145,10 +145,8 @@ pub const WORKLOAD_ARCHIVE_WINDOW_SECS: u64 = 60; /// warm-passthrough routing the DDSketch processor renames them to /// `_quantile`, and without `gorillas3` archive write the warm engine /// has nothing to look at. -pub const FRESHNESS_PROBE_METRICS: &[&str] = &[ - "http_freshness_probe_warm", - "http_freshness_probe_archive", -]; +pub const FRESHNESS_PROBE_METRICS: &[&str] = + &["http_freshness_probe_warm", "http_freshness_probe_archive"]; /// Bootstrap/replan-scope plumbing: extend an Edge stage config with /// the freshness-probe metrics (`http_freshness_probe_warm` / @@ -307,15 +305,27 @@ mod runtime_tests { #[test] fn agent_runtime_from_header_recognises_three_values() { - assert_eq!(AgentRuntime::from_header("asap-otel"), AgentRuntime::AsapOtel); - assert_eq!(AgentRuntime::from_header("asap-otap"), AgentRuntime::AsapOtap); - assert_eq!(AgentRuntime::from_header("asap-telegraf"), AgentRuntime::AsapTelegraf); + assert_eq!( + AgentRuntime::from_header("asap-otel"), + AgentRuntime::AsapOtel + ); + assert_eq!( + AgentRuntime::from_header("asap-otap"), + AgentRuntime::AsapOtap + ); + assert_eq!( + AgentRuntime::from_header("asap-telegraf"), + AgentRuntime::AsapTelegraf + ); } #[test] fn agent_runtime_from_header_short_aliases() { assert_eq!(AgentRuntime::from_header("otap"), AgentRuntime::AsapOtap); - assert_eq!(AgentRuntime::from_header("telegraf"), AgentRuntime::AsapTelegraf); + assert_eq!( + AgentRuntime::from_header("telegraf"), + AgentRuntime::AsapTelegraf + ); } #[test] @@ -326,10 +336,10 @@ mod runtime_tests { #[test] fn emit_for_runtime_default_matches_emit_edge_yaml() { - use crate::sketch_algebra::params::{DDSketchParams, SketchParams}; use crate::physical::colored_dag::emitter::{EdgeSketchProcessor, ExportTarget}; use crate::physical::colored_dag::stage_id::StageId; use crate::sketch_algebra::params::SketchKind; + use crate::sketch_algebra::params::{DDSketchParams, SketchParams}; let cfg = EdgeStageConfig { source_metric: Some("m".to_string()), @@ -348,11 +358,13 @@ mod runtime_tests { metric_to_family: std::collections::HashMap::new(), }; - let collector = emit_for_runtime( - AgentRuntime::AsapOtel, &cfg, "ws://ctrl/v1/opamp", None, - ).expect("collector emit ok"); + let collector = emit_for_runtime(AgentRuntime::AsapOtel, &cfg, "ws://ctrl/v1/opamp", None) + .expect("collector emit ok"); let direct = emit_edge_yaml(&cfg, "ws://ctrl/v1/opamp").expect("direct emit ok"); - assert_eq!(collector, direct, "AsapOtel dispatch must equal emit_edge_yaml"); + assert_eq!( + collector, direct, + "AsapOtel dispatch must equal emit_edge_yaml" + ); } #[test] @@ -371,11 +383,13 @@ mod runtime_tests { warm_passthrough_metrics: Vec::new(), metric_to_family: std::collections::HashMap::new(), }; - let yaml = emit_for_runtime( - AgentRuntime::AsapOtap, &cfg, "ws://ctrl/v1/opamp", None, - ).expect("otap emit ok"); + let yaml = emit_for_runtime(AgentRuntime::AsapOtap, &cfg, "ws://ctrl/v1/opamp", None) + .expect("otap emit ok"); // OTAP-specific token. - assert!(yaml.contains("otel_dataflow/v1"), "expected OTAP DAG version\n{yaml}"); + assert!( + yaml.contains("otel_dataflow/v1"), + "expected OTAP DAG version\n{yaml}" + ); } #[test] @@ -394,11 +408,13 @@ mod runtime_tests { warm_passthrough_metrics: Vec::new(), metric_to_family: std::collections::HashMap::new(), }; - let toml = emit_for_runtime( - AgentRuntime::AsapTelegraf, &cfg, "ws://ctrl/v1/opamp", None, - ).expect("telegraf emit ok"); + let toml = emit_for_runtime(AgentRuntime::AsapTelegraf, &cfg, "ws://ctrl/v1/opamp", None) + .expect("telegraf emit ok"); // Telegraf-specific token. - assert!(toml.contains("[[inputs.opentelemetry]]"), "expected Telegraf TOML header\n{toml}"); + assert!( + toml.contains("[[inputs.opentelemetry]]"), + "expected Telegraf TOML header\n{toml}" + ); } // ── stitching-gap regression: registry walk binds all 6 contract metrics ── @@ -415,37 +431,38 @@ mod runtime_tests { /// Mimics the pre-population loop in `main()` — turns each /// `WorkloadEntry` into a `QueryWorkload` via the shared `Analyzer`. - fn populate_store_from_registry( - registry: &WorkloadRegistry, - store: &WorkloadStore, - ) { + fn populate_store_from_registry(registry: &WorkloadRegistry, store: &WorkloadStore) { use crate::pipeline::{Analyzer, QuerySpec}; use crate::types; use crate::types_v2; let analyzer = Analyzer::new(); for entry in registry.entries() { let spec = QuerySpec { - query_string: entry.query_string.clone(), - metric_name: entry.metric_name.clone(), - label_filters: Default::default(), + query_string: entry.query_string.clone(), + metric_name: entry.metric_name.clone(), + label_filters: Default::default(), group_by_labels: vec![], - aggregations: vec!["quantile".into()], - time_window: "5m".into(), - repeat_every: None, - accuracy_sla: entry.accuracy_sla, - latency_sla: None, - sketch_type: entry.sketch_family_override.clone(), - workload: types::WorkloadCharacteristics::default(), - id: None, - language: None, - accuracy: None, - dollars: None, + aggregations: vec!["quantile".into()], + time_window: "5m".into(), + repeat_every: None, + accuracy_sla: entry.accuracy_sla, + latency_sla: None, + sketch_type: entry.sketch_family_override.clone(), + workload: types::WorkloadCharacteristics::default(), + id: None, + language: None, + accuracy: None, + dollars: None, deployment_model: None, - shape: types_v2::QueryShape::default(), - data: types_v2::DataShape::default(), + shape: types_v2::QueryShape::default(), + data: types_v2::DataShape::default(), }; if let Ok(wl) = analyzer.analyze(spec) { - store.set(&entry.metric_name, wl, types::WorkloadCharacteristics::default()); + store.set( + &entry.metric_name, + wl, + types::WorkloadCharacteristics::default(), + ); } } } @@ -503,12 +520,12 @@ mod runtime_tests { // 5 sketched metrics + http_requests_total (raw, declines binding). let expected: Vec<(&str, Option)> = vec![ - ("http_latency_ms", Some(SketchKind::DDSketch)), - ("http_requests_total", None), // raw passthrough - ("request_size_bytes", Some(SketchKind::Kll)), - ("unique_users_per_min", Some(SketchKind::Hll)), - ("top_endpoint_qps", Some(SketchKind::CountSketch)), - ("endpoint_request_freq", Some(SketchKind::Cms)), + ("http_latency_ms", Some(SketchKind::DDSketch)), + ("http_requests_total", None), // raw passthrough + ("request_size_bytes", Some(SketchKind::Kll)), + ("unique_users_per_min", Some(SketchKind::Hll)), + ("top_endpoint_qps", Some(SketchKind::CountSketch)), + ("endpoint_request_freq", Some(SketchKind::Cms)), ]; for (metric, want) in &expected { let got = map.get(*metric).cloned(); @@ -519,7 +536,10 @@ mod runtime_tests { ); } // Routing table covers all 5 sketched metrics. - assert_eq!(map.len(), 5, - "routing table should have 5 entries (5 sketches; raw declines), got: {map:?}"); + assert_eq!( + map.len(), + 5, + "routing table should have 5 entries (5 sketches; raw declines), got: {map:?}" + ); } } diff --git a/controller/src/emit/otap.rs b/controller/src/emit/otap.rs index f3e9e24e..00106327 100644 --- a/controller/src/emit/otap.rs +++ b/controller/src/emit/otap.rs @@ -38,15 +38,14 @@ use serde::Serialize; use serde_yaml::{Mapping, Value}; use std::collections::BTreeMap; -use crate::sketch_algebra::params::{SketchKind, SketchParams}; -use crate::physical::colored_dag::emitter::{EdgeStageConfig, EdgeSketchProcessor, ExportTarget}; +use crate::physical::colored_dag::emitter::{EdgeSketchProcessor, EdgeStageConfig, ExportTarget}; use crate::physical::colored_dag::stage_id::StageId; +use crate::sketch_algebra::params::{SketchKind, SketchParams}; /// Default URL for Prometheus's native OTLP HTTP receiver. /// Matches `super::stage_config::emit_edge_yaml`'s placeholder so the /// three runtime emitters agree on the wire endpoint. -pub const DEFAULT_PROMETHEUS_OTLP_URL: &str = - "http://prometheus:9090/api/v1/otlp/v1/metrics"; +pub const DEFAULT_PROMETHEUS_OTLP_URL: &str = "http://prometheus:9090/api/v1/otlp/v1/metrics"; /// URN of the OTLP HTTP exporter registered by /// `otel-arrow/rust/otap-dataflow/crates/core-nodes/src/exporters/otlp_http_exporter/`. @@ -156,8 +155,7 @@ pub fn emit_otap_dag_yaml( // multi-pipeline `pipelines:` map. Phase ε.1.5 ships the // single-pipeline case; the multi-pipeline case is an upstream // splitter concern.) - let prom_url = - prometheus_otlp_url.unwrap_or(DEFAULT_PROMETHEUS_OTLP_URL); + let prom_url = prometheus_otlp_url.unwrap_or(DEFAULT_PROMETHEUS_OTLP_URL); let exp_cfg = build_otlp_http_exporter_config(prom_url); nodes.insert( "exporter".to_string(), @@ -219,10 +217,7 @@ pub fn emit_otap_dag_yaml( } let mut pipelines = BTreeMap::new(); - pipelines.insert( - "main".to_string(), - PipelineDef { nodes, connections }, - ); + pipelines.insert("main".to_string(), PipelineDef { nodes, connections }); let mut groups = BTreeMap::new(); groups.insert("default".to_string(), Group { pipelines }); @@ -293,7 +288,10 @@ fn build_asap_sketches_config(sp: &EdgeSketchProcessor, window_secs: Option "aggregation_id".into(), Value::String(sp.aggregation_id.clone()), ); - m.insert("sketch_kind".into(), Value::String(sketch_kind_tag(&sp.sketch_kind).into())); + m.insert( + "sketch_kind".into(), + Value::String(sketch_kind_tag(&sp.sketch_kind).into()), + ); match &sp.sketch_params { SketchParams::Kll(p) => { m.insert("k".into(), Value::Number((p.k as u64).into())); @@ -336,8 +334,8 @@ fn sketch_kind_tag(kind: &SketchKind) -> &'static str { #[cfg(test)] mod tests { use super::*; - use crate::sketch_algebra::params::DDSketchParams; use crate::physical::colored_dag::emitter::{EdgeSketchProcessor, PrometheusArchiveMetric}; + use crate::sketch_algebra::params::DDSketchParams; /// Minimal struct-stub used to validate the emitted DAG parses as the /// otap-dataflow schema. We don't pull in the otap-df-config crate @@ -432,12 +430,17 @@ mod tests { /// otlp_grpc exporter to gateway. #[test] fn otap_dag_mode1_sketch_at_edge_shape() { - let yaml = - emit_otap_dag_yaml(&ddsketch_edge_cfg_mode1(), "ws://ctrl/v1/opamp", None) - .expect("emit_otap_dag_yaml ok"); + let yaml = emit_otap_dag_yaml(&ddsketch_edge_cfg_mode1(), "ws://ctrl/v1/opamp", None) + .expect("emit_otap_dag_yaml ok"); let dag: OtapDagStub = serde_yaml::from_str(&yaml).expect("DAG parses"); assert_eq!(dag.version, "otel_dataflow/v1"); - let pipe = dag.groups.get("default").unwrap().pipelines.get("main").unwrap(); + let pipe = dag + .groups + .get("default") + .unwrap() + .pipelines + .get("main") + .unwrap(); // Receiver + sketch + exporter == 3 nodes. assert_eq!(pipe.nodes.len(), 3, "expected 3 nodes\n{yaml}"); assert_eq!(pipe.nodes.get("receiver").unwrap().kind, URN_OTLP_RECEIVER); @@ -445,7 +448,10 @@ mod tests { pipe.nodes.get("sketch_0").unwrap().kind, URN_ASAP_SKETCHES_PROCESSOR ); - assert_eq!(pipe.nodes.get("exporter").unwrap().kind, URN_OTLP_GRPC_EXPORTER); + assert_eq!( + pipe.nodes.get("exporter").unwrap().kind, + URN_OTLP_GRPC_EXPORTER + ); // Connections: receiver → sketch_0 → exporter. assert_eq!(pipe.connections.len(), 2); assert_eq!(pipe.connections[0].from, "receiver"); @@ -453,7 +459,10 @@ mod tests { assert_eq!(pipe.connections[1].from, "sketch_0"); assert_eq!(pipe.connections[1].to, "exporter"); // Endpoint contains gateway:4317. - assert!(yaml.contains("gateway:4317"), "missing gateway endpoint\n{yaml}"); + assert!( + yaml.contains("gateway:4317"), + "missing gateway endpoint\n{yaml}" + ); } /// Mode 2 snapshot — raw at edge: receiver → otlp_grpc exporter. @@ -463,10 +472,23 @@ mod tests { let yaml = emit_otap_dag_yaml(&raw_edge_cfg_mode2(), "ws://ctrl/v1/opamp", None) .expect("emit_otap_dag_yaml ok"); let dag: OtapDagStub = serde_yaml::from_str(&yaml).expect("DAG parses"); - let pipe = dag.groups.get("default").unwrap().pipelines.get("main").unwrap(); - assert_eq!(pipe.nodes.len(), 2, "expected receiver + exporter only\n{yaml}"); + let pipe = dag + .groups + .get("default") + .unwrap() + .pipelines + .get("main") + .unwrap(); + assert_eq!( + pipe.nodes.len(), + 2, + "expected receiver + exporter only\n{yaml}" + ); assert_eq!(pipe.nodes.get("receiver").unwrap().kind, URN_OTLP_RECEIVER); - assert_eq!(pipe.nodes.get("exporter").unwrap().kind, URN_OTLP_GRPC_EXPORTER); + assert_eq!( + pipe.nodes.get("exporter").unwrap().kind, + URN_OTLP_GRPC_EXPORTER + ); // Direct connection. assert_eq!(pipe.connections.len(), 1); assert_eq!(pipe.connections[0].from, "receiver"); @@ -485,9 +507,18 @@ mod tests { let yaml = emit_otap_dag_yaml(&prom_edge_cfg_mode3(), "ws://ctrl/v1/opamp", None) .expect("emit_otap_dag_yaml ok"); let dag: OtapDagStub = serde_yaml::from_str(&yaml).expect("DAG parses"); - let pipe = dag.groups.get("default").unwrap().pipelines.get("main").unwrap(); + let pipe = dag + .groups + .get("default") + .unwrap() + .pipelines + .get("main") + .unwrap(); assert_eq!(pipe.nodes.len(), 2); - assert_eq!(pipe.nodes.get("exporter").unwrap().kind, URN_OTLP_HTTP_EXPORTER); + assert_eq!( + pipe.nodes.get("exporter").unwrap().kind, + URN_OTLP_HTTP_EXPORTER + ); // Path round-trips verbatim. assert!( yaml.contains("/api/v1/otlp/v1/metrics"), diff --git a/controller/src/emit/stage_config.rs b/controller/src/emit/stage_config.rs index 647ed907..3c17c67e 100644 --- a/controller/src/emit/stage_config.rs +++ b/controller/src/emit/stage_config.rs @@ -38,14 +38,14 @@ use serde_json::{json, Value as JsonValue}; use serde_yaml::{Mapping, Value}; use std::collections::HashMap; -use crate::sketch_algebra::params::{SketchKind, SketchParams}; -use crate::sketch_algebra::sketch_expr::EstimateOp; use crate::physical::colored_dag::emitter::{ AggregationInput, ArchiveTierMetric, BackendAggregation, BackendReadout, BackendStageConfig, EdgeSketchProcessor, EdgeStageConfig, ExportTarget, GatewayMergeProcessor, GatewayStageConfig, PrometheusArchiveMetric, }; use crate::physical::colored_dag::stage_id::StageId; +use crate::sketch_algebra::params::{SketchKind, SketchParams}; +use crate::sketch_algebra::sketch_expr::EstimateOp; // ── YAML structural types ───────────────────────────────────────────────────── // @@ -154,7 +154,7 @@ pub fn emit_edge_yaml(cfg: &EdgeStageConfig, opamp_endpoint: &str) -> Result Result { /// /// ```json /// { -/// "default_engine": "sketch_warm_tier", +/// "default_engine": "asap_query", /// "metrics": [ /// { "name": "http_requests_total", /// "targets": [ -/// { "engine": "thanos_archive", +/// { "engine": "thanos_query", /// "applies_to_query_shape": ["count", "topk", "rate_post_hoc", /// "histogram_quantile", "delta", "absent"] }, -/// { "engine": "sketch_warm_tier" } +/// { "engine": "asap_query" } /// ] /// } /// ] @@ -569,7 +569,7 @@ pub fn emit_backend_config_json(cfg: &BackendStageConfig) -> Result { /// * **CountMinSketch** present → warm-tier serves `point_count` / /// `count` shape (the CMS's `Estimate` readout). /// -/// The `thanos_archive` target is always added with the **archive-eligible +/// The `thanos_query` target is always added with the **archive-eligible /// shape list** — those PromQL shapes that no warm-tier sketch can /// answer at all (`histogram_quantile`, `delta`, `deriv`, `absent`, /// post-hoc / un-planned ranges). When a sketch-eligible shape is also @@ -617,27 +617,27 @@ pub fn emit_backend_storage_routing_for_tenant( } Ok(json!({ "tenant": tenant, - "default_engine": "sketch_warm_tier", + "default_engine": "asap_query", "metrics": metrics_json, })) } /// Phase ε.1 — same as [`emit_backend_storage_routing`] but also -/// emits `prometheus_remote` engine entries for Mode 3 metrics. +/// emits `thanos_query` engine entries for Mode 3 metrics. /// /// Mode-3 metrics have NO `BackendStageConfig` entry (the backend doesn't /// own the storage; Prometheus does). They surface here as plain metric -/// names paired with a single `prometheus_remote` target. The backend's +/// names paired with a single `thanos_query` target. The backend's /// HTTP query handler consults the routing table at request time and /// HTTP-forwards Mode-3 queries to /// `${ASAP_PROMETHEUS_QUERY_URL:-http://prometheus:9090}/api/v1/query`. /// -/// Phase ε.2 implements the `prometheus_remote` engine on the backend +/// Phase ε.2 implements the `thanos_query` engine on the backend /// (the HTTP forwarder); Phase ε.1 only commits the routing wire shape. /// /// `mode3_metrics` is the list of metric names the planner routed to /// Prometheus archive this cycle. Each yields a single-target row with -/// `engine: prometheus_remote` and no shape filter (Prom answers +/// `engine: thanos_query` and no shape filter (Prom answers /// everything for these metrics, exact ε = 0). pub fn emit_backend_storage_routing_with_prometheus( metric_plans: &[(String, &BackendStageConfig)], @@ -666,19 +666,19 @@ pub fn emit_backend_storage_routing_with_prometheus_for_tenant( } for metric_name in mode3_metrics { // Mode 3 — Prometheus owns the storage. Single target, - // engine=prometheus_remote, no shape filter (all PromQL shapes + // engine=thanos_query, no shape filter (all PromQL shapes // route through the backend's HTTP forwarder). metrics_json.push(json!({ "name": metric_name, "targets": [ - { "engine": "prometheus_remote" } + { "engine": "thanos_query" } ], "asap_mode": "prometheus_archive", })); } Ok(json!({ "tenant": tenant, - "default_engine": "sketch_warm_tier", + "default_engine": "asap_query", "metrics": metrics_json, })) } @@ -783,11 +783,11 @@ fn build_routing_entry(metric_name: &str, cfg: &BackendStageConfig) -> JsonValue // archive serves better. let mut targets: Vec = Vec::new(); targets.push(json!({ - "engine": "sketch_warm_tier", + "engine": "asap_query", })); if !archive_shapes.is_empty() { targets.push(json!({ - "engine": "thanos_archive", + "engine": "thanos_query", "applies_to_query_shape": archive_shapes, })); } @@ -1839,7 +1839,7 @@ mod tests { let plans: Vec<(String, &BackendStageConfig)> = vec![("http_request_duration_seconds".to_string(), &ddsketch)]; let v = emit_backend_storage_routing(&plans).expect("emit ok"); - assert_eq!(v["default_engine"], "sketch_warm_tier"); + assert_eq!(v["default_engine"], "asap_query"); let metrics = v["metrics"].as_array().expect("metrics array"); assert_eq!(metrics.len(), 1); assert_eq!(metrics[0]["name"], "http_request_duration_seconds"); @@ -1870,7 +1870,7 @@ mod tests { emit_backend_storage_routing_for_tenant("tenant-a", &[("latency".into(), &ddsketch)]) .expect("emit ok"); assert_eq!(v["tenant"], "tenant-a"); - assert_eq!(v["default_engine"], "sketch_warm_tier"); + assert_eq!(v["default_engine"], "asap_query"); // Single metric, single warm + archive target shape — the // per-tenant emit doesn't change the metric-side shape. let metrics = v["metrics"].as_array().expect("metrics array"); @@ -1887,25 +1887,25 @@ mod tests { .expect("emit ok"); assert_eq!(v["tenant"], "tenant-b"); assert_eq!(v["metrics"][0]["name"], "http_requests_total"); - assert_eq!(v["metrics"][0]["targets"][0]["engine"], "prometheus_remote"); + assert_eq!(v["metrics"][0]["targets"][0]["engine"], "thanos_query"); } #[test] - fn storage_routing_ddsketch_warm_serves_quantile_archive_serves_others() { + fn storage_routing_ddasap_query_serves_quantile_archive_serves_others() { let ddsketch = backend_cfg_with_kind(SketchKind::DDSketch); let v = emit_backend_storage_routing(&[("latency".into(), &ddsketch)]).expect("emit ok"); let metric = &v["metrics"][0]; let targets = metric["targets"].as_array().expect("targets array"); // Default slot — warm tier, no filter. - assert_eq!(targets[0]["engine"], "sketch_warm_tier"); + assert_eq!(targets[0]["engine"], "asap_query"); assert!( targets[0].get("applies_to_query_shape").is_none(), "warm slot must be the default (no filter); got {targets:?}" ); // Archive slot — must carry the predictable archive shapes. - assert_eq!(targets[1]["engine"], "thanos_archive"); + assert_eq!(targets[1]["engine"], "thanos_query"); let archive_shapes: Vec = targets[1]["applies_to_query_shape"] .as_array() .unwrap() @@ -1997,13 +1997,13 @@ mod tests { // serialises keys alphabetically, so `tenant` lands at the // end of the document. let expected = r#"{ - "default_engine": "sketch_warm_tier", + "default_engine": "asap_query", "metrics": [ { "name": "http_requests_total", "targets": [ { - "engine": "sketch_warm_tier" + "engine": "asap_query" }, { "applies_to_query_shape": [ @@ -2014,7 +2014,7 @@ mod tests { "rate_post_hoc", "count" ], - "engine": "thanos_archive" + "engine": "thanos_query" } ], "warm_tier_native_shapes": [ @@ -2030,7 +2030,7 @@ mod tests { "name": "active_users", "targets": [ { - "engine": "sketch_warm_tier" + "engine": "asap_query" }, { "applies_to_query_shape": [ @@ -2041,7 +2041,7 @@ mod tests { "rate_post_hoc", "topk" ], - "engine": "thanos_archive" + "engine": "thanos_query" } ], "warm_tier_native_shapes": [ @@ -2057,7 +2057,7 @@ mod tests { "name": "request_latency_seconds", "targets": [ { - "engine": "sketch_warm_tier" + "engine": "asap_query" }, { "applies_to_query_shape": [ @@ -2069,7 +2069,7 @@ mod tests { "topk", "count" ], - "engine": "thanos_archive" + "engine": "thanos_query" } ], "warm_tier_native_shapes": [ @@ -2091,7 +2091,7 @@ mod tests { #[test] fn storage_routing_empty_input_emits_empty_metrics_array() { let v = emit_backend_storage_routing(&[]).expect("emit ok"); - assert_eq!(v["default_engine"], "sketch_warm_tier"); + assert_eq!(v["default_engine"], "asap_query"); assert_eq!(v["metrics"].as_array().unwrap().len(), 0); } @@ -2112,8 +2112,8 @@ mod tests { assert!(metric.get("warm_tier_native_shapes").is_none()); // Targets: warm-tier default + archive default-shape list. let targets = metric["targets"].as_array().unwrap(); - assert_eq!(targets[0]["engine"], "sketch_warm_tier"); - assert_eq!(targets[1]["engine"], "thanos_archive"); + assert_eq!(targets[0]["engine"], "asap_query"); + assert_eq!(targets[1]["engine"], "thanos_query"); } // ── Phase β: emit_backend_config_json snapshot for new pattern coverage ── @@ -2234,12 +2234,12 @@ mod tests { } /// Mode 3 (Prometheus archive) — the routing emitter adds a - /// `prometheus_remote` engine target for the metric. The backend's + /// `thanos_query` engine target for the metric. The backend's /// HTTP query handler HTTP-forwards the matching PromQL queries to /// `${ASAP_PROMETHEUS_QUERY_URL}/api/v1/query`. Phase ε.2 registers /// the engine on the backend. #[test] - fn phase_eps1_mode3_storage_routing_emits_prometheus_remote() { + fn phase_eps1_mode3_storage_routing_emits_thanos_query() { // No backend-side aggregations for mode 3 — Prometheus owns it. let mode3 = vec!["http_requests_total".to_string()]; let v = emit_backend_storage_routing_with_prometheus(&[], &mode3).expect("emit ok"); @@ -2248,7 +2248,7 @@ mod tests { assert_eq!(metrics[0]["name"], "http_requests_total"); let targets = metrics[0]["targets"].as_array().unwrap(); assert_eq!(targets.len(), 1); - assert_eq!(targets[0]["engine"], "prometheus_remote"); + assert_eq!(targets[0]["engine"], "thanos_query"); // No shape filter — Prometheus serves every PromQL shape. assert!(targets[0].get("applies_to_query_shape").is_none()); // `asap_mode` annotation surfaces so operators can see why a @@ -2269,13 +2269,13 @@ mod tests { assert_eq!(metrics[0]["name"], "latency_seconds"); // Mode-1 entry — full warm/archive routing. let m1_targets = metrics[0]["targets"].as_array().unwrap(); - assert_eq!(m1_targets[0]["engine"], "sketch_warm_tier"); - assert_eq!(m1_targets[1]["engine"], "thanos_archive"); - // Mode-3 entry — single prometheus_remote target. + assert_eq!(m1_targets[0]["engine"], "asap_query"); + assert_eq!(m1_targets[1]["engine"], "thanos_query"); + // Mode-3 entry — single thanos_query target. assert_eq!(metrics[1]["name"], "http_requests_total"); let m3_targets = metrics[1]["targets"].as_array().unwrap(); assert_eq!(m3_targets.len(), 1); - assert_eq!(m3_targets[0]["engine"], "prometheus_remote"); + assert_eq!(m3_targets[0]["engine"], "thanos_query"); } /// Mode 3 emit_edge_yaml — produces a YAML with `otlphttp/prometheus` @@ -2600,7 +2600,10 @@ mod tests { /// family per the canonical workload-spec table in MVP §46. fn five_sketch_edge_cfg() -> EdgeStageConfig { let mut metric_to_family: HashMap = HashMap::new(); - metric_to_family.insert("http_requests_total_latency_ms".into(), SketchKind::DDSketch); + metric_to_family.insert( + "http_requests_total_latency_ms".into(), + SketchKind::DDSketch, + ); metric_to_family.insert("request_size_bytes".into(), SketchKind::Kll); metric_to_family.insert("unique_users_per_min".into(), SketchKind::Hll); metric_to_family.insert("top_endpoint_qps".into(), SketchKind::CountSketch); diff --git a/controller/src/emit/telegraf.rs b/controller/src/emit/telegraf.rs index 5d346646..ef04d777 100644 --- a/controller/src/emit/telegraf.rs +++ b/controller/src/emit/telegraf.rs @@ -37,17 +37,16 @@ use anyhow::{Context, Result}; -use crate::sketch_algebra::params::{SketchKind, SketchParams}; -use crate::physical::colored_dag::emitter::{EdgeStageConfig, EdgeSketchProcessor, ExportTarget}; +use crate::physical::colored_dag::emitter::{EdgeSketchProcessor, EdgeStageConfig, ExportTarget}; use crate::physical::colored_dag::stage_id::StageId; +use crate::sketch_algebra::params::{SketchKind, SketchParams}; /// Default Prometheus remote-write URL for Mode 3 — Telegraf doesn't /// support OTLP-HTTP egress, so we land in the same Prometheus archive /// via remote-write instead. The URL maps to the same Prometheus instance /// the OTel-collector emitter targets via OTLP HTTP — Prometheus accepts /// both ingest paths and stores into the same TSDB. -pub const DEFAULT_PROMETHEUS_REMOTE_WRITE_URL: &str = - "http://prometheus:9090/api/v1/write"; +pub const DEFAULT_PROMETHEUS_REMOTE_WRITE_URL: &str = "http://prometheus:9090/api/v1/write"; // ── Public API ─────────────────────────────────────────────────────────────── @@ -80,8 +79,7 @@ pub fn emit_telegraf_toml( if has_prometheus_archive { // Mode 3 — Prometheus archive: passthrough, then remote-write // to Prometheus. We do NOT include `[[processors.allsketches]]`. - let url = - prometheus_remote_write_url.unwrap_or(DEFAULT_PROMETHEUS_REMOTE_WRITE_URL); + let url = prometheus_remote_write_url.unwrap_or(DEFAULT_PROMETHEUS_REMOTE_WRITE_URL); emit_outputs_http_remote_write(&mut out, url); } else if has_sketch { // Mode 1 — sketch at edge. One `[[processors.allsketches]]` per @@ -148,15 +146,16 @@ fn emit_processors_allsketches( window_secs: Option, ) { out.push_str("[[processors.allsketches]]\n"); - let mode = if window_secs.is_some() { "window" } else { "batch" }; + let mode = if window_secs.is_some() { + "window" + } else { + "batch" + }; out.push_str(&format!(" mode = \"{mode}\"\n")); if let Some(w) = window_secs { out.push_str(&format!(" window_duration = \"{w}s\"\n")); } - out.push_str(&format!( - " aggregation_id = \"{}\"\n", - sp.aggregation_id - )); + out.push_str(&format!(" aggregation_id = \"{}\"\n", sp.aggregation_id)); out.push_str(&format!( " sketch_kind = \"{}\"\n", sketch_kind_tag(&sp.sketch_kind) @@ -250,10 +249,7 @@ mod toml_minimal { // unquoted scalar (number / fraction). if val.starts_with('"') { if !val.ends_with('"') || val.len() < 2 { - return Err(anyhow!( - "line {}: unbalanced \" in: {raw}", - lineno + 1 - )); + return Err(anyhow!("line {}: unbalanced \" in: {raw}", lineno + 1)); } } } @@ -295,8 +291,8 @@ mod toml_minimal { #[cfg(test)] mod tests { use super::*; - use crate::sketch_algebra::params::{DDSketchParams, KllParams}; use crate::physical::colored_dag::emitter::{EdgeSketchProcessor, PrometheusArchiveMetric}; + use crate::sketch_algebra::params::{DDSketchParams, KllParams}; fn ddsketch_edge_cfg_mode1() -> EdgeStageConfig { EdgeStageConfig { @@ -354,9 +350,12 @@ mod tests { /// `[[outputs.opentelemetry]]`. #[test] fn telegraf_toml_mode1_sketch_at_edge_shape() { - let toml = emit_telegraf_toml(&ddsketch_edge_cfg_mode1(), None) - .expect("emit_telegraf_toml ok"); - assert!(toml.contains("[[inputs.opentelemetry]]"), "missing input\n{toml}"); + let toml = + emit_telegraf_toml(&ddsketch_edge_cfg_mode1(), None).expect("emit_telegraf_toml ok"); + assert!( + toml.contains("[[inputs.opentelemetry]]"), + "missing input\n{toml}" + ); assert!( toml.contains("[[processors.allsketches]]"), "missing sketch processor\n{toml}" @@ -370,17 +369,25 @@ mod tests { "missing gateway endpoint\n{toml}" ); // Sketch params preserved. - assert!(toml.contains("relative_accuracy = 0.01"), "missing alpha\n{toml}"); - assert!(toml.contains("sketch_kind = \"ddsketch\""), "wrong kind\n{toml}"); + assert!( + toml.contains("relative_accuracy = 0.01"), + "missing alpha\n{toml}" + ); + assert!( + toml.contains("sketch_kind = \"ddsketch\""), + "wrong kind\n{toml}" + ); } /// Mode 2 snapshot — raw at edge. No `[[processors.allsketches]]`. /// `[[outputs.opentelemetry]]` ships raw OTLP to the gateway. #[test] fn telegraf_toml_mode2_raw_at_edge_shape() { - let toml = emit_telegraf_toml(&raw_edge_cfg_mode2(), None) - .expect("emit_telegraf_toml ok"); - assert!(toml.contains("[[inputs.opentelemetry]]"), "missing input\n{toml}"); + let toml = emit_telegraf_toml(&raw_edge_cfg_mode2(), None).expect("emit_telegraf_toml ok"); + assert!( + toml.contains("[[inputs.opentelemetry]]"), + "missing input\n{toml}" + ); assert!( !toml.contains("[[processors.allsketches]]"), "Mode 2 must not include sketch processor\n{toml}" @@ -401,9 +408,11 @@ mod tests { /// Prometheus TSDB the OTel-collector path lands in via OTLP HTTP). #[test] fn telegraf_toml_mode3_prometheus_archive_shape() { - let toml = emit_telegraf_toml(&prom_edge_cfg_mode3(), None) - .expect("emit_telegraf_toml ok"); - assert!(toml.contains("[[inputs.opentelemetry]]"), "missing input\n{toml}"); + let toml = emit_telegraf_toml(&prom_edge_cfg_mode3(), None).expect("emit_telegraf_toml ok"); + assert!( + toml.contains("[[inputs.opentelemetry]]"), + "missing input\n{toml}" + ); assert!( !toml.contains("[[processors.allsketches]]"), "Mode 3 must not include sketch processor\n{toml}" @@ -457,7 +466,10 @@ mod tests { // Parsing happens inside emit_telegraf_toml; if we got Ok, // parsing succeeded. Spot-check a handful of expected // tokens defensively. - assert!(toml.contains("inputs.opentelemetry"), "{name}: missing input header"); + assert!( + toml.contains("inputs.opentelemetry"), + "{name}: missing input header" + ); } } diff --git a/controller/src/intent_algebra/cse.rs b/controller/src/intent_algebra/cse.rs index d370e3e5..1e7fa702 100644 --- a/controller/src/intent_algebra/cse.rs +++ b/controller/src/intent_algebra/cse.rs @@ -253,10 +253,7 @@ mod tests { child: Box::new(windowed_scan()), }; - let out = dedupe_subtrees(vec![ - (QueryId::new("q1"), q1), - (QueryId::new("q2"), q2), - ]); + let out = dedupe_subtrees(vec![(QueryId::new("q1"), q1), (QueryId::new("q2"), q2)]); // One binding hoisted, two roots rewritten to `Ref { name: "shared_0" }`. assert_eq!(out.bindings.len(), 1); diff --git a/controller/src/intent_algebra/lower.rs b/controller/src/intent_algebra/lower.rs index 8912e6d5..1894e4c9 100644 --- a/controller/src/intent_algebra/lower.rs +++ b/controller/src/intent_algebra/lower.rs @@ -260,7 +260,9 @@ mod tests { } // Mid: Window match child.as_ref() { - QueryExpr::Window { kind, size, child, .. } => { + QueryExpr::Window { + kind, size, child, .. + } => { assert_eq!(*kind, WindowKind::Sliding); assert_eq!(*size, std::time::Duration::from_secs(300)); // Leaf: Scan @@ -305,10 +307,9 @@ mod tests { #[test] fn lower_promql_with_group_by() { - let parsed = parse_query( - "sum by (host) (quantile_over_time(0.95, latency{env=\"prod\"}[1m]))", - ) - .expect("parse"); + let parsed = + parse_query("sum by (host) (quantile_over_time(0.95, latency{env=\"prod\"}[1m]))") + .expect("parse"); let expr = lower_parsed_query(&parsed, AccuracyTarget::Epsilon(0.05)).expect("lower"); let schema = expr.output_schema().expect("output_schema"); // `host` is a group-by → it lands in unique_keys at position 0 @@ -319,10 +320,9 @@ mod tests { #[test] fn lower_promql_cardinality() { - let parsed = parse_query( - "count by (user_id) (count_over_time(active_users{env=\"prod\"}[5m]))", - ) - .expect("parse"); + let parsed = + parse_query("count by (user_id) (count_over_time(active_users{env=\"prod\"}[5m]))") + .expect("parse"); let expr = lower_parsed_query(&parsed, AccuracyTarget::Epsilon(0.01)).expect("lower"); let schema = expr.output_schema().expect("output_schema"); // Cardinality intent → output column named `cardinality` of dtype Int64. diff --git a/controller/src/intent_algebra/schema.rs b/controller/src/intent_algebra/schema.rs index 2e9aadd6..ffb00963 100644 --- a/controller/src/intent_algebra/schema.rs +++ b/controller/src/intent_algebra/schema.rs @@ -192,10 +192,7 @@ pub enum CseError { /// deduper has identified for the candidate binding. Single-consumer /// cases short-circuit with `InsufficientConsumers` — a `LetBinding` /// with one `Ref` is just a no-op alias and shouldn't be hoisted. -pub fn cse_reuse_is_legal( - producer_schema: &Schema, - consumer_count: usize, -) -> Result<(), CseError> { +pub fn cse_reuse_is_legal(producer_schema: &Schema, consumer_count: usize) -> Result<(), CseError> { if consumer_count < 2 { return Err(CseError::InsufficientConsumers(consumer_count)); } @@ -243,10 +240,7 @@ mod tests { /// drops on the floor"). #[test] fn cse_reuse_illegal_when_unique_keys_empty() { - let producer = Schema::new(vec![ - col("a", DataType::Int64), - col("b", DataType::Float64), - ]); + let producer = Schema::new(vec![col("a", DataType::Int64), col("b", DataType::Float64)]); assert_eq!( cse_reuse_is_legal(&producer, 2), Err(CseError::NoUniqueKeys) @@ -286,7 +280,6 @@ mod tests { ); } - #[test] fn schema_new_has_no_time_or_unique_key() { let s = Schema::new(vec![col("k", DataType::Utf8), col("v", DataType::Float64)]); @@ -325,7 +318,10 @@ mod tests { #[test] fn schema_serde_roundtrip() { let s = Schema::with_time_index( - vec![col("ts", DataType::Timestamp), col("value", DataType::Float64)], + vec![ + col("ts", DataType::Timestamp), + col("value", DataType::Float64), + ], 0, vec![vec![0]], ); diff --git a/controller/src/language_logical_plan/lower.rs b/controller/src/language_logical_plan/lower.rs index 7b10e5ab..67349e51 100644 --- a/controller/src/language_logical_plan/lower.rs +++ b/controller/src/language_logical_plan/lower.rs @@ -26,9 +26,7 @@ pub enum LoweringError { /// `query_parser::parse_query_expr` and stashed inside [`PromQLAst`]). /// We just re-tag it as the PromQL L2 variant and project the flat /// summary; no re-parsing or re-walking is required. -pub fn lower_to_logical_plan( - ast: &LanguageAst, -) -> Result { +pub fn lower_to_logical_plan(ast: &LanguageAst) -> Result { match ast { LanguageAst::PromQL(p) => Ok(lower_promql(p)), } diff --git a/controller/src/language_logical_plan/tests.rs b/controller/src/language_logical_plan/tests.rs index 4e0f9644..663c4a39 100644 --- a/controller/src/language_logical_plan/tests.rs +++ b/controller/src/language_logical_plan/tests.rs @@ -7,7 +7,9 @@ use crate::types::AggType; use crate::types_v2::QueryLanguage; fn parse_promql(src: &str) -> LanguageAst { - PromQLLanguage.parse(src).expect("PromQL parse should succeed") + PromQLLanguage + .parse(src) + .expect("PromQL parse should succeed") } #[test] @@ -22,9 +24,7 @@ fn lower_promql_to_logical_plan_basic() { #[test] fn lower_promql_quantile_preserves_summary() { - let ast = parse_promql( - "quantile_over_time(0.99, http_request_duration{env=\"prod\"}[5m])", - ); + let ast = parse_promql("quantile_over_time(0.99, http_request_duration{env=\"prod\"}[5m])"); let plan = lower_to_logical_plan(&ast).unwrap(); let s = plan.summary(); assert_eq!(s.metric_name, "http_request_duration"); @@ -38,22 +38,21 @@ fn lower_promql_quantile_preserves_summary() { fn lower_promql_keeps_algebra_tree_for_l3() { // The PromQL L2 tree IS the existing `QueryExpr`; assert that the // tree shape matches what `parse_query_expr` would have produced. - let ast = parse_promql( - "quantile_over_time(0.99, http_request_duration{env=\"prod\"}[5m])", - ); + let ast = parse_promql("quantile_over_time(0.99, http_request_duration{env=\"prod\"}[5m])"); let plan = lower_to_logical_plan(&ast).unwrap(); let tree = plan.as_promql_tree().expect("PromQL plan"); assert!(matches!( tree, - QueryExpr::WindowedAgg { agg: AggIntent::Quantile { .. }, .. } + QueryExpr::WindowedAgg { + agg: AggIntent::Quantile { .. }, + .. + } )); } #[test] fn lower_promql_topk_extracts_groupby() { - let ast = parse_promql( - "topk by (service) (10, count_over_time(requests{env=\"prod\"}[1m]))", - ); + let ast = parse_promql("topk by (service) (10, count_over_time(requests{env=\"prod\"}[1m]))"); let plan = lower_to_logical_plan(&ast).unwrap(); let s = plan.summary(); assert_eq!(s.metric_name, "requests"); @@ -68,7 +67,9 @@ fn lower_unsupported_language_errors_cleanly() { // smoke test is: stub backends fail at L1 with `Unimplemented`, and // the type system rules out passing them to L2 lowering. We assert // that contract here. - let err = crate::query_parser::language::SqlLanguage.parse("SELECT 1").unwrap_err(); + let err = crate::query_parser::language::SqlLanguage + .parse("SELECT 1") + .unwrap_err(); assert!(matches!( err, crate::query_parser::language::ParseError::Unimplemented(_) diff --git a/controller/src/main.rs b/controller/src/main.rs index bf45d3fd..cdb5fbf3 100644 --- a/controller/src/main.rs +++ b/controller/src/main.rs @@ -94,7 +94,7 @@ struct AppState { /// post path emits a single-element `metrics:[…]` document per /// `handle_plan` call, so when N metrics replan in sequence only /// the last metric's entry survives in the backend's routing table. - /// That defaults the other N-1 metrics to `sketch_warm_tier`, which + /// That defaults the other N-1 metrics to `sketch_store`, which /// has no warm-tier sketch state for archive-shape queries /// (`count`, `topk`, `rate_post_hoc`, `histogram_quantile`, /// `delta`, `deriv`, `absent`) → the backend returns empty / 404 → @@ -618,7 +618,7 @@ async fn handle_plan( // each per-metric replan erases the routing // entries for every other metric and the // backend defaults them to - // `sketch_warm_tier` (which has nothing for + // `sketch_store` (which has nothing for // archive-shape queries). That's the // `archive_miss` failure mode for // HLL/CountSketch/CountMin/KLL metrics in @@ -2499,7 +2499,7 @@ mod api_tests { // `metrics:[…]` document per call, so when the demo POSTed // `/api/v1/plan` for each of the 5 sketched contract metrics in // sequence, only the LAST metric's entry survived in the backend. - // The other 4 metrics defaulted to `sketch_warm_tier` (which has + // The other 4 metrics defaulted to `sketch_store` (which has // no warm-tier sketch state for archive-shape queries) → the // demo's accuracy reducer logged `archive_miss` for those metrics // even though gorillas3 wrote their TSDB blocks to MinIO and @@ -2598,7 +2598,7 @@ mod api_tests { // MUST list ALL 5 sketched metrics, otherwise the swap would // erase the routing entries for the metrics planned earlier // in the sequence and the backend would default them to - // `sketch_warm_tier` → archive_miss for those metrics' archive + // `sketch_store` → archive_miss for those metrics' archive // queries even though gorillas3's TSDB blocks are present in // MinIO and Thanos has them indexed. let last: serde_json::Value = @@ -2618,10 +2618,10 @@ mod api_tests { ); } - // Each metric entry must carry a `thanos_archive` target — the + // Each metric entry must carry a `thanos_query` target — the // archive-tier dispatch that lets backend forward archive-shape // queries to Thanos. Without this target the metric falls back - // to `default_engine: sketch_warm_tier` and the archive miss + // to `default_engine: sketch_store` and the archive miss // reproduces. for m in last["metrics"].as_array().unwrap() { let targets = m["targets"].as_array().expect("targets array"); @@ -2630,8 +2630,8 @@ mod api_tests { .map(|t| t["engine"].as_str().unwrap()) .collect(); assert!( - engines.contains(&"thanos_archive"), - "metric `{}` missing `thanos_archive` target; engines={engines:?}", + engines.contains(&"thanos_query"), + "metric `{}` missing `thanos_query` target; engines={engines:?}", m["name"].as_str().unwrap(), ); } diff --git a/controller/src/metrics_exposer.rs b/controller/src/metrics_exposer.rs index a825a231..fecc5d85 100644 --- a/controller/src/metrics_exposer.rs +++ b/controller/src/metrics_exposer.rs @@ -166,9 +166,13 @@ impl MetricsRegistry { registry.register(Box::new(latency_p99.clone())).unwrap(); registry.register(Box::new(memory_bytes.clone())).unwrap(); registry.register(Box::new(last_seen_unix.clone())).unwrap(); - registry.register(Box::new(batches_received.clone())).unwrap(); + registry + .register(Box::new(batches_received.clone())) + .unwrap(); registry.register(Box::new(records_stored.clone())).unwrap(); - registry.register(Box::new(records_evicted.clone())).unwrap(); + registry + .register(Box::new(records_evicted.clone())) + .unwrap(); registry.register(Box::new(decode_errors.clone())).unwrap(); Arc::new(Self { @@ -195,7 +199,9 @@ impl MetricsRegistry { // plan_id label permanently emitting a stale value. self.active_plan_id.reset(); for metric in plan_store.metrics() { - let Ok(plan) = plan_store.get(&metric) else { continue }; + let Ok(plan) = plan_store.get(&metric) else { + continue; + }; // Stable hash of the plan's debug repr — good enough for // a label value, doesn't need to be cryptographic. let mut hasher = DefaultHasher::new(); @@ -416,9 +422,8 @@ mod tests { assert!(text.contains( "asap_runtime_throughput_items_per_sec{impl=\"oxide\",sketch=\"cms\",source=\"dc-a\"}" )); - assert!(text.contains( - "asap_runtime_latency_p99_ns{impl=\"lib\",sketch=\"hll\",source=\"dc-a\"}" - )); + assert!(text + .contains("asap_runtime_latency_p99_ns{impl=\"lib\",sketch=\"hll\",source=\"dc-a\"}")); } #[test] @@ -460,7 +465,10 @@ mod tests { let plan_store = Arc::new(PlanStore::new()); plan_store.set("http_requests_total", make_plan(SketchType::DDSketch, 600)); - plan_store.set("http_requests_total_latency_ms", make_plan(SketchType::HLL, 600)); + plan_store.set( + "http_requests_total_latency_ms", + make_plan(SketchType::HLL, 600), + ); let registry = MetricsRegistry::new(); registry.refresh_plan_ids(&plan_store); diff --git a/controller/src/monitor/mod.rs b/controller/src/monitor/mod.rs index 29defa37..3ac7e69c 100644 --- a/controller/src/monitor/mod.rs +++ b/controller/src/monitor/mod.rs @@ -15,11 +15,11 @@ use crate::types::SketchType; #[derive(Debug, Clone)] pub struct CollectorMetrics { - pub agent_id: String, + pub agent_id: String, pub sketch_size_bytes: f64, pub cpu_seconds_total: f64, - pub samples_ingested: f64, - pub error_rate: f64, + pub samples_ingested: f64, + pub error_rate: f64, } #[derive(Debug, Clone, PartialEq)] @@ -33,40 +33,40 @@ impl std::fmt::Display for ViolationKind { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { ViolationKind::Bandwidth => write!(f, "bandwidth"), - ViolationKind::Accuracy => write!(f, "accuracy"), - ViolationKind::Cpu => write!(f, "cpu"), + ViolationKind::Accuracy => write!(f, "accuracy"), + ViolationKind::Cpu => write!(f, "cpu"), } } } #[derive(Debug, Clone)] pub struct Violation { - pub agent_id: String, - pub kind: ViolationKind, - pub observed: f64, + pub agent_id: String, + pub kind: ViolationKind, + pub observed: f64, pub threshold: f64, } #[derive(Debug, Clone, Copy)] pub struct Thresholds { - pub max_sketch_size_bytes: f64, - pub max_error_rate: f64, + pub max_sketch_size_bytes: f64, + pub max_error_rate: f64, pub max_cpu_micros_per_sample: f64, } impl Default for Thresholds { fn default() -> Self { Self { - max_sketch_size_bytes: 5.0 * 1024.0 * 1024.0, // 5 MB - max_error_rate: 0.02, // 2 % - max_cpu_micros_per_sample: 5.0, // 5 µs/sample + max_sketch_size_bytes: 5.0 * 1024.0 * 1024.0, // 5 MB + max_error_rate: 0.02, // 2 % + max_cpu_micros_per_sample: 5.0, // 5 µs/sample } } } #[derive(Debug, Clone)] pub struct Endpoint { - pub agent_id: String, + pub agent_id: String, pub metrics_url: String, /// The sketch type currently deployed to this agent; used to attribute /// scraped metrics to the right EMA bucket. @@ -75,50 +75,54 @@ pub struct Endpoint { impl Endpoint { pub fn new(agent_id: impl Into, metrics_url: impl Into) -> Self { - Self { agent_id: agent_id.into(), metrics_url: metrics_url.into(), sketch_type: None } + Self { + agent_id: agent_id.into(), + metrics_url: metrics_url.into(), + sketch_type: None, + } } } /// Data reported to the `on_metrics` callback after each successful scrape. #[derive(Debug, Clone)] pub struct ScrapedData { - pub agent_id: String, + pub agent_id: String, /// The sketch type configured on this endpoint at scrape time (if known). - pub sketch_type: Option, + pub sketch_type: Option, /// Current total sketch size in bytes at the agent. - pub sketch_size_bytes: f64, + pub sketch_size_bytes: f64, /// Derived µs/sample over the last scrape window; `None` on the very first /// scrape because there is no previous baseline yet. pub cpu_micros_per_sample: Option, } -pub type OnViolationFn = Arc; -pub type OnMetricsFn = Arc; +pub type OnViolationFn = Arc; +pub type OnMetricsFn = Arc; // ── Scraper ─────────────────────────────────────────────────────────────────── pub struct Scraper { - endpoints: Arc>>, - thresholds: Thresholds, + endpoints: Arc>>, + thresholds: Thresholds, on_violation: OnViolationFn, - on_metrics: Option, - interval: Duration, - client: reqwest::Client, - last: Mutex>, + on_metrics: Option, + interval: Duration, + client: reqwest::Client, + last: Mutex>, } impl Scraper { pub fn new( - endpoints: Vec, - thresholds: Thresholds, + endpoints: Vec, + thresholds: Thresholds, on_violation: OnViolationFn, - interval: Duration, + interval: Duration, ) -> Self { Self { - endpoints: Arc::new(RwLock::new(endpoints)), + endpoints: Arc::new(RwLock::new(endpoints)), thresholds, on_violation, - on_metrics: None, + on_metrics: None, interval, client: reqwest::Client::builder() .timeout(Duration::from_secs(5)) @@ -175,26 +179,29 @@ impl Scraper { let endpoints = self.endpoints.read().await.clone(); for ep in &endpoints { match self.scrape(ep).await { - Ok(m) => self.analyze(&m, ep.sketch_type.as_ref()), + Ok(m) => self.analyze(&m, ep.sketch_type.as_ref()), Err(e) => warn!(agent = %ep.agent_id, "scrape failed: {e}"), } } } async fn scrape(&self, ep: &Endpoint) -> anyhow::Result { - let text = self.client + let text = self + .client .get(&ep.metrics_url) - .send().await + .send() + .await .context("GET metrics")? - .text().await + .text() + .await .context("read body")?; let mut m = CollectorMetrics { - agent_id: ep.agent_id.clone(), + agent_id: ep.agent_id.clone(), sketch_size_bytes: 0.0, cpu_seconds_total: 0.0, - samples_ingested: 0.0, - error_rate: 0.0, + samples_ingested: 0.0, + error_rate: 0.0, }; parse_prometheus_text(&text, &mut m); Ok(m) @@ -204,9 +211,9 @@ impl Scraper { // Bandwidth / sketch size. if m.sketch_size_bytes > self.thresholds.max_sketch_size_bytes { (self.on_violation)(Violation { - agent_id: m.agent_id.clone(), - kind: ViolationKind::Bandwidth, - observed: m.sketch_size_bytes, + agent_id: m.agent_id.clone(), + kind: ViolationKind::Bandwidth, + observed: m.sketch_size_bytes, threshold: self.thresholds.max_sketch_size_bytes, }); } @@ -214,9 +221,9 @@ impl Scraper { // Accuracy / error rate. if m.error_rate > self.thresholds.max_error_rate { (self.on_violation)(Violation { - agent_id: m.agent_id.clone(), - kind: ViolationKind::Accuracy, - observed: m.error_rate, + agent_id: m.agent_id.clone(), + kind: ViolationKind::Accuracy, + observed: m.error_rate, threshold: self.thresholds.max_error_rate, }); } @@ -224,15 +231,15 @@ impl Scraper { // CPU: compare δCPU/δsamples with the previous scrape. let mut last = self.last.lock().unwrap(); let cpu_micros = if let Some(prev) = last.get(&m.agent_id) { - let delta_samples = m.samples_ingested - prev.samples_ingested; - let delta_cpu = m.cpu_seconds_total - prev.cpu_seconds_total; + let delta_samples = m.samples_ingested - prev.samples_ingested; + let delta_cpu = m.cpu_seconds_total - prev.cpu_seconds_total; if delta_samples > 0.0 { let micros_per_sample = (delta_cpu / delta_samples) * 1e6; if micros_per_sample > self.thresholds.max_cpu_micros_per_sample { (self.on_violation)(Violation { - agent_id: m.agent_id.clone(), - kind: ViolationKind::Cpu, - observed: micros_per_sample, + agent_id: m.agent_id.clone(), + kind: ViolationKind::Cpu, + observed: micros_per_sample, threshold: self.thresholds.max_cpu_micros_per_sample, }); } @@ -249,9 +256,9 @@ impl Scraper { // Fire on_metrics callback so callers can feed EMA / telemetry. if let Some(cb) = &self.on_metrics { cb(ScrapedData { - agent_id: m.agent_id.clone(), - sketch_type: sketch_type.cloned(), - sketch_size_bytes: m.sketch_size_bytes, + agent_id: m.agent_id.clone(), + sketch_type: sketch_type.cloned(), + sketch_size_bytes: m.sketch_size_bytes, cpu_micros_per_sample: cpu_micros, }); } @@ -263,20 +270,26 @@ impl Scraper { fn parse_prometheus_text(text: &str, m: &mut CollectorMetrics) { for line in text.lines() { let line = line.trim(); - if line.is_empty() || line.starts_with('#') { continue; } + if line.is_empty() || line.starts_with('#') { + continue; + } // Handle lines with optional labels: metric_name{...} value [timestamp] // Split on whitespace to get name and value parts. let parts: Vec<&str> = line.splitn(2, ' ').collect(); - if parts.len() < 2 { continue; } + if parts.len() < 2 { + continue; + } // Strip label block {…} from the metric name, if any. let name = parts[0].split('{').next().unwrap_or(parts[0]); let val_str = parts[1].split_whitespace().next().unwrap_or(""); - let Ok(val) = val_str.parse::() else { continue }; + let Ok(val) = val_str.parse::() else { + continue; + }; match name { - "otelcol_sketch_size_bytes" => m.sketch_size_bytes = val, - "process_cpu_seconds_total" => m.cpu_seconds_total = val, - "otelcol_processor_accepted_metric_points" => m.samples_ingested = val, - "otelcol_sketch_error_rate" => m.error_rate = val, + "otelcol_sketch_size_bytes" => m.sketch_size_bytes = val, + "process_cpu_seconds_total" => m.cpu_seconds_total = val, + "otelcol_processor_accepted_metric_points" => m.samples_ingested = val, + "otelcol_sketch_error_rate" => m.error_rate = val, _ => {} } } @@ -287,7 +300,7 @@ fn parse_prometheus_text(text: &str, m: &mut CollectorMetrics) { #[cfg(test)] mod tests { use super::*; - use axum::{Router, routing::get}; + use axum::{routing::get, Router}; use tokio::net::TcpListener; const NORMAL_PAYLOAD: &str = " @@ -312,11 +325,12 @@ otelcol_sketch_error_rate 0.05 "; async fn serve_metrics(payload: &'static str) -> String { - let app = Router::new().route("/metrics", - get(move || async move { payload })); + let app = Router::new().route("/metrics", get(move || async move { payload })); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); - tokio::spawn(async move { axum::serve(listener, app).await.unwrap(); }); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); format!("http://{addr}/metrics") } @@ -381,7 +395,9 @@ otelcol_sketch_error_rate 0.05 })); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); - tokio::spawn(async move { axum::serve(listener, app).await.unwrap(); }); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); let violations: Arc>> = Arc::new(Mutex::new(vec![])); let v2 = Arc::clone(&violations); @@ -395,22 +411,21 @@ otelcol_sketch_error_rate 0.05 s.scrape_all().await; // delta → CPU violation let v = violations.lock().unwrap(); - assert!(v.iter().any(|vio| vio.kind == ViolationKind::Cpu), - "expected CPU violation, got: {v:?}"); + assert!( + v.iter().any(|vio| vio.kind == ViolationKind::Cpu), + "expected CPU violation, got: {v:?}" + ); } #[tokio::test] async fn multiple_endpoints_only_bad_violates() { - let ok_url = serve_metrics(NORMAL_PAYLOAD).await; + let ok_url = serve_metrics(NORMAL_PAYLOAD).await; let bad_url = serve_metrics(HIGH_BANDWIDTH_PAYLOAD).await; let violations: Arc>> = Arc::new(Mutex::new(vec![])); let v2 = Arc::clone(&violations); let s = Arc::new(Scraper::new( - vec![ - Endpoint::new("ok", ok_url), - Endpoint::new("bad", bad_url), - ], + vec![Endpoint::new("ok", ok_url), Endpoint::new("bad", bad_url)], Thresholds::default(), Arc::new(move |v| v2.lock().unwrap().push(v)), Duration::from_secs(60), @@ -418,8 +433,10 @@ otelcol_sketch_error_rate 0.05 s.scrape_all().await; let v = violations.lock().unwrap(); - assert!(v.iter().all(|vio| vio.agent_id == "bad"), - "only bad agent should violate: {v:?}"); + assert!( + v.iter().all(|vio| vio.agent_id == "bad"), + "only bad agent should violate: {v:?}" + ); assert!(v.iter().any(|vio| vio.agent_id == "bad")); } @@ -433,8 +450,8 @@ otelcol_sketch_error_rate 0.05 #[test] fn violation_kind_display() { assert_eq!(ViolationKind::Bandwidth.to_string(), "bandwidth"); - assert_eq!(ViolationKind::Accuracy.to_string(), "accuracy"); - assert_eq!(ViolationKind::Cpu.to_string(), "cpu"); + assert_eq!(ViolationKind::Accuracy.to_string(), "accuracy"); + assert_eq!(ViolationKind::Cpu.to_string(), "cpu"); } #[tokio::test] diff --git a/controller/src/opamp/mod.rs b/controller/src/opamp/mod.rs index c3deee5c..f2507762 100644 --- a/controller/src/opamp/mod.rs +++ b/controller/src/opamp/mod.rs @@ -34,16 +34,16 @@ pub mod opamp_proto { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RemoteConfig { pub config_hash: String, - pub yaml: String, + pub yaml: String, } /// Status report sent back from an agent after applying a config. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AgentStatus { - pub agent_id: String, + pub agent_id: String, pub config_hash: String, - pub healthy: bool, - pub error: Option, + pub healthy: bool, + pub error: Option, } // ── Role ────────────────────────────────────────────────────────────────────── @@ -72,7 +72,7 @@ impl AgentRole { match value.trim().to_lowercase().as_str() { "backend" => AgentRole::Backend, "gateway" => AgentRole::Gateway, - _ => AgentRole::Agent, + _ => AgentRole::Agent, } } } @@ -81,20 +81,20 @@ impl AgentRole { type AgentMap = HashMap, AgentRole)>; -pub type OnConnectFn = Arc; -pub type OnDisconnectFn = Arc; +pub type OnConnectFn = Arc; +pub type OnDisconnectFn = Arc; pub struct OpampServer { - agents: Arc>, - on_connect: Option, + agents: Arc>, + on_connect: Option, on_disconnect: Option, } impl Default for OpampServer { fn default() -> Self { Self { - agents: Arc::new(RwLock::new(HashMap::new())), - on_connect: None, + agents: Arc::new(RwLock::new(HashMap::new())), + on_connect: None, on_disconnect: None, } } @@ -103,18 +103,23 @@ impl Default for OpampServer { impl Clone for OpampServer { fn clone(&self) -> Self { Self { - agents: Arc::clone(&self.agents), - on_connect: self.on_connect.clone(), + agents: Arc::clone(&self.agents), + on_connect: self.on_connect.clone(), on_disconnect: self.on_disconnect.clone(), } } } impl OpampServer { - pub fn new() -> Self { Self::default() } + pub fn new() -> Self { + Self::default() + } /// Register a callback invoked when an agent connects. - pub fn with_on_connect(mut self, f: impl Fn(String, AgentRole) + Send + Sync + 'static) -> Self { + pub fn with_on_connect( + mut self, + f: impl Fn(String, AgentRole) + Send + Sync + 'static, + ) -> Self { self.on_connect = Some(Arc::new(f)); self } @@ -129,7 +134,7 @@ impl OpampServer { /// Agents must include `X-Agent-ID: ` in the upgrade request. /// Optional `X-Agent-Role: agent|backend` (default: `agent`). pub async fn ws_handler( - ws: WebSocketUpgrade, + ws: WebSocketUpgrade, headers: HeaderMap, State(srv): State>, ) -> impl IntoResponse { @@ -165,17 +170,24 @@ impl OpampServer { /// Broadcasts a config to every connected agent regardless of role. pub async fn push_all(&self, cfg: RemoteConfig) { let ids: Vec = self.agents.read().await.keys().cloned().collect(); - for id in ids { self.push(&id, cfg.clone()).await; } + for id in ids { + self.push(&id, cfg.clone()).await; + } } /// Broadcasts a config only to agents matching the given role. pub async fn push_to_role(&self, role: AgentRole, cfg: RemoteConfig) { - let ids: Vec = self.agents.read().await + let ids: Vec = self + .agents + .read() + .await .iter() .filter(|(_, (_, r))| *r == role) .map(|(id, _)| id.clone()) .collect(); - for id in ids { self.push(&id, cfg.clone()).await; } + for id in ids { + self.push(&id, cfg.clone()).await; + } } /// Returns the IDs of currently connected agents (all roles). @@ -185,16 +197,26 @@ impl OpampServer { /// Returns a map of agent_id → role for all connected agents. pub async fn connected_agents_with_roles(&self) -> HashMap { - self.agents.read().await + self.agents + .read() + .await .iter() .map(|(id, (_, role))| (id.clone(), role.clone())) .collect() } } -async fn handle_socket(socket: WebSocket, agent_id: String, role: AgentRole, srv: Arc) { +async fn handle_socket( + socket: WebSocket, + agent_id: String, + role: AgentRole, + srv: Arc, +) { let (tx, mut rx) = mpsc::channel::(16); - srv.agents.write().await.insert(agent_id.clone(), (tx, role.clone())); + srv.agents + .write() + .await + .insert(agent_id.clone(), (tx, role.clone())); info!(agent = %agent_id, ?role, "agent connected"); if let Some(cb) = &srv.on_connect { @@ -226,7 +248,9 @@ async fn handle_socket(socket: WebSocket, agent_id: String, role: AgentRole, srv let mut buf = Vec::with_capacity(1 + payload.len()); buf.push(0u8); buf.extend_from_slice(&payload); - if ws_tx.send(Message::Binary(buf.into())).await.is_err() { break; } + if ws_tx.send(Message::Binary(buf.into())).await.is_err() { + break; + } info!(agent = %writer_id, hash = %cfg.config_hash, "config pushed (OpAMP protobuf)"); } }); @@ -273,7 +297,7 @@ async fn handle_socket(socket: WebSocket, agent_id: String, role: AgentRole, srv if let Some(cm) = &ec.config_map { for (name, file) in &cm.config_map { let body_preview = String::from_utf8_lossy( - &file.body[..file.body.len().min(160)] + &file.body[..file.body.len().min(160)], ); info!( agent = %agent_id, @@ -334,12 +358,16 @@ async fn handle_socket(socket: WebSocket, agent_id: String, role: AgentRole, srv info!(agent = %agent_id, healthy = health.healthy, "agent health"); } } - Err(e) => warn!(agent = %agent_id, error = %e, "failed to decode AgentToServer"), + Err(e) => { + warn!(agent = %agent_id, error = %e, "failed to decode AgentToServer") + } } } // Also accept JSON for backward compatibility. Message::Text(text) => match serde_json::from_str::(&text) { - Ok(s) => info!(agent = %s.agent_id, healthy = s.healthy, "agent status (legacy JSON)"), + Ok(s) => { + info!(agent = %s.agent_id, healthy = s.healthy, "agent status (legacy JSON)") + } Err(_) => warn!(agent = %agent_id, "unexpected text message"), }, Message::Close(_) => break, @@ -430,15 +458,21 @@ fn encode_remote_config(cfg: &RemoteConfig) -> opamp_proto::ServerToAgent { #[cfg(test)] mod tests { use super::*; - use axum::{Router, routing::get}; + use axum::{routing::get, Router}; use tokio::net::TcpListener; #[tokio::test] async fn no_agents_push_returns_false() { let srv = Arc::new(OpampServer::new()); - let sent = srv.push("unknown", RemoteConfig { - config_hash: "h".into(), yaml: "y".into() - }).await; + let sent = srv + .push( + "unknown", + RemoteConfig { + config_hash: "h".into(), + yaml: "y".into(), + }, + ) + .await; assert!(!sent); } @@ -451,8 +485,8 @@ mod tests { #[test] fn role_from_header() { assert_eq!(AgentRole::from_header("backend"), AgentRole::Backend); - assert_eq!(AgentRole::from_header("agent"), AgentRole::Agent); - assert_eq!(AgentRole::from_header(""), AgentRole::Agent); + assert_eq!(AgentRole::from_header("agent"), AgentRole::Agent); + assert_eq!(AgentRole::from_header(""), AgentRole::Agent); assert_eq!(AgentRole::from_header("BACKEND"), AgentRole::Backend); } @@ -552,10 +586,14 @@ mod tests { tokio::time::sleep(std::time::Duration::from_millis(50)).await; let yaml_payload = "ddsketch:\n mode: window\n"; - srv.push_to_role(AgentRole::Agent, RemoteConfig { - config_hash: "hash-1".into(), - yaml: yaml_payload.to_string(), - }).await; + srv.push_to_role( + AgentRole::Agent, + RemoteConfig { + config_hash: "hash-1".into(), + yaml: yaml_payload.to_string(), + }, + ) + .await; let msg = tokio::time::timeout( std::time::Duration::from_secs(2), @@ -571,10 +609,17 @@ mod tests { let sta = decode_server_to_agent_frame(data.as_ref()); let rc = sta.remote_config.expect("should have remote_config"); let config = rc.config.expect("should have config"); - let file = config.config_map.get("").expect("should have empty-key entry"); + let file = config + .config_map + .get("") + .expect("should have empty-key entry"); let yaml = String::from_utf8(file.body.clone()).unwrap(); assert_eq!(yaml, yaml_payload, "delivered yaml must match"); - assert_eq!(String::from_utf8(rc.config_hash).unwrap(), "hash-1", "delivered hash must match"); + assert_eq!( + String::from_utf8(rc.config_hash).unwrap(), + "hash-1", + "delivered hash must match" + ); } /// Phase C integration test: gateway YAML emitted from the typed L5 @@ -588,7 +633,7 @@ mod tests { async fn push_to_role_gateway_routes_only_to_gateway_role() { use futures_util::StreamExt; let (srv, addr) = start_server().await; - let mut agent_ws = connect_ws_client(addr, "agent-1", "agent").await; + let mut agent_ws = connect_ws_client(addr, "agent-1", "agent").await; let mut gateway_ws = connect_ws_client(addr, "gateway-1", "gateway").await; let mut backend_ws = connect_ws_client(addr, "backend-1", "backend").await; @@ -599,36 +644,33 @@ mod tests { // We use a stand-in YAML payload here; the emitter has its own // tests in stage_config.rs. let yaml = "extensions:\n opamp: {}\n".to_string(); - srv.push_to_role(AgentRole::Gateway, RemoteConfig { - config_hash: "hash-gw".into(), - yaml, - }).await; + srv.push_to_role( + AgentRole::Gateway, + RemoteConfig { + config_hash: "hash-gw".into(), + yaml, + }, + ) + .await; // Gateway must receive exactly one frame. - let msg = tokio::time::timeout( - std::time::Duration::from_secs(2), - gateway_ws.next(), - ) - .await - .expect("gateway timed out") - .unwrap() - .unwrap(); + let msg = tokio::time::timeout(std::time::Duration::from_secs(2), gateway_ws.next()) + .await + .expect("gateway timed out") + .unwrap() + .unwrap(); let bytes = msg.into_data(); assert!(!bytes.is_empty(), "gateway must receive a non-empty frame"); // Other roles must receive nothing within a short window. - let agent_result = tokio::time::timeout( - std::time::Duration::from_millis(200), - agent_ws.next(), - ).await; + let agent_result = + tokio::time::timeout(std::time::Duration::from_millis(200), agent_ws.next()).await; assert!( agent_result.is_err(), "agent-role client must not receive gateway-role push" ); - let backend_result = tokio::time::timeout( - std::time::Duration::from_millis(200), - backend_ws.next(), - ).await; + let backend_result = + tokio::time::timeout(std::time::Duration::from_millis(200), backend_ws.next()).await; assert!( backend_result.is_err(), "backend-role client must not receive gateway-role push" @@ -640,36 +682,34 @@ mod tests { async fn push_to_agent_role_does_not_reach_backend_role() { use futures_util::StreamExt; let (srv, addr) = start_server().await; - let mut agent_ws = connect_ws_client(addr, "agent-1", "agent").await; + let mut agent_ws = connect_ws_client(addr, "agent-1", "agent").await; let mut backend_ws = connect_ws_client(addr, "backend-1", "backend").await; tokio::time::sleep(std::time::Duration::from_millis(50)).await; - srv.push_to_role(AgentRole::Agent, RemoteConfig { - config_hash: "hash-agent".into(), - yaml: "ddsketch:\n mode: batch\n".to_string(), - }).await; + srv.push_to_role( + AgentRole::Agent, + RemoteConfig { + config_hash: "hash-agent".into(), + yaml: "ddsketch:\n mode: batch\n".to_string(), + }, + ) + .await; // Agent must receive the message. - let msg = tokio::time::timeout( - std::time::Duration::from_secs(2), - agent_ws.next(), - ) - .await - .expect("agent timed out") - .unwrap() - .unwrap(); + let msg = tokio::time::timeout(std::time::Duration::from_secs(2), agent_ws.next()) + .await + .expect("agent timed out") + .unwrap() + .unwrap(); let data = msg.into_data(); let sta = decode_server_to_agent_frame(data.as_ref()); let rc = sta.remote_config.expect("should have remote_config"); assert_eq!(String::from_utf8(rc.config_hash).unwrap(), "hash-agent"); // Backend must receive nothing within a short window. - let backend_result = tokio::time::timeout( - std::time::Duration::from_millis(200), - backend_ws.next(), - ) - .await; + let backend_result = + tokio::time::timeout(std::time::Duration::from_millis(200), backend_ws.next()).await; assert!( backend_result.is_err(), "backend-role client must not receive agent-role push" diff --git a/controller/src/optimizer/baseline.rs b/controller/src/optimizer/baseline.rs index 50ff4648..d8d7a5a5 100644 --- a/controller/src/optimizer/baseline.rs +++ b/controller/src/optimizer/baseline.rs @@ -16,8 +16,8 @@ use std::collections::HashMap; use std::sync::{Arc, RwLock}; -use crate::types::{CollectionPlan, QueryWorkload, WorkloadCharacteristics}; use crate::optimizer::cost::CostModelPlanner; +use crate::types::{CollectionPlan, QueryWorkload, WorkloadCharacteristics}; pub struct BaselinePlanner { inner: CostModelPlanner, @@ -51,7 +51,10 @@ impl BaselinePlanner { // Slow path: first request for this metric — run cost optimisation. let plan = self.inner.plan(workload, wc); - self.cache.write().unwrap().insert(key.clone(), plan.clone()); + self.cache + .write() + .unwrap() + .insert(key.clone(), plan.clone()); plan } @@ -73,23 +76,23 @@ impl BaselinePlanner { #[cfg(test)] mod tests { use super::*; + use crate::types::AggType; use std::collections::HashMap; use std::time::Duration; - use crate::types::AggType; fn workload(metric: &str) -> QueryWorkload { QueryWorkload { - metric_name: metric.into(), - label_filters: HashMap::new(), - group_by_labels: vec![], - aggregations: vec![AggType::Quantile], - time_window: Duration::from_secs(300), - repeat_every: None, - accuracy_sla: 0.01, - latency_sla: None, + metric_name: metric.into(), + label_filters: HashMap::new(), + group_by_labels: vec![], + aggregations: vec![AggType::Quantile], + time_window: Duration::from_secs(300), + repeat_every: None, + accuracy_sla: 0.01, + latency_sla: None, sketch_type_override: None, - exact_required: false, - quantiles: vec![0.99], + exact_required: false, + quantiles: vec![0.99], } } @@ -110,14 +113,13 @@ mod tests { #[test] fn second_call_returns_same_plan() { let p = planner(); - let first = p.plan(&workload("latency"), None); + let first = p.plan(&workload("latency"), None); // Change the workload — the baseline planner must ignore it. let mut w2 = workload("latency"); w2.aggregations = vec![AggType::Cardinality]; let second = p.plan(&w2, None); assert_eq!( - first.agent_config.sketch_type, - second.agent_config.sketch_type, + first.agent_config.sketch_type, second.agent_config.sketch_type, "baseline plan must not change even when workload changes" ); } @@ -143,8 +145,7 @@ mod tests { // workload and should produce an equivalent plan. let second = p.plan(&workload("latency"), None); assert_eq!( - first.agent_config.sketch_type, - second.agent_config.sketch_type, + first.agent_config.sketch_type, second.agent_config.sketch_type, "same workload after reset should produce the same sketch type" ); } diff --git a/controller/src/optimizer/cost/delta.rs b/controller/src/optimizer/cost/delta.rs index 0a31f411..5a89d110 100644 --- a/controller/src/optimizer/cost/delta.rs +++ b/controller/src/optimizer/cost/delta.rs @@ -199,13 +199,16 @@ pub fn estimate_fill_rate( w: &QueryWorkload, ) -> f64 { let flush_secs = flush_period_secs(plan, w); - let inserts_per_flush = - wc.samples_per_sec_per_series * wc.series_count as f64 * flush_secs; + let inserts_per_flush = wc.samples_per_sec_per_series * wc.series_count as f64 * flush_secs; let distinct = estimate_distinct_keys(inserts_per_flush, wc); match &plan.agent_config.sketch_params { SketchParams::CountMinSketch { cols, .. } => { let cols = *cols as f64; - if cols > 0.0 { (distinct / cols).min(1.0) } else { 0.05 } + if cols > 0.0 { + (distinct / cols).min(1.0) + } else { + 0.05 + } } SketchParams::CountSketch { .. } => { // CountSketch uses epsilon-based sizing; approximate cols ≈ 1/ε². @@ -237,10 +240,18 @@ pub fn interpolate_compression(costs: &DeltaCosts, fill_rate: f64) -> f64 { costs.compression_at_fill_1pct } else if fill_rate <= 0.05 { let t = (fill_rate - 0.01) / (0.05 - 0.01); - lerp(costs.compression_at_fill_1pct, costs.compression_at_fill_5pct, t) + lerp( + costs.compression_at_fill_1pct, + costs.compression_at_fill_5pct, + t, + ) } else if fill_rate <= 0.20 { let t = (fill_rate - 0.05) / (0.20 - 0.05); - lerp(costs.compression_at_fill_5pct, costs.compression_at_fill_20pct, t) + lerp( + costs.compression_at_fill_5pct, + costs.compression_at_fill_20pct, + t, + ) } else { // Linear extrapolation toward 1.0 at 100 % fill. let t = ((fill_rate - 0.20) / 0.80).min(1.0); @@ -268,7 +279,10 @@ pub fn raw_bytes_per_sec(wc: &WorkloadCharacteristics) -> f64 { /// The dim_multiplier in the existing `PlanScore` captures the QUERY fanout /// (how many group-by combinations exist); for bandwidth estimation we treat /// `series_count` as the total sketch instances after aggregation. -pub fn sketch_full_bytes_per_sec(wc: &WorkloadCharacteristics, bytes_per_series_per_sec: f64) -> f64 { +pub fn sketch_full_bytes_per_sec( + wc: &WorkloadCharacteristics, + bytes_per_series_per_sec: f64, +) -> f64 { wc.series_count as f64 * bytes_per_series_per_sec } @@ -309,7 +323,11 @@ pub fn decide_delta( let raw_bw = raw_bytes_per_sec(wc); let full_bw = sketch_full_bytes_per_sec(wc, bytes_per_series_per_sec); let flush_secs = flush_period_secs(plan, w); - let flush_hz = if flush_secs > 0.0 { 1.0 / flush_secs } else { 1.0 }; + let flush_hz = if flush_secs > 0.0 { + 1.0 / flush_secs + } else { + 1.0 + }; // ── 1. Workload too small for sketching ────────────────────────────────── let total_sample_rate = wc.series_count as f64 * wc.samples_per_sec_per_series; @@ -394,8 +412,7 @@ pub fn decide_delta( // ── 6. Memory overhead ─────────────────────────────────────────────────── // One snapshot per sketch instance; the number of sketch instances // equals series_count × dim_mult (each group-by partition is separate). - let snapshot_mem = - wc.series_count as f64 * dim_mult * costs.snapshot_bytes_per_sketch as f64; + let snapshot_mem = wc.series_count as f64 * dim_mult * costs.snapshot_bytes_per_sketch as f64; let summary = TransmissionCostSummary { raw_bytes_per_sec: raw_bw, @@ -620,7 +637,10 @@ mod tests { let r5 = interpolate_compression(&costs, 0.05); let r20 = interpolate_compression(&costs, 0.20); let r80 = interpolate_compression(&costs, 0.80); - assert!(r1 >= r5, "compression should decrease as fill rises: {r1} vs {r5}"); + assert!( + r1 >= r5, + "compression should decrease as fill rises: {r1} vs {r5}" + ); assert!(r5 >= r20, "{r5} vs {r20}"); assert!(r20 >= r80, "{r20} vs {r80}"); } @@ -629,7 +649,10 @@ mod tests { fn compression_at_100pct_is_near_one() { let costs = delta_benchmark_table()[&SketchType::CountMinSketch]; let r = interpolate_compression(&costs, 1.0); - assert!(r <= 1.05, "at 100 % fill compression ratio should be ~1: {r}"); + assert!( + r <= 1.05, + "at 100 % fill compression ratio should be ~1: {r}" + ); } // ── decide_delta branches ───────────────────────────────────────────────── @@ -768,8 +791,7 @@ mod tests { let (_, s10) = decide_delta(&plan_10s, &w, &default_wc(), 200.0); let (_, s60) = decide_delta(&plan_60s, &w, &default_wc(), 200.0); assert!( - s60.delta_cpu_overhead_micros_per_sample - < s10.delta_cpu_overhead_micros_per_sample, + s60.delta_cpu_overhead_micros_per_sample < s10.delta_cpu_overhead_micros_per_sample, "longer flush period should lower per-sample CPU overhead: \ 10s={:.4}µs 60s={:.4}µs", s10.delta_cpu_overhead_micros_per_sample, diff --git a/controller/src/optimizer/cost/mod.rs b/controller/src/optimizer/cost/mod.rs index c103ab9f..14baa12a 100644 --- a/controller/src/optimizer/cost/mod.rs +++ b/controller/src/optimizer/cost/mod.rs @@ -95,7 +95,11 @@ pub struct PlanScore { } /// Estimates resource costs for a given plan + workload using the provided cost table. -pub fn score_with(plan: &CollectionPlan, w: &QueryWorkload, table: &HashMap) -> PlanScore { +pub fn score_with( + plan: &CollectionPlan, + w: &QueryWorkload, + table: &HashMap, +) -> PlanScore { let st = &plan.agent_config.sketch_type; let Some(&costs) = table.get(st) else { return PlanScore { @@ -111,7 +115,11 @@ pub fn score_with(plan: &CollectionPlan, w: &QueryWorkload, table: &HashMap PlanScore { fn estimate_error(_st: &SketchType, p: &SketchParams, costs: SketchCosts) -> f64 { match p { - SketchParams::DDSketch { relative_accuracy, .. } if *relative_accuracy > 0.0 => *relative_accuracy, + SketchParams::DDSketch { + relative_accuracy, .. + } if *relative_accuracy > 0.0 => *relative_accuracy, SketchParams::KLL { k, .. } if *k > 0 => 1.0 / *k as f64, - SketchParams::HLL { precision } if *precision > 0 => 1.04 / (2.0f64.powi(*precision as i32)).sqrt(), + SketchParams::HLL { precision } if *precision > 0 => { + 1.04 / (2.0f64.powi(*precision as i32)).sqrt() + } _ => costs.relative_error_at_default, } } @@ -177,13 +189,16 @@ fn estimate_error(_st: &SketchType, p: &SketchParams, costs: SketchCosts) -> f64 /// the planner blends live EMA observations into the cost table used for scoring, /// so that real-world behaviour gradually supersedes the static benchmark defaults. pub struct CostModelPlanner { - inner: RulesPlanner, + inner: RulesPlanner, online_store: Option, } impl CostModelPlanner { pub fn new() -> Self { - Self { inner: RulesPlanner::new(), online_store: None } + Self { + inner: RulesPlanner::new(), + online_store: None, + } } pub fn with_sketch_defaults(mut self, defaults: SketchDefaults) -> Self { @@ -201,7 +216,7 @@ impl CostModelPlanner { fn cost_table(&self) -> HashMap { match &self.online_store { Some(s) => online::effective_table(s), - None => benchmark_table_pub(), + None => benchmark_table_pub(), } } @@ -212,11 +227,7 @@ impl CostModelPlanner { /// CPU / memory overhead, and raw vs. sketch bandwidth comparison. /// Pass `None` to use conservative defaults (1 000 series, 100 Hz, /// 100 B/sample, Zipf distribution, no memory budget). - pub fn plan( - &self, - w: &QueryWorkload, - wc: Option<&WorkloadCharacteristics>, - ) -> CollectionPlan { + pub fn plan(&self, w: &QueryWorkload, wc: Option<&WorkloadCharacteristics>) -> CollectionPlan { let default_wc; let wc = match wc { Some(c) => c, @@ -280,9 +291,9 @@ impl CostModelPlanner { /// Runs the delta cost model and writes the decision into the plan using a provided cost table. fn apply_delta_decision_with( - plan: &mut CollectionPlan, - w: &QueryWorkload, - wc: &WorkloadCharacteristics, + plan: &mut CollectionPlan, + w: &QueryWorkload, + wc: &WorkloadCharacteristics, table: &HashMap, ) { let bytes_per_series_per_sec = table @@ -782,7 +793,8 @@ mod workload_cost_tests { let wc = workload_cost(&plan).unwrap(); // Per-root breakdown reports the standalone cost of `q`. - let standalone = subtree_cost_standalone(&q, &HashMap::new(), &BindingScope::new()).unwrap(); + let standalone = + subtree_cost_standalone(&q, &HashMap::new(), &BindingScope::new()).unwrap(); assert_eq!(wc.per_root_breakdown.len(), 1); assert_eq!(wc.per_root_breakdown[0].0, QueryId::new("q1")); assert!((wc.per_root_breakdown[0].1 - standalone).abs() < 1e-9); @@ -819,12 +831,18 @@ mod workload_cost_tests { #[test] fn workload_cost_two_roots_shared_window_credits_once() { let shared = windowed_scan(); - let q1 = quantile_root(0.99, QueryExpr::Ref { - name: BindingName::new("w"), - }); - let q2 = quantile_root(0.95, QueryExpr::Ref { - name: BindingName::new("w"), - }); + let q1 = quantile_root( + 0.99, + QueryExpr::Ref { + name: BindingName::new("w"), + }, + ); + let q2 = quantile_root( + 0.95, + QueryExpr::Ref { + name: BindingName::new("w"), + }, + ); let plan = WorkloadCostPlan { bindings: vec![(BindingName::new("w"), &shared)], @@ -862,12 +880,18 @@ mod workload_cost_tests { #[test] fn workload_cost_three_roots_two_share_partial() { let shared = windowed_scan(); - let q1 = quantile_root(0.99, QueryExpr::Ref { - name: BindingName::new("w"), - }); - let q2 = quantile_root(0.95, QueryExpr::Ref { - name: BindingName::new("w"), - }); + let q1 = quantile_root( + 0.99, + QueryExpr::Ref { + name: BindingName::new("w"), + }, + ); + let q2 = quantile_root( + 0.95, + QueryExpr::Ref { + name: BindingName::new("w"), + }, + ); // q3 builds its own scan + window — no shared producer. let q3 = max_root(windowed_scan()); @@ -899,12 +923,18 @@ mod workload_cost_tests { #[test] fn workload_cost_three_roots_all_share_one_binding() { let shared = windowed_scan(); - let q1 = quantile_root(0.99, QueryExpr::Ref { - name: BindingName::new("w"), - }); - let q2 = quantile_root(0.95, QueryExpr::Ref { - name: BindingName::new("w"), - }); + let q1 = quantile_root( + 0.99, + QueryExpr::Ref { + name: BindingName::new("w"), + }, + ); + let q2 = quantile_root( + 0.95, + QueryExpr::Ref { + name: BindingName::new("w"), + }, + ); let q3 = max_root(QueryExpr::Ref { name: BindingName::new("w"), }); @@ -951,9 +981,12 @@ mod workload_cost_tests { /// can refuse the plan rather than under-quote it. #[test] fn workload_cost_unresolved_ref_errors() { - let q = quantile_root(0.99, QueryExpr::Ref { - name: BindingName::new("missing"), - }); + let q = quantile_root( + 0.99, + QueryExpr::Ref { + name: BindingName::new("missing"), + }, + ); let plan = WorkloadCostPlan { bindings: vec![], roots: vec![(QueryId::new("q1"), &q)], diff --git a/controller/src/optimizer/cost/online.rs b/controller/src/optimizer/cost/online.rs index ffcfd9e6..6cd84b7e 100644 --- a/controller/src/optimizer/cost/online.rs +++ b/controller/src/optimizer/cost/online.rs @@ -52,17 +52,17 @@ impl OnlineSketchCosts { fn from_benchmark(base: &SketchCosts) -> Self { Self { bw_bytes_per_series_per_sec: base.bytes_per_series_per_sec, - cpu_micros_per_sample: base.cpu_micros_per_sample, - observations: 0, + cpu_micros_per_sample: base.cpu_micros_per_sample, + observations: 0, } } /// Incorporate a new observation. pub fn update(&mut self, observed_bw: f64, observed_cpu: f64) { - self.bw_bytes_per_series_per_sec = EMA_ALPHA * observed_bw - + (1.0 - EMA_ALPHA) * self.bw_bytes_per_series_per_sec; - self.cpu_micros_per_sample = EMA_ALPHA * observed_cpu - + (1.0 - EMA_ALPHA) * self.cpu_micros_per_sample; + self.bw_bytes_per_series_per_sec = + EMA_ALPHA * observed_bw + (1.0 - EMA_ALPHA) * self.bw_bytes_per_series_per_sec; + self.cpu_micros_per_sample = + EMA_ALPHA * observed_cpu + (1.0 - EMA_ALPHA) * self.cpu_micros_per_sample; self.observations += 1; } @@ -70,14 +70,14 @@ impl OnlineSketchCosts { /// The online weight grows with observation count, capping at 70 %. pub fn effective_costs(&self, base: &SketchCosts) -> SketchCosts { let online_w = (self.observations as f64 / MIN_OBS).min(1.0) * MAX_ONLINE_WEIGHT; - let bench_w = 1.0 - online_w; + let bench_w = 1.0 - online_w; SketchCosts { bytes_per_series_per_sec: online_w * self.bw_bytes_per_series_per_sec + bench_w * base.bytes_per_series_per_sec, cpu_micros_per_sample: online_w * self.cpu_micros_per_sample + bench_w * base.cpu_micros_per_sample, - base_memory_bytes: base.base_memory_bytes, - relative_error_at_default: base.relative_error_at_default, + base_memory_bytes: base.base_memory_bytes, + relative_error_at_default: base.relative_error_at_default, } } } @@ -100,18 +100,18 @@ pub fn init_store() -> OnlineMetricsStore { /// Update the EMA for `sketch_type` with a new bandwidth/cpu observation. /// If `sketch_type` is not yet in the store it is inserted from the benchmark. pub async fn update( - store: &OnlineMetricsStore, - sketch_type: &SketchType, - observed_bw: f64, + store: &OnlineMetricsStore, + sketch_type: &SketchType, + observed_bw: f64, observed_cpu: f64, ) { let mut map = store.write().await; let entry = map.entry(sketch_type.clone()).or_insert_with(|| { let base = benchmark_table_pub(); let costs = base.get(sketch_type).cloned().unwrap_or(SketchCosts { - bytes_per_series_per_sec: 200.0, - cpu_micros_per_sample: 1.0, - base_memory_bytes: 4096.0, + bytes_per_series_per_sec: 200.0, + cpu_micros_per_sample: 1.0, + base_memory_bytes: 4096.0, relative_error_at_default: 0.01, }); OnlineSketchCosts::from_benchmark(&costs) @@ -147,9 +147,9 @@ mod tests { fn base() -> SketchCosts { SketchCosts { - bytes_per_series_per_sec: 100.0, - cpu_micros_per_sample: 1.0, - base_memory_bytes: 4096.0, + bytes_per_series_per_sec: 100.0, + cpu_micros_per_sample: 1.0, + base_memory_bytes: 4096.0, relative_error_at_default: 0.01, } } @@ -171,9 +171,11 @@ mod tests { // Online weight = 1.0 × 0.70 = 0.70 // effective = 0.70 × online_ema + 0.30 × benchmark // online_ema after 10 EMA steps from 100→200: converging toward 200 - assert!(eff.bytes_per_series_per_sec > 100.0, + assert!( + eff.bytes_per_series_per_sec > 100.0, "effective bw should exceed benchmark after high observations: {}", - eff.bytes_per_series_per_sec); + eff.bytes_per_series_per_sec + ); } #[test] @@ -183,8 +185,11 @@ mod tests { let eff = o.effective_costs(&base()); // online_weight = (1/5).min(1.0) × 0.70 = 0.14 // should only move a little from benchmark - assert!(eff.bytes_per_series_per_sec < 200.0, - "single spike should not dominate: {}", eff.bytes_per_series_per_sec); + assert!( + eff.bytes_per_series_per_sec < 200.0, + "single spike should not dominate: {}", + eff.bytes_per_series_per_sec + ); } #[tokio::test] diff --git a/controller/src/optimizer/cost/pareto.rs b/controller/src/optimizer/cost/pareto.rs index 20501faf..2c1d1e3c 100644 --- a/controller/src/optimizer/cost/pareto.rs +++ b/controller/src/optimizer/cost/pareto.rs @@ -23,9 +23,9 @@ use std::collections::HashMap; -use crate::optimizer::cost::{benchmark_table_pub, score_with, SketchCosts}; use crate::optimizer::cost::delta::decide_delta; use crate::optimizer::cost::online; +use crate::optimizer::cost::{benchmark_table_pub, score_with, SketchCosts}; use crate::optimizer::rules::{default_sketch_params, select_window_strategy, RulesPlanner}; use crate::types::*; @@ -45,7 +45,11 @@ pub struct ObjectiveWeights { impl Default for ObjectiveWeights { fn default() -> Self { - Self { bandwidth: 1.0, cpu: 0.0, memory: 0.0 } + Self { + bandwidth: 1.0, + cpu: 0.0, + memory: 0.0, + } } } @@ -53,12 +57,16 @@ impl ObjectiveWeights { fn normalised(self) -> Self { let total = self.bandwidth + self.cpu + self.memory; if total <= 0.0 { - return Self { bandwidth: 1.0, cpu: 0.0, memory: 0.0 }; + return Self { + bandwidth: 1.0, + cpu: 0.0, + memory: 0.0, + }; } Self { bandwidth: self.bandwidth / total, - cpu: self.cpu / total, - memory: self.memory / total, + cpu: self.cpu / total, + memory: self.memory / total, } } } @@ -66,13 +74,13 @@ impl ObjectiveWeights { /// A single candidate on the Pareto frontier. #[derive(Debug, Clone)] pub struct ParetoPoint { - pub plan: CollectionPlan, - pub sketch_type: SketchType, + pub plan: CollectionPlan, + pub sketch_type: SketchType, pub bandwidth_bytes_per_sec: f64, - pub cpu_micros_per_sample: f64, - pub memory_bytes: f64, - pub estimated_error: f64, - pub meets_sla: bool, + pub cpu_micros_per_sample: f64, + pub memory_bytes: f64, + pub estimated_error: f64, + pub meets_sla: bool, } // ── Frontier computation ────────────────────────────────────────────────────── @@ -85,14 +93,14 @@ pub struct ParetoPoint { /// If `online_store` is `Some` the scoring uses EMA-blended costs; otherwise /// it falls back to the static benchmark table. pub fn pareto_frontier( - workload: &QueryWorkload, - wc: &WorkloadCharacteristics, - weights: ObjectiveWeights, + workload: &QueryWorkload, + wc: &WorkloadCharacteristics, + weights: ObjectiveWeights, online_store: Option<&online::OnlineMetricsStore>, ) -> Vec { let table: HashMap = match online_store { Some(s) => online::effective_table(s), - None => benchmark_table_pub(), + None => benchmark_table_pub(), }; let candidates = all_sketch_types(); @@ -104,9 +112,9 @@ pub fn pareto_frontier( let (mode, window_duration) = select_window_strategy(workload); let mut plan = rules.plan(workload); - plan.agent_config.sketch_type = st.clone(); - plan.agent_config.sketch_params = params; - plan.agent_config.mode = mode; + plan.agent_config.sketch_type = st.clone(); + plan.agent_config.sketch_params = params; + plan.agent_config.mode = mode; plan.agent_config.window_duration = window_duration; plan.backend_config.merge_sketch_type = st.clone(); @@ -114,15 +122,17 @@ pub fn pareto_frontier( apply_delta(st.clone(), &mut plan, workload, wc, &table); let s = score_with(&plan, workload, &table); - if !s.meets_sla { continue; } + if !s.meets_sla { + continue; + } points.push(ParetoPoint { - sketch_type: st, + sketch_type: st, bandwidth_bytes_per_sec: s.bandwidth_bytes_per_sec, - cpu_micros_per_sample: s.cpu_micros_per_sample, - memory_bytes: s.memory_bytes, - estimated_error: s.estimated_error, - meets_sla: s.meets_sla, + cpu_micros_per_sample: s.cpu_micros_per_sample, + memory_bytes: s.memory_bytes, + estimated_error: s.estimated_error, + meets_sla: s.meets_sla, plan, }); } @@ -134,7 +144,8 @@ pub fn pareto_frontier( let w = weights.normalised(); let mut sorted = optimal; sorted.sort_by(|a, b| { - weighted_score(a, w).partial_cmp(&weighted_score(b, w)) + weighted_score(a, w) + .partial_cmp(&weighted_score(b, w)) .unwrap_or(std::cmp::Ordering::Equal) }); sorted @@ -145,7 +156,8 @@ pub fn pareto_frontier( pub fn select_best(frontier: &[ParetoPoint], weights: ObjectiveWeights) -> Option<&ParetoPoint> { let w = weights.normalised(); frontier.iter().min_by(|a, b| { - weighted_score(a, w).partial_cmp(&weighted_score(b, w)) + weighted_score(a, w) + .partial_cmp(&weighted_score(b, w)) .unwrap_or(std::cmp::Ordering::Equal) }) } @@ -163,11 +175,11 @@ fn all_sketch_types() -> Vec { } fn apply_delta( - st: SketchType, - plan: &mut CollectionPlan, - w: &QueryWorkload, - wc: &WorkloadCharacteristics, - table: &HashMap, + st: SketchType, + plan: &mut CollectionPlan, + w: &QueryWorkload, + wc: &WorkloadCharacteristics, + table: &HashMap, ) { let bytes_per_series_per_sec = table .get(&st) @@ -179,14 +191,14 @@ fn apply_delta( match &decision { DeltaDecision::UseDelta { threshold, .. } => { plan.agent_config.delta_transmission = true; - plan.agent_config.delta_threshold = *threshold; + plan.agent_config.delta_threshold = *threshold; } _ => { plan.agent_config.delta_transmission = false; - plan.agent_config.delta_threshold = 0.0; + plan.agent_config.delta_threshold = 0.0; } } - plan.delta_decision = decision; + plan.delta_decision = decision; plan.transmission_cost_summary = summary; } @@ -197,10 +209,10 @@ fn pareto_filter(points: &[ParetoPoint]) -> Vec { !points.iter().any(|q| { q.bandwidth_bytes_per_sec <= p.bandwidth_bytes_per_sec && q.cpu_micros_per_sample <= p.cpu_micros_per_sample - && q.memory_bytes <= p.memory_bytes + && q.memory_bytes <= p.memory_bytes && (q.bandwidth_bytes_per_sec < p.bandwidth_bytes_per_sec || q.cpu_micros_per_sample < p.cpu_micros_per_sample - || q.memory_bytes < p.memory_bytes) + || q.memory_bytes < p.memory_bytes) }) }) .cloned() @@ -223,21 +235,23 @@ mod tests { fn quantile_workload() -> QueryWorkload { QueryWorkload { - metric_name: "latency".into(), - label_filters: HashMap::new(), - group_by_labels: vec![], - aggregations: vec![AggType::Quantile], - time_window: Duration::from_secs(300), - repeat_every: None, - accuracy_sla: 0.02, - latency_sla: None, + metric_name: "latency".into(), + label_filters: HashMap::new(), + group_by_labels: vec![], + aggregations: vec![AggType::Quantile], + time_window: Duration::from_secs(300), + repeat_every: None, + accuracy_sla: 0.02, + latency_sla: None, sketch_type_override: None, - exact_required: false, - quantiles: vec![], + exact_required: false, + quantiles: vec![], } } - fn default_wc() -> WorkloadCharacteristics { WorkloadCharacteristics::default() } + fn default_wc() -> WorkloadCharacteristics { + WorkloadCharacteristics::default() + } #[test] fn quantile_frontier_non_empty_and_kll_present() { @@ -245,36 +259,65 @@ mod tests { // dominated and excluded. KLL and HLL form the frontier (HLL has // lower BW/CPU but higher memory than KLL). let f = pareto_frontier( - &quantile_workload(), &default_wc(), - ObjectiveWeights::default(), None, + &quantile_workload(), + &default_wc(), + ObjectiveWeights::default(), + None, + ); + assert!( + !f.is_empty(), + "frontier should not be empty for quantile workload" ); - assert!(!f.is_empty(), "frontier should not be empty for quantile workload"); let types: Vec<_> = f.iter().map(|p| &p.sketch_type).collect(); - assert!(types.contains(&&SketchType::KLL), "KLL expected in frontier"); + assert!( + types.contains(&&SketchType::KLL), + "KLL expected in frontier" + ); } #[test] fn all_frontier_points_meet_sla() { let f = pareto_frontier( - &quantile_workload(), &default_wc(), - ObjectiveWeights::default(), None, + &quantile_workload(), + &default_wc(), + ObjectiveWeights::default(), + None, ); for p in &f { - assert!(p.meets_sla, "{} does not meet SLA (error={})", p.sketch_type, p.estimated_error); + assert!( + p.meets_sla, + "{} does not meet SLA (error={})", + p.sketch_type, p.estimated_error + ); } } #[test] fn bandwidth_weight_selects_lowest_bw() { let f = pareto_frontier( - &quantile_workload(), &default_wc(), - ObjectiveWeights { bandwidth: 1.0, cpu: 0.0, memory: 0.0 }, None, + &quantile_workload(), + &default_wc(), + ObjectiveWeights { + bandwidth: 1.0, + cpu: 0.0, + memory: 0.0, + }, + None, ); - let best = select_best(&f, ObjectiveWeights { bandwidth: 1.0, cpu: 0.0, memory: 0.0 }).unwrap(); + let best = select_best( + &f, + ObjectiveWeights { + bandwidth: 1.0, + cpu: 0.0, + memory: 0.0, + }, + ) + .unwrap(); for p in &f { assert!( p.bandwidth_bytes_per_sec >= best.bandwidth_bytes_per_sec, - "best ({}) should have minimum bandwidth", best.sketch_type + "best ({}) should have minimum bandwidth", + best.sketch_type ); } } @@ -282,14 +325,29 @@ mod tests { #[test] fn memory_weight_selects_lowest_memory() { let f = pareto_frontier( - &quantile_workload(), &default_wc(), - ObjectiveWeights { bandwidth: 0.0, cpu: 0.0, memory: 1.0 }, None, + &quantile_workload(), + &default_wc(), + ObjectiveWeights { + bandwidth: 0.0, + cpu: 0.0, + memory: 1.0, + }, + None, ); - let best = select_best(&f, ObjectiveWeights { bandwidth: 0.0, cpu: 0.0, memory: 1.0 }).unwrap(); + let best = select_best( + &f, + ObjectiveWeights { + bandwidth: 0.0, + cpu: 0.0, + memory: 1.0, + }, + ) + .unwrap(); for p in &f { assert!( p.memory_bytes >= best.memory_bytes, - "best ({}) should have minimum memory", best.sketch_type + "best ({}) should have minimum memory", + best.sketch_type ); } } @@ -308,32 +366,42 @@ mod tests { }; let f = pareto_frontier(&w, &default_wc(), ObjectiveWeights::default(), None); for p in &f { - assert!(p.estimated_error <= 0.001, - "{} error {} exceeds 0.001 SLA", p.sketch_type, p.estimated_error); + assert!( + p.estimated_error <= 0.001, + "{} error {} exceeds 0.001 SLA", + p.sketch_type, + p.estimated_error + ); } } #[test] fn pareto_no_dominated_points() { let f = pareto_frontier( - &quantile_workload(), &default_wc(), - ObjectiveWeights::default(), None, + &quantile_workload(), + &default_wc(), + ObjectiveWeights::default(), + None, ); // Verify no point in f is dominated by another. for i in 0..f.len() { for j in 0..f.len() { - if i == j { continue; } + if i == j { + continue; + } let a = &f[i]; let b = &f[j]; - let b_dominates_a = - b.bandwidth_bytes_per_sec <= a.bandwidth_bytes_per_sec + let b_dominates_a = b.bandwidth_bytes_per_sec <= a.bandwidth_bytes_per_sec && b.cpu_micros_per_sample <= a.cpu_micros_per_sample - && b.memory_bytes <= a.memory_bytes + && b.memory_bytes <= a.memory_bytes && (b.bandwidth_bytes_per_sec < a.bandwidth_bytes_per_sec || b.cpu_micros_per_sample < a.cpu_micros_per_sample - || b.memory_bytes < a.memory_bytes); - assert!(!b_dominates_a, "{} dominates {} but both are in frontier", - b.sketch_type, a.sketch_type); + || b.memory_bytes < a.memory_bytes); + assert!( + !b_dominates_a, + "{} dominates {} but both are in frontier", + b.sketch_type, a.sketch_type + ); } } } diff --git a/controller/src/optimizer/cost/tco.rs b/controller/src/optimizer/cost/tco.rs index ef09666c..ac539d80 100644 --- a/controller/src/optimizer/cost/tco.rs +++ b/controller/src/optimizer/cost/tco.rs @@ -146,9 +146,8 @@ fn compute_before(w: &TcoWorkload, p: &CloudPricing) -> TcoBefore { }; // Ingestion: Grafana charges per 1000 active series at 1 DPM. - let ingestion = (w.series_count as f64 / 1000.0) - * p.grafana_per_1k_series_1dpm - * dpm_multiplier; + let ingestion = + (w.series_count as f64 / 1000.0) * p.grafana_per_1k_series_1dpm * dpm_multiplier; // Storage: raw bytes over retention period. let samples_per_day = w.samples_per_sec * 86_400.0; @@ -269,8 +268,14 @@ mod tests { let pricing = CloudPricing::default(); let est = estimate_tco(&workload, &pricing); - assert!(est.before.total_dollars > 0.0, "before total should be positive"); - assert!(est.after.total_dollars > 0.0, "after total should be positive"); + assert!( + est.before.total_dollars > 0.0, + "before total should be positive" + ); + assert!( + est.after.total_dollars > 0.0, + "after total should be positive" + ); assert!( est.savings_percent > 50.0, "expected >50% savings for 100K series, got {:.1}%", @@ -296,14 +301,18 @@ mod tests { let est = estimate_tco(&workload, &pricing); assert!(est.before.total_dollars > est.after.total_dollars); - assert!(est.savings_percent > 50.0, - "expected >50% savings at 1M series, got {:.1}%", est.savings_percent); + assert!( + est.savings_percent > 50.0, + "expected >50% savings at 1M series, got {:.1}%", + est.savings_percent + ); // At 1M series we need ceil(1M/100K) = 10 instances. let expected_compute = pricing.ec2_sketch_instance_per_hour * HOURS_PER_MONTH * 10.0; assert!( (est.after.compute_dollars - expected_compute).abs() < 0.01, "compute should be ~${:.2}, got ${:.2}", - expected_compute, est.after.compute_dollars + expected_compute, + est.after.compute_dollars ); } @@ -326,7 +335,10 @@ mod tests { "storage should be 0" ); // After still has a minimum 1-instance compute cost. - assert!(est.after.compute_dollars > 0.0, "compute has a 1-instance minimum"); + assert!( + est.after.compute_dollars > 0.0, + "compute has a 1-instance minimum" + ); // But sketch/s3 ingestion should be zero. assert!( est.after.sketch_ingestion_dollars.abs() < f64::EPSILON, diff --git a/controller/src/optimizer/cost/wire.rs b/controller/src/optimizer/cost/wire.rs index 8c91bd18..7e8d9106 100644 --- a/controller/src/optimizer/cost/wire.rs +++ b/controller/src/optimizer/cost/wire.rs @@ -182,11 +182,7 @@ impl WireWorkload { /// [`WorkloadCharacteristics`] + a window duration. Useful for the /// planner's wire-cost decision when the caller already has a /// `WorkloadCharacteristics` for the delta model. - pub fn from_chars( - wc: &WorkloadCharacteristics, - window_secs: u64, - accuracy_sla: f64, - ) -> Self { + pub fn from_chars(wc: &WorkloadCharacteristics, window_secs: u64, accuracy_sla: f64) -> Self { let samples_per_window = (wc.samples_per_sec_per_series * window_secs as f64).round() as u64; Self { @@ -322,8 +318,7 @@ pub fn est_wire_bytes_per_window_per_series( table: &WireCostTable, ) -> u64 { match mode { - BindMode::SketchAtEdge { family } - | BindMode::RawAtEdgeSketchAtBackend { family } => { + BindMode::SketchAtEdge { family } | BindMode::RawAtEdgeSketchAtBackend { family } => { // Same edge → backend wire footprint either way (the sketch // state crosses the gateway in mode 1, the raw samples then // sketched do in mode 2; backend ingest cost is mode-2-higher @@ -359,10 +354,18 @@ mod tests { #[test] fn break_even_table_at_50_bytes_per_sample() { let t = WireCostTable::default(); - assert_eq!(break_even_samples(t.ddsketch_delta, 50), 16, "DDSketch+delta"); + assert_eq!( + break_even_samples(t.ddsketch_delta, 50), + 16, + "DDSketch+delta" + ); assert_eq!(break_even_samples(t.kll_full, 50), 64, "KLL full"); assert_eq!(break_even_samples(t.hll_delta, 50), 204, "HLL+delta"); - assert_eq!(break_even_samples(t.count_min_delta, 50), 84, "Count-Min+delta"); + assert_eq!( + break_even_samples(t.count_min_delta, 50), + 84, + "Count-Min+delta" + ); assert_eq!( break_even_samples(t.count_sketch_delta, 50), 5_004, @@ -378,11 +381,8 @@ mod tests { let table = WireCostTable::default(); let w = WireWorkload::default_phase_eps_1(); // 60 samples × 50 B = 3 000 B raw. - let raw = est_wire_bytes_per_window_per_series( - &BindMode::RawAtEdgePrometheusArchive, - &w, - &table, - ); + let raw = + est_wire_bytes_per_window_per_series(&BindMode::RawAtEdgePrometheusArchive, &w, &table); assert_eq!(raw, 3_000); // DDSketch state — 600 + 200 = 800 B. Sketch wins. let ddsketch = est_wire_bytes_per_window_per_series( diff --git a/controller/src/optimizer/mod.rs b/controller/src/optimizer/mod.rs index 5e409cd0..4b5be375 100644 --- a/controller/src/optimizer/mod.rs +++ b/controller/src/optimizer/mod.rs @@ -24,10 +24,10 @@ //! | `planner/rules.rs` | [`rules`] (shared rule library) | //! | `planner/baseline_planner.rs` | [`baseline`] | -pub mod engine; +pub mod baseline; pub mod cost; +pub mod engine; pub mod rules; -pub mod baseline; pub mod trait_def; // Re-exports — preserve the surface that `crate::algebra::QueryOptimizer` diff --git a/controller/src/optimizer/rules/mod.rs b/controller/src/optimizer/rules/mod.rs index 60621ad8..12225100 100644 --- a/controller/src/optimizer/rules/mod.rs +++ b/controller/src/optimizer/rules/mod.rs @@ -62,11 +62,9 @@ pub fn typed_sketch_algebra_enabled() -> bool { /// the parallel `USE_TYPED_STAGE_SPLIT` gate is enabled — the bound /// `SketchExpr` is then fed into `planner::stage_split::split_typed_three_stage` /// + the per-stage emitters in `config::stage_config`. -pub fn bind_workload_typed( - w: &QueryWorkload, -) -> Option { - use crate::intent_algebra::{AggIntent as L3AggIntent, QueryExpr, Schema, Source, WindowKind}; +pub fn bind_workload_typed(w: &QueryWorkload) -> Option { use crate::intent_algebra::schema::{Column, DataType}; + use crate::intent_algebra::{AggIntent as L3AggIntent, QueryExpr, Schema, Source, WindowKind}; use crate::sketch_algebra::capability_matching::{ classify_demo_metric, is_valid_pair, pick_family, AccuracyPreference, StatisticClass, }; @@ -104,8 +102,8 @@ pub fn bind_workload_typed( // Priority: workload-spec metric-name match → AggType-driven // default. The metric-name match owns the demo contract rows; the // AggType fallback covers everything else. - let (statistic, accuracy_pref) = classify_demo_metric(&w.metric_name) - .unwrap_or_else(|| match w.aggregations[0] { + let (statistic, accuracy_pref) = + classify_demo_metric(&w.metric_name).unwrap_or_else(|| match w.aggregations[0] { AggType::Quantile => (StatisticClass::Quantile, AccuracyPreference::RelativeError), AggType::Cardinality => (StatisticClass::Cardinality, AccuracyPreference::default()), AggType::Frequency => (StatisticClass::Frequency, AccuracyPreference::default()), @@ -328,7 +326,12 @@ impl RulesPlanner { } let sketch_type = crate::physical::sketch_catalog::sketch_type_for_agg(&w.aggregations); - let sketch_params = crate::physical::sketch_catalog::build_sketch_params(&self.sketch_defaults, &sketch_type, w.accuracy_sla, &w.quantiles); + let sketch_params = crate::physical::sketch_catalog::build_sketch_params( + &self.sketch_defaults, + &sketch_type, + w.accuracy_sla, + &w.quantiles, + ); let (mode, window_duration) = select_window_strategy(w); let mut aggregate_by = w.group_by_labels.clone(); @@ -383,8 +386,7 @@ impl RulesPlanner { /// Returns a raw-passthrough plan for queries that require exact per-sample /// computation (RSI, MACD, stochastic oscillator, etc.). fn raw_passthrough_plan(&self, w: &QueryWorkload) -> CollectionPlan { - let valid_until = Utc::now() - + chrono::Duration::seconds(self.valid_for.as_secs() as i64); + let valid_until = Utc::now() + chrono::Duration::seconds(self.valid_for.as_secs() as i64); let mut label_matchers: Vec = w .label_filters @@ -395,18 +397,18 @@ impl RulesPlanner { CollectionPlan { agent_config: AgentCollectorConfig { - output_mode: OutputMode::Raw, - sketch_type: SketchType::DDSketch, // unused for raw mode - sketch_params: SketchParams::default(), - aggregate_by: vec![], + output_mode: OutputMode::Raw, + sketch_type: SketchType::DDSketch, // unused for raw mode + sketch_params: SketchParams::default(), + aggregate_by: vec![], label_matchers, - window_duration: None, - mode: ProcessorMode::Batch, + window_duration: None, + mode: ProcessorMode::Batch, enable_self_monitoring: true, - transmit_sketch: false, - drop_original: false, - delta_transmission: false, - delta_threshold: 0.0, + transmit_sketch: false, + drop_original: false, + delta_transmission: false, + delta_threshold: 0.0, enable_series_id: true, series_id_ttl_secs: 0, @@ -415,11 +417,11 @@ impl RulesPlanner { gateway_config: GatewayCollectorConfig { passthrough: true }, backend_config: BackendCollectorConfig { merge_sketch_type: SketchType::DDSketch, - group_by: vec![], + group_by: vec![], }, - precompute: vec![], + precompute: vec![], valid_until, - delta_decision: DeltaDecision::default(), + delta_decision: DeltaDecision::default(), transmission_cost_summary: TransmissionCostSummary::default(), staged_plan: None, } @@ -428,7 +430,7 @@ impl RulesPlanner { // ── Sketch selection (delegated to algebra::directory) ─────────────────────── -pub use crate::physical::sketch_catalog::{default_sketch_params, build_sketch_params}; +pub use crate::physical::sketch_catalog::{build_sketch_params, default_sketch_params}; // ── Window strategy ─────────────────────────────────────────────────────────── @@ -554,7 +556,9 @@ mod tests { w.accuracy_sla = 0.005; let plan = RulesPlanner::new().plan(&w); match &plan.agent_config.sketch_params { - SketchParams::DDSketch { relative_accuracy, .. } => assert_eq!(*relative_accuracy, 0.005), + SketchParams::DDSketch { + relative_accuracy, .. + } => assert_eq!(*relative_accuracy, 0.005), other => panic!("expected DDSketch, got {:?}", other), } } @@ -565,7 +569,9 @@ mod tests { w.accuracy_sla = 0.03; let plan = RulesPlanner::new().plan(&w); match &plan.agent_config.sketch_params { - SketchParams::HLL { precision } => assert_eq!(*precision, 10, "coarse SLA should use lower precision"), + SketchParams::HLL { precision } => { + assert_eq!(*precision, 10, "coarse SLA should use lower precision") + } other => panic!("expected HLL, got {:?}", other), } } @@ -619,9 +625,7 @@ mod tests { match expr { SketchExpr::SketchAgg { sketch_type, .. } => Some(sketch_type.clone()), SketchExpr::SketchEstimate { child, .. } => extract_family(child), - SketchExpr::SketchMerge { children, .. } => { - children.iter().find_map(extract_family) - } + SketchExpr::SketchMerge { children, .. } => children.iter().find_map(extract_family), SketchExpr::LetBinding { expr, child, .. } => { extract_family(expr).or_else(|| extract_family(child)) } @@ -777,8 +781,8 @@ mod tests { // back to the canonical CountSketch default. let mut w = workload_for("top_endpoint_qps", AggType::Frequency); w.sketch_type_override = Some(SketchType::CountMinSketch); - let bound = bind_workload_typed(&w) - .expect("CountMin override on a TopK metric should still bind"); + let bound = + bind_workload_typed(&w).expect("CountMin override on a TopK metric should still bind"); assert_eq!( extract_family(&bound), Some(SketchKind::Cms), @@ -823,11 +827,31 @@ mod tests { // ("verify each produces the expected `SketchExpr` family"). let cases: Vec<(&str, AggType, Option)> = vec![ ("http_requests_total", AggType::Frequency, None), - ("http_latency_ms", AggType::Quantile, Some(SketchKind::DDSketch)), - ("request_size_bytes", AggType::Quantile, Some(SketchKind::Kll)), - ("unique_users_per_min", AggType::Cardinality, Some(SketchKind::Hll)), - ("top_endpoint_qps", AggType::Frequency, Some(SketchKind::CountSketch)), - ("endpoint_request_freq", AggType::Frequency, Some(SketchKind::Cms)), + ( + "http_latency_ms", + AggType::Quantile, + Some(SketchKind::DDSketch), + ), + ( + "request_size_bytes", + AggType::Quantile, + Some(SketchKind::Kll), + ), + ( + "unique_users_per_min", + AggType::Cardinality, + Some(SketchKind::Hll), + ), + ( + "top_endpoint_qps", + AggType::Frequency, + Some(SketchKind::CountSketch), + ), + ( + "endpoint_request_freq", + AggType::Frequency, + Some(SketchKind::Cms), + ), ]; for (metric, agg, expected) in cases { let w = workload_for(metric, agg); diff --git a/controller/src/physical/colored_dag/allocator.rs b/controller/src/physical/colored_dag/allocator.rs index 6eae6bd6..a54e8e3d 100644 --- a/controller/src/physical/colored_dag/allocator.rs +++ b/controller/src/physical/colored_dag/allocator.rs @@ -38,9 +38,9 @@ use std::collections::HashMap; -use crate::sketch_algebra::SketchExpr; use crate::physical::colored_dag::dag::{ColoredDag, ColoredNode, NodeId}; use crate::physical::colored_dag::stage_id::{StageId, Topology}; +use crate::sketch_algebra::SketchExpr; use crate::types_v2::BindingName; /// Errors surfaced by [`StageAllocator::allocate`]. @@ -304,7 +304,10 @@ mod tests { let err = StageAllocator .allocate(&leaf, Topology::SingleStage) .unwrap_err(); - assert_eq!(err, AllocateError::UnsupportedTopology(Topology::SingleStage)); + assert_eq!( + err, + AllocateError::UnsupportedTopology(Topology::SingleStage) + ); } #[test] @@ -315,7 +318,9 @@ mod tests { SketchParams::Kll(KllParams { k: 200 }), windowed_scan(), ); - let dag = StageAllocator.allocate(&expr, Topology::ThreeStage).unwrap(); + let dag = StageAllocator + .allocate(&expr, Topology::ThreeStage) + .unwrap(); // root = SketchEstimate → Backend assert_eq!(dag.root().unwrap().stage, StageId::Backend); // node 1 = SketchAgg → Edge diff --git a/controller/src/physical/colored_dag/dag.rs b/controller/src/physical/colored_dag/dag.rs index 671c8c93..2f97753e 100644 --- a/controller/src/physical/colored_dag/dag.rs +++ b/controller/src/physical/colored_dag/dag.rs @@ -22,8 +22,8 @@ use serde::{Deserialize, Serialize}; -use crate::sketch_algebra::SketchExpr; use crate::physical::colored_dag::stage_id::{StageId, Topology}; +use crate::sketch_algebra::SketchExpr; /// Stable position-based identifier for a node within a `ColoredDag`. /// `NodeId(0)` is the root; depth-first walk order otherwise. @@ -96,7 +96,6 @@ impl Default for ColoredDag { } impl ColoredDag { - /// Root node (the original `SketchExpr` root). `None` only for the /// degenerate empty DAG. pub fn root(&self) -> Option<&ColoredNode> { @@ -145,10 +144,10 @@ impl ColoredDag { #[cfg(test)] mod tests { use super::*; - use crate::sketch_algebra::SketchExpr; + use crate::intent_algebra::QueryExpr; use crate::sketch_algebra::params::{KllParams, SketchKind, SketchParams}; use crate::sketch_algebra::sketch_expr::EstimateOp; - use crate::intent_algebra::QueryExpr; + use crate::sketch_algebra::SketchExpr; fn dummy_logical() -> SketchExpr { SketchExpr::Logical(QueryExpr::Ref { diff --git a/controller/src/physical/colored_dag/emitter.rs b/controller/src/physical/colored_dag/emitter.rs index 3b2e8d24..cc95b5c1 100644 --- a/controller/src/physical/colored_dag/emitter.rs +++ b/controller/src/physical/colored_dag/emitter.rs @@ -31,10 +31,10 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; -use crate::sketch_algebra::params::{SketchKind, SketchParams}; -use crate::sketch_algebra::sketch_expr::{EstimateOp, SketchExpr}; use crate::physical::colored_dag::dag::ColoredDag; use crate::physical::colored_dag::stage_id::{StageId, Topology}; +use crate::sketch_algebra::params::{SketchKind, SketchParams}; +use crate::sketch_algebra::sketch_expr::{EstimateOp, SketchExpr}; /// Errors surfaced by [`Emitter::emit_per_stage`]. #[derive(Debug, thiserror::Error, PartialEq)] diff --git a/controller/src/physical/colored_dag/tests.rs b/controller/src/physical/colored_dag/tests.rs index fef621e6..d5844b86 100644 --- a/controller/src/physical/colored_dag/tests.rs +++ b/controller/src/physical/colored_dag/tests.rs @@ -11,13 +11,13 @@ use std::time::Duration; use crate::intent_algebra::schema::{Column, DataType}; use crate::intent_algebra::{LabelFilter, QueryExpr, Schema, Source, WindowKind}; +use crate::physical::colored_dag::allocator::StageAllocator; +use crate::physical::colored_dag::emitter::{EmitError, Emitter, StageConfig, ThreeStageEmitter}; +use crate::physical::colored_dag::stage_id::{StageId, Topology}; use crate::sketch_algebra::params::{ DDSketchParams, HllParams, KllParams, SketchKind, SketchParams, }; use crate::sketch_algebra::sketch_expr::{EstimateOp, MergeAlgebra, SketchExpr}; -use crate::physical::colored_dag::allocator::StageAllocator; -use crate::physical::colored_dag::emitter::{EmitError, Emitter, StageConfig, ThreeStageEmitter}; -use crate::physical::colored_dag::stage_id::{StageId, Topology}; use crate::types_v2::{AccuracyTarget, BindingName}; // ── Test fixtures ───────────────────────────────────────────────────────────── diff --git a/controller/src/physical/mod.rs b/controller/src/physical/mod.rs index 24649729..ee7ca9d9 100644 --- a/controller/src/physical/mod.rs +++ b/controller/src/physical/mod.rs @@ -21,11 +21,11 @@ //! | [`topology`] | Re-exports `StageId` + `Topology` from [`colored_dag::stage_id`] — design.md §5 entry point for deployment-topology descriptors | pub mod allocator; +pub mod colored_dag; pub mod plan; pub mod planner; pub mod sketch_catalog; pub mod stage_split; -pub mod colored_dag; pub mod topology; // Convenience re-exports — preserve the surface that consumers of the diff --git a/controller/src/physical/plan.rs b/controller/src/physical/plan.rs index 1c30e044..3dafce48 100644 --- a/controller/src/physical/plan.rs +++ b/controller/src/physical/plan.rs @@ -39,10 +39,10 @@ pub enum PipelineStage { impl std::fmt::Display for PipelineStage { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let s = match self { - PipelineStage::Agent => "agent", - PipelineStage::Backend => "backend", + PipelineStage::Agent => "agent", + PipelineStage::Backend => "backend", PipelineStage::Precompute => "precompute", - PipelineStage::Db => "db", + PipelineStage::Db => "db", }; write!(f, "{s}") } @@ -112,17 +112,17 @@ pub struct NodeAnnotation { #[derive(Debug, Clone)] pub struct PlanNode { /// The logical operator at this node. - pub expr: QueryExpr, + pub expr: QueryExpr, /// Which pipeline stage executes this operator. - pub stage: PipelineStage, + pub stage: PipelineStage, /// Sketch vs. exact vs. passthrough. - pub mode: ExecutionMode, + pub mode: ExecutionMode, /// Estimated resource cost. - pub cost: CostEstimate, + pub cost: CostEstimate, /// Allocator hints for code-generation. pub annotation: NodeAnnotation, /// Child plan nodes (mirrors `expr`'s children after annotation). - pub children: Vec, + pub children: Vec, } impl PlanNode { @@ -132,9 +132,9 @@ impl PlanNode { expr, stage, mode, - cost: CostEstimate::default(), + cost: CostEstimate::default(), annotation: NodeAnnotation::default(), - children: vec![], + children: vec![], } } @@ -214,10 +214,10 @@ pub struct PlanSummary { /// One row in the [`PlanSummary::node_annotations`] table. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct NodeSummaryEntry { - pub node_kind: String, - pub stage: PipelineStage, - pub mode: ExecutionMode, - pub rationale: String, + pub node_kind: String, + pub stage: PipelineStage, + pub mode: ExecutionMode, + pub rationale: String, pub memory_bytes: f64, pub bytes_per_sec: f64, } @@ -226,18 +226,19 @@ impl PlanNode { /// Build a [`PlanSummary`] from this root node. pub fn summarise(&self, raw_bytes_per_sec: f64) -> PlanSummary { let flat = self.flatten(); - let agent_mem: f64 = flat.iter() + let agent_mem: f64 = flat + .iter() .filter(|(_, n)| n.stage == PipelineStage::Agent) .map(|(_, n)| n.cost.memory_bytes) .sum(); - let backend_mem: f64 = flat.iter() + let backend_mem: f64 = flat + .iter() .filter(|(_, n)| n.stage == PipelineStage::Backend) .map(|(_, n)| n.cost.memory_bytes) .sum(); - let plan_bw: f64 = flat.iter() - .filter(|(_, n)| matches!( - n.stage, PipelineStage::Agent | PipelineStage::Backend - )) + let plan_bw: f64 = flat + .iter() + .filter(|(_, n)| matches!(n.stage, PipelineStage::Agent | PipelineStage::Backend)) .map(|(_, n)| n.cost.bytes_per_sec) .fold(f64::INFINITY, f64::min); // min of outbound paths let saved = if raw_bytes_per_sec > plan_bw { @@ -246,16 +247,21 @@ impl PlanNode { 0.0 }; let has_demotion = flat.iter().any(|(_, n)| n.annotation.budget_demotion); - let entries = flat.iter().map(|(_, n)| { - NodeSummaryEntry { - node_kind: format!("{:?}", n.expr).split_whitespace().next().unwrap_or("?").to_string(), - stage: n.stage.clone(), - mode: n.mode.clone(), - rationale: n.annotation.rationale.clone(), + let entries = flat + .iter() + .map(|(_, n)| NodeSummaryEntry { + node_kind: format!("{:?}", n.expr) + .split_whitespace() + .next() + .unwrap_or("?") + .to_string(), + stage: n.stage.clone(), + mode: n.mode.clone(), + rationale: n.annotation.rationale.clone(), memory_bytes: n.cost.memory_bytes, bytes_per_sec: n.cost.bytes_per_sec, - } - }).collect(); + }) + .collect(); PlanSummary { bandwidth_saved_bytes_per_sec: saved, agent_memory_bytes: agent_mem, @@ -297,13 +303,18 @@ mod tests { #[test] fn nodes_at_stage_collects_correctly() { let root = PlanNode { - expr: QueryExpr::Source(SourceSpec { name: "root".into() }), - stage: PipelineStage::Agent, - mode: ExecutionMode::Sketch, - cost: CostEstimate { memory_bytes: 100.0, ..Default::default() }, + expr: QueryExpr::Source(SourceSpec { + name: "root".into(), + }), + stage: PipelineStage::Agent, + mode: ExecutionMode::Sketch, + cost: CostEstimate { + memory_bytes: 100.0, + ..Default::default() + }, annotation: NodeAnnotation::default(), - children: vec![ - source_node("child_agent", PipelineStage::Agent), + children: vec![ + source_node("child_agent", PipelineStage::Agent), source_node("child_backend", PipelineStage::Backend), ], }; @@ -318,19 +329,23 @@ mod tests { #[test] fn sketch_nodes_only_returns_sketch_mode() { let root = PlanNode { - expr: QueryExpr::Source(SourceSpec { name: "r".into() }), - stage: PipelineStage::Agent, - mode: ExecutionMode::Sketch, - cost: CostEstimate::default(), + expr: QueryExpr::Source(SourceSpec { name: "r".into() }), + stage: PipelineStage::Agent, + mode: ExecutionMode::Sketch, + cost: CostEstimate::default(), annotation: NodeAnnotation::default(), - children: vec![ + children: vec![ PlanNode::leaf( - QueryExpr::Source(SourceSpec { name: "exact_child".into() }), + QueryExpr::Source(SourceSpec { + name: "exact_child".into(), + }), PipelineStage::Db, ExecutionMode::Exact, ), PlanNode::leaf( - QueryExpr::Source(SourceSpec { name: "sketch_child".into() }), + QueryExpr::Source(SourceSpec { + name: "sketch_child".into(), + }), PipelineStage::Backend, ExecutionMode::Sketch, ), @@ -345,21 +360,25 @@ mod tests { #[test] fn stage_bandwidth_sums_nodes_at_stage() { let root = PlanNode { - expr: QueryExpr::Source(SourceSpec { name: "r".into() }), - stage: PipelineStage::Agent, - mode: ExecutionMode::Sketch, - cost: CostEstimate { bytes_per_sec: 500.0, ..Default::default() }, + expr: QueryExpr::Source(SourceSpec { name: "r".into() }), + stage: PipelineStage::Agent, + mode: ExecutionMode::Sketch, + cost: CostEstimate { + bytes_per_sec: 500.0, + ..Default::default() + }, annotation: NodeAnnotation::default(), - children: vec![ - PlanNode { - expr: QueryExpr::Source(SourceSpec { name: "c".into() }), - stage: PipelineStage::Agent, - mode: ExecutionMode::Passthrough, - cost: CostEstimate { bytes_per_sec: 200.0, ..Default::default() }, - annotation: NodeAnnotation::default(), - children: vec![], + children: vec![PlanNode { + expr: QueryExpr::Source(SourceSpec { name: "c".into() }), + stage: PipelineStage::Agent, + mode: ExecutionMode::Passthrough, + cost: CostEstimate { + bytes_per_sec: 200.0, + ..Default::default() }, - ], + annotation: NodeAnnotation::default(), + children: vec![], + }], }; assert!((root.stage_bandwidth(&PipelineStage::Agent) - 700.0).abs() < 1e-6); } @@ -377,12 +396,12 @@ mod tests { #[test] fn flatten_depth_increments_per_level() { let root = PlanNode { - expr: QueryExpr::Source(SourceSpec { name: "r".into() }), - stage: PipelineStage::Agent, - mode: ExecutionMode::Passthrough, - cost: CostEstimate::default(), + expr: QueryExpr::Source(SourceSpec { name: "r".into() }), + stage: PipelineStage::Agent, + mode: ExecutionMode::Passthrough, + cost: CostEstimate::default(), annotation: NodeAnnotation::default(), - children: vec![source_node("c1", PipelineStage::Backend)], + children: vec![source_node("c1", PipelineStage::Backend)], }; let flat = root.flatten(); assert_eq!(flat[0].0, 0); @@ -394,16 +413,16 @@ mod tests { #[test] fn summarise_reports_bandwidth_saved() { let root = PlanNode { - expr: QueryExpr::Source(SourceSpec { name: "r".into() }), - stage: PipelineStage::Agent, - mode: ExecutionMode::Sketch, - cost: CostEstimate { + expr: QueryExpr::Source(SourceSpec { name: "r".into() }), + stage: PipelineStage::Agent, + mode: ExecutionMode::Sketch, + cost: CostEstimate { bytes_per_sec: 1_000.0, - memory_bytes: 256.0, + memory_bytes: 256.0, ..Default::default() }, annotation: NodeAnnotation::default(), - children: vec![], + children: vec![], }; // Raw baseline is 10 000 B/s; plan reduces to 1 000 B/s → saved = 9 000. let summary = root.summarise(10_000.0); @@ -415,12 +434,15 @@ mod tests { #[test] fn summarise_detects_budget_demotion() { let root = PlanNode { - expr: QueryExpr::Source(SourceSpec { name: "r".into() }), - stage: PipelineStage::Backend, - mode: ExecutionMode::Sketch, - cost: CostEstimate::default(), - annotation: NodeAnnotation { budget_demotion: true, ..Default::default() }, - children: vec![], + expr: QueryExpr::Source(SourceSpec { name: "r".into() }), + stage: PipelineStage::Backend, + mode: ExecutionMode::Sketch, + cost: CostEstimate::default(), + annotation: NodeAnnotation { + budget_demotion: true, + ..Default::default() + }, + children: vec![], }; let summary = root.summarise(0.0); assert!(summary.has_budget_demotion); diff --git a/controller/src/pipeline.rs b/controller/src/pipeline.rs index 46aed67c..e3f4b757 100644 --- a/controller/src/pipeline.rs +++ b/controller/src/pipeline.rs @@ -1,7 +1,7 @@ -use std::collections::{HashMap, HashSet}; -use std::time::Duration; use anyhow::{anyhow, Context}; use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; +use std::time::Duration; use crate::query_parser; use crate::types::{AggType, QueryWorkload, SketchType, WorkloadCharacteristics}; @@ -27,32 +27,32 @@ pub struct QuerySpec { /// When provided, metric_name / aggregations / time_window may be omitted /// and will be derived from the query. #[serde(default)] - pub query_string: Option, + pub query_string: Option, /// Metric name override. Required when `query_string` is absent. #[serde(default)] - pub metric_name: String, + pub metric_name: String, #[serde(default)] - pub label_filters: HashMap, + pub label_filters: HashMap, #[serde(default)] pub group_by_labels: Vec, /// Aggregation type overrides ("quantile", "cardinality", "frequency"). /// Required when `query_string` is absent. #[serde(default)] - pub aggregations: Vec, + pub aggregations: Vec, /// Time window override (e.g. "5m"). Required when `query_string` is absent. #[serde(default)] - pub time_window: String, + pub time_window: String, #[serde(default)] - pub repeat_every: Option, - pub accuracy_sla: f64, - pub latency_sla: Option, + pub repeat_every: Option, + pub accuracy_sla: f64, + pub latency_sla: Option, /// Optional: pin a specific sketch type, bypassing the cost-model planner. - pub sketch_type: Option, + pub sketch_type: Option, /// Observable data-stream characteristics used for delta / raw-vs-sketch /// bandwidth comparison. Omit to use conservative defaults. #[serde(default)] - pub workload: WorkloadCharacteristics, + pub workload: WorkloadCharacteristics, // ── design.md alignment: new fields, defaulted for back-compat ──────── // @@ -63,7 +63,6 @@ pub struct QuerySpec { // in the controller) keeps working without supplying them. The // planner does not yet consume these — see `Analyzer::analyze` for // the L1 cross-product validation that does fire today. - /// Stable identifier preserved across replan cycles. Optional; /// auto-derived from `metric_name + accuracy_sla` if omitted /// (existing API callers don't supply this). @@ -103,17 +102,26 @@ pub struct QuerySpec { pub data: DataShape, } -fn default_query_shape() -> QueryShape { QueryShape::default() } -fn default_data_shape() -> DataShape { DataShape::default() } +fn default_query_shape() -> QueryShape { + QueryShape::default() +} +fn default_data_shape() -> DataShape { + DataShape::default() +} pub struct Analyzer; impl Analyzer { - pub fn new() -> Self { Self } + pub fn new() -> Self { + Self + } pub fn analyze(&self, spec: QuerySpec) -> anyhow::Result { if !(0.0..=1.0).contains(&spec.accuracy_sla) { - return Err(anyhow!("accuracy_sla must be in [0,1], got {}", spec.accuracy_sla)); + return Err(anyhow!( + "accuracy_sla must be in [0,1], got {}", + spec.accuracy_sla + )); } // ── design.md L1: shape × data cross-product check ───────────────── @@ -160,7 +168,9 @@ impl Analyzer { }; // ── Step 1: parse query_string if provided ───────────────────────── - let parsed = spec.query_string.as_deref() + let parsed = spec + .query_string + .as_deref() .map(|q| query_parser::parse_query(q)) .transpose() .with_context(|| "failed to parse query_string")?; @@ -171,9 +181,7 @@ impl Analyzer { } else if let Some(ref p) = parsed { p.metric_name.clone() } else { - return Err(anyhow!( - "metric_name is required (or provide query_string)" - )); + return Err(anyhow!("metric_name is required (or provide query_string)")); }; // ── Step 3: resolve aggregations ─────────────────────────────────── @@ -207,13 +215,18 @@ impl Analyzer { // ── Step 5: resolve dimensions (group_by + label_filter keys) ────── // Parsed values are the base; explicit spec fields override / extend. - let parsed_group_by = parsed.as_ref().map(|p| p.group_by_labels.as_slice()).unwrap_or(&[]); - let parsed_filters: HashMap = - parsed.as_ref().map(|p| p.label_filters.clone()).unwrap_or_default(); + let parsed_group_by = parsed + .as_ref() + .map(|p| p.group_by_labels.as_slice()) + .unwrap_or(&[]); + let parsed_filters: HashMap = parsed + .as_ref() + .map(|p| p.label_filters.clone()) + .unwrap_or_default(); let merged_filters: HashMap = { let mut m = parsed_filters; - m.extend(spec.label_filters.clone()); // explicit overrides parsed + m.extend(spec.label_filters.clone()); // explicit overrides parsed m }; @@ -224,18 +237,25 @@ impl Analyzer { ); // ── Step 6: scalar fields ────────────────────────────────────────── - let repeat_every = spec.repeat_every.as_deref() + let repeat_every = spec + .repeat_every + .as_deref() .map(parse_duration) .transpose() .with_context(|| "invalid repeat_every")?; - let latency_sla = spec.latency_sla.as_deref() + let latency_sla = spec + .latency_sla + .as_deref() .map(parse_duration) .transpose() .with_context(|| "invalid latency_sla")?; let exact_required = parsed.as_ref().map(|p| p.exact_required).unwrap_or(false); - let quantiles = parsed.as_ref().map(|p| p.quantiles.clone()).unwrap_or_default(); + let quantiles = parsed + .as_ref() + .map(|p| p.quantiles.clone()) + .unwrap_or_default(); // Note: `(planner not yet using this)` — these are populated for // downstream consumers but the planner / cost model still keys @@ -243,14 +263,20 @@ impl Analyzer { // L4-aware downstream PR will switch the cost model to read // `spec.accuracy`, the L5 stage allocator to gate on // `spec.shape`, and the leaf planner to gate on `spec.data`. - let _ = (&spec.accuracy, &spec.shape, &spec.data, - &spec.id, &spec.language, &spec.dollars, - &spec.deployment_model); + let _ = ( + &spec.accuracy, + &spec.shape, + &spec.data, + &spec.id, + &spec.language, + &spec.dollars, + &spec.deployment_model, + ); Ok(QueryWorkload { metric_name, - label_filters: merged_filters, - group_by_labels: all_group_by, + label_filters: merged_filters, + group_by_labels: all_group_by, aggregations, time_window, repeat_every, @@ -261,7 +287,6 @@ impl Analyzer { quantiles, }) } - } // ── Duration helpers (used by other modules) ────────────────────────────────── @@ -278,7 +303,8 @@ pub fn parse_duration(s: &str) -> anyhow::Result { if ch.is_ascii_digit() { current_num.push(ch); } else { - let n: u64 = current_num.parse() + let n: u64 = current_num + .parse() .map_err(|_| anyhow!("invalid number in duration {:?}", s))?; current_num.clear(); match ch { @@ -302,30 +328,41 @@ pub fn format_duration(d: Duration) -> String { let m = (s % 3600) / 60; let sec = s % 60; let mut out = String::new(); - if h > 0 { out.push_str(&format!("{}h", h)); } - if m > 0 { out.push_str(&format!("{}m", m)); } - if sec > 0 || out.is_empty() { out.push_str(&format!("{}s", sec)); } + if h > 0 { + out.push_str(&format!("{}h", h)); + } + if m > 0 { + out.push_str(&format!("{}m", m)); + } + if sec > 0 || out.is_empty() { + out.push_str(&format!("{}s", sec)); + } out } // ── Private helpers ─────────────────────────────────────────────────────────── fn parse_agg_types(raw: &[String]) -> anyhow::Result> { - raw.iter().map(|s| match s.to_lowercase().trim() { - "quantile" => Ok(AggType::Quantile), - "cardinality" => Ok(AggType::Cardinality), - "frequency" => Ok(AggType::Frequency), - other => Err(anyhow!( - "unknown aggregation type {:?} (want: quantile, cardinality, frequency)", other - )), - }).collect() + raw.iter() + .map(|s| match s.to_lowercase().trim() { + "quantile" => Ok(AggType::Quantile), + "cardinality" => Ok(AggType::Cardinality), + "frequency" => Ok(AggType::Frequency), + other => Err(anyhow!( + "unknown aggregation type {:?} (want: quantile, cardinality, frequency)", + other + )), + }) + .collect() } fn dedup_dims(a: &[String], b: &[String]) -> Vec { let mut seen = HashSet::new(); - let mut out = Vec::new(); + let mut out = Vec::new(); for v in a.iter().chain(b.iter()) { - if seen.insert(v.clone()) { out.push(v.clone()); } + if seen.insert(v.clone()) { + out.push(v.clone()); + } } out } @@ -338,25 +375,25 @@ mod tests { fn basic_spec() -> QuerySpec { QuerySpec { - query_string: None, - metric_name: "request_latency".into(), - label_filters: [("service".into(), "web".into())].into(), + query_string: None, + metric_name: "request_latency".into(), + label_filters: [("service".into(), "web".into())].into(), group_by_labels: vec!["host.name".into()], - aggregations: vec!["quantile".into()], - time_window: "5m".into(), - repeat_every: Some("1m".into()), - accuracy_sla: 0.01, - latency_sla: Some("10m".into()), - sketch_type: None, - workload: Default::default(), + aggregations: vec!["quantile".into()], + time_window: "5m".into(), + repeat_every: Some("1m".into()), + accuracy_sla: 0.01, + latency_sla: Some("10m".into()), + sketch_type: None, + workload: Default::default(), // design.md alignment: defaults preserve legacy behaviour. - id: None, - language: None, - accuracy: None, - dollars: None, + id: None, + language: None, + accuracy: None, + dollars: None, deployment_model: None, - shape: QueryShape::default(), - data: DataShape::default(), + shape: QueryShape::default(), + data: DataShape::default(), } } @@ -365,25 +402,35 @@ mod tests { let w = Analyzer::new().analyze(basic_spec()).unwrap(); assert_eq!(w.metric_name, "request_latency"); assert_eq!(w.accuracy_sla, 0.01); - assert_eq!(w.time_window, Duration::from_secs(300)); - assert_eq!(w.repeat_every, Some(Duration::from_secs(60))); - assert_eq!(w.latency_sla, Some(Duration::from_secs(600))); - assert_eq!(w.aggregations, vec![AggType::Quantile]); + assert_eq!(w.time_window, Duration::from_secs(300)); + assert_eq!(w.repeat_every, Some(Duration::from_secs(60))); + assert_eq!(w.latency_sla, Some(Duration::from_secs(600))); + assert_eq!(w.aggregations, vec![AggType::Quantile]); } #[test] fn dimension_merge_dedup() { let mut spec = basic_spec(); - spec.label_filters = [("service".into(), "api".into()), - ("host.name".into(), "h1".into())].into(); + spec.label_filters = [ + ("service".into(), "api".into()), + ("host.name".into(), "h1".into()), + ] + .into(); spec.group_by_labels = vec!["host.name".into(), "region".into()]; let w = Analyzer::new().analyze(spec).unwrap(); for dim in &["host.name", "region", "service"] { - assert!(w.group_by_labels.contains(&dim.to_string()), "missing {dim}"); + assert!( + w.group_by_labels.contains(&dim.to_string()), + "missing {dim}" + ); } // host.name must appear exactly once after dedup assert_eq!( - w.group_by_labels.iter().filter(|d| d.as_str() == "host.name").count(), 1 + w.group_by_labels + .iter() + .filter(|d| d.as_str() == "host.name") + .count(), + 1 ); } @@ -392,7 +439,10 @@ mod tests { let mut spec = basic_spec(); spec.aggregations = vec!["cardinality".into(), "frequency".into()]; let w = Analyzer::new().analyze(spec).unwrap(); - assert_eq!(w.aggregations, vec![AggType::Cardinality, AggType::Frequency]); + assert_eq!( + w.aggregations, + vec![AggType::Cardinality, AggType::Frequency] + ); } #[test] @@ -428,18 +478,23 @@ mod tests { for bad in &[-0.1f64, 1.5] { let mut spec = basic_spec(); spec.accuracy_sla = *bad; - assert!(Analyzer::new().analyze(spec).is_err(), - "expected error for accuracy_sla={bad}"); + assert!( + Analyzer::new().analyze(spec).is_err(), + "expected error for accuracy_sla={bad}" + ); } } #[test] fn parse_duration_formats() { - assert_eq!(parse_duration("30s").unwrap(), Duration::from_secs(30)); - assert_eq!(parse_duration("5m").unwrap(), Duration::from_secs(300)); - assert_eq!(parse_duration("1h").unwrap(), Duration::from_secs(3600)); - assert_eq!(parse_duration("1h30m").unwrap(), Duration::from_secs(5400)); - assert_eq!(parse_duration("1h5m30s").unwrap(),Duration::from_secs(3930)); + assert_eq!(parse_duration("30s").unwrap(), Duration::from_secs(30)); + assert_eq!(parse_duration("5m").unwrap(), Duration::from_secs(300)); + assert_eq!(parse_duration("1h").unwrap(), Duration::from_secs(3600)); + assert_eq!(parse_duration("1h30m").unwrap(), Duration::from_secs(5400)); + assert_eq!( + parse_duration("1h5m30s").unwrap(), + Duration::from_secs(3930) + ); } #[test] @@ -462,25 +517,25 @@ mod tests { /// Build a minimal QuerySpec driven entirely by a query_string. fn qs_only(query: &str) -> QuerySpec { QuerySpec { - query_string: Some(query.into()), - metric_name: "".into(), - label_filters: Default::default(), + query_string: Some(query.into()), + metric_name: "".into(), + label_filters: Default::default(), group_by_labels: vec![], - aggregations: vec![], - time_window: "".into(), - repeat_every: None, - accuracy_sla: 0.01, - latency_sla: None, - sketch_type: None, - workload: Default::default(), + aggregations: vec![], + time_window: "".into(), + repeat_every: None, + accuracy_sla: 0.01, + latency_sla: None, + sketch_type: None, + workload: Default::default(), // design.md alignment: defaults preserve legacy behaviour. - id: None, - language: None, - accuracy: None, - dollars: None, + id: None, + language: None, + accuracy: None, + dollars: None, deployment_model: None, - shape: QueryShape::default(), - data: DataShape::default(), + shape: QueryShape::default(), + data: DataShape::default(), } } @@ -489,12 +544,14 @@ mod tests { #[test] fn query_string_promql_populates_workload() { let w = Analyzer::new() - .analyze(qs_only("sum by (host) (quantile_over_time(0.99, latency[5m]))")) + .analyze(qs_only( + "sum by (host) (quantile_over_time(0.99, latency[5m]))", + )) .unwrap(); - assert_eq!(w.metric_name, "latency"); + assert_eq!(w.metric_name, "latency"); assert_eq!(w.aggregations, vec![AggType::Quantile]); - assert_eq!(w.time_window, Duration::from_secs(300)); - assert_eq!(w.quantiles, vec![0.99]); + assert_eq!(w.time_window, Duration::from_secs(300)); + assert_eq!(w.quantiles, vec![0.99]); assert!(!w.exact_required); } @@ -545,7 +602,9 @@ mod tests { #[test] fn query_string_exact_required_propagated() { let w = Analyzer::new() - .analyze(qs_only("sum by (service) (sum_over_time(request_bytes[1h]))")) + .analyze(qs_only( + "sum by (service) (sum_over_time(request_bytes[1h]))", + )) .unwrap(); assert!(w.exact_required, "sum_over_time must set exact_required"); assert_eq!(w.aggregations, vec![]); @@ -555,7 +614,9 @@ mod tests { #[test] fn query_string_quantiles_populated() { let w = Analyzer::new() - .analyze(qs_only("sum by (host) (quantile_over_time(0.5, latency[5m]))")) + .analyze(qs_only( + "sum by (host) (quantile_over_time(0.5, latency[5m]))", + )) .unwrap(); assert_eq!(w.quantiles, vec![0.5]); } @@ -565,9 +626,9 @@ mod tests { #[test] fn backward_compat_no_query_string() { let w = Analyzer::new().analyze(basic_spec()).unwrap(); - assert_eq!(w.metric_name, "request_latency"); + assert_eq!(w.metric_name, "request_latency"); assert_eq!(w.aggregations, vec![AggType::Quantile]); - assert_eq!(w.time_window, Duration::from_secs(300)); + assert_eq!(w.time_window, Duration::from_secs(300)); assert!(!w.exact_required); assert!(w.quantiles.is_empty()); } @@ -580,12 +641,16 @@ mod tests { #[test] fn typed_accuracy_overrides_legacy_accuracy_sla() { let mut spec = basic_spec(); - spec.accuracy_sla = 0.99; // legacy: ε = 0.01 - spec.accuracy = Some(AccuracyTarget::Epsilon(0.05)); + spec.accuracy_sla = 0.99; // legacy: ε = 0.01 + spec.accuracy = Some(AccuracyTarget::Epsilon(0.05)); let w = Analyzer::new().analyze(spec).unwrap(); // The resolved 1.0 - 0.05 = 0.95 must reach the QueryWorkload, not // the legacy 0.99. - assert!((w.accuracy_sla - 0.95).abs() < 1e-9, "got {}", w.accuracy_sla); + assert!( + (w.accuracy_sla - 0.95).abs() < 1e-9, + "got {}", + w.accuracy_sla + ); } /// Typed `accuracy: Some(Exact)` clamps the SLA to 1.0 regardless of @@ -594,7 +659,7 @@ mod tests { fn typed_accuracy_exact_clamps_to_one() { let mut spec = basic_spec(); spec.accuracy_sla = 0.5; - spec.accuracy = Some(AccuracyTarget::Exact); + spec.accuracy = Some(AccuracyTarget::Exact); let w = Analyzer::new().analyze(spec).unwrap(); assert_eq!(w.accuracy_sla, 1.0); } @@ -605,10 +670,12 @@ mod tests { fn l1_rejects_streaming_over_batch() { let mut spec = basic_spec(); spec.shape = QueryShape::Streaming; - spec.data = DataShape::Batch; + spec.data = DataShape::Batch; let err = Analyzer::new().analyze(spec).unwrap_err().to_string(); - assert!(err.contains("Streaming") && err.contains("Batch"), - "expected the error to name the rejected combination: {err}"); + assert!( + err.contains("Streaming") && err.contains("Batch"), + "expected the error to name the rejected combination: {err}" + ); } /// L1 rejects `(QueryShape::Streaming, DataShape::Mutable)` — no @@ -617,10 +684,12 @@ mod tests { fn l1_rejects_streaming_over_mutable() { let mut spec = basic_spec(); spec.shape = QueryShape::Streaming; - spec.data = DataShape::Mutable; + spec.data = DataShape::Mutable; let err = Analyzer::new().analyze(spec).unwrap_err().to_string(); - assert!(err.contains("Streaming") && err.contains("Mutable"), - "expected the error to name the rejected combination: {err}"); + assert!( + err.contains("Streaming") && err.contains("Mutable"), + "expected the error to name the rejected combination: {err}" + ); } /// `(QueryShape::Streaming, DataShape::AppendOnlyStream)` — the @@ -629,7 +698,7 @@ mod tests { fn l1_accepts_streaming_over_append_only_stream() { let mut spec = basic_spec(); spec.shape = QueryShape::Streaming; - spec.data = DataShape::AppendOnlyStream; + spec.data = DataShape::AppendOnlyStream; assert!(Analyzer::new().analyze(spec).is_ok()); } @@ -652,14 +721,18 @@ mod tests { assert!(spec.dollars.is_none()); assert!(spec.deployment_model.is_none()); assert_eq!(spec.shape, QueryShape::OneShot); - assert_eq!(spec.data, DataShape::AppendOnlyStream); + assert_eq!(spec.data, DataShape::AppendOnlyStream); // And the analyzer accepts it. let w = Analyzer::new().analyze(spec).unwrap(); assert_eq!(w.metric_name, "request_latency"); // Legacy accuracy_sla=0.99 round-trips through resolution // (no typed `accuracy` supplied → translate from legacy → // Epsilon(0.01) → back to 1 - 0.01 = 0.99). - assert!((w.accuracy_sla - 0.99).abs() < 1e-9, "got {}", w.accuracy_sla); + assert!( + (w.accuracy_sla - 0.99).abs() < 1e-9, + "got {}", + w.accuracy_sla + ); } /// JSON *with* the new fields parses correctly — the wire schema @@ -694,6 +767,10 @@ mod tests { let w = Analyzer::new().analyze(spec).unwrap(); // typed `accuracy: Epsilon(0.02)` overrode the legacy 0.5 → // resolved accuracy_sla in the workload is 1.0 - 0.02 = 0.98. - assert!((w.accuracy_sla - 0.98).abs() < 1e-9, "got {}", w.accuracy_sla); + assert!( + (w.accuracy_sla - 0.98).abs() < 1e-9, + "got {}", + w.accuracy_sla + ); } } diff --git a/controller/src/query_parser/language/elastic_dsl/mod.rs b/controller/src/query_parser/language/elastic_dsl/mod.rs index 8b20f3f5..19dc0991 100644 --- a/controller/src/query_parser/language/elastic_dsl/mod.rs +++ b/controller/src/query_parser/language/elastic_dsl/mod.rs @@ -4,8 +4,8 @@ //! (`controller/docs/design.md` §3 row 1). No L1 parser is shipped in //! the DC deployment build. -use super::{Language, ParseError}; use super::language_ast::LanguageAst; +use super::{Language, ParseError}; use crate::types_v2::QueryLanguage; /// ElasticDSL implementation of the [`Language`] trait. Currently stubbed. diff --git a/controller/src/query_parser/language/mod.rs b/controller/src/query_parser/language/mod.rs index bdee934a..a2b4cc8c 100644 --- a/controller/src/query_parser/language/mod.rs +++ b/controller/src/query_parser/language/mod.rs @@ -30,15 +30,15 @@ //! adding a real backend later is a localised change to the relevant //! sub-module. +pub mod elastic_dsl; pub mod language_ast; pub mod promql; pub mod sql; -pub mod elastic_dsl; +pub use elastic_dsl::ElasticDslLanguage; pub use language_ast::LanguageAst; pub use promql::PromQLLanguage; pub use sql::SqlLanguage; -pub use elastic_dsl::ElasticDslLanguage; #[cfg(test)] mod tests; @@ -71,7 +71,10 @@ pub enum ParseError { impl ParseError { /// Convenience: wrap any `Display` parser error into [`ParseError::Backend`]. pub fn backend(language: QueryLanguage, e: impl std::fmt::Display) -> Self { - ParseError::Backend { language, source_err: e.to_string() } + ParseError::Backend { + language, + source_err: e.to_string(), + } } } diff --git a/controller/src/query_parser/language/promql/ast.rs b/controller/src/query_parser/language/promql/ast.rs index 5e3993fc..7b4fce11 100644 --- a/controller/src/query_parser/language/promql/ast.rs +++ b/controller/src/query_parser/language/promql/ast.rs @@ -27,12 +27,20 @@ pub struct PromQLAst { impl PromQLAst { /// Construct from the legacy parser's two outputs. pub fn new(source: String, expr: QueryExpr, summary: ParsedQuery) -> Self { - Self { source, expr, summary } + Self { + source, + expr, + summary, + } } /// Borrow the algebra tree. - pub fn expr(&self) -> &QueryExpr { &self.expr } + pub fn expr(&self) -> &QueryExpr { + &self.expr + } /// Borrow the flat summary. - pub fn summary(&self) -> &ParsedQuery { &self.summary } + pub fn summary(&self) -> &ParsedQuery { + &self.summary + } } diff --git a/controller/src/query_parser/language/promql/mod.rs b/controller/src/query_parser/language/promql/mod.rs index 3f293930..92f763f9 100644 --- a/controller/src/query_parser/language/promql/mod.rs +++ b/controller/src/query_parser/language/promql/mod.rs @@ -7,8 +7,8 @@ pub mod ast; -use super::{Language, ParseError}; use super::language_ast::LanguageAst; +use super::{Language, ParseError}; use crate::query_parser::{parse_query, parse_query_expr}; use crate::types_v2::QueryLanguage; @@ -30,10 +30,14 @@ impl Language for PromQLLanguage { // Delegate to the existing parser — both entry points re-parse the // same string today; the cost is negligible (microseconds) and we // get the legacy `ParsedQuery` for free for back-compat callers. - let expr = parse_query_expr(source) - .map_err(|e| ParseError::backend(QueryLanguage::PromQL, e))?; - let summary = parse_query(source) - .map_err(|e| ParseError::backend(QueryLanguage::PromQL, e))?; - Ok(LanguageAst::PromQL(PromQLAst::new(source.to_string(), expr, summary))) + let expr = + parse_query_expr(source).map_err(|e| ParseError::backend(QueryLanguage::PromQL, e))?; + let summary = + parse_query(source).map_err(|e| ParseError::backend(QueryLanguage::PromQL, e))?; + Ok(LanguageAst::PromQL(PromQLAst::new( + source.to_string(), + expr, + summary, + ))) } } diff --git a/controller/src/query_parser/language/sql/mod.rs b/controller/src/query_parser/language/sql/mod.rs index 2e4b184d..b8ca1c70 100644 --- a/controller/src/query_parser/language/sql/mod.rs +++ b/controller/src/query_parser/language/sql/mod.rs @@ -5,8 +5,8 @@ //! is not exposed via the new [`Language`] trait yet. A real impl wraps //! `sqlparser` and emits a `SqlAst` analogous to [`super::promql::PromQLAst`]. -use super::{Language, ParseError}; use super::language_ast::LanguageAst; +use super::{Language, ParseError}; use crate::types_v2::QueryLanguage; /// SQL implementation of the [`Language`] trait. Currently stubbed. diff --git a/controller/src/query_parser/language/tests.rs b/controller/src/query_parser/language/tests.rs index 7468404c..1fd5e8dd 100644 --- a/controller/src/query_parser/language/tests.rs +++ b/controller/src/query_parser/language/tests.rs @@ -11,7 +11,9 @@ fn promql_language_id_is_promql() { #[test] fn promql_language_parses_basic() { - let ast = PromQLLanguage.parse("up").expect("PromQL parse should succeed"); + let ast = PromQLLanguage + .parse("up") + .expect("PromQL parse should succeed"); assert!(ast.is_promql(), "expected PromQL variant"); let promql = ast.as_promql().unwrap(); assert_eq!(promql.summary().metric_name, "up"); diff --git a/controller/src/query_parser/sql.rs b/controller/src/query_parser/sql.rs index 7eb7cecc..9b8df2d5 100644 --- a/controller/src/query_parser/sql.rs +++ b/controller/src/query_parser/sql.rs @@ -40,17 +40,16 @@ use std::time::Duration; use anyhow::{anyhow, Context}; use sqlparser::ast::{ - BinaryOperator, DuplicateTreatment, Expr, FunctionArg, FunctionArgExpr, - FunctionArgumentList, FunctionArguments, GroupByExpr, Join, JoinConstraint, - JoinOperator, LimitClause, ObjectName, OrderBy, OrderByExpr, OrderByKind, - Query, Select, SelectItem, SetExpr, SetOperator, Statement, TableFactor, - Value, ValueWithSpan, + BinaryOperator, DuplicateTreatment, Expr, FunctionArg, FunctionArgExpr, FunctionArgumentList, + FunctionArguments, GroupByExpr, Join, JoinConstraint, JoinOperator, LimitClause, ObjectName, + OrderBy, OrderByExpr, OrderByKind, Query, Select, SelectItem, SetExpr, SetOperator, Statement, + TableFactor, Value, ValueWithSpan, }; use sqlparser::dialect::GenericDialect; use crate::intent_algebra::legacy_expr::{ - AggFunc, AggItem as AlgAggItem, BinaryOpKind, ColumnRef, JoinKind, LiteralValue, - ProjectItem, QueryExpr, ScalarExpr, SetOpKind, SortKey, SourceSpec, + AggFunc, AggItem as AlgAggItem, BinaryOpKind, ColumnRef, JoinKind, LiteralValue, ProjectItem, + QueryExpr, ScalarExpr, SetOpKind, SortKey, SourceSpec, }; // ── Public entry point ──────────────────────────────────────────────────────── @@ -63,7 +62,9 @@ pub fn parse_sql_expr(sql: &str) -> anyhow::Result { let dialect = GenericDialect {}; let mut stmts = sqlparser::parser::Parser::parse_sql(&dialect, sql) .with_context(|| format!("SQL parse error: {sql:?}"))?; - let stmt = stmts.pop().ok_or_else(|| anyhow!("no SQL statement found"))?; + let stmt = stmts + .pop() + .ok_or_else(|| anyhow!("no SQL statement found"))?; let query = match stmt { Statement::Query(q) => *q, other => return Err(anyhow!("expected SELECT, got {:?}", other)), @@ -75,16 +76,24 @@ pub fn parse_sql_expr(sql: &str) -> anyhow::Result { fn extract_query_expr(query: &Query) -> anyhow::Result { let order_by: Vec = match &query.order_by { - Some(OrderBy { kind: OrderByKind::Expressions(exprs), .. }) => exprs.clone(), + Some(OrderBy { + kind: OrderByKind::Expressions(exprs), + .. + }) => exprs.clone(), _ => vec![], }; let (limit_n, offset_n) = match &query.limit_clause { - Some(LimitClause::LimitOffset { limit: Some(e), offset, .. }) => { - (Some(e.clone()), offset.as_ref().and_then(|o| expr_to_u64(&o.value))) - } - Some(LimitClause::OffsetCommaLimit { limit: e, offset, .. }) => { - (Some(e.clone()), Some(expr_to_u64(offset).unwrap_or(0))) - } + Some(LimitClause::LimitOffset { + limit: Some(e), + offset, + .. + }) => ( + Some(e.clone()), + offset.as_ref().and_then(|o| expr_to_u64(&o.value)), + ), + Some(LimitClause::OffsetCommaLimit { + limit: e, offset, .. + }) => (Some(e.clone()), Some(expr_to_u64(offset).unwrap_or(0))), _ => (None, None), }; let limit_val = limit_n.as_ref().and_then(|e| expr_to_u64(e)); @@ -95,22 +104,27 @@ fn extract_query_expr(query: &Query) -> anyhow::Result { } fn extract_set_expr_qe( - set_expr: &SetExpr, - order_by: &[OrderByExpr], - limit_n: Option, - offset_n: u64, + set_expr: &SetExpr, + order_by: &[OrderByExpr], + limit_n: Option, + offset_n: u64, ) -> anyhow::Result { match set_expr { SetExpr::Select(sel) => extract_select_qe(sel, order_by, limit_n, offset_n), SetExpr::Query(inner) => extract_query_expr(inner), // UNION / INTERSECT / EXCEPT - SetExpr::SetOperation { left, right, op, set_quantifier } => { + SetExpr::SetOperation { + left, + right, + op, + set_quantifier, + } => { use sqlparser::ast::{SetOperator, SetQuantifier}; - let left_qe = extract_set_expr_qe(left, &[], None, 0)?; + let left_qe = extract_set_expr_qe(left, &[], None, 0)?; let right_qe = extract_set_expr_qe(right, &[], None, 0)?; let kind = match op { - SetOperator::Union => SetOpKind::Union, + SetOperator::Union => SetOpKind::Union, SetOperator::Intersect => SetOpKind::Intersect, SetOperator::Except | SetOperator::Minus => SetOpKind::Except, }; @@ -118,7 +132,7 @@ fn extract_set_expr_qe( Ok(QueryExpr::SetOp { kind, all, - left: Box::new(left_qe), + left: Box::new(left_qe), right: Box::new(right_qe), }) } @@ -129,18 +143,18 @@ fn extract_set_expr_qe( // ── SELECT-level extraction ─────────────────────────────────────────────────── fn extract_select_qe( - sel: &Select, + sel: &Select, order_by: &[OrderByExpr], - limit_n: Option, + limit_n: Option, offset_n: u64, ) -> anyhow::Result { - let metric_name = extract_table_name(sel)?; - let where_scalar = sel.selection.as_ref().map(sql_expr_to_scalar); - let group_keys = extract_group_by(&sel.group_by); + let metric_name = extract_table_name(sel)?; + let where_scalar = sel.selection.as_ref().map(sql_expr_to_scalar); + let group_keys = extract_group_by(&sel.group_by); let having_scalar = sel.having.as_ref().map(sql_expr_to_scalar); - let agg_items = collect_agg_items_qe(&sel.projection); - let join_qe = extract_join_qe(sel); - let window_spec = extract_group_by_window(&sel.group_by); + let agg_items = collect_agg_items_qe(&sel.projection); + let join_qe = extract_join_qe(sel); + let window_spec = extract_group_by_window(&sel.group_by); let source = QueryExpr::Source(SourceSpec { name: metric_name.clone(), @@ -148,17 +162,20 @@ fn extract_select_qe( // WHERE → Filter let after_where = match where_scalar { - Some(pred) => QueryExpr::Filter { pred, input: Box::new(source) }, - None => source, + Some(pred) => QueryExpr::Filter { + pred, + input: Box::new(source), + }, + None => source, }; // JOIN let after_join = if let Some((inner_table, join_kind, join_pred)) = join_qe { let inner_source = QueryExpr::Source(SourceSpec { name: inner_table }); QueryExpr::Join { - kind: join_kind, - pred: join_pred, - left: Box::new(after_where), + kind: join_kind, + pred: join_pred, + left: Box::new(after_where), right: Box::new(inner_source), } } else { @@ -169,8 +186,8 @@ fn extract_select_qe( let after_window = if let Some(ws) = window_spec { QueryExpr::Window { duration: ws.size, - slide: ws.slide, - input: Box::new(after_join), + slide: ws.slide, + input: Box::new(after_join), } } else { after_join @@ -180,14 +197,17 @@ fn extract_select_qe( let after_agg = if agg_items.is_empty() { // No aggregation — bare projection with possible DISTINCT. let cols = collect_project_items(&sel.projection); - QueryExpr::Project { cols, input: Box::new(after_window) } + QueryExpr::Project { + cols, + input: Box::new(after_window), + } } else { let having = having_scalar; QueryExpr::Aggregate { - keys: group_keys, - aggs: agg_items, + keys: group_keys, + aggs: agg_items, having, - input: Box::new(after_window), + input: Box::new(after_window), } }; @@ -195,18 +215,28 @@ fn extract_select_qe( let after_sort = if order_by.is_empty() { after_agg } else { - let keys: Vec = order_by.iter().map(|o| SortKey { - col: expr_to_col_name(&o.expr).unwrap_or_else(|| "?".into()), - desc: matches!(o.options.asc, Some(false) | None), - nulls_first: None, - }).collect(); - QueryExpr::Sort { keys, input: Box::new(after_agg) } + let keys: Vec = order_by + .iter() + .map(|o| SortKey { + col: expr_to_col_name(&o.expr).unwrap_or_else(|| "?".into()), + desc: matches!(o.options.asc, Some(false) | None), + nulls_first: None, + }) + .collect(); + QueryExpr::Sort { + keys, + input: Box::new(after_agg), + } }; // LIMIT / OFFSET let result = match limit_n { - Some(n) => QueryExpr::Limit { n, offset: offset_n, input: Box::new(after_sort) }, - None => after_sort, + Some(n) => QueryExpr::Limit { + n, + offset: offset_n, + input: Box::new(after_sort), + }, + None => after_sort, }; Ok(result) @@ -218,9 +248,9 @@ fn collect_agg_items_qe(projection: &[SelectItem]) -> Vec { let mut out = Vec::new(); for item in projection { let (expr, alias) = match item { - SelectItem::UnnamedExpr(e) => (e, None), + SelectItem::UnnamedExpr(e) => (e, None), SelectItem::ExprWithAlias { expr, alias } => (expr, Some(alias.value.clone())), - _ => continue, + _ => continue, }; collect_agg_from_expr_qe(expr, alias, &mut out); } @@ -230,14 +260,22 @@ fn collect_agg_items_qe(projection: &[SelectItem]) -> Vec { fn collect_agg_from_expr_qe(expr: &Expr, alias: Option, out: &mut Vec) { match expr { Expr::Function(f) => { - let fn_name = f.name.0.last() + let fn_name = f + .name + .0 + .last() .and_then(|i| i.as_ident()) .map(|id| id.value.to_uppercase()) .unwrap_or_default(); let (distinct, args) = match &f.args { - FunctionArguments::List(FunctionArgumentList { duplicate_treatment, args, .. }) => { - let is_distinct = matches!(duplicate_treatment, Some(DuplicateTreatment::Distinct)); + FunctionArguments::List(FunctionArgumentList { + duplicate_treatment, + args, + .. + }) => { + let is_distinct = + matches!(duplicate_treatment, Some(DuplicateTreatment::Distinct)); (is_distinct, args.as_slice()) } _ => (false, &[][..]), @@ -247,23 +285,23 @@ fn collect_agg_from_expr_qe(expr: &Expr, alias: Option, out: &mut Vec AggFunc::CountDistinct, - "COUNT" => AggFunc::Count, - "SUM" => AggFunc::Sum, - "AVG" => AggFunc::Avg, - "MIN" => AggFunc::Min, - "MAX" => AggFunc::Max, - _ => return, + "COUNT" => AggFunc::Count, + "SUM" => AggFunc::Sum, + "AVG" => AggFunc::Avg, + "MIN" => AggFunc::Min, + "MAX" => AggFunc::Max, + _ => return, }; out.push(AlgAggItem { - alias: alias.unwrap_or_else(|| fn_name.to_lowercase()), + alias: alias.unwrap_or_else(|| fn_name.to_lowercase()), func, col, distinct, }); } Expr::BinaryOp { left, right, .. } => { - collect_agg_from_expr_qe(left, None, out); + collect_agg_from_expr_qe(left, None, out); collect_agg_from_expr_qe(right, None, out); } Expr::Nested(inner) => collect_agg_from_expr_qe(inner, alias, out), @@ -272,21 +310,24 @@ fn collect_agg_from_expr_qe(expr: &Expr, alias: Option, out: &mut Vec Vec { - projection.iter().filter_map(|item| match item { - SelectItem::UnnamedExpr(e) => Some(ProjectItem { - alias: None, - expr: sql_expr_to_scalar(e), - }), - SelectItem::ExprWithAlias { expr, alias } => Some(ProjectItem { - alias: Some(alias.value.clone()), - expr: sql_expr_to_scalar(expr), - }), - SelectItem::Wildcard(_) => Some(ProjectItem { - alias: None, - expr: ScalarExpr::Column("*".into()), - }), - _ => None, - }).collect() + projection + .iter() + .filter_map(|item| match item { + SelectItem::UnnamedExpr(e) => Some(ProjectItem { + alias: None, + expr: sql_expr_to_scalar(e), + }), + SelectItem::ExprWithAlias { expr, alias } => Some(ProjectItem { + alias: Some(alias.value.clone()), + expr: sql_expr_to_scalar(expr), + }), + SelectItem::Wildcard(_) => Some(ProjectItem { + alias: None, + expr: ScalarExpr::Column("*".into()), + }), + _ => None, + }) + .collect() } // ── AST helpers: aggregation arguments ─────────────────────────────────────── @@ -313,33 +354,39 @@ fn first_col_from_args(args: &[FunctionArg]) -> ColumnRef { /// Window spec extracted from a TUMBLE() or HOP() call in GROUP BY. struct SqlWindowSpec { - size: Duration, - slide: Option, + size: Duration, + slide: Option, time_col: Option, } fn extract_group_by(group_by: &GroupByExpr) -> Vec { let exprs = match group_by { - GroupByExpr::All(_) => return vec![], + GroupByExpr::All(_) => return vec![], GroupByExpr::Expressions(e, _) => e, }; - exprs.iter().filter_map(|e| match e { - Expr::Identifier(id) => Some(id.value.clone()), - Expr::CompoundIdentifier(parts) => parts.last().map(|i| i.value.clone()), - // Skip TUMBLE/HOP function calls — extracted separately. - Expr::Function(f) => { - let name = f.name.0.last() - .and_then(|i| i.as_ident()) - .map(|id| id.value.to_uppercase()) - .unwrap_or_default(); - if name == "TUMBLE" || name == "HOP" || name == "TIME_BUCKET" { - None - } else { - None // unknown function in GROUP BY — skip + exprs + .iter() + .filter_map(|e| match e { + Expr::Identifier(id) => Some(id.value.clone()), + Expr::CompoundIdentifier(parts) => parts.last().map(|i| i.value.clone()), + // Skip TUMBLE/HOP function calls — extracted separately. + Expr::Function(f) => { + let name = f + .name + .0 + .last() + .and_then(|i| i.as_ident()) + .map(|id| id.value.to_uppercase()) + .unwrap_or_default(); + if name == "TUMBLE" || name == "HOP" || name == "TIME_BUCKET" { + None + } else { + None // unknown function in GROUP BY — skip + } } - } - _ => None, - }).collect() + _ => None, + }) + .collect() } /// Extract a TUMBLE / HOP / time_bucket window from the GROUP BY clause. @@ -355,7 +402,10 @@ fn extract_group_by_window(group_by: &GroupByExpr) -> Option { }; for expr in exprs { if let Expr::Function(f) = expr { - let name = f.name.0.last() + let name = f + .name + .0 + .last() .and_then(|i| i.as_ident()) .map(|id| id.value.to_uppercase()) .unwrap_or_default(); @@ -370,20 +420,32 @@ fn extract_group_by_window(group_by: &GroupByExpr) -> Option { // TUMBLE(ts_col, interval) let time_col = func_arg_to_col_name(&args[0]); let size = func_arg_to_duration(&args[1])?; - return Some(SqlWindowSpec { size, slide: None, time_col }); + return Some(SqlWindowSpec { + size, + slide: None, + time_col, + }); } "HOP" if args.len() >= 3 => { // HOP(ts_col, slide_interval, size_interval) let time_col = func_arg_to_col_name(&args[0]); let slide = func_arg_to_duration(&args[1])?; - let size = func_arg_to_duration(&args[2])?; - return Some(SqlWindowSpec { size, slide: Some(slide), time_col }); + let size = func_arg_to_duration(&args[2])?; + return Some(SqlWindowSpec { + size, + slide: Some(slide), + time_col, + }); } "TIME_BUCKET" if args.len() >= 2 => { // time_bucket('5 minutes', ts_col) — first arg is interval string let size = func_arg_to_duration(&args[0])?; let time_col = func_arg_to_col_name(&args[1]); - return Some(SqlWindowSpec { size, slide: None, time_col }); + return Some(SqlWindowSpec { + size, + slide: None, + time_col, + }); } _ => {} } @@ -394,10 +456,10 @@ fn extract_group_by_window(group_by: &GroupByExpr) -> Option { fn func_arg_to_col_name(arg: &FunctionArg) -> Option { match arg { - FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Identifier(id))) => - Some(id.value.clone()), - FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::CompoundIdentifier(parts))) => - parts.last().map(|i| i.value.clone()), + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Identifier(id))) => Some(id.value.clone()), + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::CompoundIdentifier(parts))) => { + parts.last().map(|i| i.value.clone()) + } _ => None, } } @@ -426,17 +488,15 @@ fn expr_to_duration(expr: &Expr) -> Option { let secs = match unit { sqlparser::ast::DateTimeField::Second => val, sqlparser::ast::DateTimeField::Minute => val * 60, - sqlparser::ast::DateTimeField::Hour => val * 3600, - sqlparser::ast::DateTimeField::Day => val * 86400, + sqlparser::ast::DateTimeField::Hour => val * 3600, + sqlparser::ast::DateTimeField::Day => val * 86400, _ => return None, }; Some(Duration::from_secs(secs)) } // '5 minutes' string (time_bucket style) Expr::Value(vws) => match &vws.value { - Value::SingleQuotedString(s) | Value::DoubleQuotedString(s) => { - parse_duration_string(s) - } + Value::SingleQuotedString(s) | Value::DoubleQuotedString(s) => parse_duration_string(s), _ => None, }, _ => None, @@ -476,7 +536,8 @@ fn parse_duration_string(s: &str) -> Option { // ── AST helpers: table name ─────────────────────────────────────────────────── fn extract_table_name(sel: &Select) -> anyhow::Result { - sel.from.first() + sel.from + .first() .and_then(|t| match &t.relation { TableFactor::Table { name, .. } => Some(object_name_str(name)), _ => None, @@ -485,7 +546,8 @@ fn extract_table_name(sel: &Select) -> anyhow::Result { } fn object_name_str(name: &ObjectName) -> String { - name.0.iter() + name.0 + .iter() .map(|i| i.as_ident().map(|id| id.value.as_str()).unwrap_or("")) .collect::>() .join(".") @@ -496,37 +558,63 @@ fn object_name_str(name: &ObjectName) -> String { fn sql_expr_to_scalar(expr: &Expr) -> ScalarExpr { match expr { Expr::Identifier(id) => ScalarExpr::Column(id.value.clone()), - Expr::CompoundIdentifier(parts) => { - ScalarExpr::Column(parts.iter().map(|i| i.value.as_str()).collect::>().join(".")) - } + Expr::CompoundIdentifier(parts) => ScalarExpr::Column( + parts + .iter() + .map(|i| i.value.as_str()) + .collect::>() + .join("."), + ), Expr::Value(vws) => sql_value_to_scalar(&vws.value), Expr::BinaryOp { left, op, right } => { let lhs = sql_expr_to_scalar(left); let rhs = sql_expr_to_scalar(right); let bop = sql_binop_to_algebra(op); - ScalarExpr::BinaryOp { op: bop, lhs: Box::new(lhs), rhs: Box::new(rhs) } + ScalarExpr::BinaryOp { + op: bop, + lhs: Box::new(lhs), + rhs: Box::new(rhs), + } } Expr::IsNull(inner) => ScalarExpr::IsNull { - expr: Box::new(sql_expr_to_scalar(inner)), + expr: Box::new(sql_expr_to_scalar(inner)), negated: false, }, Expr::IsNotNull(inner) => ScalarExpr::IsNull { - expr: Box::new(sql_expr_to_scalar(inner)), + expr: Box::new(sql_expr_to_scalar(inner)), negated: true, }, - Expr::Between { expr, negated, low, high } => ScalarExpr::Between { - expr: Box::new(sql_expr_to_scalar(expr)), - low: Box::new(sql_expr_to_scalar(low)), - high: Box::new(sql_expr_to_scalar(high)), + Expr::Between { + expr, + negated, + low, + high, + } => ScalarExpr::Between { + expr: Box::new(sql_expr_to_scalar(expr)), + low: Box::new(sql_expr_to_scalar(low)), + high: Box::new(sql_expr_to_scalar(high)), negated: *negated, }, - Expr::InList { expr, list, negated } => ScalarExpr::InList { - expr: Box::new(sql_expr_to_scalar(expr)), - list: list.iter().map(sql_expr_to_scalar).collect(), + Expr::InList { + expr, + list, + negated, + } => ScalarExpr::InList { + expr: Box::new(sql_expr_to_scalar(expr)), + list: list.iter().map(sql_expr_to_scalar).collect(), negated: *negated, }, - Expr::Like { expr, pattern, negated, .. } => { - let op = if *negated { BinaryOpKind::NotLike } else { BinaryOpKind::Like }; + Expr::Like { + expr, + pattern, + negated, + .. + } => { + let op = if *negated { + BinaryOpKind::NotLike + } else { + BinaryOpKind::Like + }; ScalarExpr::BinaryOp { op, lhs: Box::new(sql_expr_to_scalar(expr)), @@ -535,7 +623,10 @@ fn sql_expr_to_scalar(expr: &Expr) -> ScalarExpr { } Expr::Nested(inner) => sql_expr_to_scalar(inner), Expr::Function(f) => { - let name = f.name.0.last() + let name = f + .name + .0 + .last() .and_then(|i| i.as_ident()) .map(|id| id.value.clone()) .unwrap_or_default(); @@ -547,8 +638,9 @@ fn sql_expr_to_scalar(expr: &Expr) -> ScalarExpr { fn sql_value_to_scalar(v: &Value) -> ScalarExpr { match v { - Value::SingleQuotedString(s) | Value::DoubleQuotedString(s) => - ScalarExpr::Literal(LiteralValue::Str(s.clone())), + Value::SingleQuotedString(s) | Value::DoubleQuotedString(s) => { + ScalarExpr::Literal(LiteralValue::Str(s.clone())) + } Value::Number(n, _) => { if let Ok(i) = n.parse::() { ScalarExpr::Literal(LiteralValue::Int(i)) @@ -559,31 +651,31 @@ fn sql_value_to_scalar(v: &Value) -> ScalarExpr { } } Value::Boolean(b) => ScalarExpr::Literal(LiteralValue::Bool(*b)), - Value::Null => ScalarExpr::Literal(LiteralValue::Null), - _ => ScalarExpr::Literal(LiteralValue::Null), + Value::Null => ScalarExpr::Literal(LiteralValue::Null), + _ => ScalarExpr::Literal(LiteralValue::Null), } } fn sql_binop_to_algebra(op: &BinaryOperator) -> BinaryOpKind { match op { - BinaryOperator::Plus => BinaryOpKind::Add, - BinaryOperator::Minus => BinaryOpKind::Sub, - BinaryOperator::Multiply => BinaryOpKind::Mul, - BinaryOperator::Divide => BinaryOpKind::Div, - BinaryOperator::Modulo => BinaryOpKind::Mod, - BinaryOperator::Eq => BinaryOpKind::Eq, - BinaryOperator::NotEq => BinaryOpKind::Ne, - BinaryOperator::Lt => BinaryOpKind::Lt, - BinaryOperator::LtEq => BinaryOpKind::Le, - BinaryOperator::Gt => BinaryOpKind::Gt, - BinaryOperator::GtEq => BinaryOpKind::Ge, - BinaryOperator::And => BinaryOpKind::And, - BinaryOperator::Or => BinaryOpKind::Or, + BinaryOperator::Plus => BinaryOpKind::Add, + BinaryOperator::Minus => BinaryOpKind::Sub, + BinaryOperator::Multiply => BinaryOpKind::Mul, + BinaryOperator::Divide => BinaryOpKind::Div, + BinaryOperator::Modulo => BinaryOpKind::Mod, + BinaryOperator::Eq => BinaryOpKind::Eq, + BinaryOperator::NotEq => BinaryOpKind::Ne, + BinaryOperator::Lt => BinaryOpKind::Lt, + BinaryOperator::LtEq => BinaryOpKind::Le, + BinaryOperator::Gt => BinaryOpKind::Gt, + BinaryOperator::GtEq => BinaryOpKind::Ge, + BinaryOperator::And => BinaryOpKind::And, + BinaryOperator::Or => BinaryOpKind::Or, BinaryOperator::BitwiseAnd => BinaryOpKind::BitAnd, - BinaryOperator::BitwiseOr => BinaryOpKind::BitOr, + BinaryOperator::BitwiseOr => BinaryOpKind::BitOr, BinaryOperator::BitwiseXor => BinaryOpKind::BitXor, BinaryOperator::StringConcat => BinaryOpKind::Concat, - _ => BinaryOpKind::Eq, // unknown → eq + _ => BinaryOpKind::Eq, // unknown → eq } } @@ -597,16 +689,11 @@ fn extract_join_qe(sel: &Select) -> Option<(String, JoinKind, Option _ => return None, }; let (kind, pred) = match &join.join_operator { - JoinOperator::Inner(c) => - (JoinKind::Inner, join_constraint_to_scalar(c)), - JoinOperator::LeftOuter(c) => - (JoinKind::LeftOuter, join_constraint_to_scalar(c)), - JoinOperator::RightOuter(c) => - (JoinKind::RightOuter, join_constraint_to_scalar(c)), - JoinOperator::FullOuter(c) => - (JoinKind::FullOuter, join_constraint_to_scalar(c)), - JoinOperator::CrossJoin(_) => - (JoinKind::Cross, None), + JoinOperator::Inner(c) => (JoinKind::Inner, join_constraint_to_scalar(c)), + JoinOperator::LeftOuter(c) => (JoinKind::LeftOuter, join_constraint_to_scalar(c)), + JoinOperator::RightOuter(c) => (JoinKind::RightOuter, join_constraint_to_scalar(c)), + JoinOperator::FullOuter(c) => (JoinKind::FullOuter, join_constraint_to_scalar(c)), + JoinOperator::CrossJoin(_) => (JoinKind::Cross, None), _ => return None, }; Some((inner_table, kind, pred)) @@ -633,7 +720,7 @@ fn expr_to_u64(expr: &Expr) -> Option { fn expr_to_col_name(expr: &Expr) -> Option { match expr { - Expr::Identifier(id) => Some(id.value.clone()), + Expr::Identifier(id) => Some(id.value.clone()), Expr::CompoundIdentifier(parts) => parts.last().map(|i| i.value.clone()), _ => None, } @@ -667,7 +754,9 @@ mod tests { #[test] fn count_star_group_by_is_frequency() { - let pq = pq("SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID"); + let pq = pq( + "SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID", + ); assert!(pq.aggregations.contains(&AggType::Frequency)); assert!(pq.group_by_labels.contains(&"AdvEngineID".to_string())); } @@ -681,10 +770,8 @@ mod tests { #[test] fn count_star_order_by_desc_limit_is_topk() { - let pq = pq( - "SELECT SearchPhrase, COUNT(*) AS c FROM hits \ - WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10", - ); + let pq = pq("SELECT SearchPhrase, COUNT(*) AS c FROM hits \ + WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10"); assert!(pq.aggregations.contains(&AggType::Frequency)); } @@ -721,7 +808,10 @@ mod tests { #[test] fn where_equality_captured() { let pq = pq("SELECT COUNT(*) FROM hits WHERE sectype = 'E' GROUP BY symbol"); - assert_eq!(pq.label_filters.get("sectype").map(String::as_str), Some("E")); + assert_eq!( + pq.label_filters.get("sectype").map(String::as_str), + Some("E") + ); } #[test] @@ -738,9 +828,18 @@ mod tests { "SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) \ FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10", ); - assert!(pq.aggregations.contains(&AggType::Cardinality), "missing cardinality"); - assert!(pq.aggregations.contains(&AggType::Frequency), "missing frequency"); - assert!(pq.aggregations.contains(&AggType::Quantile), "missing quantile"); + assert!( + pq.aggregations.contains(&AggType::Cardinality), + "missing cardinality" + ); + assert!( + pq.aggregations.contains(&AggType::Frequency), + "missing frequency" + ); + assert!( + pq.aggregations.contains(&AggType::Quantile), + "missing quantile" + ); // SUM adds exact_required alongside sketch ops assert!(pq.exact_required, "SUM should set exact_required"); } @@ -808,7 +907,10 @@ mod tests { "SELECT symbol, AVG(price) FROM trades \ GROUP BY symbol, TUMBLE(ts, INTERVAL '5' MINUTE)", ); - assert!(has_windowed_agg(&expr), "expected WindowedAgg in tree, got {expr:?}"); + assert!( + has_windowed_agg(&expr), + "expected WindowedAgg in tree, got {expr:?}" + ); } #[test] @@ -817,7 +919,10 @@ mod tests { "SELECT symbol, COUNT(*) FROM trades \ GROUP BY symbol, HOP(ts, INTERVAL '1' MINUTE, INTERVAL '5' MINUTE)", ); - assert!(has_windowed_agg(&expr), "expected WindowedAgg in tree, got {expr:?}"); + assert!( + has_windowed_agg(&expr), + "expected WindowedAgg in tree, got {expr:?}" + ); } #[test] @@ -826,7 +931,10 @@ mod tests { "SELECT symbol, AVG(price) FROM trades \ GROUP BY symbol, time_bucket('5 minutes', ts)", ); - assert!(has_windowed_agg(&expr), "expected WindowedAgg in tree, got {expr:?}"); + assert!( + has_windowed_agg(&expr), + "expected WindowedAgg in tree, got {expr:?}" + ); } #[test] @@ -838,8 +946,10 @@ mod tests { ); match &expr { QueryExpr::Aggregate { input, .. } => { - assert!(matches!(input.as_ref(), QueryExpr::Window { .. }), - "expected Window inside Aggregate, got {input:?}"); + assert!( + matches!(input.as_ref(), QueryExpr::Window { .. }), + "expected Window inside Aggregate, got {input:?}" + ); } other => panic!("expected Aggregate, got {other:?}"), } diff --git a/controller/src/replan.rs b/controller/src/replan.rs index 10ebe4ee..e1d3e3b1 100644 --- a/controller/src/replan.rs +++ b/controller/src/replan.rs @@ -45,11 +45,11 @@ fn short_hash(s: &str) -> String { // ── Replanner ───────────────────────────────────────────────────────────────── pub struct Replanner { - planner: Arc, - plan_store: Arc, + planner: Arc, + plan_store: Arc, workload_store: Arc, - opamp: Arc, - scraper: Arc, + opamp: Arc, + scraper: Arc, opamp_endpoint: String, /// Optional client for pushing newly-generated `StreamingConfig` /// YAML to the ASAPQuery-backend's `/api/v1/streaming-config` @@ -76,11 +76,11 @@ pub struct Replanner { impl Replanner { pub fn new( - planner: Arc, - plan_store: Arc, + planner: Arc, + plan_store: Arc, workload_store: Arc, - opamp: Arc, - scraper: Arc, + opamp: Arc, + scraper: Arc, opamp_endpoint: impl Into, ) -> Self { Self { @@ -122,7 +122,9 @@ impl Replanner { /// Record that `agent_id` is serving `metric`. Called from `handle_plan` /// after pushing configs so violations can be mapped back to a metric. pub async fn register_agent(&self, agent_id: impl Into, metric: impl Into) { - self.agent_to_metric.write().await + self.agent_to_metric + .write() + .await .insert(agent_id.into(), metric.into()); } @@ -203,8 +205,7 @@ impl Replanner { // skip the stitch — the legacy single-pipeline emit still // covers correctness for the metric being replanned. if let Some(registry) = self.workload_registry.as_ref() { - edge_cfg.metric_to_family = - collect_metric_to_family(registry, &self.workload_store); + edge_cfg.metric_to_family = collect_metric_to_family(registry, &self.workload_store); } // OpAMP `on_connect` doesn't expose the agent's runtime @@ -213,7 +214,13 @@ impl Replanner { // assumption `main::handle_plan`'s typed push path makes // (`push_to_role(Agent, …)` with edge YAML, no runtime // dispatch). - emit_for_runtime(AgentRuntime::AsapOtel, &edge_cfg, &self.opamp_endpoint, None).ok() + emit_for_runtime( + AgentRuntime::AsapOtel, + &edge_cfg, + &self.opamp_endpoint, + None, + ) + .ok() } /// Push the current plan config to a specific agent. @@ -237,7 +244,9 @@ impl Replanner { let metric = self.agent_to_metric.read().await.get(agent_id).cloned(); let Some(metric) = metric else { return false }; - let Ok(plan) = self.plan_store.get(&metric) else { return false }; + let Ok(plan) = self.plan_store.get(&metric) else { + return false; + }; let yaml = if stage_split::typed_stage_split_enabled() { match self.try_emit_typed_edge_yaml(&metric) { @@ -273,10 +282,15 @@ impl Replanner { } }; - self.opamp.push(agent_id, RemoteConfig { - config_hash: short_hash(&yaml), - yaml, - }).await; + self.opamp + .push( + agent_id, + RemoteConfig { + config_hash: short_hash(&yaml), + yaml, + }, + ) + .await; info!(agent = agent_id, metric = %metric, "pushed config to reconnecting agent"); true } @@ -309,7 +323,8 @@ impl Replanner { match self.try_emit_typed_edge_yaml_for_workload(&workload) { Some(y) => { info!( - metric, bytes = y.len(), + metric, + bytes = y.len(), "[USE_TYPED_STAGE_SPLIT] re-plan emitted typed edge YAML" ); Some(y) @@ -327,9 +342,13 @@ impl Replanner { generate_agent_config(&plan.agent_config, &self.opamp_endpoint).ok() }; if let Some(yaml) = agent_yaml { - let cfg = RemoteConfig { config_hash: short_hash(&yaml), yaml }; + let cfg = RemoteConfig { + config_hash: short_hash(&yaml), + yaml, + }; let agents = self.agent_to_metric.read().await; - let target_agents: Vec = agents.iter() + let target_agents: Vec = agents + .iter() .filter(|(_, m)| m.as_str() == metric) .map(|(id, _)| id.clone()) .collect(); @@ -339,10 +358,15 @@ impl Replanner { } } if let Ok(yaml) = generate_backend_config(&plan.backend_config, &self.opamp_endpoint) { - self.opamp.push_to_role( - AgentRole::Backend, - RemoteConfig { config_hash: short_hash(&yaml), yaml }, - ).await; + self.opamp + .push_to_role( + AgentRole::Backend, + RemoteConfig { + config_hash: short_hash(&yaml), + yaml, + }, + ) + .await; } // Push the ASAPQuery-backend StreamingConfig YAML via HTTP if a @@ -369,7 +393,9 @@ impl Replanner { // Update scraper endpoint sketch types for correct EMA attribution. let sketch_type = plan.agent_config.sketch_type; for agent_id in self.opamp.connected_agents().await { - self.scraper.set_sketch_type(&agent_id, sketch_type.clone()).await; + self.scraper + .set_sketch_type(&agent_id, sketch_type.clone()) + .await; } info!(metric, sketch_type = %sketch_type, "re-plan complete"); @@ -379,7 +405,9 @@ impl Replanner { /// Re-plans all metrics whose `valid_until` has already passed. pub async fn replan_expired(&self) { let expired = self.plan_store.expired(chrono::Utc::now()); - if expired.is_empty() { return; } + if expired.is_empty() { + return; + } info!(count = expired.len(), "re-planning expired metrics"); for metric in expired { self.replan_metric(&metric).await; @@ -396,7 +424,10 @@ impl Replanner { self.replan_metric(&m).await; } None => { - warn!(agent = agent_id, "SLA violation but no metric mapping found; re-planning all expired"); + warn!( + agent = agent_id, + "SLA violation but no metric mapping found; re-planning all expired" + ); self.replan_expired().await; } } @@ -429,33 +460,39 @@ mod tests { use crate::types::*; fn make_replanner() -> Arc { - let plan_store = Arc::new(PlanStore::new()); + let plan_store = Arc::new(PlanStore::new()); let workload_store = Arc::new(WorkloadStore::new()); - let planner = Arc::new(BaselinePlanner::new(CostModelPlanner::new())); - let opamp = Arc::new(crate::opamp::OpampServer::new()); - let scraper = Arc::new(crate::monitor::Scraper::new( - vec![], crate::monitor::Thresholds::default(), - Arc::new(|_| {}), Duration::from_secs(60), + let planner = Arc::new(BaselinePlanner::new(CostModelPlanner::new())); + let opamp = Arc::new(crate::opamp::OpampServer::new()); + let scraper = Arc::new(crate::monitor::Scraper::new( + vec![], + crate::monitor::Thresholds::default(), + Arc::new(|_| {}), + Duration::from_secs(60), )); Arc::new(Replanner::new( - planner, plan_store, workload_store, opamp, scraper, + planner, + plan_store, + workload_store, + opamp, + scraper, "ws://ctrl:4320/v1/opamp", )) } fn test_workload(metric: &str) -> (QueryWorkload, WorkloadCharacteristics) { let wl = QueryWorkload { - metric_name: metric.into(), - label_filters: HashMap::new(), - group_by_labels: vec![], - aggregations: vec![AggType::Quantile], - time_window: Duration::from_secs(300), - repeat_every: None, - accuracy_sla: 0.01, - latency_sla: None, + metric_name: metric.into(), + label_filters: HashMap::new(), + group_by_labels: vec![], + aggregations: vec![AggType::Quantile], + time_window: Duration::from_secs(300), + repeat_every: None, + accuracy_sla: 0.01, + latency_sla: None, sketch_type_override: None, - exact_required: false, - quantiles: vec![], + exact_required: false, + quantiles: vec![], }; (wl, WorkloadCharacteristics::default()) } @@ -465,7 +502,10 @@ mod tests { agent_config: AgentCollectorConfig { output_mode: OutputMode::Sketch, sketch_type: SketchType::DDSketch, - sketch_params: SketchParams::DDSketch { relative_accuracy: 0.01, quantiles: vec![0.5, 0.99] }, + sketch_params: SketchParams::DDSketch { + relative_accuracy: 0.01, + quantiles: vec![0.5, 0.99], + }, aggregate_by: vec![], label_matchers: vec![], window_duration: None, @@ -583,7 +623,10 @@ mod tests { let lock = TYPED_ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner()); let previous = std::env::var("USE_TYPED_STAGE_SPLIT").ok(); std::env::set_var("USE_TYPED_STAGE_SPLIT", "1"); - Self { previous, _lock: lock } + Self { + previous, + _lock: lock, + } } } impl Drop for TypedEnvGuard { @@ -674,10 +717,14 @@ mod tests { // Legacy single-pipeline DDSketch output has NONE of the // typed-path processors. - assert!(!yaml.contains("gorillas3"), - "legacy path must not emit gorillas3 processor:\n{yaml}"); - assert!(!yaml.contains("metrics/warm_passthrough"), - "legacy path must not emit warm-passthrough pipeline:\n{yaml}"); + assert!( + !yaml.contains("gorillas3"), + "legacy path must not emit gorillas3 processor:\n{yaml}" + ); + assert!( + !yaml.contains("metrics/warm_passthrough"), + "legacy path must not emit warm-passthrough pipeline:\n{yaml}" + ); // Restore. match prior { diff --git a/controller/src/sketch_algebra/capability.rs b/controller/src/sketch_algebra/capability.rs index 1aeb6816..7bdbe653 100644 --- a/controller/src/sketch_algebra/capability.rs +++ b/controller/src/sketch_algebra/capability.rs @@ -794,10 +794,8 @@ mod tests { #[test] fn is_satisfied_by_topk_handles_must_match() { let required = Capability::FrequencyTopk(SketchKindHandle::CmsWithHeap); - let indexed_with_heap = - Capability::FrequencyTopk(SketchKindHandle::CmsWithHeap); - let indexed_no_heap = - Capability::FrequencyTopk(SketchKindHandle::CountMin); + let indexed_with_heap = Capability::FrequencyTopk(SketchKindHandle::CmsWithHeap); + let indexed_no_heap = Capability::FrequencyTopk(SketchKindHandle::CountMin); assert!(required.is_satisfied_by(&indexed_with_heap)); assert!(!required.is_satisfied_by(&indexed_no_heap)); } @@ -899,7 +897,9 @@ mod tests { fn default_table_hll_serves_cardinality_intent() { let t = default_capability_table(); let cap = t.get(&SketchKind::Hll).unwrap(); - assert!(cap.supported_intents.contains(&SupportedIntent::Cardinality)); + assert!(cap + .supported_intents + .contains(&SupportedIntent::Cardinality)); } // ── load_capability_overrides ──────────────────────────────────────── diff --git a/controller/src/sketch_algebra/capability_matching.rs b/controller/src/sketch_algebra/capability_matching.rs index f0b90e98..5ab4367e 100644 --- a/controller/src/sketch_algebra/capability_matching.rs +++ b/controller/src/sketch_algebra/capability_matching.rs @@ -167,10 +167,7 @@ pub fn is_valid_pair(sketch: SketchKind, statistic: StatisticClass) -> bool { /// `sketch_family_override` (treated as `QueryWorkload::sketch_type_override` /// at the planner-rules layer) wins over the capability-matched default — /// see `planner::rules::bind_workload_typed`. -pub fn pick_family( - statistic: StatisticClass, - accuracy: AccuracyPreference, -) -> Option { +pub fn pick_family(statistic: StatisticClass, accuracy: AccuracyPreference) -> Option { use AccuracyPreference::*; use SketchKind::*; use StatisticClass::*; @@ -198,9 +195,7 @@ pub fn pick_family( /// The name-matching is exact (case-sensitive) to keep the contract row /// the single source of truth — a typo'd metric name should fall through /// to the AggType default rather than silently bind to the wrong family. -pub fn classify_demo_metric( - metric_name: &str, -) -> Option<(StatisticClass, AccuracyPreference)> { +pub fn classify_demo_metric(metric_name: &str) -> Option<(StatisticClass, AccuracyPreference)> { use AccuracyPreference::*; use StatisticClass::*; Some(match metric_name { @@ -224,11 +219,23 @@ mod tests { #[test] fn ddsketch_is_quantile_only() { - assert!(is_valid_pair(SketchKind::DDSketch, StatisticClass::Quantile)); - assert!(!is_valid_pair(SketchKind::DDSketch, StatisticClass::Cardinality)); + assert!(is_valid_pair( + SketchKind::DDSketch, + StatisticClass::Quantile + )); + assert!(!is_valid_pair( + SketchKind::DDSketch, + StatisticClass::Cardinality + )); assert!(!is_valid_pair(SketchKind::DDSketch, StatisticClass::TopK)); - assert!(!is_valid_pair(SketchKind::DDSketch, StatisticClass::Frequency)); - assert!(!is_valid_pair(SketchKind::DDSketch, StatisticClass::SumRateCount)); + assert!(!is_valid_pair( + SketchKind::DDSketch, + StatisticClass::Frequency + )); + assert!(!is_valid_pair( + SketchKind::DDSketch, + StatisticClass::SumRateCount + )); } #[test] @@ -237,7 +244,10 @@ mod tests { assert!(!is_valid_pair(SketchKind::Kll, StatisticClass::Cardinality)); assert!(!is_valid_pair(SketchKind::Kll, StatisticClass::TopK)); assert!(!is_valid_pair(SketchKind::Kll, StatisticClass::Frequency)); - assert!(!is_valid_pair(SketchKind::Kll, StatisticClass::SumRateCount)); + assert!(!is_valid_pair( + SketchKind::Kll, + StatisticClass::SumRateCount + )); } #[test] @@ -246,16 +256,31 @@ mod tests { assert!(!is_valid_pair(SketchKind::Hll, StatisticClass::Quantile)); assert!(!is_valid_pair(SketchKind::Hll, StatisticClass::TopK)); assert!(!is_valid_pair(SketchKind::Hll, StatisticClass::Frequency)); - assert!(!is_valid_pair(SketchKind::Hll, StatisticClass::SumRateCount)); + assert!(!is_valid_pair( + SketchKind::Hll, + StatisticClass::SumRateCount + )); } #[test] fn countsketch_is_topk_only() { assert!(is_valid_pair(SketchKind::CountSketch, StatisticClass::TopK)); - assert!(!is_valid_pair(SketchKind::CountSketch, StatisticClass::Quantile)); - assert!(!is_valid_pair(SketchKind::CountSketch, StatisticClass::Cardinality)); - assert!(!is_valid_pair(SketchKind::CountSketch, StatisticClass::Frequency)); - assert!(!is_valid_pair(SketchKind::CountSketch, StatisticClass::SumRateCount)); + assert!(!is_valid_pair( + SketchKind::CountSketch, + StatisticClass::Quantile + )); + assert!(!is_valid_pair( + SketchKind::CountSketch, + StatisticClass::Cardinality + )); + assert!(!is_valid_pair( + SketchKind::CountSketch, + StatisticClass::Frequency + )); + assert!(!is_valid_pair( + SketchKind::CountSketch, + StatisticClass::SumRateCount + )); } #[test] @@ -266,7 +291,10 @@ mod tests { assert!(is_valid_pair(SketchKind::Cms, StatisticClass::TopK)); assert!(!is_valid_pair(SketchKind::Cms, StatisticClass::Quantile)); assert!(!is_valid_pair(SketchKind::Cms, StatisticClass::Cardinality)); - assert!(!is_valid_pair(SketchKind::Cms, StatisticClass::SumRateCount)); + assert!(!is_valid_pair( + SketchKind::Cms, + StatisticClass::SumRateCount + )); } #[test] @@ -303,7 +331,10 @@ mod tests { #[test] fn pick_family_cardinality_picks_hll() { - for pref in [AccuracyPreference::RelativeError, AccuracyPreference::RankError] { + for pref in [ + AccuracyPreference::RelativeError, + AccuracyPreference::RankError, + ] { assert_eq!( pick_family(StatisticClass::Cardinality, pref), Some(SketchKind::Hll), diff --git a/controller/src/sketch_algebra/rules/bind_archive_only.rs b/controller/src/sketch_algebra/rules/bind_archive_only.rs index 8b6b2c8d..85556218 100644 --- a/controller/src/sketch_algebra/rules/bind_archive_only.rs +++ b/controller/src/sketch_algebra/rules/bind_archive_only.rs @@ -67,8 +67,8 @@ impl Rule for BindArchiveOnly { #[cfg(test)] mod tests { use super::*; - use crate::intent_algebra::{LabelFilter, Schema, Source, WindowKind}; use crate::intent_algebra::schema::{Column, DataType}; + use crate::intent_algebra::{LabelFilter, Schema, Source, WindowKind}; use std::time::Duration; fn ts_scan() -> QueryExpr { diff --git a/controller/src/sketch_algebra/rules/bind_cms_count.rs b/controller/src/sketch_algebra/rules/bind_cms_count.rs index e808604e..fbd0e01d 100644 --- a/controller/src/sketch_algebra/rules/bind_cms_count.rs +++ b/controller/src/sketch_algebra/rules/bind_cms_count.rs @@ -76,14 +76,8 @@ impl Rule for BindCmsOnCount { (a.min(*eps), *delta) } ( - AccuracyTarget::EpsilonDelta { - eps: a, - delta: da, - }, - AccuracyTarget::EpsilonDelta { - eps: b, - delta: db, - }, + AccuracyTarget::EpsilonDelta { eps: a, delta: da }, + AccuracyTarget::EpsilonDelta { eps: b, delta: db }, ) => (a.min(*b), da.min(*db)), }; diff --git a/controller/src/sketch_algebra/rules/bind_cms_topk.rs b/controller/src/sketch_algebra/rules/bind_cms_topk.rs index 4d9b465d..69fbe54b 100644 --- a/controller/src/sketch_algebra/rules/bind_cms_topk.rs +++ b/controller/src/sketch_algebra/rules/bind_cms_topk.rs @@ -62,14 +62,8 @@ impl Rule for BindCountSketchOnTopK { (a.min(*eps), *delta) } ( - AccuracyTarget::EpsilonDelta { - eps: a, - delta: da, - }, - AccuracyTarget::EpsilonDelta { - eps: b, - delta: db, - }, + AccuracyTarget::EpsilonDelta { eps: a, delta: da }, + AccuracyTarget::EpsilonDelta { eps: b, delta: db }, ) => (a.min(*b), da.min(*db)), }; diff --git a/controller/src/sketch_algebra/schema.rs b/controller/src/sketch_algebra/schema.rs index aae569c7..3ecbf88c 100644 --- a/controller/src/sketch_algebra/schema.rs +++ b/controller/src/sketch_algebra/schema.rs @@ -130,10 +130,8 @@ mod tests { #[test] fn kll_default_caps() { - let s = SketchStateSchema::for_kind( - SketchKind::Kll, - SketchParams::Kll(KllParams { k: 200 }), - ); + let s = + SketchStateSchema::for_kind(SketchKind::Kll, SketchParams::Kll(KllParams { k: 200 })); assert!(s.caps.mergeable); assert!(!s.caps.subtractable); assert!(!s.caps.deletable); @@ -152,28 +150,20 @@ mod tests { #[test] fn merge_compatibility_requires_matching_params() { - let a = SketchStateSchema::for_kind( - SketchKind::Kll, - SketchParams::Kll(KllParams { k: 200 }), - ); - let b = SketchStateSchema::for_kind( - SketchKind::Kll, - SketchParams::Kll(KllParams { k: 200 }), - ); - let c = SketchStateSchema::for_kind( - SketchKind::Kll, - SketchParams::Kll(KllParams { k: 400 }), - ); + let a = + SketchStateSchema::for_kind(SketchKind::Kll, SketchParams::Kll(KllParams { k: 200 })); + let b = + SketchStateSchema::for_kind(SketchKind::Kll, SketchParams::Kll(KllParams { k: 200 })); + let c = + SketchStateSchema::for_kind(SketchKind::Kll, SketchParams::Kll(KllParams { k: 400 })); assert!(a.is_compatible_for_merge(&b)); assert!(!a.is_compatible_for_merge(&c)); // different k } #[test] fn merge_compatibility_rejects_family_mismatch() { - let kll = SketchStateSchema::for_kind( - SketchKind::Kll, - SketchParams::Kll(KllParams { k: 200 }), - ); + let kll = + SketchStateSchema::for_kind(SketchKind::Kll, SketchParams::Kll(KllParams { k: 200 })); let cms = SketchStateSchema::for_kind( SketchKind::Cms, SketchParams::Cms(CmsParams { w: 2048, d: 5 }), diff --git a/controller/src/sketch_algebra/sketch_expr.rs b/controller/src/sketch_algebra/sketch_expr.rs index 480953a8..6b490c15 100644 --- a/controller/src/sketch_algebra/sketch_expr.rs +++ b/controller/src/sketch_algebra/sketch_expr.rs @@ -204,13 +204,9 @@ impl SketchExpr { #[cfg(test)] mod tests { use super::*; - use crate::intent_algebra::{ - AggIntent, LabelFilter, QueryExpr, Schema, Source, WindowKind, - }; use crate::intent_algebra::schema::{Column, DataType}; - use crate::sketch_algebra::params::{ - CountSketchParams, DDSketchParams, HllParams, KllParams, - }; + use crate::intent_algebra::{AggIntent, LabelFilter, QueryExpr, Schema, Source, WindowKind}; + use crate::sketch_algebra::params::{CountSketchParams, DDSketchParams, HllParams, KllParams}; use crate::types_v2::AccuracyTarget; use std::time::Duration; diff --git a/controller/src/sketch_algebra/tests.rs b/controller/src/sketch_algebra/tests.rs index 605c0b1f..822c1788 100644 --- a/controller/src/sketch_algebra/tests.rs +++ b/controller/src/sketch_algebra/tests.rs @@ -4,10 +4,8 @@ use std::time::Duration; -use crate::intent_algebra::{ - AggIntent, LabelFilter, QueryExpr, Schema, Source, WindowKind, -}; use crate::intent_algebra::schema::{Column, DataType}; +use crate::intent_algebra::{AggIntent, LabelFilter, QueryExpr, Schema, Source, WindowKind}; use crate::sketch_algebra::lower::bind_query_expr; use crate::sketch_algebra::params::{KllParams, SketchKind, SketchParams}; use crate::sketch_algebra::rules::{ @@ -69,9 +67,7 @@ fn agg_quantile(q: f64, accuracy: AccuracyTarget) -> QueryExpr { #[test] fn sketch_expr_serde_roundtrip() { - use crate::sketch_algebra::params::{ - CmsParams, CountSketchParams, DDSketchParams, HllParams, - }; + use crate::sketch_algebra::params::{CmsParams, CountSketchParams, DDSketchParams, HllParams}; let cases = vec![ SketchExpr::Logical(windowed_scan()), SketchExpr::SketchAgg { @@ -154,7 +150,10 @@ fn bind_kll_quantile_basic() { } => { assert_eq!(sketch_type, SketchKind::Kll); assert_eq!(params, SketchParams::Kll(KllParams { k: 200 })); - assert!(matches!(*child, SketchExpr::Logical(QueryExpr::Window { .. }))); + assert!(matches!( + *child, + SketchExpr::Logical(QueryExpr::Window { .. }) + )); } other => panic!("expected SketchAgg, got {other:?}"), } @@ -198,8 +197,8 @@ fn bind_ddsketch_quantile_basic() { #[test] fn bind_picks_ddsketch_over_kll_when_eps_explicit() { let expr = agg_quantile(0.99, AccuracyTarget::Epsilon(0.01)); - let bound = - bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)).expect("bind_query_expr should not error"); + let bound = bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)) + .expect("bind_query_expr should not error"); match bound { SketchExpr::SketchEstimate { child, .. } => match *child { SketchExpr::SketchAgg { sketch_type, .. } => { @@ -249,7 +248,10 @@ fn bind_cms_topk_basic() { assert_eq!(sketch_type, SketchKind::CountSketch); match params { SketchParams::CountSketch(p) => { - assert!(p.with_heap, "TopK binding must enable the heavy-hitter heap"); + assert!( + p.with_heap, + "TopK binding must enable the heavy-hitter heap" + ); assert!(p.w >= 2); assert!(p.d >= 1); } @@ -526,8 +528,8 @@ fn phase_b_pattern_archive_only_routes_to_archive() { /// it under the supplied accuracy target; `bind_query_expr` is the L3→L4 /// bottom-up walk. fn pipeline_l1_to_l4(query: &str, accuracy: AccuracyTarget) -> SketchExpr { - let parsed = crate::query_parser::parse_query(query) - .unwrap_or_else(|e| panic!("parse {query}: {e}")); + let parsed = + crate::query_parser::parse_query(query).unwrap_or_else(|e| panic!("parse {query}: {e}")); let qe = crate::intent_algebra::lower_parsed_query(&parsed, accuracy.clone()) .unwrap_or_else(|e| panic!("lower {query}: {e}")); bind_query_expr(&qe, accuracy).unwrap_or_else(|e| panic!("bind {query}: {e}")) @@ -540,7 +542,9 @@ fn collect_sketch_kinds(expr: &SketchExpr) -> Vec { let mut out = Vec::new(); fn walk(e: &SketchExpr, out: &mut Vec) { match e { - SketchExpr::SketchAgg { sketch_type, child, .. } => { + SketchExpr::SketchAgg { + sketch_type, child, .. + } => { out.push(sketch_type.clone()); walk(child, out); } @@ -684,7 +688,10 @@ fn phase_b_e2e_rate_falls_through_to_logical() { AccuracyTarget::Epsilon(0.01), ); assert!(collect_sketch_kinds(&bound).is_empty()); - assert!(!binding_is_archive(&bound), "Rate is warm-tier, not archive"); + assert!( + !binding_is_archive(&bound), + "Rate is warm-tier, not archive" + ); } /// `topk.yaml` — `topk(10, sum by (label) (rate(...))`. The legacy diff --git a/controller/src/store/mod.rs b/controller/src/store/mod.rs index 3889c427..f66ae587 100644 --- a/controller/src/store/mod.rs +++ b/controller/src/store/mod.rs @@ -107,22 +107,26 @@ impl PlanStore { /// Returns `None` if no previous plan exists. pub fn diff(&self, metric: &str) -> Result, StoreError> { let inner = self.inner.read().unwrap(); - let e = inner.entries.get(metric) + let e = inner + .entries + .get(metric) .ok_or_else(|| StoreError::NotFound(metric.to_string()))?; - let Some(prev) = &e.previous else { return Ok(None) }; + let Some(prev) = &e.previous else { + return Ok(None); + }; let curr = &e.current; let diff = PlanDiff { sketch_type_changed: prev.agent_config.sketch_type != curr.agent_config.sketch_type, - prev_sketch_type: prev.agent_config.sketch_type.to_string(), - curr_sketch_type: curr.agent_config.sketch_type.to_string(), - delta_transmission_changed: - prev.agent_config.delta_transmission != curr.agent_config.delta_transmission, + prev_sketch_type: prev.agent_config.sketch_type.to_string(), + curr_sketch_type: curr.agent_config.sketch_type.to_string(), + delta_transmission_changed: prev.agent_config.delta_transmission + != curr.agent_config.delta_transmission, prev_delta_transmission: prev.agent_config.delta_transmission, curr_delta_transmission: curr.agent_config.delta_transmission, mode_changed: prev.agent_config.mode != curr.agent_config.mode, - prev_mode: prev.agent_config.mode.to_string(), - curr_mode: curr.agent_config.mode.to_string(), - updated_at: e.updated_at, + prev_mode: prev.agent_config.mode.to_string(), + curr_mode: curr.agent_config.mode.to_string(), + updated_at: e.updated_at, }; Ok(Some(diff)) } @@ -131,16 +135,16 @@ impl PlanStore { /// A human-readable summary of what changed between the current and previous plan. #[derive(Debug, Clone, serde::Serialize)] pub struct PlanDiff { - pub sketch_type_changed: bool, - pub prev_sketch_type: String, - pub curr_sketch_type: String, + pub sketch_type_changed: bool, + pub prev_sketch_type: String, + pub curr_sketch_type: String, pub delta_transmission_changed: bool, - pub prev_delta_transmission: bool, - pub curr_delta_transmission: bool, - pub mode_changed: bool, - pub prev_mode: String, - pub curr_mode: String, - pub updated_at: DateTime, + pub prev_delta_transmission: bool, + pub curr_delta_transmission: bool, + pub mode_changed: bool, + pub prev_mode: String, + pub curr_mode: String, + pub updated_at: DateTime, } // ── Tests ───────────────────────────────────────────────────────────────────── diff --git a/controller/src/store/workload.rs b/controller/src/store/workload.rs index 38113a9a..149bc564 100644 --- a/controller/src/store/workload.rs +++ b/controller/src/store/workload.rs @@ -12,7 +12,9 @@ pub struct WorkloadStore { impl WorkloadStore { pub fn new() -> Self { - Self { inner: RwLock::new(HashMap::new()) } + Self { + inner: RwLock::new(HashMap::new()), + } } pub fn set(&self, metric: impl Into, wl: QueryWorkload, wc: WorkloadCharacteristics) { @@ -32,23 +34,23 @@ impl WorkloadStore { #[cfg(test)] mod tests { use super::*; + use crate::types::AggType; use std::collections::HashMap; use std::time::Duration; - use crate::types::AggType; fn wl(name: &str) -> QueryWorkload { QueryWorkload { - metric_name: name.into(), - label_filters: HashMap::new(), - group_by_labels: vec![], - aggregations: vec![AggType::Quantile], - time_window: Duration::from_secs(300), - repeat_every: None, - accuracy_sla: 0.01, - latency_sla: None, + metric_name: name.into(), + label_filters: HashMap::new(), + group_by_labels: vec![], + aggregations: vec![AggType::Quantile], + time_window: Duration::from_secs(300), + repeat_every: None, + accuracy_sla: 0.01, + latency_sla: None, sketch_type_override: None, - exact_required: false, - quantiles: vec![], + exact_required: false, + quantiles: vec![], } } diff --git a/controller/src/types.rs b/controller/src/types.rs index f5069982..e13b546d 100644 --- a/controller/src/types.rs +++ b/controller/src/types.rs @@ -330,26 +330,45 @@ impl Default for SketchDefaults { } impl Default for DDSketchDefaults { - fn default() -> Self { Self { relative_accuracy: 0.01 } } + fn default() -> Self { + Self { + relative_accuracy: 0.01, + } + } } impl Default for KLLDefaults { - fn default() -> Self { Self { min_k: 32 } } + fn default() -> Self { + Self { min_k: 32 } + } } impl Default for HLLDefaults { fn default() -> Self { - Self { precision_coarse: 10, precision_fine: 14, precision_threshold: 0.02 } + Self { + precision_coarse: 10, + precision_fine: 14, + precision_threshold: 0.02, + } } } impl Default for CountSketchDefaults { - fn default() -> Self { Self { epsilon: 0.022, delta: 0.007 } } + fn default() -> Self { + Self { + epsilon: 0.022, + delta: 0.007, + } + } } impl Default for CountMinSketchDefaults { fn default() -> Self { - Self { rows: 5, cols: 2048, metric_name: "countsketch_partition".into() } + Self { + rows: 5, + cols: 2048, + metric_name: "countsketch_partition".into(), + } } } @@ -410,8 +429,9 @@ impl SketchParams { /// Extract quantiles if this sketch type supports them. pub fn quantiles(&self) -> &[f64] { match self { - SketchParams::DDSketch { quantiles, .. } - | SketchParams::KLL { quantiles, .. } => quantiles, + SketchParams::DDSketch { quantiles, .. } | SketchParams::KLL { quantiles, .. } => { + quantiles + } _ => &[], } } @@ -472,7 +492,10 @@ pub enum AgentDataSink { /// `compression` is the transport-level codec; the canonical /// path uses `none` because the backend's tonic gRPC server /// rejects gzip-compressed bodies (returns Unimplemented). - Otlp { endpoint: String, compression: String }, + Otlp { + endpoint: String, + compression: String, + }, /// Pre-existing path: prometheus exporter at `endpoint`. Kept /// for back-compat with the legacy raw-scalar deployment. PrometheusScrape { endpoint: String }, diff --git a/controller/src/warm_tier_analysis.rs b/controller/src/warm_tier_analysis.rs index c2adef6d..92a72fc8 100644 --- a/controller/src/warm_tier_analysis.rs +++ b/controller/src/warm_tier_analysis.rs @@ -390,7 +390,10 @@ mod tests { #[test] fn analyze_quantile_over_time() { let a = analyze_promql_for_warm_tier("quantile_over_time(0.99, http_latency_ms[5m])"); - assert!(a.unsupported.is_none(), "expected no unsupported reason: {a:?}"); + assert!( + a.unsupported.is_none(), + "expected no unsupported reason: {a:?}" + ); assert_eq!(a.candidates.len(), 1); let c = &a.candidates[0]; assert_eq!(c.metric_name, "http_latency_ms"); diff --git a/controller/src/workload.rs b/controller/src/workload.rs index 725949c6..39cdda15 100644 --- a/controller/src/workload.rs +++ b/controller/src/workload.rs @@ -43,8 +43,12 @@ pub struct WorkloadEntry { pub target_path: Option, } -fn default_accuracy_sla() -> f64 { 0.01 } -fn default_role() -> String { "agent".into() } +fn default_accuracy_sla() -> f64 { + 0.01 +} +fn default_role() -> String { + "agent".into() +} /// Case-insensitive `SketchType` deserialiser. The wire YAML in /// `deploy/configs/mvp-workload.yaml` spells the variants in mixed case @@ -64,10 +68,12 @@ where "hll" => SketchType::HLL, "countsketch" => SketchType::CountSketch, "countminsketch" | "countmin" | "cms" => SketchType::CountMinSketch, - other => return Err(serde::de::Error::custom(format!( - "unknown sketch_family_override `{other}`; expected one of \ + other => { + return Err(serde::de::Error::custom(format!( + "unknown sketch_family_override `{other}`; expected one of \ DDSketch / KLL / HLL / CountSketch / CountMinSketch" - ))), + ))) + } }; Ok(Some(kind)) } @@ -117,14 +123,16 @@ impl WorkloadRegistry { /// Returns workload entries assigned to a given role. pub fn for_role(&self, role: &str) -> Vec<&WorkloadEntry> { - self.entries.iter() + self.entries + .iter() .filter(|e| e.assign_to_role.eq_ignore_ascii_case(role)) .collect() } /// Returns the first workload entry for a given role, if any. pub fn first_for_role(&self, role: &str) -> Option<&WorkloadEntry> { - self.entries.iter() + self.entries + .iter() .find(|e| e.assign_to_role.eq_ignore_ascii_case(role)) } } @@ -227,9 +235,18 @@ mod tests { assert_eq!(entries.len(), 6); assert_eq!(entries[0].sketch_family_override, Some(SketchType::KLL)); assert_eq!(entries[1].sketch_family_override, Some(SketchType::HLL)); - assert_eq!(entries[2].sketch_family_override, Some(SketchType::CountSketch)); - assert_eq!(entries[3].sketch_family_override, Some(SketchType::CountMinSketch)); - assert_eq!(entries[4].sketch_family_override, Some(SketchType::DDSketch)); + assert_eq!( + entries[2].sketch_family_override, + Some(SketchType::CountSketch) + ); + assert_eq!( + entries[3].sketch_family_override, + Some(SketchType::CountMinSketch) + ); + assert_eq!( + entries[4].sketch_family_override, + Some(SketchType::DDSketch) + ); assert_eq!(entries[5].sketch_family_override, None); } @@ -246,9 +263,18 @@ mod tests { sketch_family_override: cms "#; let entries: Vec = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(entries[0].sketch_family_override, Some(SketchType::DDSketch)); - assert_eq!(entries[1].sketch_family_override, Some(SketchType::CountMinSketch)); - assert_eq!(entries[2].sketch_family_override, Some(SketchType::CountMinSketch)); + assert_eq!( + entries[0].sketch_family_override, + Some(SketchType::DDSketch) + ); + assert_eq!( + entries[1].sketch_family_override, + Some(SketchType::CountMinSketch) + ); + assert_eq!( + entries[2].sketch_family_override, + Some(SketchType::CountMinSketch) + ); } #[test] @@ -266,29 +292,36 @@ mod tests { return; } let registry = WorkloadRegistry::load(path.to_str().unwrap()); - let by_name: std::collections::HashMap<&str, &WorkloadEntry> = - registry.entries().iter().map(|e| (e.metric_name.as_str(), e)).collect(); + let by_name: std::collections::HashMap<&str, &WorkloadEntry> = registry + .entries() + .iter() + .map(|e| (e.metric_name.as_str(), e)) + .collect(); assert_eq!( - by_name.get("request_size_bytes") + by_name + .get("request_size_bytes") .and_then(|e| e.sketch_family_override.clone()), Some(SketchType::KLL), "request_size_bytes must carry KLL override", ); assert_eq!( - by_name.get("unique_users_per_min") + by_name + .get("unique_users_per_min") .and_then(|e| e.sketch_family_override.clone()), Some(SketchType::HLL), "unique_users_per_min must carry HLL override", ); assert_eq!( - by_name.get("top_endpoint_qps") + by_name + .get("top_endpoint_qps") .and_then(|e| e.sketch_family_override.clone()), Some(SketchType::CountSketch), "top_endpoint_qps must carry CountSketch override", ); assert_eq!( - by_name.get("endpoint_request_freq") + by_name + .get("endpoint_request_freq") .and_then(|e| e.sketch_family_override.clone()), Some(SketchType::CountMinSketch), "endpoint_request_freq must carry CountMinSketch override", diff --git a/crates/asap_types/src/capability_matching.rs b/crates/asap_types/src/capability_matching.rs index 8de8e358..73f6696d 100644 --- a/crates/asap_types/src/capability_matching.rs +++ b/crates/asap_types/src/capability_matching.rs @@ -12,6 +12,11 @@ use crate::query_requirements::QueryRequirements; use crate::utils::normalize_spatial_filter; use promql_utilities::query_logics::enums::AggregationType; +pub const ENGINE_ID_ASAP_QUERY: &str = "asap_query"; +pub const ENGINE_ID_THANOS_QUERY: &str = "thanos_query"; + +pub const CANONICAL_QUERY_ENGINE_IDS: &[&str] = &[ENGINE_ID_ASAP_QUERY, ENGINE_ID_THANOS_QUERY]; + // --------------------------------------------------------------------------- // Phase-5: storage-backend capability axis // @@ -25,29 +30,28 @@ use promql_utilities::query_logics::enums::AggregationType; /// Which physical storage tier a query (or a metric configuration) routes to. /// -/// `SketchWarmTier` is the default — every existing `AggregationConfig` and +/// `SketchStore` is the default — every existing `AggregationConfig` and /// `StreamingConfig` decodes into this variant via `#[serde(default)]`, so -/// pre-Phase-5 deploys keep dispatching to `SimpleEngine` unchanged. +/// pre-Phase-5 deploys keep dispatching to `ASAPQueryEngine` unchanged. /// /// **Step-1 of the JSONL deprecation refactor** removed the /// `ColdJsonlFallback` variant. The legacy local-FS JSONL leg /// (`LocalFsColdStore`, `parse_jsonl`, the §5.2 raw-store /// fallback) was deleted at the same commit; the surviving -/// failover surface is warm-tier sketch ↔ Gorilla-S3 archive. -#[derive( - Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Default, -)] +/// failover surface is warm-tier sketch ↔ Thanos archive. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] #[serde(rename_all = "snake_case")] pub enum StorageBackend { /// Warm-tier sketch DB (today's `SimpleMapStore` + accumulators). - /// Served by `SimpleEngine`. Default for unconfigured metrics. + /// Served by `ASAPQueryEngine`. Default for unconfigured metrics. #[default] - SketchWarmTier, + SketchStore, - /// Gorilla-S3 archive. Served by `GorillaQueryEngine`, reading - /// per-hour Gorilla chunks from S3 / MinIO via the - /// `GorillaS3Store`. - GorillaS3Archive, + /// Thanos archive over MinIO/S3. The enum name is kept for + /// serde/back-compat with existing configs, but its canonical + /// query-engine identity is `thanos_query`. Gorilla is an + /// archive chunk format/storage detail, not a public query engine. + GorillaObjectStore, /// Double-write: the metric is written to both warm-tier sketches AND the /// Gorilla-S3 archive. Capability matching surfaces both options and the @@ -61,34 +65,40 @@ pub enum StorageBackend { /// Prometheus's `/api/v1/query`) under this slot so the /// controller's `RawAtEdgePrometheusArchive` mode can route a /// metric's queries to Prometheus directly. Mirrors the - /// `GorillaS3Archive` slot's "single backend, no failover" + /// `GorillaObjectStore` slot's "single backend, no failover" /// semantics — there is no warm-tier sketch to fall back on for a /// Prometheus-remote metric. PrometheusRemote, } impl StorageBackend { - /// String tag pinned for byte-comparable dispatch on the wire (mirrors + /// Canonical string tag pinned for byte-comparable dispatch on the wire (mirrors /// the `data_source: ` info-line on `QueryResult`). Engines /// register themselves under these IDs in the router. pub const fn data_source_id(self) -> &'static str { match self { - StorageBackend::SketchWarmTier => "sketch_warm", - StorageBackend::GorillaS3Archive => "gorilla_archive", + StorageBackend::SketchStore => ENGINE_ID_ASAP_QUERY, + StorageBackend::GorillaObjectStore => ENGINE_ID_THANOS_QUERY, StorageBackend::DoubleWrite => "double_write", StorageBackend::PrometheusRemote => "prometheus_remote", } } } +pub fn parse_storage_backend_engine_id(s: &str) -> Option { + match s { + ENGINE_ID_ASAP_QUERY => Some(StorageBackend::SketchStore), + ENGINE_ID_THANOS_QUERY => Some(StorageBackend::GorillaObjectStore), + _ => None, + } +} + /// Accuracy hint pushed by the controller at intent-binding time /// (`controller/docs/design.md` §6 `core::workload`). The Phase-5 capability /// router consults this to decide whether a metric configured for both warm- /// tier and Gorilla-S3 should answer from the archive (Exact) or the /// approximate warm-tier sketch. -#[derive( - Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Default, -)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] #[serde(rename_all = "snake_case")] pub enum AccuracyTarget { /// Caller demands an exact answer; warm-tier sketches are not eligible @@ -244,39 +254,39 @@ pub fn compatible_agg_types(stat: Statistic) -> &'static [AggregationType] { /// /// Routing rules (mirrors `docs/design-gorilla-s3-cold-engine.md` §8): /// -/// * Metric configured for `GorillaS3Archive`: always -/// `[GorillaS3Archive]`. Exact-on-archive subsumes approximate-on-warm, +/// * Metric configured for `GorillaObjectStore`: always +/// `[GorillaObjectStore]`. Exact-on-archive subsumes approximate-on-warm, /// so even a `Statistic::Quantile` with an `Approximate` target still /// routes to the archive when the metric is Gorilla-only — there is no /// warm-tier sketch to fall back to in that deploy shape. -/// * Metric configured for `SketchWarmTier` (or unconfigured / default): -/// `[SketchWarmTier]`. A capability miss in the warm tier surfaces +/// * Metric configured for `SketchStore` (or unconfigured / default): +/// `[SketchStore]`. A capability miss in the warm tier surfaces /// as a 404 — the previous JSONL fallback path has been deleted. /// * Metric configured for `DoubleWrite`: head depends on accuracy hint, /// tail is the failover sequence (the cost-aware `EngineRouter` picks /// the head, walks the tail on failure): -/// - `Exact` → `[GorillaS3Archive, SketchWarmTier]` -/// - `Approximate` → `[SketchWarmTier, GorillaS3Archive]` +/// - `Exact` → `[GorillaObjectStore, SketchStore]` +/// - `Approximate` → `[SketchStore, GorillaObjectStore]` pub fn compatible_storage_backends( _stat: Statistic, accuracy: AccuracyTarget, metric_storage_config: StorageBackend, ) -> Vec { match metric_storage_config { - StorageBackend::GorillaS3Archive => { - vec![StorageBackend::GorillaS3Archive] + StorageBackend::GorillaObjectStore => { + vec![StorageBackend::GorillaObjectStore] } - StorageBackend::SketchWarmTier => { - vec![StorageBackend::SketchWarmTier] + StorageBackend::SketchStore => { + vec![StorageBackend::SketchStore] } StorageBackend::DoubleWrite => match accuracy { AccuracyTarget::Exact => vec![ - StorageBackend::GorillaS3Archive, - StorageBackend::SketchWarmTier, + StorageBackend::GorillaObjectStore, + StorageBackend::SketchStore, ], AccuracyTarget::Approximate => vec![ - StorageBackend::SketchWarmTier, - StorageBackend::GorillaS3Archive, + StorageBackend::SketchStore, + StorageBackend::GorillaObjectStore, ], }, // Phase ε.2: Prometheus-remote metrics route only to the @@ -1354,22 +1364,22 @@ mod tests { let backends = compatible_storage_backends( Statistic::Sum, AccuracyTarget::Exact, - StorageBackend::GorillaS3Archive, + StorageBackend::GorillaObjectStore, ); - assert_eq!(backends, vec![StorageBackend::GorillaS3Archive]); + assert_eq!(backends, vec![StorageBackend::GorillaObjectStore]); } #[test] - fn sketch_warm_tier_metric_routes_to_simple_engine() { + fn asap_query_metric_routes_to_simple_engine() { let backends = compatible_storage_backends( Statistic::Quantile, AccuracyTarget::Approximate, - StorageBackend::SketchWarmTier, + StorageBackend::SketchStore, ); // Step-1 of the JSONL deprecation: warm-tier only routes // to itself; the previous `ColdJsonlFallback` failover slot // has been deleted. - assert_eq!(backends, vec![StorageBackend::SketchWarmTier]); + assert_eq!(backends, vec![StorageBackend::SketchStore]); } #[test] @@ -1383,8 +1393,8 @@ mod tests { assert_eq!( exact, vec![ - StorageBackend::GorillaS3Archive, - StorageBackend::SketchWarmTier, + StorageBackend::GorillaObjectStore, + StorageBackend::SketchStore, ] ); // Approximate: warm-tier head (cheaper for ε/δ-bounded @@ -1397,8 +1407,8 @@ mod tests { assert_eq!( approx, vec![ - StorageBackend::SketchWarmTier, - StorageBackend::GorillaS3Archive, + StorageBackend::SketchStore, + StorageBackend::GorillaObjectStore, ] ); } @@ -1412,38 +1422,51 @@ mod tests { let backends = compatible_storage_backends( Statistic::Quantile, AccuracyTarget::Approximate, - StorageBackend::GorillaS3Archive, + StorageBackend::GorillaObjectStore, ); - assert_eq!(backends, vec![StorageBackend::GorillaS3Archive]); + assert_eq!(backends, vec![StorageBackend::GorillaObjectStore]); } #[test] fn storage_backend_default_is_warm_tier() { // `#[serde(default)]` on `StreamingConfig.storage_backend` (and on - // `StorageBackend::default()`) MUST be `SketchWarmTier` so pre-Phase-5 + // `StorageBackend::default()`) MUST be `SketchStore` so pre-Phase-5 // configs decode without bumping deploys onto the archive. - assert_eq!(StorageBackend::default(), StorageBackend::SketchWarmTier); + assert_eq!(StorageBackend::default(), StorageBackend::SketchStore); } #[test] fn storage_backend_data_source_id_is_pinned() { // The router registers engines by these strings; dashboards // byte-compare them. Pin to catch accidental rename. - assert_eq!(StorageBackend::SketchWarmTier.data_source_id(), "sketch_warm"); assert_eq!( - StorageBackend::GorillaS3Archive.data_source_id(), - "gorilla_archive", + StorageBackend::SketchStore.data_source_id(), + ENGINE_ID_ASAP_QUERY ); assert_eq!( - StorageBackend::DoubleWrite.data_source_id(), - "double_write", + StorageBackend::GorillaObjectStore.data_source_id(), + ENGINE_ID_THANOS_QUERY, ); + assert_eq!(StorageBackend::DoubleWrite.data_source_id(), "double_write",); assert_eq!( StorageBackend::PrometheusRemote.data_source_id(), "prometheus_remote", ); } + #[test] + fn storage_backend_engine_id_parser_accepts_only_canonical_query_engines() { + assert_eq!( + parse_storage_backend_engine_id(ENGINE_ID_ASAP_QUERY), + Some(StorageBackend::SketchStore), + ); + assert_eq!( + parse_storage_backend_engine_id(ENGINE_ID_THANOS_QUERY), + Some(StorageBackend::GorillaObjectStore), + ); + assert_eq!(parse_storage_backend_engine_id("not_an_engine"), None); + } + /// Source-of-truth agreement check, mirrors /// `capability_canonical_map_agreement` for the storage axis. /// @@ -1466,8 +1489,8 @@ mod tests { ]; let accuracies = [AccuracyTarget::Exact, AccuracyTarget::Approximate]; let configs = [ - StorageBackend::SketchWarmTier, - StorageBackend::GorillaS3Archive, + StorageBackend::SketchStore, + StorageBackend::GorillaObjectStore, StorageBackend::DoubleWrite, StorageBackend::PrometheusRemote, ]; @@ -1483,26 +1506,24 @@ mod tests { ); let last = *backends.last().unwrap(); assert!( - last == StorageBackend::SketchWarmTier - || last == StorageBackend::GorillaS3Archive + last == StorageBackend::SketchStore + || last == StorageBackend::GorillaObjectStore || last == StorageBackend::PrometheusRemote, "backend list for ({stat:?}, {acc:?}, {cfg:?}) must terminate in a \ - dispatchable failover (SketchWarmTier, GorillaS3Archive, or \ + dispatchable failover (SketchStore, GorillaObjectStore, or \ PrometheusRemote); got {last:?}", ); // The expected head is determined by `(metric_storage_config, accuracy)`: let expected_head = match (cfg, acc) { - (StorageBackend::GorillaS3Archive, _) => StorageBackend::GorillaS3Archive, - (StorageBackend::SketchWarmTier, _) => StorageBackend::SketchWarmTier, + (StorageBackend::GorillaObjectStore, _) => StorageBackend::GorillaObjectStore, + (StorageBackend::SketchStore, _) => StorageBackend::SketchStore, (StorageBackend::DoubleWrite, AccuracyTarget::Exact) => { - StorageBackend::GorillaS3Archive + StorageBackend::GorillaObjectStore } (StorageBackend::DoubleWrite, AccuracyTarget::Approximate) => { - StorageBackend::SketchWarmTier - } - (StorageBackend::PrometheusRemote, _) => { - StorageBackend::PrometheusRemote + StorageBackend::SketchStore } + (StorageBackend::PrometheusRemote, _) => StorageBackend::PrometheusRemote, }; assert_eq!( backends[0], expected_head, diff --git a/crates/asap_types/src/lib.rs b/crates/asap_types/src/lib.rs index dc633d28..7d83e871 100644 --- a/crates/asap_types/src/lib.rs +++ b/crates/asap_types/src/lib.rs @@ -13,7 +13,9 @@ pub mod utils; pub use aggregation_config::*; pub use aggregation_reference::*; pub use capability_matching::{ - compatible_storage_backends, find_compatible_aggregation, AccuracyTarget, StorageBackend, + compatible_storage_backends, find_compatible_aggregation, parse_storage_backend_engine_id, + AccuracyTarget, StorageBackend, CANONICAL_QUERY_ENGINE_IDS, ENGINE_ID_ASAP_QUERY, + ENGINE_ID_THANOS_QUERY, }; pub use enums::*; pub use inference_config::*; diff --git a/crates/asap_types/src/streaming_config.rs b/crates/asap_types/src/streaming_config.rs index b6ecb7a1..ad0353cc 100644 --- a/crates/asap_types/src/streaming_config.rs +++ b/crates/asap_types/src/streaming_config.rs @@ -19,7 +19,7 @@ pub struct StreamingConfig { /// Phase-5 capability-routing axis: which storage tier serves this /// per-metric runtime config. The controller pushes this when planning /// (see `docs/design-gorilla-s3-cold-engine.md` §8); pre-Phase-5 - /// configs decode with `#[serde(default)]` to `SketchWarmTier` so + /// configs decode with `#[serde(default)]` to `SketchStore` so /// existing deploys keep dispatching to `SimpleEngine`. #[serde(default)] pub storage_backend: StorageBackend, @@ -161,19 +161,19 @@ mod tests { use super::*; /// Pre-Phase-5 deploys serialize `StreamingConfig` without the - /// `storage_backend` field; deserialize must default to `SketchWarmTier` + /// `storage_backend` field; deserialize must default to `SketchStore` /// so the router keeps dispatching to `SimpleEngine` unchanged. #[test] fn deserialize_legacy_yaml_defaults_to_warm_tier() { let yaml = "{\"aggregation_configs\":{}}"; let cfg: StreamingConfig = serde_json::from_str(yaml).expect("legacy decode"); - assert_eq!(cfg.storage_backend(), StorageBackend::SketchWarmTier); + assert_eq!(cfg.storage_backend(), StorageBackend::SketchStore); } #[test] fn deserialize_with_explicit_archive_pin() { - let yaml = "{\"aggregation_configs\":{},\"storage_backend\":\"gorilla_s3_archive\"}"; + let yaml = "{\"aggregation_configs\":{},\"storage_backend\":\"gorilla_object_store\"}"; let cfg: StreamingConfig = serde_json::from_str(yaml).expect("Phase-5 decode"); - assert_eq!(cfg.storage_backend(), StorageBackend::GorillaS3Archive); + assert_eq!(cfg.storage_backend(), StorageBackend::GorillaObjectStore); } } From 10e93ba6a2dd5099870c1be24e2f4276e662e566 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 11 May 2026 17:43:58 -0600 Subject: [PATCH 2/4] Clarify query engine module naming --- .../src/engines/asap_query/engine.rs | 37 +++++++++------- .../src/engines/asap_query/mod.rs | 2 + .../src/engines/asap_query/tests.rs | 2 +- .../{ => asap_query}/warm_tier/decoders.rs | 2 +- .../{ => asap_query}/warm_tier/delta_apply.rs | 2 +- .../engines/{ => asap_query}/warm_tier/mod.rs | 0 .../warm_tier/sketch_reducer.rs | 44 +++++++++---------- .../{ => asap_query}/warm_tier/tests.rs | 2 +- asap-query-engine/src/engines/mod.rs | 1 - .../src/engines/prometheus/forward.rs | 4 +- .../src/engines/thanos_query/forward.rs | 4 +- asap-query-engine/src/routing/mod.rs | 6 +-- ...gine_router.rs => query_engine_routing.rs} | 0 .../{query_engine.rs => archive_query.rs} | 4 +- .../src/stores/gorilla_object_store/mod.rs | 20 ++++----- .../stores/gorilla_object_store/postings.rs | 2 +- .../src/stores/gorilla_object_store/tests.rs | 2 +- crates/asap_types/src/capability_matching.rs | 6 ++- 18 files changed, 74 insertions(+), 66 deletions(-) rename asap-query-engine/src/engines/{ => asap_query}/warm_tier/decoders.rs (98%) rename asap-query-engine/src/engines/{ => asap_query}/warm_tier/delta_apply.rs (99%) rename asap-query-engine/src/engines/{ => asap_query}/warm_tier/mod.rs (100%) rename asap-query-engine/src/engines/{ => asap_query}/warm_tier/sketch_reducer.rs (96%) rename asap-query-engine/src/engines/{ => asap_query}/warm_tier/tests.rs (99%) rename asap-query-engine/src/routing/{engine_router.rs => query_engine_routing.rs} (100%) rename asap-query-engine/src/stores/gorilla_object_store/{query_engine.rs => archive_query.rs} (99%) diff --git a/asap-query-engine/src/engines/asap_query/engine.rs b/asap-query-engine/src/engines/asap_query/engine.rs index 876fd779..36795b16 100644 --- a/asap-query-engine/src/engines/asap_query/engine.rs +++ b/asap-query-engine/src/engines/asap_query/engine.rs @@ -303,7 +303,7 @@ pub struct SimpleEngine { /// When `None` (no archive engine wired), the engine returns the /// warm answer as-is; the existing `EngineRouter` failover handles /// the rest of the routing matrix. - archive_engine: Option>, + archive_engine: Option>, } /// Public production name for the warm-tier sketch query engine. @@ -502,7 +502,7 @@ impl SimpleEngine { /// When `None`, the engine returns whatever the warm tier covers. pub fn with_archive_engine( mut self, - archive: Arc, + archive: Arc, ) -> Self { self.archive_engine = Some(archive); self @@ -3507,7 +3507,7 @@ impl SimpleEngine { // to the next compatible backend. // --------------------------------------------------------------------------- -/// Adapt a [`crate::engines::warm_tier::WarmTierResult`] to the engine's +/// Adapt a [`crate::engines::asap_query::warm_tier::WarmTierResult`] to the engine's /// existing `QueryResult` shape. The reducer hands back per-series /// time-stamped scalars; we materialize them as a /// `QueryResult::Matrix` whose [`crate::engines::query_result::RangeVectorElement`]s @@ -3581,7 +3581,7 @@ fn stitch_warm_and_archive( } fn warm_tier_result_to_query_result( - result: crate::engines::warm_tier::WarmTierResult, + result: crate::engines::asap_query::warm_tier::WarmTierResult, _now_ms: u64, ) -> crate::engines::query_result::QueryResult { use crate::data_model::KeyByLabelValues; @@ -3621,7 +3621,7 @@ fn warm_tier_result_to_query_result( } #[async_trait::async_trait] -impl crate::routing::engine_router::QueryEngine for SimpleEngine { +impl crate::routing::query_engine_routing::QueryEngine for SimpleEngine { async fn execute( &self, query: &str, @@ -3701,14 +3701,15 @@ impl crate::routing::engine_router::QueryEngine for SimpleEngine { // for instant-vector candidates (range_seconds == 0). const DEFAULT_LOOKBACK_MS: u64 = 5 * 60 * 1000; - let reducer = crate::engines::warm_tier::SketchReducer::new(idx); + let reducer = crate::engines::asap_query::warm_tier::SketchReducer::new(idx); // Multi-candidate aggregation is deferred (single-result // shapes today). On the first reducer error we surface // CapabilityMiss; on Ok we keep the result for the // hybrid-stitch path below. (When more than one // candidate is supported, a follow-up will fold // per-candidate WarmTierResults.) - let mut combined_result: Option = None; + let mut combined_result: Option = + None; let mut combined_t0: u64 = u64::MAX; for candidate in &analysis.candidates { @@ -3787,7 +3788,11 @@ impl crate::routing::engine_router::QueryEngine for SimpleEngine { now_ms, ) { Ok(r) => r, - Err(crate::engines::warm_tier::WarmTierError::UnsupportedFunction(name)) => { + Err( + crate::engines::asap_query::warm_tier::WarmTierError::UnsupportedFunction( + name, + ), + ) => { return Err(crate::engines::EngineError::capability_miss( asap_types::StorageBackend::SketchStore.data_source_id(), format!( @@ -3796,7 +3801,7 @@ impl crate::routing::engine_router::QueryEngine for SimpleEngine { ), )); } - Err(crate::engines::warm_tier::WarmTierError::UnsupportedCapability { + Err(crate::engines::asap_query::warm_tier::WarmTierError::UnsupportedCapability { function, capability, }) => { @@ -3808,7 +3813,7 @@ impl crate::routing::engine_router::QueryEngine for SimpleEngine { ), )); } - Err(crate::engines::warm_tier::WarmTierError::DeserializeFailure { + Err(crate::engines::asap_query::warm_tier::WarmTierError::DeserializeFailure { sid, encoding, reason, @@ -3822,7 +3827,9 @@ impl crate::routing::engine_router::QueryEngine for SimpleEngine { ), )); } - Err(crate::engines::warm_tier::WarmTierError::NoData { metric_name: m }) => { + Err(crate::engines::asap_query::warm_tier::WarmTierError::NoData { + metric_name: m, + }) => { return Err(crate::engines::EngineError::capability_miss( asap_types::StorageBackend::SketchStore.data_source_id(), format!( @@ -3831,7 +3838,7 @@ impl crate::routing::engine_router::QueryEngine for SimpleEngine { ), )); } - Err(crate::engines::warm_tier::WarmTierError::MissingHeap { + Err(crate::engines::asap_query::warm_tier::WarmTierError::MissingHeap { sid, sketch_kind, }) => { @@ -3891,8 +3898,8 @@ impl crate::routing::engine_router::QueryEngine for SimpleEngine { } } - fn capabilities(&self) -> crate::routing::engine_router::EngineCapabilities { - crate::routing::engine_router::EngineCapabilities { + fn capabilities(&self) -> crate::routing::query_engine_routing::EngineCapabilities { + crate::routing::query_engine_routing::EngineCapabilities { data_source_id: asap_types::StorageBackend::SketchStore.data_source_id(), storage_backend: asap_types::StorageBackend::SketchStore, // Warm-tier sketches are O(sketch-size); call it 16 MiB ceiling @@ -6147,7 +6154,7 @@ mod warm_tier_classify_tests { use super::*; use crate::data_model::{CleanupPolicy, HotReloadStreamingConfig, InferenceConfig}; use crate::engines::EngineError; - use crate::routing::engine_router::QueryEngine as _; + use crate::routing::query_engine_routing::QueryEngine as _; use crate::stores::sketch_db::simple_map_store::SimpleMapStore; use crate::stores::sketch_db::sketch_index::{ AccuracyBound, Capability, SketchConfig, SketchIndex, SketchInstanceMetadata, diff --git a/asap-query-engine/src/engines/asap_query/mod.rs b/asap-query-engine/src/engines/asap_query/mod.rs index 4cf6d568..8b8bf79f 100644 --- a/asap-query-engine/src/engines/asap_query/mod.rs +++ b/asap-query-engine/src/engines/asap_query/mod.rs @@ -14,6 +14,7 @@ //! `ASAPQueryEngine`. pub mod engine; +pub mod warm_tier; #[cfg(test)] pub mod tests; @@ -22,3 +23,4 @@ pub use engine::{ ASAPQueryEngine, QueryExecutionContext, QueryMetadata, QueryTimestamps, SimpleEngine, StoreQueryParams, StoreQueryPlan, }; +pub use warm_tier::{SketchReducer, WarmTierError, WarmTierResult}; diff --git a/asap-query-engine/src/engines/asap_query/tests.rs b/asap-query-engine/src/engines/asap_query/tests.rs index 667e7c75..0775224b 100644 --- a/asap-query-engine/src/engines/asap_query/tests.rs +++ b/asap-query-engine/src/engines/asap_query/tests.rs @@ -2,7 +2,7 @@ //! //! Step-1 of the JSONL deprecation refactor moved //! `engines/simple_engine.rs` to `engines/simple/engine.rs`. The -//! engine's tests live inline in [`super::query_engine`] (~6 distinct +//! engine's tests live inline in [`super::engine`] (~6 distinct //! `#[cfg(test)] mod tests { ... }` blocks, each pinning a //! specific dispatch axis). They are exercised under //! `crate::engines::asap_query::engine::tests` rather than this file diff --git a/asap-query-engine/src/engines/warm_tier/decoders.rs b/asap-query-engine/src/engines/asap_query/warm_tier/decoders.rs similarity index 98% rename from asap-query-engine/src/engines/warm_tier/decoders.rs rename to asap-query-engine/src/engines/asap_query/warm_tier/decoders.rs index a380bb74..97a005d3 100644 --- a/asap-query-engine/src/engines/warm_tier/decoders.rs +++ b/asap-query-engine/src/engines/asap_query/warm_tier/decoders.rs @@ -1,7 +1,7 @@ //! Per-sketch-kind decoder helpers — out-of-line wrappers around //! `asap_sketchlib` deserialize / proto-decode paths. //! -//! Lifted from the inline closures in [`crate::engines::warm_tier::sketch_reducer`] +//! Lifted from the inline closures in [`crate::engines::asap_query::warm_tier::sketch_reducer`] //! once the reducer started decoding CMS / CountSketch / CMS-with-heap //! payloads in addition to DDSketch / KLL / HLL. The CMS / CountSketch //! / CMS-with-heap decoders mirror diff --git a/asap-query-engine/src/engines/warm_tier/delta_apply.rs b/asap-query-engine/src/engines/asap_query/warm_tier/delta_apply.rs similarity index 99% rename from asap-query-engine/src/engines/warm_tier/delta_apply.rs rename to asap-query-engine/src/engines/asap_query/warm_tier/delta_apply.rs index ed185278..b7b09f0a 100644 --- a/asap-query-engine/src/engines/warm_tier/delta_apply.rs +++ b/asap-query-engine/src/engines/asap_query/warm_tier/delta_apply.rs @@ -9,7 +9,7 @@ //! format ships a sparse-but-mergeable sketch fragment. //! //! Two reducer modes, picked by the PromQL function name in -//! [`crate::engines::warm_tier::sketch_reducer`]: +//! [`crate::engines::asap_query::warm_tier::sketch_reducer`]: //! //! * **per-window** (`quantile`, `histogram_quantile`, //! `cardinality_estimate`): emit one scalar per window. A `Full` diff --git a/asap-query-engine/src/engines/warm_tier/mod.rs b/asap-query-engine/src/engines/asap_query/warm_tier/mod.rs similarity index 100% rename from asap-query-engine/src/engines/warm_tier/mod.rs rename to asap-query-engine/src/engines/asap_query/warm_tier/mod.rs diff --git a/asap-query-engine/src/engines/warm_tier/sketch_reducer.rs b/asap-query-engine/src/engines/asap_query/warm_tier/sketch_reducer.rs similarity index 96% rename from asap-query-engine/src/engines/warm_tier/sketch_reducer.rs rename to asap-query-engine/src/engines/asap_query/warm_tier/sketch_reducer.rs index b45002b4..019453e9 100644 --- a/asap-query-engine/src/engines/warm_tier/sketch_reducer.rs +++ b/asap-query-engine/src/engines/asap_query/warm_tier/sketch_reducer.rs @@ -57,11 +57,11 @@ use asap_sketchlib::sketches::ddsketch::DdSketch; use asap_sketchlib::sketches::hll::HllSketch; use asap_sketchlib::sketches::kll::KllSketch; -use crate::engines::warm_tier::decoders::{ +use crate::engines::asap_query::warm_tier::decoders::{ decode_cms_from_msgpack, decode_cms_from_proto, decode_cms_with_heap_from_msgpack, decode_cs_from_msgpack, decode_cs_from_proto, }; -use crate::engines::warm_tier::delta_apply::{ +use crate::engines::asap_query::warm_tier::delta_apply::{ cumulative_evaluate, per_window_evaluate, DeltaSketchKind, }; use crate::stores::sketch_db::sketch_index::{ @@ -345,8 +345,7 @@ impl<'a> SketchReducer<'a> { // entry point, which the current `&[f64]` signature can't carry. if family == QueryFamily::FrequencyEstimate { for ts in series_list { - let mut samples_out: Vec<(i64, f64)> = - Vec::with_capacity(ts.samples.len()); + let mut samples_out: Vec<(i64, f64)> = Vec::with_capacity(ts.samples.len()); for (w_end, state) in ts.samples.iter() { any_window = true; let w = if *w_end >= 0 { *w_end as u64 } else { 0 }; @@ -356,8 +355,7 @@ impl<'a> SketchReducer<'a> { if w > cov_hi { cov_hi = w; } - let total = - decode_frequency_total(sid, meta.sketch_kind, state)?; + let total = decode_frequency_total(sid, meta.sketch_kind, state)?; samples_out.push((*w_end, total)); } out_series.push((ts.series_label_values, samples_out)); @@ -391,8 +389,7 @@ impl<'a> SketchReducer<'a> { cov_hi = w_end_u64; } let cms_heap = match meta.sketch_kind { - SketchKindHandle::CmsWithHeap - | SketchKindHandle::CountSketchWithHeap => { + SketchKindHandle::CmsWithHeap | SketchKindHandle::CountSketchWithHeap => { // Both heap-bearing variants serialize the // outer `CountMinSketchWithHeap` envelope via // msgpack (`CountSketchWithHeap` reuses the @@ -876,19 +873,19 @@ fn decode_frequency_total( }; match sketch_kind { SketchKindHandle::CountMin => { - let cms = match state.encoding { - SketchEncoding::ProtoFull => { - decode_cms_from_proto(&state.bytes).map_err(|e| to_err(e, state.encoding))? - } - SketchEncoding::MsgpackFull => decode_cms_from_msgpack(&state.bytes) - .map_err(|e| to_err(e, state.encoding))?, - SketchEncoding::ProtoDelta | SketchEncoding::MsgpackDelta => { - return Err(to_err( - "CMS delta encodings not implemented in warm-tier reducer".to_string(), - state.encoding, - )); - } - }; + let cms = + match state.encoding { + SketchEncoding::ProtoFull => decode_cms_from_proto(&state.bytes) + .map_err(|e| to_err(e, state.encoding))?, + SketchEncoding::MsgpackFull => decode_cms_from_msgpack(&state.bytes) + .map_err(|e| to_err(e, state.encoding))?, + SketchEncoding::ProtoDelta | SketchEncoding::MsgpackDelta => { + return Err(to_err( + "CMS delta encodings not implemented in warm-tier reducer".to_string(), + state.encoding, + )); + } + }; Ok(row0_sum_cms(&cms)) } SketchKindHandle::CountSketch => { @@ -896,8 +893,9 @@ fn decode_frequency_total( SketchEncoding::ProtoFull => { decode_cs_from_proto(&state.bytes).map_err(|e| to_err(e, state.encoding))? } - SketchEncoding::MsgpackFull => decode_cs_from_msgpack(&state.bytes) - .map_err(|e| to_err(e, state.encoding))?, + SketchEncoding::MsgpackFull => { + decode_cs_from_msgpack(&state.bytes).map_err(|e| to_err(e, state.encoding))? + } SketchEncoding::ProtoDelta | SketchEncoding::MsgpackDelta => { return Err(to_err( "CountSketch delta encodings not implemented in warm-tier reducer" diff --git a/asap-query-engine/src/engines/warm_tier/tests.rs b/asap-query-engine/src/engines/asap_query/warm_tier/tests.rs similarity index 99% rename from asap-query-engine/src/engines/warm_tier/tests.rs rename to asap-query-engine/src/engines/asap_query/warm_tier/tests.rs index 540b6adf..18725566 100644 --- a/asap-query-engine/src/engines/warm_tier/tests.rs +++ b/asap-query-engine/src/engines/asap_query/warm_tier/tests.rs @@ -15,7 +15,7 @@ use std::collections::{BTreeMap, BTreeSet}; use asap_sketchlib::sketches::ddsketch::DdSketch; use asap_sketchlib::sketches::hll::{HllSketch, HllVariant}; -use crate::engines::warm_tier::{SketchReducer, WarmTierError}; +use crate::engines::asap_query::warm_tier::{SketchReducer, WarmTierError}; use crate::stores::sketch_db::sketch_index::{ AccuracyBound, Capability, SketchConfig, SketchEncoding, SketchIndex, SketchInstanceMetadata, SketchKindHandle, SketchSampleState, diff --git a/asap-query-engine/src/engines/mod.rs b/asap-query-engine/src/engines/mod.rs index 43e263f1..f8ad6915 100644 --- a/asap-query-engine/src/engines/mod.rs +++ b/asap-query-engine/src/engines/mod.rs @@ -26,7 +26,6 @@ pub mod prometheus; pub mod query_result; pub mod thanos_query; pub mod timeline_dispatch; -pub mod warm_tier; pub mod window_merger; pub use asap_query::{ASAPQueryEngine, SimpleEngine}; diff --git a/asap-query-engine/src/engines/prometheus/forward.rs b/asap-query-engine/src/engines/prometheus/forward.rs index 87ded7a9..3973054c 100644 --- a/asap-query-engine/src/engines/prometheus/forward.rs +++ b/asap-query-engine/src/engines/prometheus/forward.rs @@ -45,7 +45,7 @@ use tracing::{debug, warn}; use crate::data_model::KeyByLabelValues; use crate::engines::query_result::{InstantVectorElement, QueryResult, RangeVectorElement}; -use crate::routing::engine_router::{EngineCapabilities, QueryEngine}; +use crate::routing::query_engine_routing::{EngineCapabilities, QueryEngine}; use crate::stores::sketch_db::accuracy::{AccuracyEnvelope, AccuracyProfile}; // --------------------------------------------------------------------------- @@ -667,7 +667,7 @@ mod tests { }; use super::*; use crate::engines::query_result::QueryResult; - use crate::routing::engine_router::{EngineRouter, QueryEngine as RouterQueryEngine}; + use crate::routing::query_engine_routing::{EngineRouter, QueryEngine as RouterQueryEngine}; use std::sync::Arc; fn config_for(url: &str) -> PrometheusForwardConfig { diff --git a/asap-query-engine/src/engines/thanos_query/forward.rs b/asap-query-engine/src/engines/thanos_query/forward.rs index 6e28dca7..ba9bfb9f 100644 --- a/asap-query-engine/src/engines/thanos_query/forward.rs +++ b/asap-query-engine/src/engines/thanos_query/forward.rs @@ -41,7 +41,7 @@ use tracing::{debug, warn}; use crate::data_model::KeyByLabelValues; use crate::engines::query_result::{InstantVectorElement, QueryResult, RangeVectorElement}; -use crate::routing::engine_router::{EngineCapabilities, QueryEngine}; +use crate::routing::query_engine_routing::{EngineCapabilities, QueryEngine}; use crate::stores::sketch_db::accuracy::{AccuracyEnvelope, AccuracyProfile}; // --------------------------------------------------------------------------- @@ -559,7 +559,7 @@ pub fn engine_from_env() -> Result, ThanosQueryError> pub mod test_support { //! Test-only helpers for spinning up an in-process mock //! `thanos-query` sidecar. Used by the unit + integration - //! tests below and by the `routing/engine_router.rs` tests + //! tests below and by the `routing/query_engine_routing.rs` tests //! once they grow Path A2 coverage. use std::net::SocketAddr; diff --git a/asap-query-engine/src/routing/mod.rs b/asap-query-engine/src/routing/mod.rs index 6cc06f57..68c50ac2 100644 --- a/asap-query-engine/src/routing/mod.rs +++ b/asap-query-engine/src/routing/mod.rs @@ -10,7 +10,7 @@ //! Loaded once at backend startup from //! `deploy/configs/backend-storage-routing.yaml`; queried on //! every HTTP request. -//! * [`engine_router`] — the engine dispatcher. Holds a small map +//! * [`query_engine_routing`] — the engine dispatcher. Holds a small map //! of `data_source_id → Arc` and walks the //! compatibility list returned by //! [`asap_types::compatible_storage_backends`] to pick which @@ -23,14 +23,14 @@ //! instead of straddling two unrelated module trees. pub mod backend_storage_routing; -pub mod engine_router; pub mod freshness_probe_cache; +pub mod query_engine_routing; pub use backend_storage_routing::{ classify_query_shape, routing_table_hash, BackendStorageRouting, HotReloadBackendStorageRouting, QueryShape, RoutingTarget, DEFAULT_TENANT, }; -pub use engine_router::{EngineCapabilities, EngineRouter, EngineRouterError, QueryEngine}; pub use freshness_probe_cache::{ is_freshness_probe, now_ms as freshness_probe_now_ms, FreshnessProbeCache, ProbeSample, }; +pub use query_engine_routing::{EngineCapabilities, EngineRouter, EngineRouterError, QueryEngine}; diff --git a/asap-query-engine/src/routing/engine_router.rs b/asap-query-engine/src/routing/query_engine_routing.rs similarity index 100% rename from asap-query-engine/src/routing/engine_router.rs rename to asap-query-engine/src/routing/query_engine_routing.rs diff --git a/asap-query-engine/src/stores/gorilla_object_store/query_engine.rs b/asap-query-engine/src/stores/gorilla_object_store/archive_query.rs similarity index 99% rename from asap-query-engine/src/stores/gorilla_object_store/query_engine.rs rename to asap-query-engine/src/stores/gorilla_object_store/archive_query.rs index 7543a0f0..f8f27ad9 100644 --- a/asap-query-engine/src/stores/gorilla_object_store/query_engine.rs +++ b/asap-query-engine/src/stores/gorilla_object_store/archive_query.rs @@ -1,9 +1,9 @@ -//! `GorillaQueryEngine` planner + per-statistic exact executor. +//! Gorilla archive query planner + per-statistic exact executor. //! //! Step-1 of the JSONL-deprecation refactor merged the previous //! `query_planner.rs` (PromQL → [`QueryPlan`]) and `exact_executor.rs` //! (per-statistic `[`ExactExecutor`]` dispatch) into a single -//! `engine.rs` so the engine's data flow is readable top-to-bottom +//! `archive_query.rs` so the archive data flow is readable top-to-bottom //! in one file: parse PromQL → plan → execute. //! //! The two halves keep their existing structure inside the merged diff --git a/asap-query-engine/src/stores/gorilla_object_store/mod.rs b/asap-query-engine/src/stores/gorilla_object_store/mod.rs index a1be91f7..a6d368cc 100644 --- a/asap-query-engine/src/stores/gorilla_object_store/mod.rs +++ b/asap-query-engine/src/stores/gorilla_object_store/mod.rs @@ -7,7 +7,7 @@ //! //! ## Module layout (post Step-1 refactor) //! -//! * [`query_engine`] — query planner + per-statistic exact executor (the +//! * [`archive_query`] — query planner + per-statistic exact executor (the //! merged form of the previous `query_planner.rs` + //! `exact_executor.rs`). //! * [`store`] — `GorillaS3Store` (the only `Store` impl after @@ -30,7 +30,7 @@ //! //! ## Two execution strategies //! -//! Per-statistic dispatch in [`query_engine::ExactExecutor`]: +//! Per-statistic dispatch in [`archive_query::ExactExecutor`]: //! //! * **Streaming-additive** — `Sum`, `Count`, `Min`, `Max`, `Rate`, //! `Increase` (and `Avg` derived as Sum/Count). One chunk at a @@ -41,8 +41,8 @@ //! [`GorillaEngineConfig::max_buffered_samples`]; over-budget //! queries fail fast with [`EngineError::TooManySamples`]. +pub mod archive_query; pub mod postings; -pub mod query_engine; pub mod s3_cost; pub mod store; @@ -60,7 +60,7 @@ use crate::data_model::KeyByLabelValues; use crate::engines::query_result::{InstantVectorElement, QueryResult}; use crate::stores::sketch_db::accuracy::{AccuracyEnvelope, AccuracyProfile}; -pub use query_engine::{ +pub use archive_query::{ plan_query, plan_query_at, AdditiveOp, ExactExecutor, LabelMatcher, QueryPlan, QueryStatistic, }; pub use postings::PostingsHits; @@ -105,7 +105,7 @@ impl Default for GorillaEngineConfig { #[derive(Debug, Error)] pub enum EngineError { /// PromQL string failed to parse, or used a construct outside - /// the engine's supported surface (see [`query_engine`]). + /// the archive query supported surface (see [`archive_query`]). #[error("query planning failed: {0}")] Plan(String), /// Archive-store fetch / decode failed. @@ -181,7 +181,7 @@ impl GorillaQueryEngine { /// Execute a parsed PromQL query against the archive tier. /// - /// The query string is parsed via [`query_engine::plan_query`], + /// The query string is parsed via [`archive_query::plan_query`], /// the resulting plan dispatches to either the streaming /// additive or the buffered execution path, and the answer is /// wrapped with the exact-accuracy envelope + the @@ -206,7 +206,7 @@ impl GorillaQueryEngine { } async fn execute_inner(&self, query: &str, now_ms: i64) -> Result { - let plan = query_engine::plan_query_at(query, now_ms).map_err(EngineError::Plan)?; + let plan = archive_query::plan_query_at(query, now_ms).map_err(EngineError::Plan)?; debug!( metric = plan.metric.as_str(), stat = ?plan.statistic, @@ -326,7 +326,7 @@ impl ExecutionOutcome { // --------------------------------------------------------------------------- #[async_trait::async_trait] -impl crate::routing::engine_router::QueryEngine for GorillaQueryEngine { +impl crate::routing::query_engine_routing::QueryEngine for GorillaQueryEngine { async fn execute(&self, query: &str) -> Result { match GorillaQueryEngine::execute(self, query).await { Ok(result) => Ok(result), @@ -341,8 +341,8 @@ impl crate::routing::engine_router::QueryEngine for GorillaQueryEngine { } } - fn capabilities(&self) -> crate::routing::engine_router::EngineCapabilities { - crate::routing::engine_router::EngineCapabilities { + fn capabilities(&self) -> crate::routing::query_engine_routing::EngineCapabilities { + crate::routing::query_engine_routing::EngineCapabilities { data_source_id: asap_types::StorageBackend::GorillaObjectStore.data_source_id(), storage_backend: asap_types::StorageBackend::GorillaObjectStore, // The buffered-aggregate budget gives a natural ceiling: each diff --git a/asap-query-engine/src/stores/gorilla_object_store/postings.rs b/asap-query-engine/src/stores/gorilla_object_store/postings.rs index ba9de4b7..e2eb86e1 100644 --- a/asap-query-engine/src/stores/gorilla_object_store/postings.rs +++ b/asap-query-engine/src/stores/gorilla_object_store/postings.rs @@ -5,7 +5,7 @@ //! `gorillas3processor` alongside each per-hour `index.json`: //! `//YYYY/MM/DD/HH/postings-v1.json`. It maps //! `(label_name, label_value) → [series_id, ...]` so the -//! [`super::query_engine::ExactExecutor`] can prune chunks by `label_hash` +//! [`super::archive_query::ExactExecutor`] can prune chunks by `label_hash` //! without paying the chunk-body GET cost. //! //! Step-1 of the JSONL deprecation refactor pulled this code out diff --git a/asap-query-engine/src/stores/gorilla_object_store/tests.rs b/asap-query-engine/src/stores/gorilla_object_store/tests.rs index 6fbaa431..d0485f01 100644 --- a/asap-query-engine/src/stores/gorilla_object_store/tests.rs +++ b/asap-query-engine/src/stores/gorilla_object_store/tests.rs @@ -17,7 +17,7 @@ use tokio::time::sleep; use crate::engines::query_result::QueryResult; use crate::stores::sketch_db::accuracy::{AccuracyKind, AccuracyProfile}; -use super::query_engine::{plan_query_at, QueryStatistic}; +use super::archive_query::{plan_query_at, QueryStatistic}; use super::store::{ChunkRef, RawSample, Store, StoreError}; use super::{ wrap_result, EngineError, ExactExecutor, ExecutionOutcome, GorillaEngineConfig, diff --git a/crates/asap_types/src/capability_matching.rs b/crates/asap_types/src/capability_matching.rs index 73f6696d..6556e285 100644 --- a/crates/asap_types/src/capability_matching.rs +++ b/crates/asap_types/src/capability_matching.rs @@ -1353,7 +1353,7 @@ mod tests { // ----------------------------------------------------------------------- // Phase-5: storage-backend routing // - // The Phase-5 `EngineRouter` (see `asap-query-engine/src/engines/router.rs`) + // The Phase-5 `EngineRouter` (see `asap-query-engine/src/routing/query_engine_routing.rs`) // consults `compatible_storage_backends(stat, accuracy, metric_storage)` // to pick a backend. These tests pin the routing matrix so the dispatcher // stays in lock-step with the design doc §8. @@ -1515,7 +1515,9 @@ mod tests { ); // The expected head is determined by `(metric_storage_config, accuracy)`: let expected_head = match (cfg, acc) { - (StorageBackend::GorillaObjectStore, _) => StorageBackend::GorillaObjectStore, + (StorageBackend::GorillaObjectStore, _) => { + StorageBackend::GorillaObjectStore + } (StorageBackend::SketchStore, _) => StorageBackend::SketchStore, (StorageBackend::DoubleWrite, AccuracyTarget::Exact) => { StorageBackend::GorillaObjectStore From 8d127a5cbb362a1044aec20d9c68dcdb3cbdd862 Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 11 May 2026 17:46:10 -0600 Subject: [PATCH 3/4] Rename engines directory to query-engines --- README.md | 2 +- asap-query-engine/src/lib.rs | 1 + .../{engines => query-engines}/asap_query/engine.rs | 0 .../src/{engines => query-engines}/asap_query/mod.rs | 0 .../{engines => query-engines}/asap_query/tests.rs | 2 +- .../asap_query/warm_tier/decoders.rs | 0 .../asap_query/warm_tier/delta_apply.rs | 0 .../asap_query/warm_tier/mod.rs | 0 .../asap_query/warm_tier/sketch_reducer.rs | 0 .../asap_query/warm_tier/tests.rs | 0 .../src/{engines => query-engines}/mod.rs | 0 .../{engines => query-engines}/no_data_archive.rs | 0 .../{engines => query-engines}/prometheus/forward.rs | 0 .../src/{engines => query-engines}/prometheus/mod.rs | 0 .../src/{engines => query-engines}/query_result.rs | 0 .../thanos_query/forward.rs | 0 .../{engines => query-engines}/thanos_query/mod.rs | 0 .../{engines => query-engines}/timeline_dispatch.rs | 0 .../src/{engines => query-engines}/window_merger.rs | 0 asap-query-engine/src/routing/mod.rs | 2 +- .../src/stores/gorilla_object_store/store.rs | 2 +- controller/src/lib.rs | 3 +-- docs/01-getting-started/architecture.md | 2 +- docs/design-controller-into-backend.md | 2 +- docs/design-sketch-db-core.md | 4 ++-- docs/design-sketch-db.md | 2 +- docs/proofs.md | 12 ++++++------ 27 files changed, 17 insertions(+), 17 deletions(-) rename asap-query-engine/src/{engines => query-engines}/asap_query/engine.rs (100%) rename asap-query-engine/src/{engines => query-engines}/asap_query/mod.rs (100%) rename asap-query-engine/src/{engines => query-engines}/asap_query/tests.rs (87%) rename asap-query-engine/src/{engines => query-engines}/asap_query/warm_tier/decoders.rs (100%) rename asap-query-engine/src/{engines => query-engines}/asap_query/warm_tier/delta_apply.rs (100%) rename asap-query-engine/src/{engines => query-engines}/asap_query/warm_tier/mod.rs (100%) rename asap-query-engine/src/{engines => query-engines}/asap_query/warm_tier/sketch_reducer.rs (100%) rename asap-query-engine/src/{engines => query-engines}/asap_query/warm_tier/tests.rs (100%) rename asap-query-engine/src/{engines => query-engines}/mod.rs (100%) rename asap-query-engine/src/{engines => query-engines}/no_data_archive.rs (100%) rename asap-query-engine/src/{engines => query-engines}/prometheus/forward.rs (100%) rename asap-query-engine/src/{engines => query-engines}/prometheus/mod.rs (100%) rename asap-query-engine/src/{engines => query-engines}/query_result.rs (100%) rename asap-query-engine/src/{engines => query-engines}/thanos_query/forward.rs (100%) rename asap-query-engine/src/{engines => query-engines}/thanos_query/mod.rs (100%) rename asap-query-engine/src/{engines => query-engines}/timeline_dispatch.rs (100%) rename asap-query-engine/src/{engines => query-engines}/window_merger.rs (100%) diff --git a/README.md b/README.md index 4b4be24d..2ceddbad 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ ASAPQuery-backend/ │ │ # in ASAPCollector) │ ├── bin/ # auxiliary binaries (offline tests, │ │ # logical-plan dumper) -│ ├── engines/ +│ ├── query-engines/ │ │ ├── simple/ # warm tier — SimpleEngine │ │ │ # (33 PromQL pattern matchers) │ │ └── gorilla/ # archive tier diff --git a/asap-query-engine/src/lib.rs b/asap-query-engine/src/lib.rs index c7279a2b..c8f16773 100644 --- a/asap-query-engine/src/lib.rs +++ b/asap-query-engine/src/lib.rs @@ -1,5 +1,6 @@ pub mod data_model; pub mod drivers; +#[path = "query-engines/mod.rs"] pub mod engines; pub mod precompute_engine; pub mod precompute_operators; diff --git a/asap-query-engine/src/engines/asap_query/engine.rs b/asap-query-engine/src/query-engines/asap_query/engine.rs similarity index 100% rename from asap-query-engine/src/engines/asap_query/engine.rs rename to asap-query-engine/src/query-engines/asap_query/engine.rs diff --git a/asap-query-engine/src/engines/asap_query/mod.rs b/asap-query-engine/src/query-engines/asap_query/mod.rs similarity index 100% rename from asap-query-engine/src/engines/asap_query/mod.rs rename to asap-query-engine/src/query-engines/asap_query/mod.rs diff --git a/asap-query-engine/src/engines/asap_query/tests.rs b/asap-query-engine/src/query-engines/asap_query/tests.rs similarity index 87% rename from asap-query-engine/src/engines/asap_query/tests.rs rename to asap-query-engine/src/query-engines/asap_query/tests.rs index 0775224b..7bc593f1 100644 --- a/asap-query-engine/src/engines/asap_query/tests.rs +++ b/asap-query-engine/src/query-engines/asap_query/tests.rs @@ -1,7 +1,7 @@ //! Placeholder for simple-engine tests. //! //! Step-1 of the JSONL deprecation refactor moved -//! `engines/simple_engine.rs` to `engines/simple/engine.rs`. The +//! `query-engines/simple_engine.rs` to `query-engines/asap_query/engine.rs`. The //! engine's tests live inline in [`super::engine`] (~6 distinct //! `#[cfg(test)] mod tests { ... }` blocks, each pinning a //! specific dispatch axis). They are exercised under diff --git a/asap-query-engine/src/engines/asap_query/warm_tier/decoders.rs b/asap-query-engine/src/query-engines/asap_query/warm_tier/decoders.rs similarity index 100% rename from asap-query-engine/src/engines/asap_query/warm_tier/decoders.rs rename to asap-query-engine/src/query-engines/asap_query/warm_tier/decoders.rs diff --git a/asap-query-engine/src/engines/asap_query/warm_tier/delta_apply.rs b/asap-query-engine/src/query-engines/asap_query/warm_tier/delta_apply.rs similarity index 100% rename from asap-query-engine/src/engines/asap_query/warm_tier/delta_apply.rs rename to asap-query-engine/src/query-engines/asap_query/warm_tier/delta_apply.rs diff --git a/asap-query-engine/src/engines/asap_query/warm_tier/mod.rs b/asap-query-engine/src/query-engines/asap_query/warm_tier/mod.rs similarity index 100% rename from asap-query-engine/src/engines/asap_query/warm_tier/mod.rs rename to asap-query-engine/src/query-engines/asap_query/warm_tier/mod.rs diff --git a/asap-query-engine/src/engines/asap_query/warm_tier/sketch_reducer.rs b/asap-query-engine/src/query-engines/asap_query/warm_tier/sketch_reducer.rs similarity index 100% rename from asap-query-engine/src/engines/asap_query/warm_tier/sketch_reducer.rs rename to asap-query-engine/src/query-engines/asap_query/warm_tier/sketch_reducer.rs diff --git a/asap-query-engine/src/engines/asap_query/warm_tier/tests.rs b/asap-query-engine/src/query-engines/asap_query/warm_tier/tests.rs similarity index 100% rename from asap-query-engine/src/engines/asap_query/warm_tier/tests.rs rename to asap-query-engine/src/query-engines/asap_query/warm_tier/tests.rs diff --git a/asap-query-engine/src/engines/mod.rs b/asap-query-engine/src/query-engines/mod.rs similarity index 100% rename from asap-query-engine/src/engines/mod.rs rename to asap-query-engine/src/query-engines/mod.rs diff --git a/asap-query-engine/src/engines/no_data_archive.rs b/asap-query-engine/src/query-engines/no_data_archive.rs similarity index 100% rename from asap-query-engine/src/engines/no_data_archive.rs rename to asap-query-engine/src/query-engines/no_data_archive.rs diff --git a/asap-query-engine/src/engines/prometheus/forward.rs b/asap-query-engine/src/query-engines/prometheus/forward.rs similarity index 100% rename from asap-query-engine/src/engines/prometheus/forward.rs rename to asap-query-engine/src/query-engines/prometheus/forward.rs diff --git a/asap-query-engine/src/engines/prometheus/mod.rs b/asap-query-engine/src/query-engines/prometheus/mod.rs similarity index 100% rename from asap-query-engine/src/engines/prometheus/mod.rs rename to asap-query-engine/src/query-engines/prometheus/mod.rs diff --git a/asap-query-engine/src/engines/query_result.rs b/asap-query-engine/src/query-engines/query_result.rs similarity index 100% rename from asap-query-engine/src/engines/query_result.rs rename to asap-query-engine/src/query-engines/query_result.rs diff --git a/asap-query-engine/src/engines/thanos_query/forward.rs b/asap-query-engine/src/query-engines/thanos_query/forward.rs similarity index 100% rename from asap-query-engine/src/engines/thanos_query/forward.rs rename to asap-query-engine/src/query-engines/thanos_query/forward.rs diff --git a/asap-query-engine/src/engines/thanos_query/mod.rs b/asap-query-engine/src/query-engines/thanos_query/mod.rs similarity index 100% rename from asap-query-engine/src/engines/thanos_query/mod.rs rename to asap-query-engine/src/query-engines/thanos_query/mod.rs diff --git a/asap-query-engine/src/engines/timeline_dispatch.rs b/asap-query-engine/src/query-engines/timeline_dispatch.rs similarity index 100% rename from asap-query-engine/src/engines/timeline_dispatch.rs rename to asap-query-engine/src/query-engines/timeline_dispatch.rs diff --git a/asap-query-engine/src/engines/window_merger.rs b/asap-query-engine/src/query-engines/window_merger.rs similarity index 100% rename from asap-query-engine/src/engines/window_merger.rs rename to asap-query-engine/src/query-engines/window_merger.rs diff --git a/asap-query-engine/src/routing/mod.rs b/asap-query-engine/src/routing/mod.rs index 68c50ac2..798e2993 100644 --- a/asap-query-engine/src/routing/mod.rs +++ b/asap-query-engine/src/routing/mod.rs @@ -17,7 +17,7 @@ //! engine answers a given `(query, metric_storage)` pair. //! //! Step-1 of the JSONL deprecation refactor lifted these out of -//! `data_model/backend_storage_routing.rs` and `engines/router.rs` +//! `data_model/backend_storage_routing.rs` and `query-engines/router.rs` //! into this dedicated `routing/` directory so the HTTP handler's //! dispatch surface is a single import (`use crate::routing::*`) //! instead of straddling two unrelated module trees. diff --git a/asap-query-engine/src/stores/gorilla_object_store/store.rs b/asap-query-engine/src/stores/gorilla_object_store/store.rs index e97bf9dc..e658d218 100644 --- a/asap-query-engine/src/stores/gorilla_object_store/store.rs +++ b/asap-query-engine/src/stores/gorilla_object_store/store.rs @@ -6,7 +6,7 @@ //! selected `GORILLA1` chunks via the [`asap_gorilla`] crate //! (`ASAPCollector` PR #281). //! -//! Step-1 refactor (`refactor: tier-co-locate engines/{simple,gorilla}/`) +//! Step-1 refactor (`refactor: tier-co-locate query-engines/{simple,gorilla}/`) //! folded the previous `ColdStore` trait + `RawSample`/`ChunkRef` //! types into this module. The legacy JSONL leg //! (`LocalFsColdStore`, `parse_jsonl`, `ColdJsonlFallback`) was diff --git a/controller/src/lib.rs b/controller/src/lib.rs index 567e0f9f..8fdd438d 100644 --- a/controller/src/lib.rs +++ b/controller/src/lib.rs @@ -90,7 +90,6 @@ pub mod workload; // follow-up task. /// PromQL → warm-tier candidate analyzer. Phase-9 unification of the /// per-`Capability` dispatch knowledge that previously lived in -/// `asap-query-engine/src/engines/warm_tier/promql_extract.rs`. See +/// `asap-query-engine/src/query-engines/asap_query/warm_tier/promql_extract.rs`. See /// the module docs for the full PromQL shape coverage matrix. pub mod warm_tier_analysis; - diff --git a/docs/01-getting-started/architecture.md b/docs/01-getting-started/architecture.md index 821d2d29..b488b95e 100644 --- a/docs/01-getting-started/architecture.md +++ b/docs/01-getting-started/architecture.md @@ -252,7 +252,7 @@ ASAPQuery/ ├── asap-query-engine/ # Rust query processor │ ├── src/ │ │ ├── drivers/ # Ingest, query adapters, servers -│ │ ├── engines/ # Query execution (SimpleEngine) +│ │ ├── query-engines/ # Query execution (SimpleEngine) │ │ ├── stores/ # Data storage (SimpleMapStore) │ │ ├── data_model/ # Core data structures │ │ ├── precompute_operators/ # Sketch operators diff --git a/docs/design-controller-into-backend.md b/docs/design-controller-into-backend.md index c53f54fd..f33beaaf 100644 --- a/docs/design-controller-into-backend.md +++ b/docs/design-controller-into-backend.md @@ -205,7 +205,7 @@ matches the metric+labels+capability), fall through to Thanos. - `asap-query-engine/src/routing/backend_storage_routing.rs` — swap "shape allow-list" for "warm-first, archive-fallthrough". -- `asap-query-engine/src/engines/router.rs` (EngineRouter) — add +- `asap-query-engine/src/routing/query_engine_routing.rs` (EngineRouter) — add a `query_with_fallthrough` path. **Acceptance test:** Same PromQL `count(http_requests_total)` and diff --git a/docs/design-sketch-db-core.md b/docs/design-sketch-db-core.md index 1d9330f9..431ecb27 100644 --- a/docs/design-sketch-db-core.md +++ b/docs/design-sketch-db-core.md @@ -605,7 +605,7 @@ timeline_for_metric("latency", day1-1h, day1+1h) Query engine uses this to dispatch per-segment. DB provides it from an in-memory BTree; cost is one HashMap lookup + BTree range scan, nanoseconds. Implemented at `schema.rs`; used by -`engines/timeline_dispatch.rs`. +`query-engines/timeline_dispatch.rs`. ### 7.3 Per-segment query dispatch @@ -650,7 +650,7 @@ fall-through to the exact DB for the missing segment. Crucially, this failure mode is **explicit** — the user knows they are seeing a schema-change artifact. -Implemented in `engines/timeline_dispatch.rs`. Coverage-driven branching +Implemented in `query-engines/timeline_dispatch.rs`. Coverage-driven branching (the `match` in the snippet above) is present in skeleton form; the `Coverage::BackfillInProgress` → wait-or-fallback policy is still a follow-up (see roadmap §16 "Phase 5f"). diff --git a/docs/design-sketch-db.md b/docs/design-sketch-db.md index 9ab57d81..f515e96d 100644 --- a/docs/design-sketch-db.md +++ b/docs/design-sketch-db.md @@ -103,7 +103,7 @@ partially wired; "❌ not started" means spec only. | 6.3 | Write-side schema barrier `is_writable(agg_id)` | ✅ | called in `ingest_handler.rs` | | 6.4 | `AccuracyProfile` on schema | ✅ | `stores/sketch_db/accuracy.rs` | | 7.2 | `timeline_for_metric(...)` | ✅ | `schema.rs:594` | -| 7.3 | Cross-schema query combiner | ✅ | `engines/timeline_dispatch.rs` | +| 7.3 | Cross-schema query combiner | ✅ | `query-engines/timeline_dispatch.rs` | | 8 | Incremental ingest (OTLP / Prometheus / VictoriaMetrics / Kafka drivers) | ✅ | `drivers/ingest/` | | 8.4 | Watermark + lateness policy | ✅ | `allowed_lateness_ms` + `LateSampleHandlingPolicy` | | 9 | Semantic compaction (LSM levels) | ❌ | comment-level only | diff --git a/docs/proofs.md b/docs/proofs.md index a0b06e3f..c95afec4 100644 --- a/docs/proofs.md +++ b/docs/proofs.md @@ -22,7 +22,7 @@ Cross-links to design docs: - §1 / §2 reference the schema timeline of [`design-sketch-db.md`](./design-sketch-db.md) §7 and the - combinability table of `engines/timeline_dispatch.rs`. + combinability table of `query-engines/timeline_dispatch.rs`. - §3 references the §6.3 write barrier of [`design-sketch-db-core.md`](./design-sketch-db-core.md). - §4 references the §10.5 deterministic-rebuild contract of @@ -72,7 +72,7 @@ $[t_0, t_1), [t_1, t_2), \dots, [t_{n-1}, t_n)$, each owned by a single `AggSchema` with its own sketch parameters. The query engine evaluates the statistic per segment and feeds the per- segment scalars to `combine_statistic` -(`asap-query-engine/src/engines/timeline_dispatch.rs`). +(`asap-query-engine/src/query-engines/timeline_dispatch.rs`). ### 2.1 Statement @@ -147,7 +147,7 @@ Idempotence + associativity together justify the pointwise fold. **L2.4 — Triangle inequality on real numbers.** Standard. **L2.5 — Combiner implementation.** `combine_statistic` -(`asap-query-engine/src/engines/timeline_dispatch.rs:combine_statistic`) +(`asap-query-engine/src/query-engines/timeline_dispatch.rs:combine_statistic`) folds segments via `fold(0.0, +)` for `Count` / `Sum`, `fold(None, |a,v| Some(a.map_or(v, |a| a.min(v))))` for `Min` (mut. mut. for `Max`), and returns `None` (so a `Partial` wrapper) for @@ -255,7 +255,7 @@ nothing further to prove. ### 2.5 Code anchors -- `asap-query-engine/src/engines/timeline_dispatch.rs` +- `asap-query-engine/src/query-engines/timeline_dispatch.rs` - `combine_statistic` — the per-statistic fold (additive arm, `Min` / `Max` arm, non-combinable arm). - `CombinedResult::{Full, Partial}` — the `Full` / `Partial` @@ -265,7 +265,7 @@ nothing further to prove. - `SchemaRegistry::timeline_for_metric` — disjoint covering (L2.1). - `AggSchema::config` — pinned-at-creation invariant (caveat 1). -- `asap-query-engine/src/engines/simple_engine.rs` +- `asap-query-engine/src/query-engines/asap_query/engine.rs` - `SimpleEngine::try_handle_query_promql_via_timeline` — caller that wires per-segment evaluation into `combine_statistic`. - `asap-query-engine/src/stores/sketch_db/accuracy.rs` @@ -683,7 +683,7 @@ the paper's §theory chapter can cite both at once. `asap-query-engine/src/stores/sketch_db/accuracy.rs` (`AccuracyProfile::derive`). - **`combine_statistic` correctness (§2)** → - `asap-query-engine/src/engines/timeline_dispatch.rs` + `asap-query-engine/src/query-engines/timeline_dispatch.rs` (`CombinedResult`, `combine_statistic`). - **Write-barrier safety (§3)** → `asap-query-engine/src/stores/sketch_db/schema.rs` From 392cc4fe2d1c3550f7368ae963eba637d868f56e Mon Sep 17 00:00:00 2001 From: zz_y Date: Mon, 11 May 2026 21:35:29 -0600 Subject: [PATCH 4/4] fix(routing): correct serde tags + sort order in PR #137 test fixtures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After splitting StorageBackend enum names (SketchStore / GorillaObjectStore), several routing tests still referenced the pre-rename engine-id strings ('asap_query' / 'thanos_query') where they should use the snake-case serde tags ('sketch_store' / 'gorilla_object_store') — YAML parses through serde, which only accepts the canonical snake_case form. Also: - engine_by_id sort-order assertion now lists ids in ascending order (matching ids.sort() output). - parse_storage_backend_engine_id now recognises 'double_write' / 'prometheus_remote' (the JSON 'engine:' string for Mode-3 / double-write configurations), so json_payload_prometheus_remote_* resolves the engine correctly. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../src/routing/backend_storage_routing.rs | 58 +++++++++---------- .../src/routing/query_engine_routing.rs | 2 +- crates/asap_types/src/capability_matching.rs | 2 + 3 files changed, 32 insertions(+), 30 deletions(-) diff --git a/asap-query-engine/src/routing/backend_storage_routing.rs b/asap-query-engine/src/routing/backend_storage_routing.rs index b3382027..41fb363a 100644 --- a/asap-query-engine/src/routing/backend_storage_routing.rs +++ b/asap-query-engine/src/routing/backend_storage_routing.rs @@ -46,27 +46,27 @@ //! //! ```yaml //! # v6.1 form (single-target): -//! default: asap_query +//! default: sketch_store //! metrics: -//! audit_events: thanos_query +//! audit_events: gorilla_object_store //! ``` //! //! ```yaml //! # v7 form (multi-target with query-shape selection): -//! default: asap_query +//! default: sketch_store //! routes: //! - metric: http_requests_total //! targets: -//! - backend: asap_query +//! - backend: sketch_store //! # default — predictable / planned queries land here -//! - backend: thanos_query +//! - backend: gorilla_object_store //! applies_to_query_shape: [count, topk, rate_post_hoc] //! - metric: http_freshness_probe_warm //! targets: -//! - backend: asap_query +//! - backend: sketch_store //! - metric: http_freshness_probe_archive //! targets: -//! - backend: thanos_query +//! - backend: gorilla_object_store //! ``` //! //! The two shapes can be mixed in the same YAML — metrics under @@ -75,9 +75,9 @@ //! in BOTH wins from `routes:` (multi-target overrides single-target). //! //! Valid `StorageBackend` values mirror the snake-cased serde tags on -//! `asap_types::StorageBackend`: `asap_query`, -//! `thanos_query`, `double_write`. (Step-1 of the JSONL -//! deprecation refactor removed the `cold_jsonl_fallback` tag.) +//! `asap_types::StorageBackend`: `sketch_store`, +//! `gorilla_object_store`, `double_write`, `prometheus_remote`. (Step-1 +//! of the JSONL deprecation refactor removed the `cold_jsonl_fallback` tag.) //! //! Loaded once at backend startup (CLI flag `--backend-storage-routing` //! on `precompute_engine`) and stored in `AppState`. Lookup is @@ -1104,10 +1104,10 @@ mod tests { fn yaml_with_per_metric_override_routes_correctly_v6_1_form() { // v6.1 form: `metrics:` map. Each value is a single backend. let yaml = r#" -default: asap_query +default: sketch_store metrics: - http_requests_total: thanos_query - audit_events: thanos_query + http_requests_total: gorilla_object_store + audit_events: gorilla_object_store "#; let r = BackendStorageRouting::from_yaml_str(yaml).expect("parse"); assert_eq!( @@ -1121,7 +1121,7 @@ metrics: #[test] fn yaml_default_only_routes_all_metrics_to_default() { - let yaml = "default: thanos_query\n"; + let yaml = "default: gorilla_object_store\n"; let r = BackendStorageRouting::from_yaml_str(yaml).expect("parse"); assert_eq!(r.lookup("anything"), StorageBackend::GorillaObjectStore); assert!(r.is_empty()); @@ -1130,7 +1130,7 @@ metrics: #[test] fn yaml_omitted_default_falls_back_to_asap_query() { - let yaml = "metrics:\n foo: thanos_query\n"; + let yaml = "metrics:\n foo: gorilla_object_store\n"; let r = BackendStorageRouting::from_yaml_str(yaml).expect("parse"); assert_eq!(r.lookup("foo"), StorageBackend::GorillaObjectStore); assert_eq!(r.lookup("bar"), StorageBackend::SketchStore); @@ -1157,19 +1157,19 @@ metrics: // targets — the default warm-tier slot and a cold-archive // slot scoped to count/topk/rate_post_hoc. let yaml = r#" -default: asap_query +default: sketch_store routes: - metric: http_requests_total targets: - - backend: asap_query - - backend: thanos_query + - backend: sketch_store + - backend: gorilla_object_store applies_to_query_shape: [count, topk, rate_post_hoc] - metric: http_freshness_probe_warm targets: - - backend: asap_query + - backend: sketch_store - metric: http_freshness_probe_archive targets: - - backend: thanos_query + - backend: gorilla_object_store "#; let r = BackendStorageRouting::from_yaml_str(yaml).expect("parse"); @@ -1218,7 +1218,7 @@ routes: // to the same backend (no dual-routing). let yaml = r#" metrics: - audit_events: thanos_query + audit_events: gorilla_object_store "#; let r = BackendStorageRouting::from_yaml_str(yaml).expect("parse"); for shape in [ @@ -1241,14 +1241,14 @@ metrics: // Both `metrics:` and `routes:` populated; a metric in BOTH // wins from `routes:` (multi-target overrides single-target). let yaml = r#" -default: asap_query +default: sketch_store metrics: - http_requests_total: thanos_query + http_requests_total: gorilla_object_store routes: - metric: http_requests_total targets: - - backend: asap_query - - backend: thanos_query + - backend: sketch_store + - backend: gorilla_object_store applies_to_query_shape: [count] "#; let r = BackendStorageRouting::from_yaml_str(yaml).expect("parse"); @@ -1695,9 +1695,9 @@ routes: // Existing YAMLs in the wild don't have `tenant:` — they // must keep parsing and resolve to [`DEFAULT_TENANT`]. let yaml = r#" -default: asap_query +default: sketch_store metrics: - http_requests_total: thanos_query + http_requests_total: gorilla_object_store "#; let r = BackendStorageRouting::from_yaml_str(yaml).expect("parse"); assert_eq!(r.tenant(), DEFAULT_TENANT); @@ -1707,9 +1707,9 @@ metrics: fn yaml_tenant_field_is_picked_up_when_present() { let yaml = r#" tenant: tenant-b -default: asap_query +default: sketch_store metrics: - http_requests_total: thanos_query + http_requests_total: gorilla_object_store "#; let r = BackendStorageRouting::from_yaml_str(yaml).expect("parse"); assert_eq!(r.tenant(), "tenant-b"); diff --git a/asap-query-engine/src/routing/query_engine_routing.rs b/asap-query-engine/src/routing/query_engine_routing.rs index a8cab7ad..b25ef805 100644 --- a/asap-query-engine/src/routing/query_engine_routing.rs +++ b/asap-query-engine/src/routing/query_engine_routing.rs @@ -453,7 +453,7 @@ mod tests { // Iter exposes every registered id. let mut ids: Vec<&str> = router.registered_ids().collect(); ids.sort(); - assert_eq!(ids, vec!["thanos_query", "asap_query"]); + assert_eq!(ids, vec!["asap_query", "thanos_query"]); } #[tokio::test] diff --git a/crates/asap_types/src/capability_matching.rs b/crates/asap_types/src/capability_matching.rs index 6556e285..26a2e7d1 100644 --- a/crates/asap_types/src/capability_matching.rs +++ b/crates/asap_types/src/capability_matching.rs @@ -89,6 +89,8 @@ pub fn parse_storage_backend_engine_id(s: &str) -> Option { match s { ENGINE_ID_ASAP_QUERY => Some(StorageBackend::SketchStore), ENGINE_ID_THANOS_QUERY => Some(StorageBackend::GorillaObjectStore), + "double_write" => Some(StorageBackend::DoubleWrite), + "prometheus_remote" => Some(StorageBackend::PrometheusRemote), _ => None, } }