From a906308b34c2fefc05abda87f152b66cd9428916 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 21:26:09 -0600 Subject: [PATCH 1/5] docs: define VictoriaMetrics MetricsQL architecture --- docs/developer_docs/README.md | 1 + .../victoriametrics-metricsql-support.md | 54 +++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 docs/developer_docs/query-engine/victoriametrics-metricsql-support.md diff --git a/docs/developer_docs/README.md b/docs/developer_docs/README.md index 926d382f6..d1db4235e 100644 --- a/docs/developer_docs/README.md +++ b/docs/developer_docs/README.md @@ -13,6 +13,7 @@ cross-repository interfaces live in - [Routing, readout, and query-engine implementation](query-engine/query-engine.md) - [Catalog-backed physical-plan runtime](query-engine/catalog-physical-plan-runtime.md) - [Extension points](query-engine/extension-points.md) +- [VictoriaMetrics and MetricsQL support](query-engine/victoriametrics-metricsql-support.md) ## Ingest engine diff --git a/docs/developer_docs/query-engine/victoriametrics-metricsql-support.md b/docs/developer_docs/query-engine/victoriametrics-metricsql-support.md new file mode 100644 index 000000000..060ed3b97 --- /dev/null +++ b/docs/developer_docs/query-engine/victoriametrics-metricsql-support.md @@ -0,0 +1,54 @@ +# VictoriaMetrics and MetricsQL query support + +This document is for backend developers extending the query surface. + +## Architecture + +VictoriaMetrics uses its Prometheus-compatible HTTP API, but its query language +is MetricsQL. The protocol boundary is therefore independent while successful +canonical bindings reuse the existing query runtime: + +```text +VictoriaMetrics HTTP request + | +MetricsQL binder + |-- PromQL-compatible subset --> canonical QueryExpr + | --> ASAPPlanner physical plan + | --> shared DAG executor + | --> SDS lookup --> SummaryStore + | --> VictoriaMetrics result adapter + | + `-- unsupported MetricsQL syntax --> VictoriaMetrics exact fallback +``` + +The VictoriaMetrics adapter accepts instant and range GET/POST requests at +`/api/v1/query` and `/api/v1/query_range`. It preserves the Prometheus-compatible +JSON result and error envelope used by VictoriaMetrics clients and Grafana. + +## Language boundary + +ASAPPlanner's independent MetricsQL frontend parses a MetricsQL AST and lowers +it directly to the canonical query DAG. Selectors, matchers, range selectors, +aggregates, grouping, binary expressions, and supported calls share the existing +physical planner and DAG executor. `default_rollup(metric[range])` lowers to +`LastOverTime` over `TimeRange`. An implicit `default_rollup(metric)` needs the +runtime evaluation step, and `keep_metric_names` needs metric-name lineage that +the canonical IR does not represent; both fail closed to the configured +VictoriaMetrics backend with the original expression and parameters. + +This boundary adds no variants to PromQL, SDS, QueryExpr, or the shared physical +DAG. Native acceleration for additional MetricsQL syntax must be added in a +MetricsQL-specific frontend and lowered to existing canonical operations only +when the equivalence is defined. + +## Configuration + +Run an independent listener with `--victoriametrics-http-port` and select the +exact backend with `--victoriametrics-url`. The ordinary Prometheus listener and +its fallback remain unchanged. + +## Verification + +Focused tests cover request parsing, response compatibility, canonical binding +of the common subset, fail-closed handling of MetricsQL-only syntax, and exact +fallback forwarding for instant and range queries. From 275796be569f3db4f3abafec815a55111bfc9afe Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 21:26:19 -0600 Subject: [PATCH 2/5] feat: add VictoriaMetrics MetricsQL query surface --- Cargo.lock | 18 ++- control_plane/Cargo.toml | 6 +- crates/asap_types/Cargo.toml | 2 +- data_plane/Cargo.toml | 5 +- .../src/drivers/query/adapters/config.rs | 12 ++ data_plane/src/drivers/query/adapters/mod.rs | 2 + .../query/adapters/victoriametrics_http.rs | 151 ++++++++++++++++++ data_plane/src/drivers/query/fallback/mod.rs | 2 + .../drivers/query/fallback/victoriametrics.rs | 90 +++++++++++ data_plane/src/drivers/query/servers/http.rs | 28 +++- data_plane/src/main.rs | 24 +++ .../metricsql_binder.rs | 75 +++++++++ .../asap_victoriametrics_query_engine/mod.rs | 5 + data_plane/src/query_engines/mod.rs | 1 + 14 files changed, 410 insertions(+), 11 deletions(-) create mode 100644 data_plane/src/drivers/query/adapters/victoriametrics_http.rs create mode 100644 data_plane/src/drivers/query/fallback/victoriametrics.rs create mode 100644 data_plane/src/query_engines/asap_victoriametrics_query_engine/metricsql_binder.rs create mode 100644 data_plane/src/query_engines/asap_victoriametrics_query_engine/mod.rs diff --git a/Cargo.lock b/Cargo.lock index dac8d6ac1..a716dba29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -343,7 +343,7 @@ dependencies = [ [[package]] name = "asap-aware-mapping" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=0deceda3e776216c5542d638d958b159f22e27ce#0deceda3e776216c5542d638d958b159f22e27ce" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=4ce032623d4fa9a5df6055104e848c40940b52c1#4ce032623d4fa9a5df6055104e848c40940b52c1" dependencies = [ "asap-types", "serde", @@ -351,10 +351,21 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "asap-frontend-metricsql" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=4ce032623d4fa9a5df6055104e848c40940b52c1#4ce032623d4fa9a5df6055104e848c40940b52c1" +dependencies = [ + "asap-frontend-promql", + "asap-types", + "promql-parser 0.10.0", + "thiserror 2.0.18", +] + [[package]] name = "asap-frontend-promql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=0deceda3e776216c5542d638d958b159f22e27ce#0deceda3e776216c5542d638d958b159f22e27ce" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=4ce032623d4fa9a5df6055104e848c40940b52c1#4ce032623d4fa9a5df6055104e848c40940b52c1" dependencies = [ "asap-types", "promql-parser 0.10.0", @@ -374,7 +385,7 @@ dependencies = [ [[package]] name = "asap-types" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=0deceda3e776216c5542d638d958b159f22e27ce#0deceda3e776216c5542d638d958b159f22e27ce" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=4ce032623d4fa9a5df6055104e848c40940b52c1#4ce032623d4fa9a5df6055104e848c40940b52c1" dependencies = [ "serde", "serde_json", @@ -983,6 +994,7 @@ dependencies = [ "arc-swap", "arrow", "asap-aware-mapping", + "asap-frontend-metricsql", "asap-precompute-rs", "asap-types", "asap_otel_proto", diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index fcda64036..8385f18d3 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -91,8 +91,8 @@ asap_types.workspace = true # scaffolding, unaware that `data_plane`'s `summary_executor.rs` in *this* # repo is a real one. Vendored locally instead of chased upstream -- see # `data_plane/src/query_engines/asap_query_engine/summary_exec.rs`. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "0deceda3e776216c5542d638d958b159f22e27ce" } -asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "0deceda3e776216c5542d638d958b159f22e27ce" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "4ce032623d4fa9a5df6055104e848c40940b52c1" } +asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "4ce032623d4fa9a5df6055104e848c40940b52c1" } # L1 adoption (design-target-architecture.md Part B): the PromQL front # end itself, replacing control_plane's own query_parser/promql.rs. @@ -100,7 +100,7 @@ asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = # `planner-types`/`asap-aware-mapping` above -- these three MUST move # together (two revs of the same upstream repo's types in one workspace # resolve to distinct Rust types that won't unify). -asap-frontend-promql = { git = "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/ProjectASAP/ASAPPlanner", rev = "0deceda3e776216c5542d638d958b159f22e27ce" } +asap-frontend-promql = { git = "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/ProjectASAP/ASAPPlanner", rev = "4ce032623d4fa9a5df6055104e848c40940b52c1" } [dev-dependencies] tokio = { version = "1", features = ["full", "test-util"] } diff --git a/crates/asap_types/Cargo.toml b/crates/asap_types/Cargo.toml index 67087b454..14fd95dfa 100644 --- a/crates/asap_types/Cargo.toml +++ b/crates/asap_types/Cargo.toml @@ -32,4 +32,4 @@ sha2 = "0.10" # exactly (`control_plane/Cargo.toml`) -- two different revs of the same # git dependency in one workspace resolve to two distinct Rust types that # won't unify. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "0deceda3e776216c5542d638d958b159f22e27ce" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "4ce032623d4fa9a5df6055104e848c40940b52c1" } diff --git a/data_plane/Cargo.toml b/data_plane/Cargo.toml index d21c55d7d..8ef5672f1 100644 --- a/data_plane/Cargo.toml +++ b/data_plane/Cargo.toml @@ -38,7 +38,8 @@ control_plane = { path = "../control_plane" } # reduction: Reduction, .. }`) are `pre_asap` types, in the same crate now # (not a separate `asap-ir` import). Query serving consumes the compiled # QueryPlan; these types are used at physical-plan compilation boundaries. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "0deceda3e776216c5542d638d958b159f22e27ce" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "4ce032623d4fa9a5df6055104e848c40940b52c1" } +asap-frontend-metricsql = { git = "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/ProjectASAP/ASAPPlanner", rev = "4ce032623d4fa9a5df6055104e848c40940b52c1" } # Shared external (workspace) serde.workspace = true @@ -129,7 +130,7 @@ crc32fast = "1.4" # none of them. [dev-dependencies] -asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "0deceda3e776216c5542d638d958b159f22e27ce" } +asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "4ce032623d4fa9a5df6055104e848c40940b52c1" } tempfile = "3.20.0" criterion = { version = "0.5", features = ["html_reports"] } tokio-tungstenite = "0.21" diff --git a/data_plane/src/drivers/query/adapters/config.rs b/data_plane/src/drivers/query/adapters/config.rs index b2a7c40f6..465b249a6 100644 --- a/data_plane/src/drivers/query/adapters/config.rs +++ b/data_plane/src/drivers/query/adapters/config.rs @@ -59,4 +59,16 @@ impl AdapterConfig { fallback, ) } + + /// Configuration used by the independent VictoriaMetrics listener. The + /// wire protocol is Prometheus-compatible; MetricsQL binding is handled at + /// the query-language boundary. + pub fn victoriametrics_metricsql(fallback_url: String) -> Self { + use crate::drivers::query::fallback::VictoriaMetricsHttpFallback; + Self::new( + QueryProtocol::PrometheusHttp, + QueryLanguage::promql, + Some(Arc::new(VictoriaMetricsHttpFallback::new(fallback_url))), + ) + } } diff --git a/data_plane/src/drivers/query/adapters/mod.rs b/data_plane/src/drivers/query/adapters/mod.rs index 420bad377..59f623a4a 100644 --- a/data_plane/src/drivers/query/adapters/mod.rs +++ b/data_plane/src/drivers/query/adapters/mod.rs @@ -2,6 +2,7 @@ pub mod config; pub mod factory; pub mod prometheus_http; pub mod traits; +pub mod victoriametrics_http; // Re-export main types pub use config::AdapterConfig; @@ -11,3 +12,4 @@ pub use traits::{ AdapterError, HttpProtocolAdapter, ParsedQueryRequest, ParsedRangeQueryRequest, QueryExecutionResult, QueryRequestAdapter, QueryResponseAdapter, }; +pub use victoriametrics_http::VictoriaMetricsHttpAdapter; diff --git a/data_plane/src/drivers/query/adapters/victoriametrics_http.rs b/data_plane/src/drivers/query/adapters/victoriametrics_http.rs new file mode 100644 index 000000000..b3a1b4198 --- /dev/null +++ b/data_plane/src/drivers/query/adapters/victoriametrics_http.rs @@ -0,0 +1,151 @@ +use super::{ + AdapterConfig, AdapterError, HttpProtocolAdapter, ParsedQueryRequest, ParsedRangeQueryRequest, + PrometheusHttpAdapter, QueryExecutionResult, QueryRequestAdapter, QueryResponseAdapter, +}; +use asap_types::KeyByLabelNames; +use async_trait::async_trait; +use axum::{ + body::Bytes, + extract::{Form, Query}, + http::StatusCode, + response::{Json, Response}, +}; +use serde_json::Value; +use std::{collections::HashMap, sync::Arc}; + +/// VictoriaMetrics HTTP adapter. VictoriaMetrics intentionally uses the +/// Prometheus-compatible wire envelope; language classification remains in the +/// MetricsQL binder and engine fallback path. +pub struct VictoriaMetricsHttpAdapter { + wire: PrometheusHttpAdapter, +} + +impl VictoriaMetricsHttpAdapter { + pub fn new(config: AdapterConfig) -> Self { + Self { + wire: PrometheusHttpAdapter::new(config), + } + } +} + +#[async_trait] +impl QueryRequestAdapter for VictoriaMetricsHttpAdapter { + async fn parse_get_request( + &self, + params: Query>, + ) -> Result { + self.wire.parse_get_request(params).await + } + async fn parse_post_request( + &self, + params: Form>, + ) -> Result { + self.wire.parse_post_request(params).await + } + async fn parse_json_post_request( + &self, + body: Bytes, + ) -> Result { + self.wire.parse_json_post_request(body).await + } + fn get_query_endpoint(&self) -> &'static str { + "/api/v1/query" + } + async fn parse_range_get_request( + &self, + params: Query>, + ) -> Result { + self.wire.parse_range_get_request(params).await + } + async fn parse_range_post_request( + &self, + params: Form>, + ) -> Result { + self.wire.parse_range_post_request(params).await + } + fn get_range_query_endpoint(&self) -> &'static str { + "/api/v1/query_range" + } +} + +#[async_trait] +impl QueryResponseAdapter for VictoriaMetricsHttpAdapter { + async fn format_success_response( + &self, + result: &QueryExecutionResult, + ) -> Result { + self.wire.format_success_response(result).await + } + async fn format_range_success_response( + &self, + result: &crate::query_engines::QueryResult, + labels: &KeyByLabelNames, + ) -> Result { + self.wire + .format_range_success_response(result, labels) + .await + } + async fn format_error_response(&self, error: &AdapterError) -> Result { + self.wire.format_error_response(error).await + } + async fn format_unsupported_query_response(&self) -> Result { + self.wire.format_unsupported_query_response().await + } +} + +#[async_trait] +impl HttpProtocolAdapter for VictoriaMetricsHttpAdapter { + fn adapter_name(&self) -> &'static str { + "VictoriaMetrics HTTP / MetricsQL" + } + fn get_runtime_info_path(&self) -> &'static str { + "/api/v1/status/runtimeinfo" + } + async fn handle_runtime_info( + &self, + index: Arc, + ) -> Result, StatusCode> { + self.wire.handle_runtime_info(index).await + } + async fn handle_runtime_info_with_headers( + &self, + index: Arc, + headers: HashMap, + ) -> Result, StatusCode> { + self.wire + .handle_runtime_info_with_headers(index, headers) + .await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::storage_engines::types::enums::{QueryLanguage, QueryProtocol}; + + fn adapter() -> VictoriaMetricsHttpAdapter { + VictoriaMetricsHttpAdapter::new(AdapterConfig::new( + QueryProtocol::PrometheusHttp, + QueryLanguage::promql, + None, + )) + } + + #[tokio::test] + async fn preserves_metricsql_expression_for_binding_or_fallback() { + let mut params = HashMap::new(); + params.insert( + "query".into(), + "rate(requests[5m]) keep_metric_names".into(), + ); + params.insert("time".into(), "1700000000".into()); + let parsed = adapter().parse_get_request(Query(params)).await.unwrap(); + assert_eq!(parsed.query, "rate(requests[5m]) keep_metric_names"); + } + + #[test] + fn exposes_victoriametrics_query_endpoints() { + assert_eq!(adapter().get_query_endpoint(), "/api/v1/query"); + assert_eq!(adapter().get_range_query_endpoint(), "/api/v1/query_range"); + } +} diff --git a/data_plane/src/drivers/query/fallback/mod.rs b/data_plane/src/drivers/query/fallback/mod.rs index 2f4c50d59..21840602e 100644 --- a/data_plane/src/drivers/query/fallback/mod.rs +++ b/data_plane/src/drivers/query/fallback/mod.rs @@ -117,7 +117,9 @@ pub trait FallbackClient: Send + Sync { } mod prometheus; +mod victoriametrics; pub mod metrics; pub use prometheus::PrometheusHttpFallback; +pub use victoriametrics::VictoriaMetricsHttpFallback; diff --git a/data_plane/src/drivers/query/fallback/victoriametrics.rs b/data_plane/src/drivers/query/fallback/victoriametrics.rs new file mode 100644 index 000000000..1973b1e4b --- /dev/null +++ b/data_plane/src/drivers/query/fallback/victoriametrics.rs @@ -0,0 +1,90 @@ +use super::{FallbackClient, FallbackResponse, PrometheusHttpFallback}; +use crate::drivers::query::adapters::{ParsedQueryRequest, ParsedRangeQueryRequest}; +use async_trait::async_trait; +use axum::http::StatusCode; +use std::collections::HashMap; + +/// Exact MetricsQL backend. VictoriaMetrics' query API is wire-compatible with +/// Prometheus, so transport is shared while this type keeps backend selection +/// independent from the Prometheus listener. +pub struct VictoriaMetricsHttpFallback(PrometheusHttpFallback); + +impl VictoriaMetricsHttpFallback { + pub fn new(base_url: String) -> Self { + Self(PrometheusHttpFallback::new(base_url)) + } +} + +#[async_trait] +impl FallbackClient for VictoriaMetricsHttpFallback { + async fn execute_query( + &self, + request: &ParsedQueryRequest, + ) -> Result { + self.0.execute_query(request).await + } + async fn execute_query_with_headers( + &self, + request: &ParsedQueryRequest, + headers: HashMap, + ) -> Result { + self.0.execute_query_with_headers(request, headers).await + } + async fn execute_range_query( + &self, + request: &ParsedRangeQueryRequest, + ) -> Result { + self.0.execute_range_query(request).await + } + async fn execute_range_query_with_headers( + &self, + request: &ParsedRangeQueryRequest, + headers: HashMap, + ) -> Result { + self.0 + .execute_range_query_with_headers(request, headers) + .await + } + async fn get_runtime_info(&self) -> Result { + Ok(serde_json::json!({})) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::{extract::Query, routing::get, Json, Router}; + + #[tokio::test] + async fn exact_fallback_preserves_metricsql_expression() { + let app = Router::new().route( + "/api/v1/query", + get(|Query(params): Query>| async move { + Json(serde_json::json!({ + "status": "success", + "data": {"received": params["query"]} + })) + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + + let client = VictoriaMetricsHttpFallback::new(format!("http://{address}")); + let response = client + .execute_query(&ParsedQueryRequest { + query: "rate(requests[5m]) keep_metric_names".into(), + time: 1_700_000_000.0, + timeout: Some("5s".into()), + }) + .await + .unwrap(); + let FallbackResponse::Json(payload) = response else { + panic!("VictoriaMetrics fallback must return JSON") + }; + assert_eq!( + payload["data"]["received"], + "rate(requests[5m]) keep_metric_names" + ); + } +} diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index 80109e2c5..722c645a7 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -132,6 +132,7 @@ pub struct HttpServerConfig { #[derive(Clone)] pub struct HttpServer { config: HttpServerConfig, + adapter_override: Option>, query_engine: Arc, /// Phase-5/6 capability router. Built from `query_engine` at /// construction time (`ASAPQueryEngine` registered as the ASAP-tier @@ -238,6 +239,7 @@ impl HttpServer { let query_router = Arc::new(router); Self { config, + adapter_override: None, query_engine, query_router, sketch_index, @@ -253,6 +255,22 @@ impl HttpServer { } } + /// Use a language-specific protocol adapter without extending the shared + /// protocol enum. This keeps VictoriaMetrics transport policy at its own + /// listener boundary. + pub fn with_protocol_adapter(mut self, adapter: Arc) -> Self { + self.adapter_override = Some(adapter); + self + } + + /// Clone the fully configured query service onto another query listener. + pub fn with_query_listener(mut self, port: u16, adapter_config: AdapterConfig) -> Self { + self.config.port = port; + self.config.adapter_config = adapter_config; + self.remote_write = None; + self + } + /// Enable Prometheus Remote Write v1 on the same public HTTP listener. pub fn with_remote_write( mut self, @@ -395,7 +413,10 @@ impl HttpServer { srv_metrics::register_all(); // Create adapter using factory - let adapter = create_http_adapter(self.config.adapter_config.clone()); + let adapter = self + .adapter_override + .clone() + .unwrap_or_else(|| create_http_adapter(self.config.adapter_config.clone())); let query_endpoint = adapter.get_query_endpoint(); let runtime_info_path = adapter.get_runtime_info_path(); @@ -507,7 +528,10 @@ impl HttpServer { /// the regular `start()` method. pub async fn start_test_server(&self) -> Result> { // Create adapter using factory - let adapter = create_http_adapter(self.config.adapter_config.clone()); + let adapter = self + .adapter_override + .clone() + .unwrap_or_else(|| create_http_adapter(self.config.adapter_config.clone())); let query_endpoint = adapter.get_query_endpoint(); let runtime_info_path = adapter.get_runtime_info_path(); diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index e4780f345..8889df929 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -86,6 +86,14 @@ struct Args { #[arg(long, alias = "query-port", default_value = "8088")] http_port: u16, + /// Independent VictoriaMetrics-compatible MetricsQL query listener. + #[arg(long)] + victoriametrics_http_port: Option, + + /// VictoriaMetrics base URL used for exact MetricsQL fallback. + #[arg(long, default_value = "http://localhost:8428")] + victoriametrics_url: String, + /// Deprecated/no-op: the backend's only HTTP listener is the /// PromQL query surface (`--http-port` / `--query-port`). The /// old PRW ingest port was deleted in PR #100; this flag is @@ -1292,6 +1300,18 @@ async fn main() -> Result<()> { info!("Starting HTTP server on port {}", args.http_port); + let victoria_server = args.victoriametrics_http_port.map(|port| { + use data_plane::drivers::query::adapters::VictoriaMetricsHttpAdapter; + let config = AdapterConfig::victoriametrics_metricsql(args.victoriametrics_url.clone()); + info!("Starting VictoriaMetrics MetricsQL listener on port {port}"); + server + .clone() + .with_query_listener(port, config.clone()) + .with_protocol_adapter(Arc::new(VictoriaMetricsHttpAdapter::new(config))) + }); + + let victoria_task = victoria_server.map(|server| tokio::spawn(server.run())); + // Wait for shutdown signal tokio::select! { result = server.run() => { @@ -1304,6 +1324,10 @@ async fn main() -> Result<()> { } } + if let Some(task) = victoria_task { + task.abort(); + } + // Cleanup - gracefully shutdown background tasks if let Some(handle) = backfill_service_handle { info!("Shutting down backfill service..."); diff --git a/data_plane/src/query_engines/asap_victoriametrics_query_engine/metricsql_binder.rs b/data_plane/src/query_engines/asap_victoriametrics_query_engine/metricsql_binder.rs new file mode 100644 index 000000000..76405e22e --- /dev/null +++ b/data_plane/src/query_engines/asap_victoriametrics_query_engine/metricsql_binder.rs @@ -0,0 +1,75 @@ +use control_plane::physical::post_asap::{bind_query_expr, PhysicalExpr}; +use control_plane::types_v2::AccuracyTarget; +use planner_types::pre_asap::QueryExpr; +use thiserror::Error; + +/// A MetricsQL request proven safe for the shared canonical runtime. +pub struct MetricsQlBinding { + pub canonical: QueryExpr, + pub physical: PhysicalExpr, +} + +#[derive(Debug, Error)] +pub enum MetricsQlBindingError { + #[error("MetricsQL expression is outside the accelerated PromQL-compatible subset: {0}")] + Unsupported(String), + #[error("canonical expression cannot be bound to a physical plan: {0}")] + Physical(String), +} + +/// Bind the PromQL-compatible subset of MetricsQL to the existing canonical +/// query and physical DAG. MetricsQL-only syntax deliberately returns +/// `Unsupported`, which routes the original request to VictoriaMetrics. +pub fn bind_metricsql( + query: &str, + accuracy: AccuracyTarget, +) -> Result { + let canonical = asap_frontend_metricsql::lower_metricsql(query, accuracy.clone()) + .map_err(|error| MetricsQlBindingError::Unsupported(error.to_string()))?; + let physical = bind_query_expr(&canonical, accuracy) + .map_err(|error| MetricsQlBindingError::Physical(error.to_string()))?; + Ok(MetricsQlBinding { + canonical, + physical, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn accuracy() -> AccuracyTarget { + AccuracyTarget::Epsilon(0.01) + } + + #[test] + fn promql_compatible_metricsql_reaches_shared_planning() { + bind_metricsql("sum(rate(http_requests_total[5m]))", accuracy()) + .expect("PromQL-compatible MetricsQL must bind"); + } + + #[test] + fn explicit_metricsql_rollup_reaches_shared_planning() { + bind_metricsql("default_rollup(cpu_usage[5m])", accuracy()) + .expect("explicit MetricsQL rollup must bind"); + } + + #[test] + fn implicit_metricsql_rollup_fails_closed_without_runtime_step() { + assert!(matches!( + bind_metricsql("default_rollup(cpu_usage)", accuracy()), + Err(MetricsQlBindingError::Unsupported(_)) + )); + } + + #[test] + fn metricsql_modifier_fails_closed() { + assert!(matches!( + bind_metricsql( + "rate(http_requests_total[5m]) keep_metric_names", + accuracy() + ), + Err(MetricsQlBindingError::Unsupported(_)) + )); + } +} diff --git a/data_plane/src/query_engines/asap_victoriametrics_query_engine/mod.rs b/data_plane/src/query_engines/asap_victoriametrics_query_engine/mod.rs new file mode 100644 index 000000000..2a77ab44b --- /dev/null +++ b/data_plane/src/query_engines/asap_victoriametrics_query_engine/mod.rs @@ -0,0 +1,5 @@ +//! VictoriaMetrics-specific MetricsQL binding boundary. + +pub mod metricsql_binder; + +pub use metricsql_binder::{bind_metricsql, MetricsQlBinding, MetricsQlBindingError}; diff --git a/data_plane/src/query_engines/mod.rs b/data_plane/src/query_engines/mod.rs index 722fc96dd..76828458d 100644 --- a/data_plane/src/query_engines/mod.rs +++ b/data_plane/src/query_engines/mod.rs @@ -18,6 +18,7 @@ //! `crate::query_engines::routing::QueryEngine` impl returns. pub mod asap_query_engine; +pub mod asap_victoriametrics_query_engine; pub mod no_data_archive; pub mod query_result; pub mod routing; From 4762afeef4e8e8a36d485120534acbd37477ca80 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 22:55:42 -0600 Subject: [PATCH 3/5] feat: publish and execute MetricsQL sidecar plans --- Cargo.lock | 162 ++++++++++- control_plane/Cargo.toml | 7 +- control_plane/src/lib.rs | 1 + control_plane/src/main.rs | 93 ++++++- control_plane/src/metricsql_plan.rs | 91 +++++++ control_plane/src/physical/compiler.rs | 83 +++++- control_plane/src/physical/publication.rs | 33 +++ control_plane/src/physical/workload_cost.rs | 23 +- control_plane/src/query_plan.rs | 75 +++++ crates/asap_types/Cargo.toml | 2 +- data_plane/Cargo.toml | 6 +- .../drivers/ingest/prometheus_remote_write.rs | 3 + .../src/drivers/query/adapters/traits.rs | 6 + .../query/adapters/victoriametrics_http.rs | 5 + data_plane/src/drivers/query/fallback/mod.rs | 19 +- .../drivers/query/fallback/victoriametrics.rs | 170 +++++++++++- data_plane/src/drivers/query/servers/http.rs | 256 +++++++++++++++++- data_plane/src/main.rs | 5 + .../query_engines/asap_query_engine/engine.rs | 140 ++++++++++ .../metricsql_binder.rs | 8 + .../types/hot_reload_config.rs | 4 + .../asapquery_compatibility_process_e2e.rs | 1 + .../victoriametrics-metricsql-support.md | 44 ++- 23 files changed, 1194 insertions(+), 43 deletions(-) create mode 100644 control_plane/src/metricsql_plan.rs diff --git a/Cargo.lock b/Cargo.lock index a716dba29..70a1fcd8a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,6 +18,7 @@ dependencies = [ "const-random", "getrandom 0.3.4", "once_cell", + "serde", "version_check", "zerocopy", ] @@ -343,7 +344,7 @@ dependencies = [ [[package]] name = "asap-aware-mapping" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=4ce032623d4fa9a5df6055104e848c40940b52c1#4ce032623d4fa9a5df6055104e848c40940b52c1" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=bda8cd314934e81e41c23260225e0fb0dc81baae#bda8cd314934e81e41c23260225e0fb0dc81baae" dependencies = [ "asap-types", "serde", @@ -354,18 +355,17 @@ dependencies = [ [[package]] name = "asap-frontend-metricsql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=4ce032623d4fa9a5df6055104e848c40940b52c1#4ce032623d4fa9a5df6055104e848c40940b52c1" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=bda8cd314934e81e41c23260225e0fb0dc81baae#bda8cd314934e81e41c23260225e0fb0dc81baae" dependencies = [ - "asap-frontend-promql", "asap-types", - "promql-parser 0.10.0", + "metricsql_parser", "thiserror 2.0.18", ] [[package]] name = "asap-frontend-promql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=4ce032623d4fa9a5df6055104e848c40940b52c1#4ce032623d4fa9a5df6055104e848c40940b52c1" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=bda8cd314934e81e41c23260225e0fb0dc81baae#bda8cd314934e81e41c23260225e0fb0dc81baae" dependencies = [ "asap-types", "promql-parser 0.10.0", @@ -385,7 +385,7 @@ dependencies = [ [[package]] name = "asap-types" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=4ce032623d4fa9a5df6055104e848c40940b52c1#4ce032623d4fa9a5df6055104e848c40940b52c1" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=bda8cd314934e81e41c23260225e0fb0dc81baae#bda8cd314934e81e41c23260225e0fb0dc81baae" dependencies = [ "serde", "serde_json", @@ -570,6 +570,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "beef" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" + [[package]] name = "bincode" version = "1.3.3" @@ -797,6 +803,7 @@ version = "0.1.0" dependencies = [ "anyhow", "asap-aware-mapping", + "asap-frontend-metricsql", "asap-frontend-promql", "asap-types", "asap_types", @@ -1088,6 +1095,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enquote" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06c36cb11dbde389f4096111698d8b567c0720e3452fd5ac3e6b4e47e1939932" +dependencies = [ + "thiserror 1.0.69", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1989,6 +2005,39 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "logos" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7251356ef8cb7aec833ddf598c6cb24d17b689d20b993f9d11a3d764e34e6458" +dependencies = [ + "logos-derive", +] + +[[package]] +name = "logos-codegen" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59f80069600c0d66734f5ff52cc42f2dabd6b29d205f333d61fd7832e9e9963f" +dependencies = [ + "beef", + "fnv", + "lazy_static", + "proc-macro2", + "quote", + "regex-syntax", + "syn 2.0.117", +] + +[[package]] +name = "logos-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24fb722b06a9dc12adb0963ed585f19fc61dc5413e6a9be9422ef92c091e731d" +dependencies = [ + "logos-codegen", +] + [[package]] name = "lrlex" version = "0.13.10" @@ -2079,6 +2128,37 @@ dependencies = [ "libc", ] +[[package]] +name = "metricsql_common" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=bda8cd314934e81e41c23260225e0fb0dc81baae#bda8cd314934e81e41c23260225e0fb0dc81baae" +dependencies = [ + "chrono", +] + +[[package]] +name = "metricsql_parser" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=bda8cd314934e81e41c23260225e0fb0dc81baae#bda8cd314934e81e41c23260225e0fb0dc81baae" +dependencies = [ + "ahash", + "chrono", + "enquote", + "logos", + "logos-derive", + "metricsql_common", + "num-traits", + "phf", + "regex", + "scopeguard", + "serde", + "strum", + "strum_macros", + "thiserror 1.0.69", + "tinyvec", + "xxhash-rust", +] + [[package]] name = "mime" version = "0.3.17" @@ -2294,6 +2374,48 @@ dependencies = [ "indexmap 2.14.0", ] +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand 0.8.6", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.1.12" @@ -3265,6 +3387,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + [[package]] name = "slab" version = "0.4.12" @@ -3363,6 +3491,28 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.117", +] + [[package]] name = "subtle" version = "2.6.1" diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index 8385f18d3..d4da0775d 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -91,8 +91,8 @@ asap_types.workspace = true # scaffolding, unaware that `data_plane`'s `summary_executor.rs` in *this* # repo is a real one. Vendored locally instead of chased upstream -- see # `data_plane/src/query_engines/asap_query_engine/summary_exec.rs`. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "4ce032623d4fa9a5df6055104e848c40940b52c1" } -asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "4ce032623d4fa9a5df6055104e848c40940b52c1" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "bda8cd314934e81e41c23260225e0fb0dc81baae" } +asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "bda8cd314934e81e41c23260225e0fb0dc81baae" } # L1 adoption (design-target-architecture.md Part B): the PromQL front # end itself, replacing control_plane's own query_parser/promql.rs. @@ -100,7 +100,8 @@ asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = # `planner-types`/`asap-aware-mapping` above -- these three MUST move # together (two revs of the same upstream repo's types in one workspace # resolve to distinct Rust types that won't unify). -asap-frontend-promql = { git = "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/ProjectASAP/ASAPPlanner", rev = "4ce032623d4fa9a5df6055104e848c40940b52c1" } +asap-frontend-promql = { git = "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/ProjectASAP/ASAPPlanner", rev = "bda8cd314934e81e41c23260225e0fb0dc81baae" } +asap-frontend-metricsql = { git = "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/ProjectASAP/ASAPPlanner", rev = "bda8cd314934e81e41c23260225e0fb0dc81baae" } [dev-dependencies] tokio = { version = "1", features = ["full", "test-util"] } diff --git a/control_plane/src/lib.rs b/control_plane/src/lib.rs index 023b70216..734a4cbbf 100644 --- a/control_plane/src/lib.rs +++ b/control_plane/src/lib.rs @@ -61,6 +61,7 @@ pub mod backend_client; pub mod emit; pub mod epsilon_alloc; pub mod metrics_exposer; +pub mod metricsql_plan; pub mod monitor; pub mod opamp; pub mod physical; diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index 5187d623a..4e1ae466c 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -17,7 +17,7 @@ use control_plane::workload; use axum::{ extract::{Path, State}, http::{HeaderMap, StatusCode}, - response::IntoResponse, + response::{IntoResponse, Response}, routing::{get, post}, Json, Router, }; @@ -550,10 +550,18 @@ async fn main() { "/api/v1/physical-plan/cost-manifests", post(handle_workload_cost_manifests), ) + .route( + "/api/v1/metricsql/physical-plan/cost-manifests", + post(handle_metricsql_workload_cost_manifests), + ) .route( "/api/v1/physical-plan/compile-and-publish", post(handle_compile_and_publish_physical_plan), ) + .route( + "/api/v1/metricsql/physical-plan/compile-and-publish", + post(handle_compile_and_publish_metricsql_physical_plan), + ) .route("/api/v1/plan/auto", post(handle_plan_auto)) .route("/api/v1/plan/pareto", post(handle_pareto)) .route("/api/v1/plan/:metric", get(handle_get_plan)) @@ -622,6 +630,39 @@ fn default_physical_plan_timeout_ms() -> u64 { 10_000 } +#[derive(Clone, Copy)] +enum PhysicalQueryFrontend { + PromQl, + MetricsQl, +} + +impl PhysicalQueryFrontend { + fn parse( + self, + query: &str, + accuracy: types_v2::AccuracyTarget, + ) -> Result { + match self { + Self::PromQl => parse_query_expr_canonical(query, accuracy) + .map_err(|e| format!("frontend.promql: {e}")), + Self::MetricsQl => asap_frontend_metricsql::lower_metricsql(query, accuracy) + .map_err(|e| format!("frontend.metricsql: {e}")), + } + } + fn compile( + self, + request: physical::compiler::PlanningRequest, + environment: physical::compiler::DeploymentEnvironment, + ) -> Result { + match self { + Self::PromQl => physical::compiler::PhysicalCompiler.compile(request, environment), + Self::MetricsQl => { + physical::compiler::PhysicalCompiler.compile_metricsql(request, environment) + } + } + } +} + #[derive(Debug, Serialize)] struct CompileAndPublishPhysicalPlanResponse { cost_comparison: Option, @@ -638,9 +679,24 @@ struct CompileAndPublishPhysicalPlanResponse { async fn handle_compile_and_publish_physical_plan( State(st): State, Json(request): Json, -) -> impl IntoResponse { +) -> Response { + compile_and_publish_physical_plan(st, request, PhysicalQueryFrontend::PromQl).await +} + +async fn handle_compile_and_publish_metricsql_physical_plan( + State(st): State, + Json(request): Json, +) -> Response { + compile_and_publish_physical_plan(st, request, PhysicalQueryFrontend::MetricsQl).await +} + +async fn compile_and_publish_physical_plan( + st: AppState, + request: CompileAndPublishPhysicalPlanRequest, + frontend: PhysicalQueryFrontend, +) -> Response { let (bundle, collector_ids, apply_timeout, adaptation_evidence, _) = - match compile_physical_plan_request(request, false) { + match compile_physical_plan_request(request, false, frontend) { Ok((Some(bundle), ids, timeout, adaptation, manifests)) => { (bundle, ids, timeout, adaptation, manifests) } @@ -749,6 +805,7 @@ async fn handle_compile_and_publish_physical_plan( fn compile_physical_plan_request( request: CompileAndPublishPhysicalPlanRequest, manifests_only: bool, + frontend: PhysicalQueryFrontend, ) -> Result< ( Option, @@ -800,7 +857,7 @@ fn compile_physical_plan_request( "query_id, metric, and window_secs must be non-empty/non-zero".to_string(), )); } - let expr = match parse_query_expr_canonical(&query.query_string, query.accuracy.clone()) { + let expr = match frontend.parse(&query.query_string, query.accuracy.clone()) { Ok(expr) => expr, Err(error) => return Err((StatusCode::UNPROCESSABLE_ENTITY, error.to_string())), }; @@ -864,7 +921,7 @@ fn compile_physical_plan_request( let manifests: Vec<_> = candidates .iter() .filter_map(|candidate| { - physical::compiler::PhysicalCompiler + frontend .compile(candidate.clone(), environment.clone()) .and_then(|plan| physical::workload_cost::manifest(&plan, &candidate.queries)) .ok() @@ -889,8 +946,15 @@ fn compile_physical_plan_request( )); } let compiled = match request.workload_cost_evidence { - Some(evidence) => physical::workload_cost::select(candidates, environment, &evidence), - None => physical::compiler::PhysicalCompiler.compile(planning_request, environment), + Some(evidence) => match frontend { + PhysicalQueryFrontend::PromQl => { + physical::workload_cost::select(candidates, environment, &evidence) + } + PhysicalQueryFrontend::MetricsQl => { + physical::workload_cost::select_metricsql(candidates, environment, &evidence) + } + }, + None => frontend.compile(planning_request, environment), }; let bundle = match compiled { Ok(bundle) => bundle, @@ -909,6 +973,19 @@ fn compile_physical_plan_request( async fn handle_workload_cost_manifests( Json(request): Json, ) -> impl IntoResponse { + workload_cost_manifests(request, PhysicalQueryFrontend::PromQl) +} + +async fn handle_metricsql_workload_cost_manifests( + Json(request): Json, +) -> impl IntoResponse { + workload_cost_manifests(request, PhysicalQueryFrontend::MetricsQl) +} + +fn workload_cost_manifests( + request: CompileAndPublishPhysicalPlanRequest, + frontend: PhysicalQueryFrontend, +) -> Response { if request.workload_cost_evidence.is_some() { return ( StatusCode::UNPROCESSABLE_ENTITY, @@ -916,7 +993,7 @@ async fn handle_workload_cost_manifests( ) .into_response(); } - match compile_physical_plan_request(request, true) { + match compile_physical_plan_request(request, true, frontend) { Ok((_, _, _, _, manifests)) => Json(manifests).into_response(), Err(error) => error.into_response(), } diff --git a/control_plane/src/metricsql_plan.rs b/control_plane/src/metricsql_plan.rs new file mode 100644 index 000000000..627b350d5 --- /dev/null +++ b/control_plane/src/metricsql_plan.rs @@ -0,0 +1,91 @@ +use crate::query_plan::{ExecutableQueryPlan, QueryPlanError}; +use asap_types::PolicyFingerprint; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct MetricsQlPlanEntry { + pub query_id: String, + pub canonical_metricsql: String, + pub executable: ExecutableQueryPlan, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::query_plan::{FallbackPolicy, InstantExecution, QueryNodeId, QueryPlanNode}; + + #[test] + fn sidecar_serde_never_invents_a_promql_identity() { + let identity = "default_rollup(cpu[5m])".to_string(); + let catalog = MetricsQlPlanCatalog { + plan_id: 7, + plan_version: 3, + entries: BTreeMap::from([( + identity.clone(), + MetricsQlPlanEntry { + query_id: "vm-q".into(), + canonical_metricsql: identity, + executable: ExecutableQueryPlan { + root: QueryNodeId(0), + nodes: BTreeMap::from([( + QueryNodeId(0), + QueryPlanNode::ExactFallback { + reason: "fixture".into(), + }, + )]), + instant: InstantExecution { + lookback_ms: 300_000, + full_history: false, + cumulative_readout: false, + }, + fallback: FallbackPolicy::ExactBackend, + }, + }, + )]), + }; + let json = serde_json::to_string(&catalog).unwrap(); + assert!(json.contains("canonical_metricsql")); + assert!(!json.contains("canonical_promql")); + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct MetricsQlPlanCatalog { + pub plan_id: u64, + pub plan_version: u64, + pub entries: BTreeMap, +} + +impl MetricsQlPlanCatalog { + pub fn empty() -> Self { + Self { + plan_id: 0, + plan_version: 0, + entries: BTreeMap::new(), + } + } + + pub fn lookup(&self, identity: &str) -> Result<&MetricsQlPlanEntry, QueryPlanError> { + self.entries + .get(identity) + .ok_or_else(|| QueryPlanError::QueryNotPlanned(identity.into())) + } + + pub fn validate(&self, available: &BTreeSet) -> Result<(), QueryPlanError> { + for (identity, entry) in &self.entries { + if identity != &entry.canonical_metricsql { + return Err(QueryPlanError::Invalid( + "MetricsQL catalog key disagrees with its AST identity".into(), + )); + } + entry + .executable + .execution_view(entry.query_id.clone(), String::new()) + .validate(available)?; + } + Ok(()) + } +} diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 91cc83cd9..3bee9e047 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -737,6 +737,7 @@ pub struct PhysicalPlan { pub precompute_plan: PrecomputePlan, pub transmission_plan: TransmissionPlan, pub query_plan: QueryPlan, + pub metricsql_plan_catalog: Option, /// Lifecycle component only, not a complete physical-plan comparison. pub lifecycle_estimates: Vec, pub cost_comparison: Option, @@ -2049,9 +2050,26 @@ fn preserve_invalid_exact_fallback_roots( impl PhysicalCompiler { pub fn compile( + &self, + request: PlanningRequest, + environment: DeploymentEnvironment, + ) -> Result { + self.compile_language(request, environment, false) + } + + pub fn compile_metricsql( + &self, + request: PlanningRequest, + environment: DeploymentEnvironment, + ) -> Result { + self.compile_language(request, environment, true) + } + + fn compile_language( &self, mut request: PlanningRequest, environment: DeploymentEnvironment, + metricsql: bool, ) -> Result { if request.hybrid_execution && environment.target != PhysicalDeploymentTarget::BackendLocalRemoteWrite @@ -2489,8 +2507,18 @@ impl PhysicalCompiler { .map(asap_types::PrecomputeMaterialization::policy_fingerprint) .collect(); let mut query_entries = BTreeMap::new(); + let mut metricsql_entries = BTreeMap::new(); for query in &request.queries { - let canonical = canonical_promql(&query.query_string)?; + let canonical = if metricsql { + asap_frontend_metricsql::canonical_metricsql(&query.query_string).map_err( + |error| CompileError::Query { + query_id: query.query_id.clone(), + reason: format!("frontend.metricsql.identity: {error}"), + }, + )? + } else { + canonical_promql(&query.query_string)? + }; let binding = |node: &SummaryNode, node_family: &SummaryFamilyType| -> Result { summary_agg_metric(node).ok_or_else(|| { crate::query_plan::QueryPlanError::Invalid( @@ -2583,7 +2611,22 @@ impl PhysicalCompiler { // backend-local range index leaf. crate::query_plan::logical::finalize_residuals(&mut entry)?; } - if query_entries.insert(canonical.clone(), entry).is_some() { + if metricsql { + let metrics_entry = crate::metricsql_plan::MetricsQlPlanEntry { + query_id: query.query_id.clone(), + canonical_metricsql: canonical.clone(), + executable: entry.executable(), + }; + if metricsql_entries + .insert(canonical.clone(), metrics_entry) + .is_some() + { + return Err(CompileError::Query { + query_id: query.query_id.clone(), + reason: format!("duplicate canonical MetricsQL identity `{canonical}`"), + }); + } + } else if query_entries.insert(canonical.clone(), entry).is_some() { return Err(CompileError::Query { query_id: query.query_id.clone(), reason: format!("duplicate canonical query identity `{canonical}`"), @@ -2595,12 +2638,24 @@ impl PhysicalCompiler { plan_version: envelope.plan_version, entries: query_entries, }; + let metricsql_plan_catalog = + metricsql.then_some(crate::metricsql_plan::MetricsQlPlanCatalog { + plan_id, + plan_version: envelope.plan_version, + entries: metricsql_entries, + }); for materialization in &mut precompute_plan.materializations { let fingerprint = materialization.policy_fingerprint(); let max_lookback_ms = query_plan .entries .values() .flat_map(QueryPlanEntry::materialization_bindings) + .chain( + metricsql_plan_catalog + .iter() + .flat_map(|catalog| catalog.entries.values()) + .flat_map(|entry| entry.executable.materialization_bindings()), + ) .filter(|binding| binding.materialization.fingerprint() == fingerprint) .filter_map(|binding| binding.readout_lookback_ms) .max(); @@ -2629,6 +2684,9 @@ impl PhysicalCompiler { .unwrap_or(DEFAULT_RETAINED_SUMMARY_MEMORY_BUDGET_BYTES), )?; query_plan.validate(&materialization_fingerprints)?; + if let Some(catalog) = &metricsql_plan_catalog { + catalog.validate(&materialization_fingerprints)?; + } let summary_catalog = super::summary_catalog::SummaryCatalog::from_materializations( envelope.plan_id, envelope.plan_version, @@ -2670,6 +2728,7 @@ impl PhysicalCompiler { precompute_plan, transmission_plan, query_plan, + metricsql_plan_catalog, lifecycle_estimates: lifecycle_estimates.into_values().collect(), cost_comparison: None, }) @@ -4298,6 +4357,26 @@ mod tests { request_with_evidence(query_id, promql, None).expect("post-ASAP selection") } + #[test] + fn metricsql_compilation_publishes_only_the_independent_sidecar() { + let query = "default_rollup(m[1m])"; + let mut workload = request("vm-q", "last_over_time(m[1m])"); + let accuracy = workload.queries[0].accuracy.clone(); + let canonical = asap_frontend_metricsql::lower_metricsql(query, accuracy.clone()).unwrap(); + workload.queries[0].query_string = query.into(); + workload.queries[0].post_asap = + crate::planner_selection::keep_pre_asap(&canonical).unwrap(); + let plan = PhysicalCompiler + .compile_metricsql(workload, environment(10_000)) + .unwrap(); + assert!(plan.query_plan.entries.is_empty()); + let sidecar = plan.metricsql_plan_catalog.unwrap(); + let identity = asap_frontend_metricsql::canonical_metricsql(query).unwrap(); + assert_eq!(sidecar.lookup(&identity).unwrap().query_id, "vm-q"); + let encoded = serde_json::to_string(&sidecar).unwrap(); + assert!(!encoded.contains("canonical_promql")); + } + #[test] fn hybrid_erp_capability_miss_preserves_exact_subtree() { let mut workload = request("q", "quantile_over_time(0.9, m[1m])"); diff --git a/control_plane/src/physical/publication.rs b/control_plane/src/physical/publication.rs index 3c45f6676..8c8b201f6 100644 --- a/control_plane/src/physical/publication.rs +++ b/control_plane/src/physical/publication.rs @@ -1,6 +1,7 @@ //! Canonical publication document for one catalog generation. use super::compiler::{CollectorPlan, PhysicalPlan, PrecomputePlan, TransmissionPlan}; use super::summary_catalog::SummaryCatalog; +use crate::metricsql_plan::MetricsQlPlanCatalog; use crate::query_plan::QueryPlan; use serde::{Deserialize, Serialize}; @@ -12,6 +13,8 @@ pub struct PhysicalPlanPublication { pub collector_plans: Vec, pub transmission_plan: TransmissionPlan, pub query_plan: QueryPlan, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub metricsql_plan_catalog: Option, } impl PhysicalPlanPublication { /// Validate every plan against the shared catalog snapshot. @@ -29,6 +32,20 @@ impl PhysicalPlanPublication { self.query_plan .validate_against_catalog(catalog) .map_err(|e| e.to_string())?; + if let Some(metricsql) = &self.metricsql_plan_catalog { + if (metricsql.plan_id, metricsql.plan_version) + != (self.query_plan.plan_id, self.query_plan.plan_version) + { + return Err("MetricsQL catalog generation differs from QueryPlan".into()); + } + let available = self + .precompute_plan + .materializations + .iter() + .map(|config| config.policy_fingerprint()) + .collect(); + metricsql.validate(&available).map_err(|e| e.to_string())?; + } let materializations = self .precompute_plan .materializations @@ -49,6 +66,21 @@ impl PhysicalPlanPublication { } } } + if let Some(metricsql) = &self.metricsql_plan_catalog { + for entry in metricsql.entries.values() { + for binding in entry.executable.materialization_bindings() { + let config = materializations + .get(&binding.materialization.fingerprint()) + .copied() + .ok_or("MetricsQL binding has no precompute materialization")?; + if config.slide_interval.checked_mul(1000) != Some(binding.window_ms) { + return Err( + "MetricsQL pane differs from precompute emission interval".into() + ); + } + } + } + } let mut collectors = std::collections::BTreeSet::new(); for collector in &self.collector_plans { if collector.envelope != self.precompute_plan.envelope @@ -95,6 +127,7 @@ impl PhysicalPlan { collector_plans: self.collector_plans.clone(), transmission_plan: self.transmission_plan.clone(), query_plan: self.query_plan.clone(), + metricsql_plan_catalog: self.metricsql_plan_catalog.clone(), }; artifact.validate()?; Ok(artifact) diff --git a/control_plane/src/physical/workload_cost.rs b/control_plane/src/physical/workload_cost.rs index f280452a2..c90f5f40a 100644 --- a/control_plane/src/physical/workload_cost.rs +++ b/control_plane/src/physical/workload_cost.rs @@ -405,6 +405,23 @@ pub fn select( candidates: Vec, env: DeploymentEnvironment, evidence: &WorkloadCostEvidence, +) -> Result { + select_with_frontend(candidates, env, evidence, false) +} + +pub fn select_metricsql( + candidates: Vec, + env: DeploymentEnvironment, + evidence: &WorkloadCostEvidence, +) -> Result { + select_with_frontend(candidates, env, evidence, true) +} + +fn select_with_frontend( + candidates: Vec, + env: DeploymentEnvironment, + evidence: &WorkloadCostEvidence, + metricsql: bool, ) -> Result { evidence.validate(&env)?; if candidates.is_empty() || candidates.len() > 64 { @@ -434,7 +451,11 @@ pub fn select( )> = None; for candidate in candidates { let queries = candidate.queries.clone(); - let plan = match PhysicalCompiler.compile(candidate, env.clone()) { + let plan = match if metricsql { + PhysicalCompiler.compile_metricsql(candidate, env.clone()) + } else { + PhysicalCompiler.compile(candidate, env.clone()) + } { Ok(plan) => plan, Err(error) => { alternatives.push(AlternativeCost { diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs index c60d3a33e..599718c91 100644 --- a/control_plane/src/query_plan.rs +++ b/control_plane/src/query_plan.rs @@ -150,6 +150,55 @@ pub struct QueryPlanEntry { pub fallback: FallbackPolicy, } +/// Language-neutral executable projection of a compiled physical query DAG. +/// +/// This is deliberately separate from [`QueryPlanEntry`], whose serialized +/// `canonical_promql` identity remains part of the stable PromQL contract. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct ExecutableQueryPlan { + pub root: QueryNodeId, + pub nodes: BTreeMap, + pub instant: InstantExecution, + pub fallback: FallbackPolicy, +} + +impl QueryPlanEntry { + pub fn executable(&self) -> ExecutableQueryPlan { + ExecutableQueryPlan { + root: self.root, + nodes: self.nodes.clone(), + instant: self.instant.clone(), + fallback: self.fallback.clone(), + } + } +} + +impl ExecutableQueryPlan { + pub fn materialization_bindings(&self) -> Vec<&MaterializationBinding> { + self.nodes + .values() + .filter_map(|node| match node { + QueryPlanNode::ReadMaterialization { binding } => Some(binding), + _ => None, + }) + .collect() + } + + /// Internal compatibility view for the existing executor. The supplied + /// identity is never serialized into the PromQL plan catalog. + pub fn execution_view(&self, query_id: String, source: String) -> QueryPlanEntry { + QueryPlanEntry { + query_id, + canonical_promql: source, + root: self.root, + nodes: self.nodes.clone(), + instant: self.instant.clone(), + fallback: self.fallback.clone(), + } + } +} + #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct InstantExecution { @@ -1128,6 +1177,32 @@ mod tests { ); } + #[test] + fn executable_extraction_preserves_promql_entry_serde() { + let entry = QueryPlanEntry { + query_id: "q".into(), + canonical_promql: canonical_promql("up").unwrap(), + root: QueryNodeId(0), + nodes: BTreeMap::from([( + QueryNodeId(0), + QueryPlanNode::ExactFallback { + reason: "fixture".into(), + }, + )]), + instant: InstantExecution { + lookback_ms: 1, + full_history: false, + cumulative_readout: false, + }, + fallback: FallbackPolicy::ExactBackend, + }; + let before = serde_json::to_value(&entry).unwrap(); + let _payload = entry.executable(); + assert_eq!(before, serde_json::to_value(&entry).unwrap()); + assert!(before.get("canonical_promql").is_some()); + assert!(before.get("executable").is_none()); + } + #[test] fn graph_validation_rejects_cycles() { let mut nodes = BTreeMap::new(); diff --git a/crates/asap_types/Cargo.toml b/crates/asap_types/Cargo.toml index 14fd95dfa..85c50e861 100644 --- a/crates/asap_types/Cargo.toml +++ b/crates/asap_types/Cargo.toml @@ -32,4 +32,4 @@ sha2 = "0.10" # exactly (`control_plane/Cargo.toml`) -- two different revs of the same # git dependency in one workspace resolve to two distinct Rust types that # won't unify. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "4ce032623d4fa9a5df6055104e848c40940b52c1" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "bda8cd314934e81e41c23260225e0fb0dc81baae" } diff --git a/data_plane/Cargo.toml b/data_plane/Cargo.toml index 8ef5672f1..e58dccf28 100644 --- a/data_plane/Cargo.toml +++ b/data_plane/Cargo.toml @@ -38,8 +38,8 @@ control_plane = { path = "../control_plane" } # reduction: Reduction, .. }`) are `pre_asap` types, in the same crate now # (not a separate `asap-ir` import). Query serving consumes the compiled # QueryPlan; these types are used at physical-plan compilation boundaries. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "4ce032623d4fa9a5df6055104e848c40940b52c1" } -asap-frontend-metricsql = { git = "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/ProjectASAP/ASAPPlanner", rev = "4ce032623d4fa9a5df6055104e848c40940b52c1" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "bda8cd314934e81e41c23260225e0fb0dc81baae" } +asap-frontend-metricsql = { git = "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/ProjectASAP/ASAPPlanner", rev = "bda8cd314934e81e41c23260225e0fb0dc81baae" } # Shared external (workspace) serde.workspace = true @@ -130,7 +130,7 @@ crc32fast = "1.4" # none of them. [dev-dependencies] -asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "4ce032623d4fa9a5df6055104e848c40940b52c1" } +asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "bda8cd314934e81e41c23260225e0fb0dc81baae" } tempfile = "3.20.0" criterion = { version = "0.5", features = ["html_reports"] } tokio-tungstenite = "0.21" diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index 265ba9c69..5cedd5bdd 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -803,6 +803,9 @@ mod tests { }, runtime_config: Arc::new(streaming), query_plan: Arc::new(control_plane::query_plan::QueryPlan::empty()), + metricsql_plan_catalog: Arc::new( + control_plane::metricsql_plan::MetricsQlPlanCatalog::empty(), + ), storage_routing: Arc::new(BackendStorageRouting::empty()), }; HotReloadStreamingConfig::from_active(HotReloadActivePhysicalPlan::new(active)) diff --git a/data_plane/src/drivers/query/adapters/traits.rs b/data_plane/src/drivers/query/adapters/traits.rs index 29446bab6..5ad232b23 100644 --- a/data_plane/src/drivers/query/adapters/traits.rs +++ b/data_plane/src/drivers/query/adapters/traits.rs @@ -140,6 +140,12 @@ pub trait HttpProtocolAdapter: QueryRequestAdapter + QueryResponseAdapter + Send /// Get a descriptive name for this adapter (for logging/debugging) fn adapter_name(&self) -> &'static str; + /// Language-owned canonical identity for an independently published plan + /// catalog. Protocols without a sidecar catalog return `None`. + fn canonical_plan_identity(&self, _query: &str) -> Result, AdapterError> { + Ok(None) + } + /// Get the path for the runtime info endpoint /// /// Example: "/api/v1/status/runtimeinfo" for Prometheus diff --git a/data_plane/src/drivers/query/adapters/victoriametrics_http.rs b/data_plane/src/drivers/query/adapters/victoriametrics_http.rs index b3a1b4198..8c5bd4332 100644 --- a/data_plane/src/drivers/query/adapters/victoriametrics_http.rs +++ b/data_plane/src/drivers/query/adapters/victoriametrics_http.rs @@ -98,6 +98,11 @@ impl HttpProtocolAdapter for VictoriaMetricsHttpAdapter { fn adapter_name(&self) -> &'static str { "VictoriaMetrics HTTP / MetricsQL" } + fn canonical_plan_identity(&self, query: &str) -> Result, AdapterError> { + asap_frontend_metricsql::canonical_metricsql(query) + .map(Some) + .map_err(|error| AdapterError::ParseError(format!("frontend.metricsql: {error}"))) + } fn get_runtime_info_path(&self) -> &'static str { "/api/v1/status/runtimeinfo" } diff --git a/data_plane/src/drivers/query/fallback/mod.rs b/data_plane/src/drivers/query/fallback/mod.rs index 21840602e..92a3d4630 100644 --- a/data_plane/src/drivers/query/fallback/mod.rs +++ b/data_plane/src/drivers/query/fallback/mod.rs @@ -1,6 +1,7 @@ use async_trait::async_trait; use axum::{ - http::StatusCode, + body::Bytes, + http::{HeaderMap, StatusCode}, response::{IntoResponse, Json, Response}, }; use serde_json::Value; @@ -15,6 +16,13 @@ pub enum FallbackResponse { Json(Value), /// Plain text response. Text(String), + /// Exact upstream HTTP response for protocols whose status and response + /// metadata are part of the compatibility contract. + Forwarded { + status: StatusCode, + headers: HeaderMap, + body: Bytes, + }, } impl IntoResponse for FallbackResponse { @@ -32,6 +40,15 @@ impl IntoResponse for FallbackResponse { ) .into_response() } + FallbackResponse::Forwarded { + status, + headers, + body, + } => { + let mut response = (status, body).into_response(); + *response.headers_mut() = headers; + response + } }; response.headers_mut().insert( "x-asap-execution", diff --git a/data_plane/src/drivers/query/fallback/victoriametrics.rs b/data_plane/src/drivers/query/fallback/victoriametrics.rs index 1973b1e4b..6f0f55de7 100644 --- a/data_plane/src/drivers/query/fallback/victoriametrics.rs +++ b/data_plane/src/drivers/query/fallback/victoriametrics.rs @@ -1,4 +1,4 @@ -use super::{FallbackClient, FallbackResponse, PrometheusHttpFallback}; +use super::{FallbackClient, FallbackResponse}; use crate::drivers::query::adapters::{ParsedQueryRequest, ParsedRangeQueryRequest}; use async_trait::async_trait; use axum::http::StatusCode; @@ -7,11 +7,68 @@ use std::collections::HashMap; /// Exact MetricsQL backend. VictoriaMetrics' query API is wire-compatible with /// Prometheus, so transport is shared while this type keeps backend selection /// independent from the Prometheus listener. -pub struct VictoriaMetricsHttpFallback(PrometheusHttpFallback); +pub struct VictoriaMetricsHttpFallback { + client: reqwest::Client, + base_url: String, +} impl VictoriaMetricsHttpFallback { pub fn new(base_url: String) -> Self { - Self(PrometheusHttpFallback::new(base_url)) + Self { + client: reqwest::Client::new(), + base_url, + } + } + + fn endpoint(&self, tenant: Option<&str>, range: bool) -> String { + let suffix = if range { "query_range" } else { "query" }; + match tenant { + Some(tenant) => format!( + "{}/select/{tenant}/prometheus/api/v1/{suffix}", + self.base_url.trim_end_matches('/') + ), + None => format!("{}/api/v1/{suffix}", self.base_url.trim_end_matches('/')), + } + } + + async fn send( + &self, + endpoint: String, + params: Vec<(&str, String)>, + mut headers: HashMap, + ) -> Result { + headers.remove("x-asap-tenant"); + let mut request = self.client.get(endpoint).query(¶ms); + for (name, value) in headers { + request = request.header(name, value); + } + let response = request + .timeout(std::time::Duration::from_secs(30)) + .send() + .await + .map_err(|_| StatusCode::BAD_GATEWAY)?; + let status = StatusCode::from_u16(response.status().as_u16()) + .map_err(|_| StatusCode::BAD_GATEWAY)?; + let mut response_headers = axum::http::HeaderMap::new(); + for name in ["content-type", "cache-control", "warning", "retry-after"] { + if let Some(value) = response.headers().get(name) { + if let Ok(value) = axum::http::HeaderValue::from_bytes(value.as_bytes()) { + response_headers.insert( + axum::http::HeaderName::from_bytes(name.as_bytes()).unwrap(), + value, + ); + } + } + } + let body = response + .bytes() + .await + .map_err(|_| StatusCode::BAD_GATEWAY)?; + Ok(FallbackResponse::Forwarded { + status, + headers: response_headers, + body, + }) } } @@ -21,28 +78,48 @@ impl FallbackClient for VictoriaMetricsHttpFallback { &self, request: &ParsedQueryRequest, ) -> Result { - self.0.execute_query(request).await + self.execute_query_with_headers(request, HashMap::new()) + .await } async fn execute_query_with_headers( &self, request: &ParsedQueryRequest, headers: HashMap, ) -> Result { - self.0.execute_query_with_headers(request, headers).await + let tenant = headers.get("x-asap-tenant").cloned(); + let mut params = vec![ + ("query", request.query.clone()), + ("time", request.time.to_string()), + ]; + if let Some(timeout) = &request.timeout { + params.push(("timeout", timeout.clone())); + } + self.send(self.endpoint(tenant.as_deref(), false), params, headers) + .await } async fn execute_range_query( &self, request: &ParsedRangeQueryRequest, ) -> Result { - self.0.execute_range_query(request).await + self.execute_range_query_with_headers(request, HashMap::new()) + .await } async fn execute_range_query_with_headers( &self, request: &ParsedRangeQueryRequest, headers: HashMap, ) -> Result { - self.0 - .execute_range_query_with_headers(request, headers) + let tenant = headers.get("x-asap-tenant").cloned(); + let mut params = vec![ + ("query", request.query.clone()), + ("start", request.start.to_string()), + ("end", request.end.to_string()), + ("step", request.step.to_string()), + ]; + if let Some(timeout) = &request.timeout { + params.push(("timeout", timeout.clone())); + } + self.send(self.endpoint(tenant.as_deref(), true), params, headers) .await } async fn get_runtime_info(&self) -> Result { @@ -53,7 +130,7 @@ impl FallbackClient for VictoriaMetricsHttpFallback { #[cfg(test)] mod tests { use super::*; - use axum::{extract::Query, routing::get, Json, Router}; + use axum::{extract::Query, response::IntoResponse, routing::get, Json, Router}; #[tokio::test] async fn exact_fallback_preserves_metricsql_expression() { @@ -79,12 +156,83 @@ mod tests { }) .await .unwrap(); - let FallbackResponse::Json(payload) = response else { - panic!("VictoriaMetrics fallback must return JSON") + let FallbackResponse::Forwarded { status, body, .. } = response else { + panic!("VictoriaMetrics fallback must forward the upstream response") }; + assert_eq!(status, StatusCode::OK); + let payload: serde_json::Value = serde_json::from_slice(&body).unwrap(); assert_eq!( payload["data"]["received"], "rate(requests[5m]) keep_metric_names" ); } + + #[tokio::test] + async fn cluster_fallback_uses_tenant_prefixed_path() { + let app = Router::new().route( + "/select/42/prometheus/api/v1/query_range", + get(|Query(params): Query>| async move { + Json(serde_json::json!({ + "status": "success", + "data": {"received": params["query"]} + })) + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + + let client = VictoriaMetricsHttpFallback::new(format!("http://{address}")); + let response = client + .execute_range_query_with_headers( + &ParsedRangeQueryRequest { + query: "sum(rate(requests[5m]))".into(), + start: 1.0, + end: 2.0, + step: 1.0, + timeout: None, + }, + HashMap::from([("x-asap-tenant".into(), "42".into())]), + ) + .await + .unwrap(); + let FallbackResponse::Forwarded { status, body, .. } = response else { + panic!("VictoriaMetrics cluster fallback must forward the upstream response") + }; + assert_eq!(status, StatusCode::OK); + let payload: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(payload["data"]["received"], "sum(rate(requests[5m]))"); + } + + #[tokio::test] + async fn cluster_fallback_preserves_upstream_status_and_protocol_headers() { + let app = Router::new().route( + "/select/42/prometheus/api/v1/query", + get(|| async { + ( + StatusCode::UNPROCESSABLE_ENTITY, + [("content-type", "application/json"), ("retry-after", "7")], + r#"{"status":"error","error":"bad MetricsQL"}"#, + ) + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let response = VictoriaMetricsHttpFallback::new(format!("http://{address}")) + .execute_query_with_headers( + &ParsedQueryRequest { + query: "invalid".into(), + time: 1.0, + timeout: None, + }, + HashMap::from([("x-asap-tenant".into(), "42".into())]), + ) + .await + .unwrap() + .into_response(); + assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY); + assert_eq!(response.headers()["retry-after"], "7"); + assert_eq!(response.headers()["x-asap-execution"], "exact_fallback"); + } } diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index 722c645a7..08d15cba0 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -1,7 +1,7 @@ use crate::drivers::query::adapters::{ParsedQueryRequest, ParsedRangeQueryRequest}; use axum::{ body::Bytes, - extract::{DefaultBodyLimit, Form, Query, State}, + extract::{DefaultBodyLimit, Form, Path, Query, State}, http::{HeaderMap, StatusCode}, response::{IntoResponse, Json, Response}, routing::{get, post}, @@ -508,8 +508,20 @@ impl HttpServer { .route( "/api/v1/db/backfill/jobs/:job_id", get(handle_get_backfill_job).delete(handle_delete_backfill_job), + ); + let app = if adapter.adapter_name() == "VictoriaMetrics HTTP / MetricsQL" { + app.route( + "/select/:tenant/prometheus/api/v1/query", + get(handle_vm_cluster_instant).post(handle_vm_cluster_instant_post), + ) + .route( + "/select/:tenant/prometheus/api/v1/query_range", + get(handle_vm_cluster_range).post(handle_vm_cluster_range_post), ) - .with_state(app_state); + } else { + app + }; + let app = app.with_state(app_state); let listener = TcpListener::bind(format!("0.0.0.0:{}", self.config.port)).await?; info!("HTTP server listening on port {}", self.config.port); @@ -615,8 +627,20 @@ impl HttpServer { .route( "/api/v1/db/backfill/jobs/:job_id", get(handle_get_backfill_job).delete(handle_delete_backfill_job), + ); + let app = if adapter.adapter_name() == "VictoriaMetrics HTTP / MetricsQL" { + app.route( + "/select/:tenant/prometheus/api/v1/query", + get(handle_vm_cluster_instant).post(handle_vm_cluster_instant_post), + ) + .route( + "/select/:tenant/prometheus/api/v1/query_range", + get(handle_vm_cluster_range).post(handle_vm_cluster_range_post), ) - .with_state(app_state); + } else { + app + }; + let app = app.with_state(app_state); let listener = TcpListener::bind("127.0.0.1:0").await?; let actual_port = listener.local_addr()?.port(); @@ -697,6 +721,54 @@ async fn process_query_request( return process_via_named_engine(state, parsed_request, start_time, override_id).await; } + let language_identity = state.adapter.canonical_plan_identity(&parsed_request.query); + if state.adapter.adapter_name() == "VictoriaMetrics HTTP / MetricsQL" + && language_identity.is_err() + { + if let Some(fallback) = &state.fallback { + return match fallback + .execute_query_with_headers(parsed_request, headers) + .await + { + Ok(response) => response.into_response(), + Err(status) => status.into_response(), + }; + } + } + if let Ok(Some(identity)) = language_identity { + let evaluation_ms = if parsed_request.time > 0.0 { + (parsed_request.time * 1_000.0) as u64 + } else { + unix_time_ms() + }; + if let Ok(query_result) = state + .query_engine + .execute_metricsql_at(&identity, evaluation_ms) + .await + { + let result = crate::drivers::query::adapters::QueryExecutionResult { + query_output_labels: asap_types::KeyByLabelNames::default(), + query_result, + }; + return match state.adapter.format_success_response(&result).await { + Ok(response) => { + annotate_data_source(response, StorageBackend::SketchStore.data_source_id()) + .await + } + Err(status) => status.into_response(), + }; + } + if let Some(fallback) = &state.fallback { + return match fallback + .execute_query_with_headers(parsed_request, headers) + .await + { + Ok(response) => response.into_response(), + Err(status) => status.into_response(), + }; + } + } + // Issue #46 â‘¥ — freshness-probe short-circuit. // // The MVP demo's freshness criterion polls @@ -1724,6 +1796,61 @@ async fn annotate_data_source(response: Response, data_source_id: &'static str) Response::from_parts(parts, axum::body::Body::from(serialized)) } +fn vm_tenant_headers(tenant: String, mut headers: HeaderMap) -> Result { + let value = axum::http::HeaderValue::from_str(&tenant) + .map_err(|_| (StatusCode::BAD_REQUEST, "invalid VictoriaMetrics tenant").into_response())?; + headers.insert(TENANT_HEADER, value); + Ok(headers) +} + +async fn handle_vm_cluster_instant( + Path(tenant): Path, + query: Query>, + headers: HeaderMap, + state: State, +) -> Response { + let Ok(headers) = vm_tenant_headers(tenant, headers) else { + return (StatusCode::BAD_REQUEST, "invalid VictoriaMetrics tenant").into_response(); + }; + handle_instant_query(query, headers, state).await +} + +async fn handle_vm_cluster_instant_post( + Path(tenant): Path, + state: State, + headers: HeaderMap, + body: Bytes, +) -> Response { + let Ok(headers) = vm_tenant_headers(tenant, headers) else { + return (StatusCode::BAD_REQUEST, "invalid VictoriaMetrics tenant").into_response(); + }; + handle_instant_query_post(state, headers, body).await +} + +async fn handle_vm_cluster_range( + Path(tenant): Path, + query: Query>, + headers: HeaderMap, + state: State, +) -> Response { + let Ok(headers) = vm_tenant_headers(tenant, headers) else { + return (StatusCode::BAD_REQUEST, "invalid VictoriaMetrics tenant").into_response(); + }; + handle_range_query(query, headers, state).await +} + +async fn handle_vm_cluster_range_post( + Path(tenant): Path, + state: State, + headers: HeaderMap, + body: Bytes, +) -> Response { + let Ok(headers) = vm_tenant_headers(tenant, headers) else { + return (StatusCode::BAD_REQUEST, "invalid VictoriaMetrics tenant").into_response(); + }; + handle_range_query_post(state, headers, body).await +} + async fn handle_instant_query( query_params: Query>, headers: axum::http::HeaderMap, @@ -2136,6 +2263,49 @@ async fn process_range_query_request( let end_ms = (parsed_request.end * 1000.0) as u64; let step_ms = (parsed_request.step * 1000.0) as u64; + let language_identity = state.adapter.canonical_plan_identity(&parsed_request.query); + if state.adapter.adapter_name() == "VictoriaMetrics HTTP / MetricsQL" + && language_identity.is_err() + { + if let Some(fallback) = &state.fallback { + return match fallback + .execute_range_query_with_headers(parsed_request, forwarding_headers) + .await + { + Ok(response) => response.into_response(), + Err(status) => status.into_response(), + }; + } + } + if let Ok(Some(identity)) = language_identity { + if let Ok(result) = state + .query_engine + .execute_metricsql_range(&identity, start_ms, end_ms, step_ms) + .await + { + return match state + .adapter + .format_range_success_response(&result, &asap_types::KeyByLabelNames::default()) + .await + { + Ok(response) => { + annotate_data_source(response, StorageBackend::SketchStore.data_source_id()) + .await + } + Err(status) => status.into_response(), + }; + } + if let Some(fallback) = &state.fallback { + return match fallback + .execute_range_query_with_headers(parsed_request, forwarding_headers) + .await + { + Ok(response) => response.into_response(), + Err(status) => status.into_response(), + }; + } + } + let metric_storage = resolve_metric_storage(state, &parsed_request.query, tenant); // Statistic is not currently used by the routing policy. Accuracy is @@ -2543,6 +2713,9 @@ mod tests { plan_version: 1, entries: Default::default(), }), + metricsql_plan_catalog: Arc::new( + control_plane::metricsql_plan::MetricsQlPlanCatalog::empty(), + ), storage_routing: Arc::new( crate::storage_engines::types::BackendStorageRouting::empty(), ), @@ -2668,6 +2841,40 @@ mod tests { .expect("Failed to start test server") } + async fn setup_victoriametrics_test_server() -> u16 { + use crate::drivers::query::adapters::VictoriaMetricsHttpAdapter; + let adapter_config = + AdapterConfig::victoriametrics_metricsql("http://127.0.0.1:9999".to_string()); + let server = HttpServer::new( + HttpServerConfig { + port: 0, + handle_http_requests: true, + adapter_config: adapter_config.clone(), + }, + Arc::new(ASAPQueryEngine::new( + Arc::new(StreamingConfig::default()), + 15_000, + )), + Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()), + ) + .with_protocol_adapter(Arc::new(VictoriaMetricsHttpAdapter::new(adapter_config))); + server.start_test_server().await.unwrap() + } + + #[tokio::test] + async fn victoriametrics_cluster_routes_are_registered() { + let port = setup_victoriametrics_test_server().await; + let response = Client::new() + .get(format!( + "http://127.0.0.1:{port}/select/42/prometheus/api/v1/query" + )) + .query(&[("query", "default_rollup(cpu[5m])"), ("time", "1")]) + .send() + .await + .unwrap(); + assert_ne!(response.status(), reqwest::StatusCode::NOT_FOUND); + } + #[tokio::test] async fn test_get_endpoint_plus_symbol_decoding() { // Enable debug logging for this test @@ -5801,6 +6008,8 @@ pub struct PhysicalPlanInstallRequest { pub precompute_plan: control_plane::physical::compiler::PrecomputePlan, pub transmission_plan: control_plane::physical::compiler::TransmissionPlan, pub query_plan: control_plane::query_plan::QueryPlan, + #[serde(default)] + pub metricsql_plan_catalog: Option, pub storage_routing: Option, #[serde(default)] pub adaptation_evidence: Vec, @@ -5825,6 +6034,13 @@ pub fn build_active_physical_plan( .query_plan .validate_against_catalog(&request.summary_catalog) .map_err(|error| format!("QueryPlan catalog validation error: {error}"))?; + if let Some(metricsql) = &request.metricsql_plan_catalog { + if (metricsql.plan_id, metricsql.plan_version) + != (request.query_plan.plan_id, request.query_plan.plan_version) + { + return Err("MetricsQL catalog generation differs from QueryPlan".into()); + } + } for collector in &request.collector_plans { collector .validate_against_catalog(&request.summary_catalog) @@ -5852,6 +6068,25 @@ pub fn build_active_physical_plan( } } } + if let Some(metricsql) = &request.metricsql_plan_catalog { + for entry in metricsql.entries.values() { + for binding in entry.executable.materialization_bindings() { + let materialization = request + .precompute_plan + .materializations + .iter() + .find(|config| { + config.policy_fingerprint() == binding.materialization.fingerprint() + }) + .ok_or_else(|| "MetricsQL binding has no precompute definition".to_string())?; + if binding.window_ms != materialization.slide_interval.saturating_mul(1_000) { + return Err( + "MetricsQL pane differs from installed precompute definition".into(), + ); + } + } + } + } let runtime_materializations = request .precompute_plan .runtime_materializations() @@ -5879,6 +6114,11 @@ pub fn build_active_physical_plan( .query_plan .validate(&typed_fps) .map_err(|error| format!("QueryPlan validation error: {error}"))?; + if let Some(metricsql) = &request.metricsql_plan_catalog { + metricsql + .validate(&typed_fps) + .map_err(|error| format!("MetricsQL plan validation error: {error}"))?; + } let storage_routing = match request.storage_routing.as_ref() { Some(value) => Arc::new( crate::storage_engines::types::BackendStorageRouting::from_json_payload(value) @@ -5893,6 +6133,11 @@ pub fn build_active_physical_plan( transmission_plan: request.transmission_plan, runtime_config: Arc::new(runtime_config), query_plan: Arc::new(request.query_plan), + metricsql_plan_catalog: Arc::new( + request + .metricsql_plan_catalog + .unwrap_or_else(control_plane::metricsql_plan::MetricsQlPlanCatalog::empty), + ), storage_routing, }) } @@ -5971,6 +6216,7 @@ async fn handle_post_physical_plan( } let plan_id = active.plan_id(); let materialization_count = active.precompute_plan.materializations.len(); + let metricsql_query_count = active.metricsql_plan_catalog.entries.len(); let plan_version = active.plan_version(); let now = unix_time_ms(); if let Err(error) = lifecycle.stage(active, now) { @@ -5984,7 +6230,8 @@ async fn handle_post_physical_plan( StatusCode::ACCEPTED, axum::Json(serde_json::json!({ "status": "staged", "plan_id": plan_id, "plan_version": plan_version, - "materialization_count": materialization_count + "materialization_count": materialization_count, + "metricsql_query_count": metricsql_query_count })), ) .into_response() @@ -6900,6 +7147,7 @@ mod catalog_install_tests { precompute_plan: plan.precompute_plan, transmission_plan: plan.transmission_plan, query_plan: plan.query_plan, + metricsql_plan_catalog: plan.metricsql_plan_catalog, storage_routing: None, adaptation_evidence: vec![], } diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index 8889df929..3fb281512 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -512,6 +512,7 @@ async fn main() -> Result<()> { precompute_plan: plan.precompute_plan, transmission_plan: plan.transmission_plan, query_plan: plan.query_plan, + metricsql_plan_catalog: plan.metricsql_plan_catalog, storage_routing: None, adaptation_evidence: Vec::new(), }, @@ -725,6 +726,9 @@ async fn main() -> Result<()> { transmission_plan: initial_transmission_plan, runtime_config: streaming_config.clone(), query_plan: Arc::new(control_plane::query_plan::QueryPlan::empty()), + metricsql_plan_catalog: Arc::new( + control_plane::metricsql_plan::MetricsQlPlanCatalog::empty(), + ), storage_routing: Arc::new( data_plane::storage_engines::types::BackendStorageRouting::empty(), ), @@ -1113,6 +1117,7 @@ async fn main() -> Result<()> { transmission_plan: current.transmission_plan.clone(), runtime_config: current.runtime_config.clone(), query_plan: current.query_plan.clone(), + metricsql_plan_catalog: current.metricsql_plan_catalog.clone(), storage_routing: Arc::new(bootstrap_routing), }); } diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index 54f675847..d4211b061 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -124,6 +124,71 @@ pub struct ASAPQueryEngine { } impl ASAPQueryEngine { + pub async fn execute_metricsql_at( + &self, + identity: &str, + now_ms: u64, + ) -> Result + { + let physical = self.physical_plan_snapshot().ok_or_else(|| { + crate::query_engines::EngineError::capability_miss( + "metricsql_catalog", + "no active physical plan", + ) + })?; + let planned = physical + .metricsql_plan_catalog + .lookup(identity) + .map_err(|error| { + crate::query_engines::EngineError::capability_miss( + "metricsql_catalog", + error.to_string(), + ) + })?; + let entry = planned.executable.execution_view( + planned.query_id.clone(), + planned.canonical_metricsql.clone(), + ); + let leaves = self.prepare_logical(&physical, &entry, &[now_ms]).await?; + let (mut result, mut stats) = + self.execute_logical_entry(&physical, &entry, &leaves, now_ms)?; + stats.remote_evaluations = leaves.values().map(|leaf| leaf.remote_evaluations).sum(); + stats.remote_rpcs = leaves.values().map(|leaf| leaf.remote_rpcs).sum(); + annotate_logical_execution(&mut result, &stats); + Ok(result) + } + + pub async fn execute_metricsql_range( + &self, + identity: &str, + start_ms: u64, + end_ms: u64, + step_ms: u64, + ) -> Result + { + let physical = self.physical_plan_snapshot().ok_or_else(|| { + crate::query_engines::EngineError::capability_miss( + "metricsql_catalog", + "no active physical plan", + ) + })?; + let planned = physical + .metricsql_plan_catalog + .lookup(identity) + .map_err(|error| { + crate::query_engines::EngineError::capability_miss( + "metricsql_catalog", + error.to_string(), + ) + })?; + let entry = planned.executable.execution_view( + planned.query_id.clone(), + planned.canonical_metricsql.clone(), + ); + self.execute_logical_range(&physical, &entry, start_ms, end_ms, step_ms) + .await + } + /// Construct a `ASAPQueryEngine` with a static `Arc`. /// Wraps the config in a fresh `HotReloadStreamingConfig` internally /// — callers that need to share the hot-reload handle with the HTTP @@ -3649,4 +3714,79 @@ mod range_stitch_tests { "warm-only result must carry just the warm window sample: {ts:?}" ); } + + #[tokio::test] + async fn active_metricsql_sidecar_reaches_the_shared_dag_executor() { + use control_plane::metricsql_plan::{MetricsQlPlanCatalog, MetricsQlPlanEntry}; + use control_plane::query_plan::{ + ExecutableQueryPlan, FallbackPolicy, InstantExecution, QueryNodeId, QueryPlanNode, + }; + let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = + serde_json::from_str(include_str!( + "../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" + )) + .unwrap(); + let plan = snapshot.compile().unwrap(); + let identity = asap_frontend_metricsql::canonical_metricsql("1 + 2").unwrap(); + let sidecar = MetricsQlPlanCatalog { + plan_id: plan.envelope.plan_id, + plan_version: plan.envelope.plan_version, + entries: std::collections::BTreeMap::from([( + identity.clone(), + MetricsQlPlanEntry { + query_id: "vm-scalar".into(), + canonical_metricsql: identity.clone(), + executable: ExecutableQueryPlan { + root: QueryNodeId(2), + nodes: std::collections::BTreeMap::from([ + (QueryNodeId(0), QueryPlanNode::Scalar { value: 1.0 }), + (QueryNodeId(1), QueryPlanNode::Scalar { value: 2.0 }), + ( + QueryNodeId(2), + QueryPlanNode::Binary { + inputs: [QueryNodeId(0), QueryNodeId(1)], + operator: planner_types::pre_asap::ArithmeticOpKind::Add, + }, + ), + ]), + instant: InstantExecution { + lookback_ms: 1, + full_history: false, + cumulative_readout: false, + }, + fallback: FallbackPolicy::ExactBackend, + }, + }, + )]), + }; + let mut active = crate::drivers::query::servers::http::build_active_physical_plan( + crate::drivers::query::servers::http::PhysicalPlanInstallRequest { + summary_catalog: plan.summary_catalog, + collector_plans: plan.collector_plans, + precompute_plan: plan.precompute_plan, + transmission_plan: plan.transmission_plan, + query_plan: plan.query_plan, + metricsql_plan_catalog: Some(sidecar), + storage_routing: None, + adaptation_evidence: vec![], + }, + Arc::new(crate::storage_engines::types::BackendStorageRouting::empty()), + ) + .unwrap(); + active.envelope.expiry_unix_ms = None; + let active = crate::storage_engines::types::HotReloadActivePhysicalPlan::new(active); + let hot = HotReloadStreamingConfig::from_active(active.clone()); + let engine = + ASAPQueryEngine::new_with_hot_reload(hot, 15).with_active_physical_plan(active); + let error = engine + .execute_metricsql_at(&identity, 1_000) + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("bound subtree requires one explicit positive window"), + "{error}" + ); + } } diff --git a/data_plane/src/query_engines/asap_victoriametrics_query_engine/metricsql_binder.rs b/data_plane/src/query_engines/asap_victoriametrics_query_engine/metricsql_binder.rs index 76405e22e..7139ccf65 100644 --- a/data_plane/src/query_engines/asap_victoriametrics_query_engine/metricsql_binder.rs +++ b/data_plane/src/query_engines/asap_victoriametrics_query_engine/metricsql_binder.rs @@ -72,4 +72,12 @@ mod tests { Err(MetricsQlBindingError::Unsupported(_)) )); } + + #[test] + fn invalid_multi_argument_aggregate_fails_closed_without_dropping_arguments() { + assert!(matches!( + bind_metricsql("sum(foo, bar)", accuracy()), + Err(MetricsQlBindingError::Unsupported(_)) + )); + } } diff --git a/data_plane/src/storage_engines/types/hot_reload_config.rs b/data_plane/src/storage_engines/types/hot_reload_config.rs index f356c4953..97565feb5 100644 --- a/data_plane/src/storage_engines/types/hot_reload_config.rs +++ b/data_plane/src/storage_engines/types/hot_reload_config.rs @@ -95,6 +95,7 @@ pub struct ActivePhysicalPlan { pub transmission_plan: control_plane::physical::compiler::TransmissionPlan, pub runtime_config: Arc, pub query_plan: Arc, + pub metricsql_plan_catalog: Arc, pub storage_routing: Arc, } @@ -692,6 +693,9 @@ mod tests { plan_version, entries: Default::default(), }), + metricsql_plan_catalog: Arc::new( + control_plane::metricsql_plan::MetricsQlPlanCatalog::empty(), + ), storage_routing: Arc::new(crate::storage_engines::types::BackendStorageRouting::empty()), } } diff --git a/data_plane/tests/asapquery_compatibility_process_e2e.rs b/data_plane/tests/asapquery_compatibility_process_e2e.rs index c0e6a61dc..f587e1a54 100644 --- a/data_plane/tests/asapquery_compatibility_process_e2e.rs +++ b/data_plane/tests/asapquery_compatibility_process_e2e.rs @@ -486,6 +486,7 @@ async fn registered_temporal_topk(algorithm: planner_types::post_asap::SketchAlg precompute_plan: plan.precompute_plan, transmission_plan: plan.transmission_plan, query_plan: plan.query_plan, + metricsql_plan_catalog: plan.metricsql_plan_catalog, storage_routing: None, adaptation_evidence: vec![], }; diff --git a/docs/developer_docs/query-engine/victoriametrics-metricsql-support.md b/docs/developer_docs/query-engine/victoriametrics-metricsql-support.md index 060ed3b97..fe1a51869 100644 --- a/docs/developer_docs/query-engine/victoriametrics-metricsql-support.md +++ b/docs/developer_docs/query-engine/victoriametrics-metricsql-support.md @@ -47,8 +47,46 @@ Run an independent listener with `--victoriametrics-http-port` and select the exact backend with `--victoriametrics-url`. The ordinary Prometheus listener and its fallback remain unchanged. +The listener also accepts VictoriaMetrics cluster paths +`/select/{tenant}/prometheus/api/v1/query` and +`/select/{tenant}/prometheus/api/v1/query_range`. The path tenant scopes the +installed routing snapshot and is preserved in the exact fallback URL. + +The control plane exposes an independent MetricsQL physical-plan publication +endpoint. It lowers the MetricsQL AST to `QueryExpr`, invokes the shared +ASAP-aware physical compiler, and publishes a `MetricsQlPlanCatalog` sidecar. +Each sidecar entry owns `canonical_metricsql` and a language-neutral executable +payload. The existing `QueryPlanEntry.canonical_promql`, its wire encoding, and +its lookup rules remain unchanged. + +The backend stages and activates the sidecar atomically with the SDS catalog, +precompute plan, transmission plan, and PromQL query plan. A VictoriaMetrics +request can execute only a matching entry in the active sidecar. The executable +payload uses the shared DAG validator, descriptor resolver, SummaryStore +readout, and executor. A catalog miss, incomplete coverage, validation failure, +or execution failure routes the original request to VictoriaMetrics. + +## MetricsQL operator coverage + +| Construct | Canonical acceleration | Boundary behavior | +| --- | --- | --- | +| metric selectors and one matcher set | yes | AST lowers to a time-series scan | +| explicit positive range selectors | yes | lowers to `TimeRange` | +| `default_rollup` with explicit range; `last/first/avg/min/max/sum/count/stddev/stdvar_over_time`; `rate`, `irate`, `increase`, `changes`, `delta`, `idelta`, `deriv`, `resets`, `mad`, `present`, `absent`, and numeric `quantile_over_time` | yes | exact arity is required | +| `sum`, `avg`, `min`, `max`, `count`, `stddev`, `stdvar`, `group`, numeric `quantile`, with `by`/`without` | yes | exact arity is required | +| scalar unary and binary arithmetic/comparison/set operators without vector modifiers | yes | lowers to canonical binary nodes | +| `keep_metric_names`, aggregate `limit`, implicit `default_rollup`, offsets, `@`, subquery/inherited steps, OR-delimited matcher groups, binary vector matching, `if`, `ifnot`, `default`, non-rollup functions, unsupported aggregates, malformed or extra arguments | no | typed frontend rejection, then exact VictoriaMetrics fallback | + +The vendored upstream parser currently has 21 known upstream-baseline failures +and three parser-support compatibility failures in its broader internal suite. +They cover WITH expansion, OR matcher/tokenization, filter pushdown, and +simplifier behavior. Those constructs are outside the accelerated subset and +remain in the fail-closed fallback denominator; they are not reported as +accelerated queries. + ## Verification -Focused tests cover request parsing, response compatibility, canonical binding -of the common subset, fail-closed handling of MetricsQL-only syntax, and exact -fallback forwarding for instant and range queries. +Focused tests cover request parsing, response compatibility, canonical binding, +strict aggregate arity, independent sidecar serialization and validation, +unchanged PromQL entry serialization, atomic installation, tenant-prefixed +fallback, upstream status/header preservation, and instant/range fallback. From f3f1b003b333841f862d948e66dcb89e0f4b1af7 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 9 Sep 2026 22:57:59 -0600 Subject: [PATCH 4/5] chore: retain workspace sketchlib lock version --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 70a1fcd8a..c609bc442 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -405,7 +405,7 @@ dependencies = [ [[package]] name = "asap_sketchlib" -version = "0.3.0" +version = "0.2.2" dependencies = [ "bytes", "prost", From 4c9c9024ad997bb9860dcfa72f86f18b5787f36a Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 10 Sep 2026 00:10:53 -0600 Subject: [PATCH 5/5] refactor: publish MetricsQL in the shared query plan --- Cargo.lock | 12 +- control_plane/Cargo.toml | 8 +- control_plane/src/lib.rs | 1 - control_plane/src/metricsql_plan.rs | 91 --------------- control_plane/src/physical/compiler.rs | 49 ++------ control_plane/src/physical/publication.rs | 33 ------ control_plane/src/physical/workload_cost.rs | 2 +- control_plane/src/query_plan.rs | 110 +++++++----------- control_plane/src/query_plan/logical.rs | 11 +- crates/asap_types/Cargo.toml | 2 +- data_plane/Cargo.toml | 6 +- .../drivers/ingest/prometheus_remote_write.rs | 3 - data_plane/src/drivers/query/servers/http.rs | 68 ++++------- data_plane/src/main.rs | 5 - .../query_engines/asap_query_engine/engine.rs | 104 +++++++---------- .../asap_query_engine/exact_subqueries.rs | 3 +- .../asap_query_engine/live_serve.rs | 3 +- .../asap_query_engine/logical_dag.rs | 6 +- .../asap_query_engine/physical_dag.rs | 3 +- .../asap_query_engine/post_asap_readout.rs | 6 +- .../types/hot_reload_config.rs | 4 - .../asapquery_compatibility_process_e2e.rs | 1 - ...e2e_controller_plans_and_backend_serves.rs | 2 +- data_plane/tests/support/physical_fixture.rs | 3 +- .../victoriametrics-metricsql-support.md | 34 +++--- 25 files changed, 178 insertions(+), 392 deletions(-) delete mode 100644 control_plane/src/metricsql_plan.rs diff --git a/Cargo.lock b/Cargo.lock index c609bc442..97c6fad16 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -344,7 +344,7 @@ dependencies = [ [[package]] name = "asap-aware-mapping" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=bda8cd314934e81e41c23260225e0fb0dc81baae#bda8cd314934e81e41c23260225e0fb0dc81baae" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=5f37a850b367f522c7a4bda25d1727e8b44e73ee#5f37a850b367f522c7a4bda25d1727e8b44e73ee" dependencies = [ "asap-types", "serde", @@ -355,7 +355,7 @@ dependencies = [ [[package]] name = "asap-frontend-metricsql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=bda8cd314934e81e41c23260225e0fb0dc81baae#bda8cd314934e81e41c23260225e0fb0dc81baae" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=5f37a850b367f522c7a4bda25d1727e8b44e73ee#5f37a850b367f522c7a4bda25d1727e8b44e73ee" dependencies = [ "asap-types", "metricsql_parser", @@ -365,7 +365,7 @@ dependencies = [ [[package]] name = "asap-frontend-promql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=bda8cd314934e81e41c23260225e0fb0dc81baae#bda8cd314934e81e41c23260225e0fb0dc81baae" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=5f37a850b367f522c7a4bda25d1727e8b44e73ee#5f37a850b367f522c7a4bda25d1727e8b44e73ee" dependencies = [ "asap-types", "promql-parser 0.10.0", @@ -385,7 +385,7 @@ dependencies = [ [[package]] name = "asap-types" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=bda8cd314934e81e41c23260225e0fb0dc81baae#bda8cd314934e81e41c23260225e0fb0dc81baae" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=5f37a850b367f522c7a4bda25d1727e8b44e73ee#5f37a850b367f522c7a4bda25d1727e8b44e73ee" dependencies = [ "serde", "serde_json", @@ -2131,7 +2131,7 @@ dependencies = [ [[package]] name = "metricsql_common" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=bda8cd314934e81e41c23260225e0fb0dc81baae#bda8cd314934e81e41c23260225e0fb0dc81baae" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=5f37a850b367f522c7a4bda25d1727e8b44e73ee#5f37a850b367f522c7a4bda25d1727e8b44e73ee" dependencies = [ "chrono", ] @@ -2139,7 +2139,7 @@ dependencies = [ [[package]] name = "metricsql_parser" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=bda8cd314934e81e41c23260225e0fb0dc81baae#bda8cd314934e81e41c23260225e0fb0dc81baae" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=5f37a850b367f522c7a4bda25d1727e8b44e73ee#5f37a850b367f522c7a4bda25d1727e8b44e73ee" dependencies = [ "ahash", "chrono", diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index d4da0775d..c8940e47b 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -91,8 +91,8 @@ asap_types.workspace = true # scaffolding, unaware that `data_plane`'s `summary_executor.rs` in *this* # repo is a real one. Vendored locally instead of chased upstream -- see # `data_plane/src/query_engines/asap_query_engine/summary_exec.rs`. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "bda8cd314934e81e41c23260225e0fb0dc81baae" } -asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "bda8cd314934e81e41c23260225e0fb0dc81baae" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "5f37a850b367f522c7a4bda25d1727e8b44e73ee" } +asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "5f37a850b367f522c7a4bda25d1727e8b44e73ee" } # L1 adoption (design-target-architecture.md Part B): the PromQL front # end itself, replacing control_plane's own query_parser/promql.rs. @@ -100,8 +100,8 @@ asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = # `planner-types`/`asap-aware-mapping` above -- these three MUST move # together (two revs of the same upstream repo's types in one workspace # resolve to distinct Rust types that won't unify). -asap-frontend-promql = { git = "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/ProjectASAP/ASAPPlanner", rev = "bda8cd314934e81e41c23260225e0fb0dc81baae" } -asap-frontend-metricsql = { git = "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/ProjectASAP/ASAPPlanner", rev = "bda8cd314934e81e41c23260225e0fb0dc81baae" } +asap-frontend-promql = { git = "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/ProjectASAP/ASAPPlanner", rev = "5f37a850b367f522c7a4bda25d1727e8b44e73ee" } +asap-frontend-metricsql = { git = "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/ProjectASAP/ASAPPlanner", rev = "5f37a850b367f522c7a4bda25d1727e8b44e73ee" } [dev-dependencies] tokio = { version = "1", features = ["full", "test-util"] } diff --git a/control_plane/src/lib.rs b/control_plane/src/lib.rs index 734a4cbbf..023b70216 100644 --- a/control_plane/src/lib.rs +++ b/control_plane/src/lib.rs @@ -61,7 +61,6 @@ pub mod backend_client; pub mod emit; pub mod epsilon_alloc; pub mod metrics_exposer; -pub mod metricsql_plan; pub mod monitor; pub mod opamp; pub mod physical; diff --git a/control_plane/src/metricsql_plan.rs b/control_plane/src/metricsql_plan.rs deleted file mode 100644 index 627b350d5..000000000 --- a/control_plane/src/metricsql_plan.rs +++ /dev/null @@ -1,91 +0,0 @@ -use crate::query_plan::{ExecutableQueryPlan, QueryPlanError}; -use asap_types::PolicyFingerprint; -use serde::{Deserialize, Serialize}; -use std::collections::{BTreeMap, BTreeSet}; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct MetricsQlPlanEntry { - pub query_id: String, - pub canonical_metricsql: String, - pub executable: ExecutableQueryPlan, -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::query_plan::{FallbackPolicy, InstantExecution, QueryNodeId, QueryPlanNode}; - - #[test] - fn sidecar_serde_never_invents_a_promql_identity() { - let identity = "default_rollup(cpu[5m])".to_string(); - let catalog = MetricsQlPlanCatalog { - plan_id: 7, - plan_version: 3, - entries: BTreeMap::from([( - identity.clone(), - MetricsQlPlanEntry { - query_id: "vm-q".into(), - canonical_metricsql: identity, - executable: ExecutableQueryPlan { - root: QueryNodeId(0), - nodes: BTreeMap::from([( - QueryNodeId(0), - QueryPlanNode::ExactFallback { - reason: "fixture".into(), - }, - )]), - instant: InstantExecution { - lookback_ms: 300_000, - full_history: false, - cumulative_readout: false, - }, - fallback: FallbackPolicy::ExactBackend, - }, - }, - )]), - }; - let json = serde_json::to_string(&catalog).unwrap(); - assert!(json.contains("canonical_metricsql")); - assert!(!json.contains("canonical_promql")); - } -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct MetricsQlPlanCatalog { - pub plan_id: u64, - pub plan_version: u64, - pub entries: BTreeMap, -} - -impl MetricsQlPlanCatalog { - pub fn empty() -> Self { - Self { - plan_id: 0, - plan_version: 0, - entries: BTreeMap::new(), - } - } - - pub fn lookup(&self, identity: &str) -> Result<&MetricsQlPlanEntry, QueryPlanError> { - self.entries - .get(identity) - .ok_or_else(|| QueryPlanError::QueryNotPlanned(identity.into())) - } - - pub fn validate(&self, available: &BTreeSet) -> Result<(), QueryPlanError> { - for (identity, entry) in &self.entries { - if identity != &entry.canonical_metricsql { - return Err(QueryPlanError::Invalid( - "MetricsQL catalog key disagrees with its AST identity".into(), - )); - } - entry - .executable - .execution_view(entry.query_id.clone(), String::new()) - .validate(available)?; - } - Ok(()) - } -} diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 3bee9e047..3d32c8b2f 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -737,7 +737,6 @@ pub struct PhysicalPlan { pub precompute_plan: PrecomputePlan, pub transmission_plan: TransmissionPlan, pub query_plan: QueryPlan, - pub metricsql_plan_catalog: Option, /// Lifecycle component only, not a complete physical-plan comparison. pub lifecycle_estimates: Vec, pub cost_comparison: Option, @@ -2507,7 +2506,6 @@ impl PhysicalCompiler { .map(asap_types::PrecomputeMaterialization::policy_fingerprint) .collect(); let mut query_entries = BTreeMap::new(); - let mut metricsql_entries = BTreeMap::new(); for query in &request.queries { let canonical = if metricsql { asap_frontend_metricsql::canonical_metricsql(&query.query_string).map_err( @@ -2612,21 +2610,9 @@ impl PhysicalCompiler { crate::query_plan::logical::finalize_residuals(&mut entry)?; } if metricsql { - let metrics_entry = crate::metricsql_plan::MetricsQlPlanEntry { - query_id: query.query_id.clone(), - canonical_metricsql: canonical.clone(), - executable: entry.executable(), - }; - if metricsql_entries - .insert(canonical.clone(), metrics_entry) - .is_some() - { - return Err(CompileError::Query { - query_id: query.query_id.clone(), - reason: format!("duplicate canonical MetricsQL identity `{canonical}`"), - }); - } - } else if query_entries.insert(canonical.clone(), entry).is_some() { + entry.language = crate::query_plan::QueryLanguage::MetricsQl; + } + if query_entries.insert(canonical.clone(), entry).is_some() { return Err(CompileError::Query { query_id: query.query_id.clone(), reason: format!("duplicate canonical query identity `{canonical}`"), @@ -2638,24 +2624,12 @@ impl PhysicalCompiler { plan_version: envelope.plan_version, entries: query_entries, }; - let metricsql_plan_catalog = - metricsql.then_some(crate::metricsql_plan::MetricsQlPlanCatalog { - plan_id, - plan_version: envelope.plan_version, - entries: metricsql_entries, - }); for materialization in &mut precompute_plan.materializations { let fingerprint = materialization.policy_fingerprint(); let max_lookback_ms = query_plan .entries .values() .flat_map(QueryPlanEntry::materialization_bindings) - .chain( - metricsql_plan_catalog - .iter() - .flat_map(|catalog| catalog.entries.values()) - .flat_map(|entry| entry.executable.materialization_bindings()), - ) .filter(|binding| binding.materialization.fingerprint() == fingerprint) .filter_map(|binding| binding.readout_lookback_ms) .max(); @@ -2684,9 +2658,6 @@ impl PhysicalCompiler { .unwrap_or(DEFAULT_RETAINED_SUMMARY_MEMORY_BUDGET_BYTES), )?; query_plan.validate(&materialization_fingerprints)?; - if let Some(catalog) = &metricsql_plan_catalog { - catalog.validate(&materialization_fingerprints)?; - } let summary_catalog = super::summary_catalog::SummaryCatalog::from_materializations( envelope.plan_id, envelope.plan_version, @@ -2728,7 +2699,6 @@ impl PhysicalCompiler { precompute_plan, transmission_plan, query_plan, - metricsql_plan_catalog, lifecycle_estimates: lifecycle_estimates.into_values().collect(), cost_comparison: None, }) @@ -4358,7 +4328,7 @@ mod tests { } #[test] - fn metricsql_compilation_publishes_only_the_independent_sidecar() { + fn metricsql_compilation_publishes_a_language_tagged_query_entry() { let query = "default_rollup(m[1m])"; let mut workload = request("vm-q", "last_over_time(m[1m])"); let accuracy = workload.queries[0].accuracy.clone(); @@ -4369,12 +4339,13 @@ mod tests { let plan = PhysicalCompiler .compile_metricsql(workload, environment(10_000)) .unwrap(); - assert!(plan.query_plan.entries.is_empty()); - let sidecar = plan.metricsql_plan_catalog.unwrap(); let identity = asap_frontend_metricsql::canonical_metricsql(query).unwrap(); - assert_eq!(sidecar.lookup(&identity).unwrap().query_id, "vm-q"); - let encoded = serde_json::to_string(&sidecar).unwrap(); - assert!(!encoded.contains("canonical_promql")); + let entry = plan + .query_plan + .lookup_canonical(crate::query_plan::QueryLanguage::MetricsQl, &identity) + .unwrap(); + assert_eq!(entry.query_id, "vm-q"); + assert_eq!(entry.language, crate::query_plan::QueryLanguage::MetricsQl); } #[test] diff --git a/control_plane/src/physical/publication.rs b/control_plane/src/physical/publication.rs index 8c8b201f6..3c45f6676 100644 --- a/control_plane/src/physical/publication.rs +++ b/control_plane/src/physical/publication.rs @@ -1,7 +1,6 @@ //! Canonical publication document for one catalog generation. use super::compiler::{CollectorPlan, PhysicalPlan, PrecomputePlan, TransmissionPlan}; use super::summary_catalog::SummaryCatalog; -use crate::metricsql_plan::MetricsQlPlanCatalog; use crate::query_plan::QueryPlan; use serde::{Deserialize, Serialize}; @@ -13,8 +12,6 @@ pub struct PhysicalPlanPublication { pub collector_plans: Vec, pub transmission_plan: TransmissionPlan, pub query_plan: QueryPlan, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub metricsql_plan_catalog: Option, } impl PhysicalPlanPublication { /// Validate every plan against the shared catalog snapshot. @@ -32,20 +29,6 @@ impl PhysicalPlanPublication { self.query_plan .validate_against_catalog(catalog) .map_err(|e| e.to_string())?; - if let Some(metricsql) = &self.metricsql_plan_catalog { - if (metricsql.plan_id, metricsql.plan_version) - != (self.query_plan.plan_id, self.query_plan.plan_version) - { - return Err("MetricsQL catalog generation differs from QueryPlan".into()); - } - let available = self - .precompute_plan - .materializations - .iter() - .map(|config| config.policy_fingerprint()) - .collect(); - metricsql.validate(&available).map_err(|e| e.to_string())?; - } let materializations = self .precompute_plan .materializations @@ -66,21 +49,6 @@ impl PhysicalPlanPublication { } } } - if let Some(metricsql) = &self.metricsql_plan_catalog { - for entry in metricsql.entries.values() { - for binding in entry.executable.materialization_bindings() { - let config = materializations - .get(&binding.materialization.fingerprint()) - .copied() - .ok_or("MetricsQL binding has no precompute materialization")?; - if config.slide_interval.checked_mul(1000) != Some(binding.window_ms) { - return Err( - "MetricsQL pane differs from precompute emission interval".into() - ); - } - } - } - } let mut collectors = std::collections::BTreeSet::new(); for collector in &self.collector_plans { if collector.envelope != self.precompute_plan.envelope @@ -127,7 +95,6 @@ impl PhysicalPlan { collector_plans: self.collector_plans.clone(), transmission_plan: self.transmission_plan.clone(), query_plan: self.query_plan.clone(), - metricsql_plan_catalog: self.metricsql_plan_catalog.clone(), }; artifact.validate()?; Ok(artifact) diff --git a/control_plane/src/physical/workload_cost.rs b/control_plane/src/physical/workload_cost.rs index c90f5f40a..c74334c7e 100644 --- a/control_plane/src/physical/workload_cost.rs +++ b/control_plane/src/physical/workload_cost.rs @@ -254,7 +254,7 @@ pub fn manifest( for node_id in entry.topological_order()? { add( format!("query:{}:{}", entry.query_id, node_id.0), - json!({"node": entry.nodes[&node_id], "query": entry.canonical_promql, "instant": entry.instant}), + json!({"node": entry.nodes[&node_id], "query": entry.canonical_query, "instant": entry.instant}), "query_evaluation", evaluations, ); diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs index 599718c91..4039dcf82 100644 --- a/control_plane/src/query_plan.rs +++ b/control_plane/src/query_plan.rs @@ -36,9 +36,18 @@ impl QueryPlan { pub fn lookup(&self, promql: &str) -> Result<&QueryPlanEntry, QueryPlanError> { let identity = canonical_promql(promql)?; + self.lookup_canonical(QueryLanguage::PromQl, &identity) + } + + pub fn lookup_canonical( + &self, + language: QueryLanguage, + identity: &str, + ) -> Result<&QueryPlanEntry, QueryPlanError> { self.entries - .get(&identity) - .ok_or(QueryPlanError::QueryNotPlanned(identity)) + .get(identity) + .filter(|entry| entry.language == language) + .ok_or_else(|| QueryPlanError::QueryNotPlanned(identity.into())) } /// Validate semantic bindings against the authoritative snapshot before use. @@ -121,10 +130,10 @@ impl QueryPlan { )); } for (identity, entry) in &self.entries { - if identity != &entry.canonical_promql { + if identity != &entry.canonical_query { return Err(QueryPlanError::Invalid(format!( "query map key `{identity}` differs from entry identity `{}`", - entry.canonical_promql + entry.canonical_query ))); } entry.validate(available)?; @@ -133,6 +142,14 @@ impl QueryPlan { } } +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum QueryLanguage { + #[default] + PromQl, + MetricsQl, +} + /// Stable identity inside one query entry. Edges are IDs so common /// subexpressions remain shared after serialization. #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -142,63 +159,17 @@ pub struct QueryNodeId(pub u64); #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct QueryPlanEntry { + #[serde(default)] + pub language: QueryLanguage, pub query_id: String, - pub canonical_promql: String, + #[serde(alias = "canonical_promql")] + pub canonical_query: String, pub root: QueryNodeId, pub nodes: BTreeMap, pub instant: InstantExecution, pub fallback: FallbackPolicy, } -/// Language-neutral executable projection of a compiled physical query DAG. -/// -/// This is deliberately separate from [`QueryPlanEntry`], whose serialized -/// `canonical_promql` identity remains part of the stable PromQL contract. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct ExecutableQueryPlan { - pub root: QueryNodeId, - pub nodes: BTreeMap, - pub instant: InstantExecution, - pub fallback: FallbackPolicy, -} - -impl QueryPlanEntry { - pub fn executable(&self) -> ExecutableQueryPlan { - ExecutableQueryPlan { - root: self.root, - nodes: self.nodes.clone(), - instant: self.instant.clone(), - fallback: self.fallback.clone(), - } - } -} - -impl ExecutableQueryPlan { - pub fn materialization_bindings(&self) -> Vec<&MaterializationBinding> { - self.nodes - .values() - .filter_map(|node| match node { - QueryPlanNode::ReadMaterialization { binding } => Some(binding), - _ => None, - }) - .collect() - } - - /// Internal compatibility view for the existing executor. The supplied - /// identity is never serialized into the PromQL plan catalog. - pub fn execution_view(&self, query_id: String, source: String) -> QueryPlanEntry { - QueryPlanEntry { - query_id, - canonical_promql: source, - root: self.root, - nodes: self.nodes.clone(), - instant: self.instant.clone(), - fallback: self.fallback.clone(), - } - } -} - #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct InstantExecution { @@ -223,7 +194,7 @@ impl QueryPlanEntry { pub fn compile_bound( query_id: String, - canonical_promql: String, + canonical_query: String, root: &Rc, instant: InstantExecution, fallback: FallbackPolicy, @@ -244,8 +215,9 @@ impl QueryPlanEntry { }; let root = compiler.lower(root)?; Ok(Self { + language: QueryLanguage::PromQl, query_id, - canonical_promql, + canonical_query, root, nodes: compiler.nodes, instant, @@ -257,7 +229,7 @@ impl QueryPlanEntry { /// This is a distinct physical alternative; native execution remains available. pub fn compile_bound_composable( query_id: String, - canonical_promql: String, + canonical_query: String, root: &Rc, instant: InstantExecution, fallback: FallbackPolicy, @@ -274,12 +246,13 @@ impl QueryPlanEntry { nodes: BTreeMap::new(), seen: BTreeMap::new(), bind: &mut bind, - logical_source: Some(canonical_promql.clone()), + logical_source: Some(canonical_query.clone()), }; let root = compiler.lower(root)?; let mut entry = Self { + language: QueryLanguage::PromQl, query_id, - canonical_promql, + canonical_query, root, nodes: compiler.nodes, instant, @@ -1178,10 +1151,11 @@ mod tests { } #[test] - fn executable_extraction_preserves_promql_entry_serde() { + fn language_tag_preserves_query_entry_serde() { let entry = QueryPlanEntry { + language: crate::query_plan::QueryLanguage::PromQl, query_id: "q".into(), - canonical_promql: canonical_promql("up").unwrap(), + canonical_query: canonical_promql("up").unwrap(), root: QueryNodeId(0), nodes: BTreeMap::from([( QueryNodeId(0), @@ -1197,9 +1171,8 @@ mod tests { fallback: FallbackPolicy::ExactBackend, }; let before = serde_json::to_value(&entry).unwrap(); - let _payload = entry.executable(); assert_eq!(before, serde_json::to_value(&entry).unwrap()); - assert!(before.get("canonical_promql").is_some()); + assert!(before.get("canonical_query").is_some()); assert!(before.get("executable").is_none()); } @@ -1213,8 +1186,9 @@ mod tests { }, ); let entry = QueryPlanEntry { + language: crate::query_plan::QueryLanguage::PromQl, query_id: "q".into(), - canonical_promql: "up".into(), + canonical_query: "up".into(), root: QueryNodeId(0), nodes, instant: InstantExecution { @@ -1237,8 +1211,9 @@ mod tests { reason: "prepared".into(), }; let entry = QueryPlanEntry { + language: crate::query_plan::QueryLanguage::PromQl, query_id: "q".into(), - canonical_promql: "topk(2, rate(m[5m]))".into(), + canonical_query: "topk(2, rate(m[5m]))".into(), root: QueryNodeId(2), nodes: BTreeMap::from([ (QueryNodeId(0), leaf.clone()), @@ -1306,8 +1281,9 @@ mod catalog_binding_tests { config.pane_origin_ms = Some(0); let catalog = SummaryCatalog::from_materializations(7, 2, &[config.clone()]).unwrap(); let entry = QueryPlanEntry { + language: crate::query_plan::QueryLanguage::PromQl, query_id: "q".into(), - canonical_promql: "sum_over_time(m[1m])".into(), + canonical_query: "sum_over_time(m[1m])".into(), root: QueryNodeId(1), nodes: BTreeMap::from([( QueryNodeId(1), @@ -1333,7 +1309,7 @@ mod catalog_binding_tests { QueryPlan { plan_id: 7, plan_version: 2, - entries: BTreeMap::from([(entry.canonical_promql.clone(), entry)]), + entries: BTreeMap::from([(entry.canonical_query.clone(), entry)]), }, catalog, ) diff --git a/control_plane/src/query_plan/logical.rs b/control_plane/src/query_plan/logical.rs index 4c488a1b1..162c80743 100644 --- a/control_plane/src/query_plan/logical.rs +++ b/control_plane/src/query_plan/logical.rs @@ -407,19 +407,20 @@ impl QueryPlanEntry { /// Callers retain a separate external-native alternative for cost comparison. pub fn compile_logical( query_id: String, - canonical_promql: String, + canonical_query: String, instant: InstantExecution, fallback: FallbackPolicy, ) -> Result { - let expr = parser::parse(&canonical_promql).map_err(|e| invalid(e.to_string()))?; + let expr = parser::parse(&canonical_query).map_err(|e| invalid(e.to_string()))?; let mut lower = Lower { nodes: BTreeMap::new(), seen: BTreeMap::new(), }; let root = lower.lower(&expr)?; let entry = Self { + language: super::QueryLanguage::PromQl, query_id, - canonical_promql, + canonical_query, root, nodes: lower.nodes, instant, @@ -442,7 +443,7 @@ impl QueryPlanEntry { } Self::compile_logical( self.query_id.clone(), - self.canonical_promql.clone(), + self.canonical_query.clone(), self.instant, self.fallback, ) @@ -1373,7 +1374,7 @@ pub fn externalize_residuals(entry: &mut QueryPlanEntry) -> Result<(), QueryPlan _ => {} } } - let expr = parser::parse(&entry.canonical_promql).map_err(|e| invalid(e.to_string()))?; + let expr = parser::parse(&entry.canonical_query).map_err(|e| invalid(e.to_string()))?; let mut expressions = Vec::new(); gather(&expr, &mut expressions); let mut witnesses = BTreeMap::new(); diff --git a/crates/asap_types/Cargo.toml b/crates/asap_types/Cargo.toml index 85c50e861..32667b2f4 100644 --- a/crates/asap_types/Cargo.toml +++ b/crates/asap_types/Cargo.toml @@ -32,4 +32,4 @@ sha2 = "0.10" # exactly (`control_plane/Cargo.toml`) -- two different revs of the same # git dependency in one workspace resolve to two distinct Rust types that # won't unify. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "bda8cd314934e81e41c23260225e0fb0dc81baae" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "5f37a850b367f522c7a4bda25d1727e8b44e73ee" } diff --git a/data_plane/Cargo.toml b/data_plane/Cargo.toml index e58dccf28..42680a320 100644 --- a/data_plane/Cargo.toml +++ b/data_plane/Cargo.toml @@ -38,8 +38,8 @@ control_plane = { path = "../control_plane" } # reduction: Reduction, .. }`) are `pre_asap` types, in the same crate now # (not a separate `asap-ir` import). Query serving consumes the compiled # QueryPlan; these types are used at physical-plan compilation boundaries. -planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "bda8cd314934e81e41c23260225e0fb0dc81baae" } -asap-frontend-metricsql = { git = "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/ProjectASAP/ASAPPlanner", rev = "bda8cd314934e81e41c23260225e0fb0dc81baae" } +planner-types = { package = "asap-types", git = "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/ProjectASAP/ASAPPlanner", rev = "5f37a850b367f522c7a4bda25d1727e8b44e73ee" } +asap-frontend-metricsql = { git = "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/ProjectASAP/ASAPPlanner", rev = "5f37a850b367f522c7a4bda25d1727e8b44e73ee" } # Shared external (workspace) serde.workspace = true @@ -130,7 +130,7 @@ crc32fast = "1.4" # none of them. [dev-dependencies] -asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "bda8cd314934e81e41c23260225e0fb0dc81baae" } +asap-aware-mapping = { git = "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/ProjectASAP/ASAPPlanner", rev = "5f37a850b367f522c7a4bda25d1727e8b44e73ee" } tempfile = "3.20.0" criterion = { version = "0.5", features = ["html_reports"] } tokio-tungstenite = "0.21" diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index 5cedd5bdd..265ba9c69 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -803,9 +803,6 @@ mod tests { }, runtime_config: Arc::new(streaming), query_plan: Arc::new(control_plane::query_plan::QueryPlan::empty()), - metricsql_plan_catalog: Arc::new( - control_plane::metricsql_plan::MetricsQlPlanCatalog::empty(), - ), storage_routing: Arc::new(BackendStorageRouting::empty()), }; HotReloadStreamingConfig::from_active(HotReloadActivePhysicalPlan::new(active)) diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index 08d15cba0..65d2af0c5 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -2713,9 +2713,6 @@ mod tests { plan_version: 1, entries: Default::default(), }), - metricsql_plan_catalog: Arc::new( - control_plane::metricsql_plan::MetricsQlPlanCatalog::empty(), - ), storage_routing: Arc::new( crate::storage_engines::types::BackendStorageRouting::empty(), ), @@ -6008,8 +6005,6 @@ pub struct PhysicalPlanInstallRequest { pub precompute_plan: control_plane::physical::compiler::PrecomputePlan, pub transmission_plan: control_plane::physical::compiler::TransmissionPlan, pub query_plan: control_plane::query_plan::QueryPlan, - #[serde(default)] - pub metricsql_plan_catalog: Option, pub storage_routing: Option, #[serde(default)] pub adaptation_evidence: Vec, @@ -6034,13 +6029,6 @@ pub fn build_active_physical_plan( .query_plan .validate_against_catalog(&request.summary_catalog) .map_err(|error| format!("QueryPlan catalog validation error: {error}"))?; - if let Some(metricsql) = &request.metricsql_plan_catalog { - if (metricsql.plan_id, metricsql.plan_version) - != (request.query_plan.plan_id, request.query_plan.plan_version) - { - return Err("MetricsQL catalog generation differs from QueryPlan".into()); - } - } for collector in &request.collector_plans { collector .validate_against_catalog(&request.summary_catalog) @@ -6068,25 +6056,6 @@ pub fn build_active_physical_plan( } } } - if let Some(metricsql) = &request.metricsql_plan_catalog { - for entry in metricsql.entries.values() { - for binding in entry.executable.materialization_bindings() { - let materialization = request - .precompute_plan - .materializations - .iter() - .find(|config| { - config.policy_fingerprint() == binding.materialization.fingerprint() - }) - .ok_or_else(|| "MetricsQL binding has no precompute definition".to_string())?; - if binding.window_ms != materialization.slide_interval.saturating_mul(1_000) { - return Err( - "MetricsQL pane differs from installed precompute definition".into(), - ); - } - } - } - } let runtime_materializations = request .precompute_plan .runtime_materializations() @@ -6114,11 +6083,6 @@ pub fn build_active_physical_plan( .query_plan .validate(&typed_fps) .map_err(|error| format!("QueryPlan validation error: {error}"))?; - if let Some(metricsql) = &request.metricsql_plan_catalog { - metricsql - .validate(&typed_fps) - .map_err(|error| format!("MetricsQL plan validation error: {error}"))?; - } let storage_routing = match request.storage_routing.as_ref() { Some(value) => Arc::new( crate::storage_engines::types::BackendStorageRouting::from_json_payload(value) @@ -6133,11 +6097,6 @@ pub fn build_active_physical_plan( transmission_plan: request.transmission_plan, runtime_config: Arc::new(runtime_config), query_plan: Arc::new(request.query_plan), - metricsql_plan_catalog: Arc::new( - request - .metricsql_plan_catalog - .unwrap_or_else(control_plane::metricsql_plan::MetricsQlPlanCatalog::empty), - ), storage_routing, }) } @@ -6216,7 +6175,12 @@ async fn handle_post_physical_plan( } let plan_id = active.plan_id(); let materialization_count = active.precompute_plan.materializations.len(); - let metricsql_query_count = active.metricsql_plan_catalog.entries.len(); + let metricsql_query_count = active + .query_plan + .entries + .values() + .filter(|entry| entry.language == control_plane::query_plan::QueryLanguage::MetricsQl) + .count(); let plan_version = active.plan_version(); let now = unix_time_ms(); if let Err(error) = lifecycle.stage(active, now) { @@ -7147,7 +7111,6 @@ mod catalog_install_tests { precompute_plan: plan.precompute_plan, transmission_plan: plan.transmission_plan, query_plan: plan.query_plan, - metricsql_plan_catalog: plan.metricsql_plan_catalog, storage_routing: None, adaptation_evidence: vec![], } @@ -7224,4 +7187,23 @@ mod catalog_install_tests { binding.window_ms += 1; assert!(install(request).unwrap_err().contains("pane duration")); } + + #[test] + fn catalog_install_applies_pane_origin_validation_to_metricsql_entries() { + let mut request = request(); + let entry = request.query_plan.entries.values_mut().next().unwrap(); + entry.language = control_plane::query_plan::QueryLanguage::MetricsQl; + let binding = entry + .nodes + .values_mut() + .find_map(|node| match node { + control_plane::query_plan::QueryPlanNode::ReadMaterialization { binding } => { + Some(binding) + } + _ => None, + }) + .expect("demo has maintained summaries"); + binding.pane_origin_ms = Some(1); + assert!(install(request).unwrap_err().contains("pane origin")); + } } diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index 3fb281512..8889df929 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -512,7 +512,6 @@ async fn main() -> Result<()> { precompute_plan: plan.precompute_plan, transmission_plan: plan.transmission_plan, query_plan: plan.query_plan, - metricsql_plan_catalog: plan.metricsql_plan_catalog, storage_routing: None, adaptation_evidence: Vec::new(), }, @@ -726,9 +725,6 @@ async fn main() -> Result<()> { transmission_plan: initial_transmission_plan, runtime_config: streaming_config.clone(), query_plan: Arc::new(control_plane::query_plan::QueryPlan::empty()), - metricsql_plan_catalog: Arc::new( - control_plane::metricsql_plan::MetricsQlPlanCatalog::empty(), - ), storage_routing: Arc::new( data_plane::storage_engines::types::BackendStorageRouting::empty(), ), @@ -1117,7 +1113,6 @@ async fn main() -> Result<()> { transmission_plan: current.transmission_plan.clone(), runtime_config: current.runtime_config.clone(), query_plan: current.query_plan.clone(), - metricsql_plan_catalog: current.metricsql_plan_catalog.clone(), storage_routing: Arc::new(bootstrap_routing), }); } diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index d4211b061..046299e0b 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -132,26 +132,22 @@ impl ASAPQueryEngine { { let physical = self.physical_plan_snapshot().ok_or_else(|| { crate::query_engines::EngineError::capability_miss( - "metricsql_catalog", + "query_plan", "no active physical plan", ) })?; let planned = physical - .metricsql_plan_catalog - .lookup(identity) + .query_plan + .lookup_canonical( + control_plane::query_plan::QueryLanguage::MetricsQl, + identity, + ) .map_err(|error| { - crate::query_engines::EngineError::capability_miss( - "metricsql_catalog", - error.to_string(), - ) + crate::query_engines::EngineError::capability_miss("query_plan", error.to_string()) })?; - let entry = planned.executable.execution_view( - planned.query_id.clone(), - planned.canonical_metricsql.clone(), - ); - let leaves = self.prepare_logical(&physical, &entry, &[now_ms]).await?; + let leaves = self.prepare_logical(&physical, planned, &[now_ms]).await?; let (mut result, mut stats) = - self.execute_logical_entry(&physical, &entry, &leaves, now_ms)?; + self.execute_logical_entry(&physical, planned, &leaves, now_ms)?; stats.remote_evaluations = leaves.values().map(|leaf| leaf.remote_evaluations).sum(); stats.remote_rpcs = leaves.values().map(|leaf| leaf.remote_rpcs).sum(); annotate_logical_execution(&mut result, &stats); @@ -168,24 +164,20 @@ impl ASAPQueryEngine { { let physical = self.physical_plan_snapshot().ok_or_else(|| { crate::query_engines::EngineError::capability_miss( - "metricsql_catalog", + "query_plan", "no active physical plan", ) })?; let planned = physical - .metricsql_plan_catalog - .lookup(identity) + .query_plan + .lookup_canonical( + control_plane::query_plan::QueryLanguage::MetricsQl, + identity, + ) .map_err(|error| { - crate::query_engines::EngineError::capability_miss( - "metricsql_catalog", - error.to_string(), - ) + crate::query_engines::EngineError::capability_miss("query_plan", error.to_string()) })?; - let entry = planned.executable.execution_view( - planned.query_id.clone(), - planned.canonical_metricsql.clone(), - ); - self.execute_logical_range(&physical, &entry, start_ms, end_ms, step_ms) + self.execute_logical_range(&physical, planned, start_ms, end_ms, step_ms) .await } @@ -3716,49 +3708,44 @@ mod range_stitch_tests { } #[tokio::test] - async fn active_metricsql_sidecar_reaches_the_shared_dag_executor() { - use control_plane::metricsql_plan::{MetricsQlPlanCatalog, MetricsQlPlanEntry}; + async fn active_metricsql_entry_reaches_the_shared_dag_executor() { use control_plane::query_plan::{ - ExecutableQueryPlan, FallbackPolicy, InstantExecution, QueryNodeId, QueryPlanNode, + FallbackPolicy, InstantExecution, QueryLanguage, QueryNodeId, QueryPlanEntry, + QueryPlanNode, }; let snapshot: control_plane::physical::compiler::BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( "../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" )) .unwrap(); - let plan = snapshot.compile().unwrap(); + let mut plan = snapshot.compile().unwrap(); let identity = asap_frontend_metricsql::canonical_metricsql("1 + 2").unwrap(); - let sidecar = MetricsQlPlanCatalog { - plan_id: plan.envelope.plan_id, - plan_version: plan.envelope.plan_version, - entries: std::collections::BTreeMap::from([( - identity.clone(), - MetricsQlPlanEntry { - query_id: "vm-scalar".into(), - canonical_metricsql: identity.clone(), - executable: ExecutableQueryPlan { - root: QueryNodeId(2), - nodes: std::collections::BTreeMap::from([ - (QueryNodeId(0), QueryPlanNode::Scalar { value: 1.0 }), - (QueryNodeId(1), QueryPlanNode::Scalar { value: 2.0 }), - ( - QueryNodeId(2), - QueryPlanNode::Binary { - inputs: [QueryNodeId(0), QueryNodeId(1)], - operator: planner_types::pre_asap::ArithmeticOpKind::Add, - }, - ), - ]), - instant: InstantExecution { - lookback_ms: 1, - full_history: false, - cumulative_readout: false, + plan.query_plan.entries.insert( + identity.clone(), + QueryPlanEntry { + language: QueryLanguage::MetricsQl, + query_id: "vm-scalar".into(), + canonical_query: identity.clone(), + root: QueryNodeId(2), + nodes: std::collections::BTreeMap::from([ + (QueryNodeId(0), QueryPlanNode::Scalar { value: 1.0 }), + (QueryNodeId(1), QueryPlanNode::Scalar { value: 2.0 }), + ( + QueryNodeId(2), + QueryPlanNode::Binary { + inputs: [QueryNodeId(0), QueryNodeId(1)], + operator: planner_types::pre_asap::ArithmeticOpKind::Add, }, - fallback: FallbackPolicy::ExactBackend, - }, + ), + ]), + instant: InstantExecution { + lookback_ms: 1, + full_history: false, + cumulative_readout: false, }, - )]), - }; + fallback: FallbackPolicy::ExactBackend, + }, + ); let mut active = crate::drivers::query::servers::http::build_active_physical_plan( crate::drivers::query::servers::http::PhysicalPlanInstallRequest { summary_catalog: plan.summary_catalog, @@ -3766,7 +3753,6 @@ mod range_stitch_tests { precompute_plan: plan.precompute_plan, transmission_plan: plan.transmission_plan, query_plan: plan.query_plan, - metricsql_plan_catalog: Some(sidecar), storage_routing: None, adaptation_evidence: vec![], }, diff --git a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs index 749aecab3..507591161 100644 --- a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs +++ b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs @@ -207,8 +207,9 @@ mod tests { use control_plane::query_plan::{FallbackPolicy, InstantExecution}; fn entry(nodes: BTreeMap) -> QueryPlanEntry { QueryPlanEntry { + language: control_plane::query_plan::QueryLanguage::PromQl, query_id: "remote-cut".into(), - canonical_promql: "a / b".into(), + canonical_query: "a / b".into(), root: QueryNodeId(0), nodes, instant: InstantExecution { diff --git a/data_plane/src/query_engines/asap_query_engine/live_serve.rs b/data_plane/src/query_engines/asap_query_engine/live_serve.rs index 4427e7435..47350af82 100644 --- a/data_plane/src/query_engines/asap_query_engine/live_serve.rs +++ b/data_plane/src/query_engines/asap_query_engine/live_serve.rs @@ -448,8 +448,9 @@ mod tests { ); } let entry = control_plane::query_plan::QueryPlanEntry { + language: control_plane::query_plan::QueryLanguage::PromQl, query_id: "q-sum".into(), - canonical_promql: "sum_over_time(bytes[1s])".into(), + canonical_query: "sum_over_time(bytes[1s])".into(), root: control_plane::query_plan::QueryNodeId(0), nodes: BTreeMap::from([ ( diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs index fbf5f19b2..5a1ca434b 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs @@ -853,8 +853,9 @@ mod topk_tests { let summary = QueryNodeId(0); let root = QueryNodeId(1); let entry = QueryPlanEntry { + language: control_plane::query_plan::QueryLanguage::PromQl, query_id: "summary-rate-topk".into(), - canonical_promql: "topk(2, rate(requests_total[5m]))".into(), + canonical_query: "topk(2, rate(requests_total[5m]))".into(), root, nodes: BTreeMap::from([ ( @@ -980,8 +981,9 @@ mod topk_tests { let value_id = QueryNodeId(1); let root = QueryNodeId(2); let entry = QueryPlanEntry { + language: control_plane::query_plan::QueryLanguage::PromQl, query_id: "candidate-topk".into(), - canonical_promql: "topk(1, rate(requests_total[5m]))".into(), + canonical_query: "topk(1, rate(requests_total[5m]))".into(), root, nodes: BTreeMap::from([ ( diff --git a/data_plane/src/query_engines/asap_query_engine/physical_dag.rs b/data_plane/src/query_engines/asap_query_engine/physical_dag.rs index 2d87bf24d..96dd70dae 100644 --- a/data_plane/src/query_engines/asap_query_engine/physical_dag.rs +++ b/data_plane/src/query_engines/asap_query_engine/physical_dag.rs @@ -133,8 +133,9 @@ mod tests { .into_iter() .collect(); let entry = QueryPlanEntry { + language: control_plane::query_plan::QueryLanguage::PromQl, query_id: "q".into(), - canonical_promql: "up".into(), + canonical_query: "up".into(), root, nodes, instant: InstantExecution { diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index c1018bbd7..35fa95880 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -1026,8 +1026,9 @@ mod tests { } let entry = control_plane::query_plan::QueryPlanEntry { + language: control_plane::query_plan::QueryLanguage::PromQl, query_id: "q-rate".into(), - canonical_promql: "rate(requests_total[1m])".into(), + canonical_query: "rate(requests_total[1m])".into(), root: control_plane::query_plan::QueryNodeId(0), nodes: BTreeMap::from([ ( @@ -1120,8 +1121,9 @@ mod tests { idx.append_precompute(7, BTreeMap::new(), (0, 60_000), Box::new(accumulator)); let entry = control_plane::query_plan::QueryPlanEntry { + language: control_plane::query_plan::QueryLanguage::PromQl, query_id: "q-rate".into(), - canonical_promql: "rate(requests_total[1m])".into(), + canonical_query: "rate(requests_total[1m])".into(), root: control_plane::query_plan::QueryNodeId(0), nodes: BTreeMap::from([ ( diff --git a/data_plane/src/storage_engines/types/hot_reload_config.rs b/data_plane/src/storage_engines/types/hot_reload_config.rs index 97565feb5..f356c4953 100644 --- a/data_plane/src/storage_engines/types/hot_reload_config.rs +++ b/data_plane/src/storage_engines/types/hot_reload_config.rs @@ -95,7 +95,6 @@ pub struct ActivePhysicalPlan { pub transmission_plan: control_plane::physical::compiler::TransmissionPlan, pub runtime_config: Arc, pub query_plan: Arc, - pub metricsql_plan_catalog: Arc, pub storage_routing: Arc, } @@ -693,9 +692,6 @@ mod tests { plan_version, entries: Default::default(), }), - metricsql_plan_catalog: Arc::new( - control_plane::metricsql_plan::MetricsQlPlanCatalog::empty(), - ), storage_routing: Arc::new(crate::storage_engines::types::BackendStorageRouting::empty()), } } diff --git a/data_plane/tests/asapquery_compatibility_process_e2e.rs b/data_plane/tests/asapquery_compatibility_process_e2e.rs index f587e1a54..c0e6a61dc 100644 --- a/data_plane/tests/asapquery_compatibility_process_e2e.rs +++ b/data_plane/tests/asapquery_compatibility_process_e2e.rs @@ -486,7 +486,6 @@ async fn registered_temporal_topk(algorithm: planner_types::post_asap::SketchAlg precompute_plan: plan.precompute_plan, transmission_plan: plan.transmission_plan, query_plan: plan.query_plan, - metricsql_plan_catalog: plan.metricsql_plan_catalog, storage_routing: None, adaptation_evidence: vec![], }; diff --git a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs index 28fc86ac9..323057c44 100644 --- a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs +++ b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs @@ -86,7 +86,7 @@ async fn post_full_config(client: &reqwest::Client, stack: &FullStack, json: &Js .query_plan .entries .values_mut() - .filter(|e| e.canonical_promql.starts_with("count(")) + .filter(|e| e.canonical_query.starts_with("count(")) { entry.instant.lookback_ms = 1000; for node in entry.nodes.values_mut() { diff --git a/data_plane/tests/support/physical_fixture.rs b/data_plane/tests/support/physical_fixture.rs index 018af8883..4ea617204 100644 --- a/data_plane/tests/support/physical_fixture.rs +++ b/data_plane/tests/support/physical_fixture.rs @@ -103,8 +103,9 @@ pub fn artifact(config: &StreamingConfig) -> PhysicalPlanInstallRequest { query_plan.entries.insert( canonical.clone(), QueryPlanEntry { + language: control_plane::query_plan::QueryLanguage::PromQl, query_id: canonical.clone(), - canonical_promql: canonical, + canonical_query: canonical, root: QueryNodeId(1), nodes: BTreeMap::from([ ( diff --git a/docs/developer_docs/query-engine/victoriametrics-metricsql-support.md b/docs/developer_docs/query-engine/victoriametrics-metricsql-support.md index fe1a51869..91befe966 100644 --- a/docs/developer_docs/query-engine/victoriametrics-metricsql-support.md +++ b/docs/developer_docs/query-engine/victoriametrics-metricsql-support.md @@ -36,8 +36,9 @@ runtime evaluation step, and `keep_metric_names` needs metric-name lineage that the canonical IR does not represent; both fail closed to the configured VictoriaMetrics backend with the original expression and parameters. -This boundary adds no variants to PromQL, SDS, QueryExpr, or the shared physical -DAG. Native acceleration for additional MetricsQL syntax must be added in a +This boundary adds no variants to SDS, QueryExpr, or the shared physical DAG. +`QueryPlanEntry.language` records which frontend produced the canonical query +identity. Native acceleration for additional MetricsQL syntax must be added in a MetricsQL-specific frontend and lowered to existing canonical operations only when the equivalence is defined. @@ -52,19 +53,18 @@ The listener also accepts VictoriaMetrics cluster paths `/select/{tenant}/prometheus/api/v1/query_range`. The path tenant scopes the installed routing snapshot and is preserved in the exact fallback URL. -The control plane exposes an independent MetricsQL physical-plan publication -endpoint. It lowers the MetricsQL AST to `QueryExpr`, invokes the shared -ASAP-aware physical compiler, and publishes a `MetricsQlPlanCatalog` sidecar. -Each sidecar entry owns `canonical_metricsql` and a language-neutral executable -payload. The existing `QueryPlanEntry.canonical_promql`, its wire encoding, and -its lookup rules remain unchanged. +The control plane lowers the MetricsQL AST to `QueryExpr`, invokes the shared +ASAP-aware physical compiler, and publishes a language-tagged entry in the +authoritative `QueryPlan`. `canonical_query` is the frontend's AST identity; +the language tag prevents a MetricsQL request from resolving a PromQL entry. -The backend stages and activates the sidecar atomically with the SDS catalog, -precompute plan, transmission plan, and PromQL query plan. A VictoriaMetrics -request can execute only a matching entry in the active sidecar. The executable -payload uses the shared DAG validator, descriptor resolver, SummaryStore -readout, and executor. A catalog miss, incomplete coverage, validation failure, -or execution failure routes the original request to VictoriaMetrics. +The backend stages and activates that `QueryPlan` atomically with the SDS +catalog, precompute plan, and transmission plan. MetricsQL bindings therefore +receive the same SummaryCatalog validation, including pane duration and pane +origin, as PromQL bindings. The executor reads the installed entry directly; +it does not clone the DAG into a compatibility view. A plan miss, incomplete +coverage, validation failure, or execution failure routes the original request +to VictoriaMetrics. ## MetricsQL operator coverage @@ -87,6 +87,6 @@ accelerated queries. ## Verification Focused tests cover request parsing, response compatibility, canonical binding, -strict aggregate arity, independent sidecar serialization and validation, -unchanged PromQL entry serialization, atomic installation, tenant-prefixed -fallback, upstream status/header preservation, and instant/range fallback. +strict aggregate arity, language-tagged QueryPlan serialization and validation, +atomic installation, tenant-prefixed fallback, upstream status/header +preservation, and instant/range fallback.