diff --git a/Cargo.lock b/Cargo.lock index 441160eaa..c2f2ca101 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -416,30 +416,6 @@ dependencies = [ "tonic-build", ] -[[package]] -name = "asap_planner" -version = "0.1.0" -dependencies = [ - "anyhow", - "asap_types", - "chrono", - "clap 4.6.1", - "indexmap 2.14.0", - "pretty_assertions", - "promql-parser", - "promql_utilities", - "reqwest 0.11.27", - "serde", - "serde_json", - "serde_yaml", - "sql_utilities", - "sqlparser 0.59.0", - "tempfile", - "thiserror 1.0.69", - "tracing", - "tracing-subscriber", -] - [[package]] name = "asap_sketchlib" version = "0.1.0" @@ -1626,12 +1602,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "diff" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" - [[package]] name = "digest" version = "0.10.7" @@ -3387,16 +3357,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "pretty_assertions" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" -dependencies = [ - "diff", - "yansi", -] - [[package]] name = "prettyplease" version = "0.2.37" @@ -3647,7 +3607,6 @@ dependencies = [ "asap-gorilla", "asap-precompute-rs", "asap_otel_proto", - "asap_planner", "asap_sketchlib", "asap_types", "async-trait", @@ -5980,12 +5939,6 @@ dependencies = [ "lzma-sys", ] -[[package]] -name = "yansi" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" - [[package]] name = "yoke" version = "0.8.2" diff --git a/Cargo.toml b/Cargo.toml index 720b70aca..edac794f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,6 @@ members = [ "asap-common/tests/compare_matched_tokens/rust_tests", "asap-common/tests/compare_patterns", "asap-query-engine", - "asap-planner-rs", ] [workspace.package] @@ -54,5 +53,4 @@ sql_utilities = { path = "asap-common/dependencies/rs/sql_utilities" } asap_types = { path = "asap-common/dependencies/rs/asap_types" } datafusion_summary_library = { path = "asap-common/dependencies/rs/datafusion_summary_library" } elastic_dsl_utilities = { path = "asap-common/dependencies/rs/elastic_dsl_utilities" } -asap_planner = { path = "asap-planner-rs" } indexmap = { version = "2.0", features = ["serde"] } diff --git a/README.md b/README.md index bfacb171a..1bb66a24c 100644 --- a/README.md +++ b/README.md @@ -99,11 +99,6 @@ ASAPQuery-backend/ │ # (Arroyo pipelines for users who │ # prefer it over canonical OTLP │ # ingest from ASAPCollector) -├── asap-planner-rs/ # CLI tool — `bin/asap-planner` only. -│ # Library code that previously planned -│ # sketch placement was migrated into -│ # the ASAPCollector controller (see -│ # "Where the planner lives" below). └── docs/ # Design docs ``` @@ -126,10 +121,12 @@ each push without restart. ASAPQuery-backend is therefore mostly an **executor** — it ingests sketches, evaluates queries, and dispatches based on the controller-emitted routing table. -The legacy `asap-planner-rs` library (which had its own pattern -matchers) was migrated into the controller. The CLI binary -(`bin/asap-planner`) is retained for external users who script -against the planner from the command line. +The legacy `asap-planner-rs` workspace member (library + CLI) was +deleted in Phase γ — its 5 PromQL pattern matchers and 11 +archive-only intents now live in the ASAPCollector controller's L3 / +L4 stages. External users who previously scripted against +`bin/asap-planner` should drive the controller directly via its HTTP +API (or its own CLI surface — separate work item). ## Quick start diff --git a/asap-planner-rs/Cargo.toml b/asap-planner-rs/Cargo.toml deleted file mode 100644 index 802b916d6..000000000 --- a/asap-planner-rs/Cargo.toml +++ /dev/null @@ -1,34 +0,0 @@ -[package] -name = "asap_planner" -version.workspace = true -edition.workspace = true - -[lib] -name = "asap_planner" -path = "src/lib.rs" - -[[bin]] -name = "asap-planner" -path = "src/main.rs" - -[dependencies] -asap_types.workspace = true -promql_utilities.workspace = true -sql_utilities.workspace = true -sqlparser = "0.59.0" -serde.workspace = true -serde_json.workspace = true -serde_yaml.workspace = true -thiserror.workspace = true -anyhow.workspace = true -tracing.workspace = true -tracing-subscriber.workspace = true -clap.workspace = true -indexmap.workspace = true -chrono.workspace = true -promql-parser = "0.5.0" -reqwest = { version = "0.11", default-features = false, features = ["blocking", "json", "rustls-tls"] } - -[dev-dependencies] -tempfile = "3.20" -pretty_assertions = "1.4" diff --git a/asap-planner-rs/Dockerfile b/asap-planner-rs/Dockerfile deleted file mode 100644 index ca7b183f8..000000000 --- a/asap-planner-rs/Dockerfile +++ /dev/null @@ -1,49 +0,0 @@ -FROM rust:1.89 AS builder - -LABEL maintainer="SketchDB Team" -LABEL description="ASAP Planner (Rust) for SketchDB" - -WORKDIR /code - -# Required by prost-build dependencies during cargo build -RUN apt-get update && apt-get install -y --no-install-recommends \ - protobuf-compiler \ - && rm -rf /var/lib/apt/lists/* - -COPY asap-common ./asap-common -COPY Cargo.toml ./ -COPY Cargo.lock ./ -COPY asap-query-engine/Cargo.toml ./asap-query-engine/ -COPY asap-planner-rs/Cargo.toml ./asap-planner-rs/ - -# Create dummy source files so Cargo can resolve all workspace members -RUN mkdir -p asap-query-engine/src && echo "fn main() {}" > asap-query-engine/src/main.rs && \ - mkdir -p asap-query-engine/benches && echo "fn main() {}" > asap-query-engine/benches/simple_store_bench.rs && \ - mkdir -p asap-planner-rs/src && echo "fn main() {}" > asap-planner-rs/src/main.rs && \ - echo "pub fn placeholder() {}" >> asap-planner-rs/src/lib.rs - -# Build dependencies (this layer will be cached) -WORKDIR /code/asap-planner-rs -RUN cargo build --release && rm -rf src/ - -# Copy source code -COPY asap-planner-rs/src ./src - -# Build the actual application -RUN touch src/main.rs src/lib.rs && cargo build --release - -FROM ubuntu:24.04 - -WORKDIR /app - -RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates \ - libssl3 \ - zlib1g \ - && rm -rf /var/lib/apt/lists/* - -RUN mkdir -p /app/input /app/output - -COPY --from=builder /code/target/release/asap-planner /usr/local/bin/asap-planner - -ENTRYPOINT ["asap-planner"] diff --git a/asap-planner-rs/controller-cli-compose.yml.j2 b/asap-planner-rs/controller-cli-compose.yml.j2 deleted file mode 100644 index 87d0a1816..000000000 --- a/asap-planner-rs/controller-cli-compose.yml.j2 +++ /dev/null @@ -1,19 +0,0 @@ -# Docker compose Jinja2 template to be rendered and used by asap-cli - -services: - controller: - image: sketchdb-controller:latest # Change to 'asap' prefix - container_name: asap-controller - networks: - - asap-network - volumes: - - {{ input_config_path }}:/app/input/config.yaml:ro - - {{ output_dir }}:/app/outputs - command: [ - "--input_config", "/app/input/config.yaml", - "--output_dir", "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/app/outputs", - "--prometheus_scrape_interval", "{{ prometheus_scrape_interval }}", - "--streaming_engine", "{{ streaming_engine }}"{% if punting %}, - "--enable-punting"{% endif %} - ] - restart: no diff --git a/asap-planner-rs/docker-compose.yml.j2 b/asap-planner-rs/docker-compose.yml.j2 deleted file mode 100644 index 06d42d2df..000000000 --- a/asap-planner-rs/docker-compose.yml.j2 +++ /dev/null @@ -1,16 +0,0 @@ -services: - controller: - image: sketchdb-controller:latest - container_name: {{ container_name }} - volumes: - - {{ input_config_path }}:/app/input/config.yaml:ro - - {{ output_dir }}:/app/output - command: [ - "--input_config", "/app/input/config.yaml", - "--output_dir", "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/app/output", - "--prometheus_scrape_interval", "{{ prometheus_scrape_interval }}", - "--streaming_engine", "{{ streaming_engine }}", - "--prometheus-url", "{{ prometheus_url }}"{% if punting %}, - "--enable-punting"{% endif %} - ] - restart: no diff --git a/asap-planner-rs/installation/install.sh b/asap-planner-rs/installation/install.sh deleted file mode 100755 index 8824b665b..000000000 --- a/asap-planner-rs/installation/install.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -set -e - -THIS_DIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") -PARENT_DIR=$(dirname "$THIS_DIR") -WORKSPACE_DIR=$(dirname "$PARENT_DIR") - -source "$HOME/.cargo/env" - -echo "Building asap-planner-rs Rust binary..." -cd "$WORKSPACE_DIR" -cargo build --release -p asap_planner - -echo "Building asap-planner-rs Docker image..." -docker build . -f asap-planner-rs/Dockerfile -t sketchdb-controller:latest - -echo "asap-planner-rs Docker image built successfully: sketchdb-controller:latest" diff --git a/asap-planner-rs/src/config/input.rs b/asap-planner-rs/src/config/input.rs deleted file mode 100644 index 4683cd0ea..000000000 --- a/asap-planner-rs/src/config/input.rs +++ /dev/null @@ -1,124 +0,0 @@ -use asap_types::PromQLSchema; -use promql_utilities::data_model::KeyByLabelNames; -use serde::Deserialize; - -#[derive(Debug, Clone, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ControllerConfig { - pub query_groups: Vec, - pub sketch_parameters: Option, - pub aggregate_cleanup: Option, - /// Optional hint: per-metric label sets used as a fallback when Prometheus - /// returns no series for a metric. Prometheus-inferred labels take priority. - #[serde(default)] - pub metrics: Option>, -} - -impl ControllerConfig { - /// Build a `PromQLSchema` from the `metrics` hints in this config. - /// Returns an empty schema if no hints are present. - pub fn schema_from_hints(&self) -> PromQLSchema { - let mut schema = PromQLSchema::new(); - if let Some(metrics) = &self.metrics { - for m in metrics { - schema = - schema.add_metric(m.metric.clone(), KeyByLabelNames::new(m.labels.clone())); - } - } - schema - } -} - -#[derive(Debug, Clone, Deserialize)] -pub struct QueryGroup { - pub id: Option, - pub queries: Vec, - pub repetition_delay: u64, - #[serde(default)] - pub controller_options: ControllerOptions, - /// Per-group step override (seconds). Falls back to `RuntimeOptions::step` when None. - #[serde(default)] - pub step: Option, - /// Per-group range_duration override (seconds). Falls back to `RuntimeOptions::range_duration` when None. - #[serde(default)] - pub range_duration: Option, -} - -#[derive(Debug, Clone, Deserialize, Default)] -pub struct ControllerOptions { - pub accuracy_sla: f64, - pub latency_sla: f64, -} - -#[derive(Debug, Clone, Deserialize)] -pub struct MetricDefinition { - pub metric: String, - pub labels: Vec, -} - -#[derive(Debug, Clone, Deserialize)] -pub struct AggregateCleanupConfig { - pub policy: Option, -} - -#[derive(Debug, Clone, Deserialize, Default)] -pub struct SketchParameterOverrides { - #[serde(rename = "CountMinSketch")] - pub count_min_sketch: Option, - #[serde(rename = "CountMinSketchWithHeap")] - pub count_min_sketch_with_heap: Option, - #[serde(rename = "DatasketchesKLL")] - pub datasketches_kll: Option, - #[serde(rename = "HydraKLL")] - pub hydra_kll: Option, -} - -#[derive(Debug, Clone, Deserialize)] -pub struct CmsParams { - pub depth: u64, - pub width: u64, -} - -#[derive(Debug, Clone, Deserialize)] -pub struct CmsHeapParams { - pub depth: u64, - pub width: u64, - pub heap_multiplier: Option, -} - -#[derive(Debug, Clone, Deserialize)] -pub struct KllParams { - #[serde(rename = "K")] - pub k: u64, -} - -#[derive(Debug, Clone, Deserialize)] -pub struct HydraParams { - pub row_num: u64, - pub col_num: u64, - pub k: u64, -} - -#[derive(Debug, Clone, Deserialize)] -pub struct SQLControllerConfig { - pub query_groups: Vec, - pub tables: Vec, - pub sketch_parameters: Option, - pub aggregate_cleanup: Option, -} - -#[derive(Debug, Clone, Deserialize)] -pub struct SQLQueryGroup { - pub id: Option, - pub queries: Vec, - pub repetition_delay: u64, - pub controller_options: ControllerOptions, -} - -#[derive(Debug, Clone, Deserialize)] -pub struct TableDefinition { - pub name: String, - pub time_column: String, - pub value_columns: Vec, - pub metadata_columns: Vec, -} diff --git a/asap-planner-rs/src/config/mod.rs b/asap-planner-rs/src/config/mod.rs deleted file mode 100644 index 53b0e3243..000000000 --- a/asap-planner-rs/src/config/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod input; -pub use input::*; diff --git a/asap-planner-rs/src/error.rs b/asap-planner-rs/src/error.rs deleted file mode 100644 index 027488050..000000000 --- a/asap-planner-rs/src/error.rs +++ /dev/null @@ -1,23 +0,0 @@ -use thiserror::Error; - -#[derive(Debug, Error)] -pub enum ControllerError { - #[error("IO error: {0}")] - Io(#[from] std::io::Error), - #[error("YAML parse error: {0}")] - YamlParse(#[from] serde_yaml::Error), - #[error("PromQL parse error: {0}")] - PromQLParse(String), - #[error("Duplicate query: {0}")] - DuplicateQuery(String), - #[error("Planner error: {0}")] - PlannerError(String), - #[error("Unknown metric: {0}")] - UnknownMetric(String), - #[error("SQL parse error: {0}")] - SqlParse(String), - #[error("Unknown table: {0}")] - UnknownTable(String), - #[error("Prometheus client error: {0}")] - PrometheusClient(String), -} diff --git a/asap-planner-rs/src/lib.rs b/asap-planner-rs/src/lib.rs deleted file mode 100644 index dbdeb4078..000000000 --- a/asap-planner-rs/src/lib.rs +++ /dev/null @@ -1,458 +0,0 @@ -pub mod config; -pub mod error; -pub mod output; -pub mod planner; -pub mod prometheus_client; -pub mod query_log; - -use asap_types::enums::QueryLanguage; -use asap_types::inference_config::InferenceConfig; -use asap_types::streaming_config::StreamingConfig; -use promql_utilities::data_model::KeyByLabelNames; -use serde_yaml::Value as YamlValue; -use std::path::Path; -use tracing::debug; - -pub use asap_types::PromQLSchema; -pub use config::input::ControllerConfig; -pub use config::input::SQLControllerConfig; -pub use error::ControllerError; -pub use output::generator::{GeneratorOutput, PuntedQuery}; -pub use output::sql_generator::SQLRuntimeOptions; -pub use prometheus_client::build_schema_from_prometheus; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum StreamingEngine { - Arroyo, - Flink, - Precompute, -} - -#[derive(Debug, Clone)] -pub struct RuntimeOptions { - pub prometheus_scrape_interval: u64, - pub streaming_engine: StreamingEngine, - pub enable_punting: bool, - pub range_duration: u64, - pub step: u64, -} - -pub struct Controller { - config: ControllerConfig, - schema: PromQLSchema, - options: RuntimeOptions, -} - -/// Output of the planning process — contains the two YAML configs -pub struct PlannerOutput { - pub punted_queries: Vec, - streaming_yaml: YamlValue, - inference_yaml: YamlValue, - aggregation_count: usize, - query_count: usize, -} - -impl PlannerOutput { - pub fn streaming_aggregation_count(&self) -> usize { - self.aggregation_count - } - - pub fn inference_query_count(&self) -> usize { - self.query_count - } - - pub fn has_aggregation_type(&self, t: &str) -> bool { - if let YamlValue::Mapping(root) = &self.streaming_yaml { - if let Some(YamlValue::Sequence(aggs)) = root.get("aggregations") { - return aggs.iter().any(|agg| { - if let YamlValue::Mapping(m) = agg { - if let Some(YamlValue::String(agg_type)) = m.get("aggregationType") { - return agg_type == t; - } - } - false - }); - } - } - false - } - - pub fn all_tumbling_window_sizes_eq(&self, s: u64) -> bool { - self.check_tumbling_window_sizes(|size| size == s) - } - - pub fn all_tumbling_window_sizes_leq(&self, s: u64) -> bool { - self.check_tumbling_window_sizes(|size| size <= s) - } - - fn check_tumbling_window_sizes(&self, predicate: impl Fn(u64) -> bool) -> bool { - if let YamlValue::Mapping(root) = &self.streaming_yaml { - if let Some(YamlValue::Sequence(aggs)) = root.get("aggregations") { - return aggs.iter().all(|agg| { - if let YamlValue::Mapping(m) = agg { - if let Some(val) = m.get("windowSize") { - let size = match val { - YamlValue::Number(n) => n.as_u64().unwrap_or(0), - _ => 0, - }; - return predicate(size); - } - } - false - }); - } - } - false - } - - /// Returns the sorted labels for the first aggregation matching `agg_type`, - /// for the given `label_kind` ("rollup", "grouping", or "aggregated"). - pub fn aggregation_labels(&self, agg_type: &str, label_kind: &str) -> Vec { - if let YamlValue::Mapping(root) = &self.streaming_yaml { - if let Some(YamlValue::Sequence(aggs)) = root.get("aggregations") { - for agg in aggs { - if let YamlValue::Mapping(m) = agg { - if let Some(YamlValue::String(t)) = m.get("aggregationType") { - if t == agg_type { - if let Some(YamlValue::Mapping(labels)) = m.get("labels") { - if let Some(YamlValue::Sequence(seq)) = labels.get(label_kind) { - let mut result: Vec = seq - .iter() - .filter_map(|v| { - if let YamlValue::String(s) = v { - Some(s.clone()) - } else { - None - } - }) - .collect(); - result.sort(); - return result; - } - } - } - } - } - } - } - } - vec![] - } - - /// Returns the cleanup param (read_count_threshold or num_aggregates_to_retain) - /// for the first aggregation entry of the given query string. - pub fn inference_cleanup_param(&self, query: &str) -> Option { - if let YamlValue::Mapping(root) = &self.inference_yaml { - if let Some(YamlValue::Sequence(queries)) = root.get("queries") { - for q in queries { - if let YamlValue::Mapping(qm) = q { - if let Some(YamlValue::String(qs)) = qm.get("query") { - if qs == query { - if let Some(YamlValue::Sequence(aggs)) = qm.get("aggregations") { - if let Some(YamlValue::Mapping(agg)) = aggs.first() { - for key in - ["read_count_threshold", "num_aggregates_to_retain"] - { - if let Some(YamlValue::Number(n)) = agg.get(key) { - return n.as_u64(); - } - } - } - } - } - } - } - } - } - } - None - } - - pub fn to_streaming_yaml_string(&self) -> Result { - Ok(serde_yaml::to_string(&self.streaming_yaml)?) - } - - pub fn to_inference_yaml_string(&self) -> Result { - Ok(serde_yaml::to_string(&self.inference_yaml)?) - } - - pub fn to_streaming_config( - &self, - query_language: QueryLanguage, - ) -> Result { - let inference_config = self.to_inference_config(query_language)?; - StreamingConfig::from_yaml_data(&self.streaming_yaml, Some(&inference_config)) - } - - pub fn to_inference_config( - &self, - query_language: QueryLanguage, - ) -> Result { - InferenceConfig::from_yaml_data(&self.inference_yaml, query_language) - } - - /// Returns the table_name field of the first aggregation matching agg_type. - pub fn aggregation_table_name(&self, agg_type: &str) -> Option { - if let YamlValue::Mapping(root) = &self.streaming_yaml { - if let Some(YamlValue::Sequence(aggs)) = root.get("aggregations") { - for agg in aggs { - if let YamlValue::Mapping(m) = agg { - if let Some(YamlValue::String(t)) = m.get("aggregationType") { - if t == agg_type { - if let Some(YamlValue::String(name)) = m.get("table_name") { - return Some(name.clone()); - } - } - } - } - } - } - } - None - } - - /// Returns the value_column field of the first aggregation matching agg_type. - pub fn aggregation_value_column(&self, agg_type: &str) -> Option { - if let YamlValue::Mapping(root) = &self.streaming_yaml { - if let Some(YamlValue::Sequence(aggs)) = root.get("aggregations") { - for agg in aggs { - if let YamlValue::Mapping(m) = agg { - if let Some(YamlValue::String(t)) = m.get("aggregationType") { - if t == agg_type { - if let Some(YamlValue::String(col)) = m.get("value_column") { - return Some(col.clone()); - } - } - } - } - } - } - } - None - } - - /// Returns true if any aggregation has the matching type AND sub_type. - pub fn has_aggregation_type_and_sub_type(&self, agg_type: &str, sub_type: &str) -> bool { - if let YamlValue::Mapping(root) = &self.streaming_yaml { - if let Some(YamlValue::Sequence(aggs)) = root.get("aggregations") { - return aggs.iter().any(|agg| { - if let YamlValue::Mapping(m) = agg { - let type_matches = m.get("aggregationType").and_then(|v| { - if let YamlValue::String(s) = v { - Some(s.as_str()) - } else { - None - } - }) == Some(agg_type); - let sub_matches = m.get("aggregationSubType").and_then(|v| { - if let YamlValue::String(s) = v { - Some(s.as_str()) - } else { - None - } - }) == Some(sub_type); - type_matches && sub_matches - } else { - false - } - }); - } - } - false - } -} - -pub struct SQLController { - config: SQLControllerConfig, - options: SQLRuntimeOptions, -} - -impl SQLController { - pub fn new(config: SQLControllerConfig, options: SQLRuntimeOptions) -> Self { - Self { config, options } - } - - pub fn from_file(path: &Path, opts: SQLRuntimeOptions) -> Result { - let yaml_str = std::fs::read_to_string(path)?; - Self::from_yaml(&yaml_str, opts) - } - - pub fn from_yaml(yaml: &str, opts: SQLRuntimeOptions) -> Result { - let config: SQLControllerConfig = serde_yaml::from_str(yaml)?; - Ok(Self { - config, - options: opts, - }) - } - - pub fn generate(&self) -> Result { - let output = output::sql_generator::generate_sql_plan(&self.config, &self.options)?; - Ok(PlannerOutput { - punted_queries: output.punted_queries, - streaming_yaml: output.streaming_yaml, - inference_yaml: output.inference_yaml, - aggregation_count: output.aggregation_count, - query_count: output.query_count, - }) - } - - pub fn generate_to_dir(&self, dir: &Path) -> Result { - let output = self.generate()?; - std::fs::create_dir_all(dir)?; - let streaming_str = serde_yaml::to_string(&output.streaming_yaml)?; - let inference_str = serde_yaml::to_string(&output.inference_yaml)?; - std::fs::write(dir.join("streaming_config.yaml"), streaming_str)?; - std::fs::write(dir.join("inference_config.yaml"), inference_str)?; - Ok(output) - } -} - -impl Controller { - pub fn new(config: ControllerConfig, schema: PromQLSchema, options: RuntimeOptions) -> Self { - Self { - config, - schema, - options, - } - } - - /// Build a `Controller` from a config file, fetching metric labels from Prometheus. - /// - /// `prometheus_url` is queried via `GET /api/v1/series?match[]=` for each metric - /// name found in the config's PromQL queries. - pub fn from_file( - path: &Path, - opts: RuntimeOptions, - prometheus_url: &str, - ) -> Result { - let yaml_str = std::fs::read_to_string(path)?; - let config: ControllerConfig = serde_yaml::from_str(&yaml_str)?; - let all_queries: Vec = config - .query_groups - .iter() - .flat_map(|qg| qg.queries.clone()) - .collect(); - let mut schema = - prometheus_client::build_schema_from_prometheus(prometheus_url, &all_queries)?; - // For any metric that Prometheus had no series for, fall back to the - // `metrics` hint in the config file (if present). - if let Some(metric_hints) = &config.metrics { - for hint in metric_hints { - if !schema.config.contains_key(&hint.metric) { - debug!( - "Prometheus had no series for '{}'; falling back to config-file hint with labels {:?}", - hint.metric, hint.labels - ); - schema = schema.add_metric( - hint.metric.clone(), - KeyByLabelNames::new(hint.labels.clone()), - ); - } - } - } - Ok(Self { - config, - schema, - options: opts, - }) - } - - /// Build a `Controller` from a config file with a caller-supplied `PromQLSchema`. - /// - /// Use this when the schema is available without querying Prometheus (e.g. in tests - /// or when the schema is constructed in-process by the caller). - pub fn from_file_with_schema( - path: &Path, - schema: PromQLSchema, - opts: RuntimeOptions, - ) -> Result { - let yaml_str = std::fs::read_to_string(path)?; - let config: ControllerConfig = serde_yaml::from_str(&yaml_str)?; - Ok(Self { - config, - schema, - options: opts, - }) - } - - /// Build a `Controller` from a YAML string with a caller-supplied `PromQLSchema`. - pub fn from_yaml_with_schema( - yaml: &str, - schema: PromQLSchema, - opts: RuntimeOptions, - ) -> Result { - let config: ControllerConfig = serde_yaml::from_str(yaml)?; - Ok(Self { - config, - schema, - options: opts, - }) - } - - /// Build a `Controller` from a Prometheus query log file, fetching metric labels from - /// Prometheus. - /// - /// - `log_path`: newline-delimited JSON query log (Prometheus `--query.log-file` output) - /// - `prometheus_url`: base URL queried for label discovery - pub fn from_query_log( - log_path: &Path, - opts: RuntimeOptions, - prometheus_url: &str, - ) -> Result { - let entries = query_log::parse_log_file(log_path)?; - let (instants, ranges) = - query_log::infer_queries(&entries, opts.prometheus_scrape_interval); - let config = query_log::to_controller_config(instants, ranges); - let all_queries: Vec = config - .query_groups - .iter() - .flat_map(|qg| qg.queries.clone()) - .collect(); - let schema = prometheus_client::build_schema_from_prometheus(prometheus_url, &all_queries)?; - Ok(Self { - config, - schema, - options: opts, - }) - } - - /// Build a `Controller` from a Prometheus query log file with a caller-supplied `PromQLSchema`. - /// - /// Use this when the schema is available without querying Prometheus (e.g. in tests). - pub fn from_query_log_with_schema( - log_path: &Path, - schema: PromQLSchema, - opts: RuntimeOptions, - ) -> Result { - let entries = query_log::parse_log_file(log_path)?; - let (instants, ranges) = - query_log::infer_queries(&entries, opts.prometheus_scrape_interval); - let config = query_log::to_controller_config(instants, ranges); - Ok(Self { - config, - schema, - options: opts, - }) - } - - pub fn generate(&self) -> Result { - let output = output::generator::generate_plan(&self.config, &self.schema, &self.options)?; - Ok(PlannerOutput { - punted_queries: output.punted_queries, - streaming_yaml: output.streaming_yaml, - inference_yaml: output.inference_yaml, - aggregation_count: output.aggregation_count, - query_count: output.query_count, - }) - } - - pub fn generate_to_dir(&self, dir: &Path) -> Result { - let output = self.generate()?; - std::fs::create_dir_all(dir)?; - let streaming_str = serde_yaml::to_string(&output.streaming_yaml)?; - let inference_str = serde_yaml::to_string(&output.inference_yaml)?; - std::fs::write(dir.join("streaming_config.yaml"), streaming_str)?; - std::fs::write(dir.join("inference_config.yaml"), inference_str)?; - Ok(output) - } -} diff --git a/asap-planner-rs/src/main.rs b/asap-planner-rs/src/main.rs deleted file mode 100644 index 3ffe68301..000000000 --- a/asap-planner-rs/src/main.rs +++ /dev/null @@ -1,134 +0,0 @@ -use asap_planner::{Controller, RuntimeOptions, SQLController, SQLRuntimeOptions, StreamingEngine}; -use asap_types::enums::QueryLanguage; -use clap::Parser; -use std::path::PathBuf; - -#[derive(Parser, Debug)] -#[command(name = "asap-planner", about = "ASAP Query Planner")] -struct Args { - /// Path to a hand-authored YAML workload config. Mutually exclusive with --query-log. - #[arg(long = "input_config", conflicts_with = "query_log")] - input_config: Option, - - /// Path to a Prometheus query log file (newline-delimited JSON). Mutually exclusive with --input_config. - #[arg(long = "query-log", conflicts_with = "input_config")] - query_log: Option, - - #[arg(long = "output_dir")] - output_dir: PathBuf, - - #[arg(long = "prometheus_scrape_interval", required = false)] - prometheus_scrape_interval: Option, - - /// Base URL of the Prometheus instance used to auto-infer metric label sets. - /// Optional: when provided, the planner queries Prometheus for label discovery. - /// When absent, labels are taken from the `metrics` hint in the config file. - /// Example: http://localhost:9090 - #[arg(long = "prometheus-url", required = false)] - prometheus_url: Option, - - #[arg(long = "streaming_engine", value_enum)] - streaming_engine: EngineArg, - - #[arg(long = "enable-punting", default_value = "false")] - enable_punting: bool, - - #[arg(long = "range-duration", default_value = "0")] - range_duration: u64, - - #[arg(long = "step", default_value = "0")] - step: u64, - - #[arg(long = "query-language", value_enum, default_value = "promql")] - query_language: QueryLanguage, - - #[arg(long = "data-ingestion-interval", required = false)] - data_ingestion_interval: Option, - - #[arg(short, long, action = clap::ArgAction::Count)] - verbose: u8, -} - -#[derive(clap::ValueEnum, Debug, Clone, Copy)] -enum EngineArg { - Arroyo, - Flink, - Precompute, -} - -fn main() -> anyhow::Result<()> { - let args = Args::parse(); - - tracing_subscriber::fmt() - .with_max_level(if args.verbose > 0 { - tracing::Level::DEBUG - } else { - tracing::Level::WARN - }) - .init(); - - let engine = match args.streaming_engine { - EngineArg::Arroyo => StreamingEngine::Arroyo, - EngineArg::Flink => StreamingEngine::Flink, - EngineArg::Precompute => StreamingEngine::Precompute, - }; - - match args.query_language { - QueryLanguage::promql => { - let scrape_interval = args.prometheus_scrape_interval.ok_or_else(|| { - anyhow::anyhow!("--prometheus_scrape_interval is required for PromQL mode") - })?; - let opts = RuntimeOptions { - prometheus_scrape_interval: scrape_interval, - streaming_engine: engine, - enable_punting: args.enable_punting, - range_duration: args.range_duration, - step: args.step, - }; - let controller = match (args.input_config, args.query_log, args.prometheus_url) { - (Some(config_path), None, Some(url)) => { - Controller::from_file(&config_path, opts, &url)? - } - (Some(config_path), None, None) => { - let yaml_str = std::fs::read_to_string(&config_path)?; - let config: asap_planner::ControllerConfig = serde_yaml::from_str(&yaml_str)?; - let schema = config.schema_from_hints(); - Controller::from_file_with_schema(&config_path, schema, opts)? - } - (None, Some(log_path), Some(url)) => { - Controller::from_query_log(&log_path, opts, &url)? - } - (None, Some(_log_path), None) => { - anyhow::bail!( - "--prometheus-url is required when using --query-log \ - (query logs have no metrics hint to fall back on)" - ) - } - _ => anyhow::bail!( - "exactly one of --input_config or --query-log must be provided for PromQL mode" - ), - }; - controller.generate_to_dir(&args.output_dir)?; - } - QueryLanguage::sql | QueryLanguage::elastic_sql => { - let interval = args.data_ingestion_interval.ok_or_else(|| { - anyhow::anyhow!("--data-ingestion-interval is required for SQL mode") - })?; - let config_path = args - .input_config - .ok_or_else(|| anyhow::anyhow!("--input_config is required for SQL mode"))?; - let opts = SQLRuntimeOptions { - streaming_engine: engine, - query_evaluation_time: None, - data_ingestion_interval: interval, - }; - SQLController::from_file(&config_path, opts)?.generate_to_dir(&args.output_dir)?; - } - QueryLanguage::elastic_querydsl => { - anyhow::bail!("ElasticQueryDSL is not yet supported"); - } - } - - println!("Generated configs in {}", args.output_dir.display()); - Ok(()) -} diff --git a/asap-planner-rs/src/output/generator.rs b/asap-planner-rs/src/output/generator.rs deleted file mode 100644 index 6da86ab04..000000000 --- a/asap-planner-rs/src/output/generator.rs +++ /dev/null @@ -1,441 +0,0 @@ -use indexmap::IndexMap; -use serde_json::Value as JsonValue; -use serde_yaml::Value as YamlValue; -use std::collections::HashMap; - -use asap_types::enums::CleanupPolicy; -use asap_types::PromQLSchema; -use promql_utilities::data_model::KeyByLabelNames; - -use crate::config::input::ControllerConfig; -use crate::error::ControllerError; -use crate::planner::single_query::{BinaryArm, IntermediateAggConfig, SingleQueryProcessor}; -use crate::RuntimeOptions; - -/// `(query_string, Vec<(identifying_key, cleanup_param)>)` pairs produced by binary leaf decomposition. -type LeafEntries = Vec<(String, Vec<(String, Option)>)>; - -/// Run the full planning pipeline and produce YAML outputs -pub fn generate_plan( - controller_config: &ControllerConfig, - schema: &PromQLSchema, - opts: &RuntimeOptions, -) -> Result { - let metric_schema = schema.clone(); - - // Determine cleanup policy - let cleanup_policy_str = controller_config - .aggregate_cleanup - .as_ref() - .and_then(|c| c.policy.as_deref()) - .unwrap_or("read_based"); - let cleanup_policy = cleanup_policy_str.parse::().map_err(|_| { - ControllerError::PlannerError(format!("Unknown cleanup policy: {}", cleanup_policy_str)) - })?; - - // Validate no duplicate queries - let mut seen_queries = std::collections::HashSet::new(); - for qg in &controller_config.query_groups { - for q in &qg.queries { - if !seen_queries.insert(q.clone()) { - return Err(ControllerError::DuplicateQuery(q.clone())); - } - } - } - - // Deduplication map: identifying_key -> (agg_config, assigned_id_placeholder) - let mut dedup_map: IndexMap = IndexMap::new(); - // query_string -> Vec<(key, cleanup_param)> - let mut query_keys_map: IndexMap)>> = IndexMap::new(); - - let mut punted_queries: Vec = Vec::new(); - - for qg in &controller_config.query_groups { - for query_string in &qg.queries { - let processor = SingleQueryProcessor::new( - query_string.clone(), - qg.repetition_delay, - opts.prometheus_scrape_interval, - metric_schema.clone(), - opts.streaming_engine, - controller_config.sketch_parameters.clone(), - qg.range_duration.unwrap_or(opts.range_duration), - qg.step.unwrap_or(opts.step), - cleanup_policy, - ); - - let mut should_process = processor.is_supported(); - if opts.enable_punting && should_process { - should_process = should_process && processor.should_be_performant(); - if !should_process { - punted_queries.push(PuntedQuery { - query: query_string.clone(), - }); - } - } - - if should_process { - match processor.get_streaming_aggregation_configs() { - Ok((configs, cleanup_param)) => { - let mut keys_for_query = Vec::new(); - for config in configs { - let key = config.identifying_key(); - keys_for_query.push((key.clone(), cleanup_param)); - dedup_map.entry(key).or_insert(config); - } - query_keys_map.insert(query_string.clone(), keys_for_query); - } - Err(ControllerError::UnknownMetric(ref metric)) => { - tracing::warn!( - query = %query_string, - metric = %metric, - "skipping query referencing unknown metric" - ); - } - Err(e) => return Err(e), - } - } else if let Some(arm_entries) = - collect_binary_leaf_entries(&processor, &mut dedup_map)? - { - // Binary arithmetic: register each leaf arm in dedup_map and query_keys_map - for (arm_query, keys_for_arm) in arm_entries { - // Use `entry` so a standalone query that duplicates an arm wins - query_keys_map.entry(arm_query).or_insert(keys_for_arm); - } - } - } - } - - // Assign sequential IDs (1-indexed, insertion order) - let mut id_map: HashMap = HashMap::new(); - for (idx, key) in dedup_map.keys().enumerate() { - id_map.insert(key.clone(), idx as u32 + 1); - } - - // Build streaming_config YAML - let streaming_yaml = build_streaming_yaml(&dedup_map, &id_map, &metric_schema)?; - - // Build inference_config YAML - let inference_yaml = build_inference_yaml( - cleanup_policy, - cleanup_policy_str, - &query_keys_map, - &id_map, - &metric_schema, - )?; - - Ok(GeneratorOutput { - punted_queries, - streaming_yaml, - inference_yaml, - aggregation_count: dedup_map.len(), - query_count: query_keys_map.len(), - }) -} - -/// Recursively collect (arm_query_string, Vec<(dedup_key, cleanup_param)>) pairs -/// from a binary arithmetic expression, registering new configs in `dedup_map`. -/// -/// Returns `Some(Vec<...>)` when every leaf arm is acceleratable. -/// Returns `None` if any arm is unsupported (caller should skip the query). -/// Returns `Err` only on internal planner errors. -fn collect_binary_leaf_entries( - processor: &SingleQueryProcessor, - dedup_map: &mut IndexMap, -) -> Result, ControllerError> { - let arms = match processor.get_binary_arm_queries() { - Some(arms) => arms, - None => return Ok(None), // not a binary expression - }; - - let mut all_entries: LeafEntries = Vec::new(); - - for arm in [arms.0, arms.1] { - match arm { - BinaryArm::Scalar(_) => { - // Scalar literals need no aggregation config — skip silently. - } - BinaryArm::Query(arm_query) => { - let arm_processor = processor.make_arm_processor(arm_query.clone()); - - if arm_processor.is_supported() { - // Leaf arm: gather its streaming aggregation configs. - let (configs, cleanup_param) = - arm_processor.get_streaming_aggregation_configs()?; - let mut keys_for_arm = Vec::new(); - for config in configs { - let key = config.identifying_key(); - keys_for_arm.push((key.clone(), cleanup_param)); - dedup_map.entry(key).or_insert(config); - } - all_entries.push((arm_query, keys_for_arm)); - } else { - // The arm might itself be a binary expression — recurse. - match collect_binary_leaf_entries(&arm_processor, dedup_map)? { - Some(sub_entries) => { - all_entries.extend(sub_entries); - } - None => { - // Arm is neither a supported leaf nor a binary expression. - // This entire query cannot be accelerated. - return Ok(None); - } - } - } - } - } - } - - Ok(Some(all_entries)) -} - -pub fn key_by_labels_to_yaml(labels: &KeyByLabelNames) -> YamlValue { - YamlValue::Sequence( - labels - .labels - .iter() - .map(|l| YamlValue::String(l.clone())) - .collect(), - ) -} - -pub fn build_aggregation_entry(id: u32, cfg: &IntermediateAggConfig) -> YamlValue { - let mut map = serde_yaml::Mapping::new(); - map.insert( - YamlValue::String("aggregationId".to_string()), - YamlValue::Number(id.into()), - ); - map.insert( - YamlValue::String("aggregationSubType".to_string()), - YamlValue::String(cfg.aggregation_sub_type.clone()), - ); - map.insert( - YamlValue::String("aggregationType".to_string()), - YamlValue::String(cfg.aggregation_type.to_string()), - ); - - let mut labels_map = serde_yaml::Mapping::new(); - labels_map.insert( - YamlValue::String("aggregated".to_string()), - key_by_labels_to_yaml(&cfg.aggregated_labels), - ); - labels_map.insert( - YamlValue::String("grouping".to_string()), - key_by_labels_to_yaml(&cfg.grouping_labels), - ); - labels_map.insert( - YamlValue::String("rollup".to_string()), - key_by_labels_to_yaml(&cfg.rollup_labels), - ); - map.insert( - YamlValue::String("labels".to_string()), - YamlValue::Mapping(labels_map), - ); - - map.insert( - YamlValue::String("metric".to_string()), - YamlValue::String(cfg.metric.clone()), - ); - map.insert( - YamlValue::String("parameters".to_string()), - params_to_yaml(&cfg.parameters), - ); - map.insert( - YamlValue::String("slideInterval".to_string()), - YamlValue::Number(cfg.slide_interval.into()), - ); - map.insert( - YamlValue::String("spatialFilter".to_string()), - YamlValue::String(cfg.spatial_filter.clone()), - ); - map.insert( - YamlValue::String("table_name".to_string()), - match &cfg.table_name { - Some(t) => YamlValue::String(t.clone()), - None => YamlValue::Null, - }, - ); - map.insert( - YamlValue::String("value_column".to_string()), - match &cfg.value_column { - Some(v) => YamlValue::String(v.clone()), - None => YamlValue::Null, - }, - ); - map.insert( - YamlValue::String("windowSize".to_string()), - YamlValue::Number(cfg.window_size.into()), - ); - map.insert( - YamlValue::String("windowType".to_string()), - YamlValue::String(cfg.window_type.to_string()), - ); - - YamlValue::Mapping(map) -} - -pub fn build_queries_yaml( - cleanup_policy: CleanupPolicy, - query_keys_map: &IndexMap)>>, - id_map: &HashMap, -) -> Vec { - query_keys_map - .iter() - .map(|(query_str, keys)| { - let aggregations: Vec = keys - .iter() - .map(|(key, cleanup_param)| { - let agg_id = id_map[key]; - let mut agg_map = serde_yaml::Mapping::new(); - agg_map.insert( - YamlValue::String("aggregation_id".to_string()), - YamlValue::Number(agg_id.into()), - ); - if let Some(param) = cleanup_param { - match cleanup_policy { - CleanupPolicy::CircularBuffer => { - agg_map.insert( - YamlValue::String("num_aggregates_to_retain".to_string()), - YamlValue::Number((*param).into()), - ); - } - CleanupPolicy::ReadBased => { - agg_map.insert( - YamlValue::String("read_count_threshold".to_string()), - YamlValue::Number((*param).into()), - ); - } - CleanupPolicy::NoCleanup => {} - } - } - YamlValue::Mapping(agg_map) - }) - .collect(); - - let mut q_map = serde_yaml::Mapping::new(); - q_map.insert( - YamlValue::String("aggregations".to_string()), - YamlValue::Sequence(aggregations), - ); - q_map.insert( - YamlValue::String("query".to_string()), - YamlValue::String(query_str.clone()), - ); - YamlValue::Mapping(q_map) - }) - .collect() -} - -pub fn params_to_yaml(params: &HashMap) -> YamlValue { - if params.is_empty() { - return YamlValue::Mapping(serde_yaml::Mapping::new()); - } - let mut map = serde_yaml::Mapping::new(); - // Sort for determinism - let mut sorted: Vec<_> = params.iter().collect(); - sorted.sort_by_key(|(k, _)| k.as_str()); - for (k, v) in sorted { - let yaml_val = match v { - JsonValue::Number(n) => { - if let Some(i) = n.as_u64() { - YamlValue::Number(serde_yaml::Number::from(i)) - } else if let Some(f) = n.as_f64() { - YamlValue::Number(serde_yaml::Number::from(f)) - } else { - YamlValue::String(n.to_string()) - } - } - JsonValue::String(s) => YamlValue::String(s.clone()), - JsonValue::Bool(b) => YamlValue::Bool(*b), - other => YamlValue::String(other.to_string()), - }; - map.insert(YamlValue::String(k.clone()), yaml_val); - } - YamlValue::Mapping(map) -} - -fn build_streaming_yaml( - dedup_map: &IndexMap, - id_map: &HashMap, - metric_schema: &asap_types::PromQLSchema, -) -> Result { - let aggregations: Vec = dedup_map - .iter() - .map(|(key, cfg)| build_aggregation_entry(id_map[key], cfg)) - .collect(); - - // Build metrics section - let mut metrics_map = serde_yaml::Mapping::new(); - for (metric_name, labels) in &metric_schema.config { - metrics_map.insert( - YamlValue::String(metric_name.clone()), - key_by_labels_to_yaml(labels), - ); - } - - let mut root = serde_yaml::Mapping::new(); - root.insert( - YamlValue::String("aggregations".to_string()), - YamlValue::Sequence(aggregations), - ); - root.insert( - YamlValue::String("metrics".to_string()), - YamlValue::Mapping(metrics_map), - ); - - Ok(YamlValue::Mapping(root)) -} - -fn build_inference_yaml( - cleanup_policy: CleanupPolicy, - cleanup_policy_str: &str, - query_keys_map: &IndexMap)>>, - id_map: &HashMap, - metric_schema: &asap_types::PromQLSchema, -) -> Result { - let mut cleanup_map = serde_yaml::Mapping::new(); - cleanup_map.insert( - YamlValue::String("name".to_string()), - YamlValue::String(cleanup_policy_str.to_string()), - ); - - let queries = build_queries_yaml(cleanup_policy, query_keys_map, id_map); - - // Build metrics section - let mut metrics_map = serde_yaml::Mapping::new(); - for (metric_name, labels) in &metric_schema.config { - metrics_map.insert( - YamlValue::String(metric_name.clone()), - key_by_labels_to_yaml(labels), - ); - } - - let mut root = serde_yaml::Mapping::new(); - root.insert( - YamlValue::String("cleanup_policy".to_string()), - YamlValue::Mapping(cleanup_map), - ); - root.insert( - YamlValue::String("metrics".to_string()), - YamlValue::Mapping(metrics_map), - ); - root.insert( - YamlValue::String("queries".to_string()), - YamlValue::Sequence(queries), - ); - - Ok(YamlValue::Mapping(root)) -} - -#[derive(Debug, Clone)] -pub struct PuntedQuery { - pub query: String, -} - -pub struct GeneratorOutput { - pub punted_queries: Vec, - pub streaming_yaml: YamlValue, - pub inference_yaml: YamlValue, - pub aggregation_count: usize, - pub query_count: usize, -} diff --git a/asap-planner-rs/src/output/mod.rs b/asap-planner-rs/src/output/mod.rs deleted file mode 100644 index 63c14cbb9..000000000 --- a/asap-planner-rs/src/output/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod generator; -pub mod sql_generator; -pub use generator::*; diff --git a/asap-planner-rs/src/output/sql_generator.rs b/asap-planner-rs/src/output/sql_generator.rs deleted file mode 100644 index 73851f032..000000000 --- a/asap-planner-rs/src/output/sql_generator.rs +++ /dev/null @@ -1,202 +0,0 @@ -use asap_types::enums::CleanupPolicy; -use indexmap::IndexMap; -use serde_yaml::Value as YamlValue; -use std::collections::HashMap; -use std::time::{SystemTime, UNIX_EPOCH}; - -use crate::config::input::SQLControllerConfig; -use crate::error::ControllerError; -use crate::output::generator::{build_aggregation_entry, build_queries_yaml, GeneratorOutput}; -use crate::planner::single_query::IntermediateAggConfig; -use crate::planner::sql_single_query::SQLSingleQueryProcessor; -use crate::StreamingEngine; - -pub struct SQLRuntimeOptions { - pub streaming_engine: StreamingEngine, - pub query_evaluation_time: Option, - pub data_ingestion_interval: u64, -} - -pub fn generate_sql_plan( - config: &SQLControllerConfig, - opts: &SQLRuntimeOptions, -) -> Result { - let eval_time: f64 = opts.query_evaluation_time.unwrap_or_else(|| { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_secs_f64() - }); - - let cleanup_policy_str = config - .aggregate_cleanup - .as_ref() - .and_then(|c| c.policy.as_deref()) - .unwrap_or("read_based"); - let cleanup_policy = cleanup_policy_str.parse::().map_err(|_| { - ControllerError::PlannerError(format!("Unknown cleanup policy: {}", cleanup_policy_str)) - })?; - - // Validate T % data_ingestion_interval == 0 - for qg in &config.query_groups { - if qg.repetition_delay % opts.data_ingestion_interval != 0 { - return Err(ControllerError::PlannerError(format!( - "repetition_delay {} is not a multiple of data_ingestion_interval {}", - qg.repetition_delay, opts.data_ingestion_interval - ))); - } - } - - // Check for duplicate queries - let mut seen_queries = std::collections::HashSet::new(); - for qg in &config.query_groups { - for q in &qg.queries { - if !seen_queries.insert(q.clone()) { - return Err(ControllerError::DuplicateQuery(q.clone())); - } - } - } - - // Dedup map: identifying_key -> IntermediateAggConfig - let mut dedup_map: IndexMap = IndexMap::new(); - // query_string -> Vec<(key, cleanup_param)> - let mut query_keys_map: IndexMap)>> = IndexMap::new(); - - for qg in &config.query_groups { - for query_string in &qg.queries { - let processor = SQLSingleQueryProcessor::new( - query_string.clone(), - qg.repetition_delay, - opts.data_ingestion_interval, - config.tables.clone(), - opts.streaming_engine, - config.sketch_parameters.clone(), - cleanup_policy, - ); - - let (configs, cleanup_param) = - processor.get_streaming_aggregation_configs(eval_time)?; - - let mut keys_for_query = Vec::new(); - for config_item in configs { - let key = config_item.identifying_key(); - keys_for_query.push((key.clone(), cleanup_param)); - dedup_map.entry(key).or_insert(config_item); - } - query_keys_map.insert(query_string.clone(), keys_for_query); - } - } - - // Assign sequential IDs - let mut id_map: HashMap = HashMap::new(); - for (idx, key) in dedup_map.keys().enumerate() { - id_map.insert(key.clone(), idx as u32 + 1); - } - - let streaming_yaml = build_sql_streaming_yaml(config, &dedup_map, &id_map)?; - let inference_yaml = build_sql_inference_yaml( - config, - cleanup_policy, - cleanup_policy_str, - &query_keys_map, - &id_map, - )?; - - Ok(GeneratorOutput { - punted_queries: Vec::new(), - streaming_yaml, - inference_yaml, - aggregation_count: dedup_map.len(), - query_count: query_keys_map.len(), - }) -} - -fn build_tables_yaml(config: &SQLControllerConfig) -> Vec { - config - .tables - .iter() - .map(|t| { - let mut map = serde_yaml::Mapping::new(); - map.insert( - YamlValue::String("name".to_string()), - YamlValue::String(t.name.clone()), - ); - map.insert( - YamlValue::String("time_column".to_string()), - YamlValue::String(t.time_column.clone()), - ); - map.insert( - YamlValue::String("value_columns".to_string()), - YamlValue::Sequence( - t.value_columns - .iter() - .map(|c| YamlValue::String(c.clone())) - .collect(), - ), - ); - map.insert( - YamlValue::String("metadata_columns".to_string()), - YamlValue::Sequence( - t.metadata_columns - .iter() - .map(|c| YamlValue::String(c.clone())) - .collect(), - ), - ); - YamlValue::Mapping(map) - }) - .collect() -} - -fn build_sql_streaming_yaml( - config: &SQLControllerConfig, - dedup_map: &IndexMap, - id_map: &HashMap, -) -> Result { - let aggregations: Vec = dedup_map - .iter() - .map(|(key, cfg)| build_aggregation_entry(id_map[key], cfg)) - .collect(); - - let mut root = serde_yaml::Mapping::new(); - root.insert( - YamlValue::String("aggregations".to_string()), - YamlValue::Sequence(aggregations), - ); - root.insert( - YamlValue::String("tables".to_string()), - YamlValue::Sequence(build_tables_yaml(config)), - ); - - Ok(YamlValue::Mapping(root)) -} - -fn build_sql_inference_yaml( - config: &SQLControllerConfig, - cleanup_policy: CleanupPolicy, - cleanup_policy_str: &str, - query_keys_map: &IndexMap)>>, - id_map: &HashMap, -) -> Result { - let mut cleanup_map = serde_yaml::Mapping::new(); - cleanup_map.insert( - YamlValue::String("name".to_string()), - YamlValue::String(cleanup_policy_str.to_string()), - ); - - let mut root = serde_yaml::Mapping::new(); - root.insert( - YamlValue::String("cleanup_policy".to_string()), - YamlValue::Mapping(cleanup_map), - ); - root.insert( - YamlValue::String("queries".to_string()), - YamlValue::Sequence(build_queries_yaml(cleanup_policy, query_keys_map, id_map)), - ); - root.insert( - YamlValue::String("tables".to_string()), - YamlValue::Sequence(build_tables_yaml(config)), - ); - - Ok(YamlValue::Mapping(root)) -} diff --git a/asap-planner-rs/src/planner/logics.rs b/asap-planner-rs/src/planner/logics.rs deleted file mode 100644 index 3e415df76..000000000 --- a/asap-planner-rs/src/planner/logics.rs +++ /dev/null @@ -1,477 +0,0 @@ -use crate::config::input::SketchParameterOverrides; -use asap_types::enums::{CleanupPolicy, WindowType}; -use promql_utilities::ast_matching::PromQLMatchResult; -use promql_utilities::data_model::KeyByLabelNames; -use promql_utilities::query_logics::enums::{AggregationType, QueryPatternType, Statistic}; -use promql_utilities::query_logics::logics::does_precompute_operator_support_subpopulations; -use std::collections::HashMap; - -// Default sketch parameters -const DEFAULT_CMS_DEPTH: u64 = 3; -const DEFAULT_CMS_WIDTH: u64 = 1024; -const DEFAULT_CMS_HEAP_MULT: u64 = 4; -const DEFAULT_KLL_K: u64 = 20; -const DEFAULT_HYDRA_ROW: u64 = 3; -const DEFAULT_HYDRA_COL: u64 = 1024; -const DEFAULT_HYDRA_K: u64 = 20; - -pub fn get_effective_repeat(t_repeat: u64, step: u64) -> u64 { - if step > 0 { - t_repeat.min(step) - } else { - t_repeat - } -} - -pub fn should_use_sliding_window( - _query_pattern_type: QueryPatternType, - _aggregation_type: &str, -) -> bool { - // HARDCODED: sliding windows crash Arroyo - false -} - -pub fn set_window_parameters( - query_pattern_type: QueryPatternType, - t_repeat: u64, - prometheus_scrape_interval: u64, - aggregation_type: &str, - step: u64, - config: &mut IntermediateWindowConfig, -) { - let effective_repeat = get_effective_repeat(t_repeat, step); - let _use_sliding = should_use_sliding_window(query_pattern_type, aggregation_type); - // use_sliding is always false, so always tumbling - set_tumbling_window_parameters( - query_pattern_type, - effective_repeat, - prometheus_scrape_interval, - config, - ); -} - -fn set_tumbling_window_parameters( - query_pattern_type: QueryPatternType, - effective_repeat: u64, - prometheus_scrape_interval: u64, - config: &mut IntermediateWindowConfig, -) { - match query_pattern_type { - QueryPatternType::OnlyTemporal | QueryPatternType::OneTemporalOneSpatial => { - config.window_size = effective_repeat; - config.slide_interval = effective_repeat; - config.window_type = WindowType::Tumbling; - } - QueryPatternType::OnlySpatial => { - config.window_size = prometheus_scrape_interval; - config.slide_interval = prometheus_scrape_interval; - config.window_type = WindowType::Tumbling; - } - } -} - -/// A mutable window config holder used during planning -#[derive(Debug, Clone, Default)] -pub struct IntermediateWindowConfig { - pub window_size: u64, - pub slide_interval: u64, - pub window_type: WindowType, -} - -/// Shared sketch parameter builder used by both PromQL and SQL paths. -/// -/// `topk_k` is only required for `CountMinSketchWithHeap`: PromQL supplies it -/// from the `topk(k, …)` query argument; SQL passes `None` (SQL never produces -/// this operator today, so the `None` branch is unreachable in practice). -pub fn build_sketch_parameters( - aggregation_type: AggregationType, - aggregation_sub_type: &str, - topk_k: Option, - sketch_params: Option<&SketchParameterOverrides>, -) -> Result, String> { - match aggregation_type { - AggregationType::Increase - | AggregationType::MinMax - | AggregationType::Sum - | AggregationType::MultipleIncrease - | AggregationType::MultipleMinMax - | AggregationType::MultipleSum - | AggregationType::DeltaSetAggregator - | AggregationType::SetAggregator => Ok(HashMap::new()), - - AggregationType::CountMinSketch => { - let depth = sketch_params - .and_then(|p| p.count_min_sketch.as_ref()) - .map(|p| p.depth) - .unwrap_or(DEFAULT_CMS_DEPTH); - let width = sketch_params - .and_then(|p| p.count_min_sketch.as_ref()) - .map(|p| p.width) - .unwrap_or(DEFAULT_CMS_WIDTH); - let mut m = HashMap::new(); - m.insert("depth".to_string(), serde_json::Value::Number(depth.into())); - m.insert("width".to_string(), serde_json::Value::Number(width.into())); - Ok(m) - } - - AggregationType::CountMinSketchWithHeap => { - if aggregation_sub_type != "topk" { - return Err(format!( - "Aggregation sub-type {} for CountMinSketchWithHeap not supported", - aggregation_sub_type - )); - } - let k = topk_k - .ok_or_else(|| "CountMinSketchWithHeap requires a topk k value".to_string())?; - let depth = sketch_params - .and_then(|p| p.count_min_sketch_with_heap.as_ref()) - .map(|p| p.depth) - .unwrap_or(DEFAULT_CMS_DEPTH); - let width = sketch_params - .and_then(|p| p.count_min_sketch_with_heap.as_ref()) - .map(|p| p.width) - .unwrap_or(DEFAULT_CMS_WIDTH); - let heap_mult = sketch_params - .and_then(|p| p.count_min_sketch_with_heap.as_ref()) - .and_then(|p| p.heap_multiplier) - .unwrap_or(DEFAULT_CMS_HEAP_MULT); - let mut m = HashMap::new(); - m.insert("depth".to_string(), serde_json::Value::Number(depth.into())); - m.insert("width".to_string(), serde_json::Value::Number(width.into())); - m.insert( - "heapsize".to_string(), - serde_json::Value::Number((k * heap_mult).into()), - ); - Ok(m) - } - - AggregationType::DatasketchesKLL => { - let k = sketch_params - .and_then(|p| p.datasketches_kll.as_ref()) - .map(|p| p.k) - .unwrap_or(DEFAULT_KLL_K); - let mut m = HashMap::new(); - m.insert("K".to_string(), serde_json::Value::Number(k.into())); - Ok(m) - } - - AggregationType::HydraKLL => { - let row_num = sketch_params - .and_then(|p| p.hydra_kll.as_ref()) - .map(|p| p.row_num) - .unwrap_or(DEFAULT_HYDRA_ROW); - let col_num = sketch_params - .and_then(|p| p.hydra_kll.as_ref()) - .map(|p| p.col_num) - .unwrap_or(DEFAULT_HYDRA_COL); - let k = sketch_params - .and_then(|p| p.hydra_kll.as_ref()) - .map(|p| p.k) - .unwrap_or(DEFAULT_HYDRA_K); - let mut m = HashMap::new(); - m.insert( - "row_num".to_string(), - serde_json::Value::Number(row_num.into()), - ); - m.insert( - "col_num".to_string(), - serde_json::Value::Number(col_num.into()), - ); - m.insert("k".to_string(), serde_json::Value::Number(k.into())); - Ok(m) - } - - other => Err(format!("Aggregation type {} not supported", other)), - } -} - -/// PromQL wrapper: extracts the topk `k` from the match result when needed, -/// then delegates to `build_sketch_parameters`. -pub fn build_sketch_parameters_from_promql( - aggregation_type: AggregationType, - aggregation_sub_type: &str, - match_result: &PromQLMatchResult, - sketch_params: Option<&SketchParameterOverrides>, -) -> Result, String> { - let topk_k = if aggregation_type == AggregationType::CountMinSketchWithHeap { - let k: u64 = match_result - .tokens - .get("aggregation") - .and_then(|t| t.aggregation.as_ref()) - .and_then(|a| a.param.as_ref()) - .and_then(|p| p.parse::().ok()) - .map(|f| f as u64) - .ok_or_else(|| "topk query missing required 'k' parameter".to_string())?; - Some(k) - } else { - None - }; - build_sketch_parameters( - aggregation_type, - aggregation_sub_type, - topk_k, - sketch_params, - ) -} - -pub fn get_cleanup_param( - cleanup_policy: CleanupPolicy, - query_pattern_type: QueryPatternType, - match_result: &PromQLMatchResult, - t_repeat: u64, - window_type: WindowType, - range_duration: u64, - step: u64, -) -> Result { - // Validation - if (range_duration == 0) != (step == 0) { - return Err(format!( - "range_duration and step must both be 0 or both > 0. Got range_duration={}, step={}", - range_duration, step - )); - } - - let is_range_query = step > 0; - - let t_lookback: u64 = if query_pattern_type == QueryPatternType::OnlySpatial { - t_repeat - } else { - match_result - .get_range_duration() - .map(|d| d.num_seconds() as u64) - .ok_or_else(|| "No range_vector token found".to_string())? - }; - - if window_type == WindowType::Sliding { - let result = if is_range_query { - range_duration / step + 1 - } else { - 1 - }; - return Ok(result); - } - - // Tumbling - let effective_repeat = get_effective_repeat(t_repeat, step); - - let result = match cleanup_policy { - CleanupPolicy::CircularBuffer => { - // ceil((t_lookback + range_duration) / effective_repeat) - let numerator = t_lookback + range_duration; - numerator.div_ceil(effective_repeat) - } - CleanupPolicy::ReadBased => { - // ceil(t_lookback / effective_repeat) * (range_duration / step + 1) - let lookback_buckets = t_lookback.div_ceil(effective_repeat); - let num_steps = if is_range_query { - range_duration / step + 1 - } else { - 1 - }; - lookback_buckets * num_steps - } - CleanupPolicy::NoCleanup => { - return Err("NoCleanup policy should not call get_cleanup_param".to_string()); - } - }; - - Ok(result) -} - -pub fn set_subpopulation_labels( - statistic: Statistic, - aggregation_type: AggregationType, - subpopulation_labels: &KeyByLabelNames, - rollup_labels: &mut KeyByLabelNames, - grouping_labels: &mut KeyByLabelNames, - aggregated_labels: &mut KeyByLabelNames, -) { - // rollup is set by caller before calling this function - let _ = rollup_labels; // not modified here - if does_precompute_operator_support_subpopulations(statistic, aggregation_type) { - *grouping_labels = KeyByLabelNames::empty(); - *aggregated_labels = subpopulation_labels.clone(); - } else { - *grouping_labels = subpopulation_labels.clone(); - *aggregated_labels = KeyByLabelNames::empty(); - } -} - -/// SQL cleanup param — SQL queries are always instant (no range_duration/step). -pub fn get_sql_cleanup_param( - cleanup_policy: CleanupPolicy, - t_lookback: u64, - t_repeat: u64, -) -> Result { - match cleanup_policy { - CleanupPolicy::CircularBuffer | CleanupPolicy::ReadBased => { - Ok(t_lookback.div_ceil(t_repeat)) - } - CleanupPolicy::NoCleanup => { - Err("NoCleanup policy should not call get_sql_cleanup_param".to_string()) - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::planner::patterns::build_patterns; - - use promql_utilities::ast_matching::PromQLMatchResult; - use promql_utilities::query_logics::enums::QueryPatternType; - - fn match_query(query: &str) -> (QueryPatternType, PromQLMatchResult) { - let ast = promql_parser::parser::parse(query).unwrap(); - let patterns = build_patterns(); - for (pt, pattern) in &patterns { - let result = pattern.matches(&ast); - if result.matches { - return (*pt, result); - } - } - panic!("no pattern matched query: {}", query); - } - - // --- get_effective_repeat --- - - #[test] - fn effective_repeat_no_step() { - assert_eq!(get_effective_repeat(300, 0), 300); - } - - #[test] - fn effective_repeat_step_smaller_than_t_repeat() { - assert_eq!(get_effective_repeat(300, 30), 30); - } - - #[test] - fn effective_repeat_step_larger_than_t_repeat() { - assert_eq!(get_effective_repeat(30, 300), 30); - } - - // --- get_cleanup_param --- - - #[test] - fn cleanup_param_circular_buffer_spatial_instant_query() { - let (pt, mr) = match_query("sum(some_metric)"); - assert_eq!(pt, QueryPatternType::OnlySpatial); - // t_lookback = t_repeat = 300 (OnlySpatial path) - // effective_repeat = 300 (step=0) - // ceil((300 + 0) / 300) = 1 - let result = get_cleanup_param( - CleanupPolicy::CircularBuffer, - pt, - &mr, - 300, - WindowType::Tumbling, - 0, - 0, - ) - .unwrap(); - assert_eq!(result, 1); - } - - #[test] - fn cleanup_param_circular_buffer_spatial_range_query() { - let (pt, mr) = match_query("sum(some_metric)"); - // t_lookback = t_repeat = 300, effective_repeat = min(300, 30) = 30 - // ceil((300 + 3600) / 30) = ceil(130) = 130 - let result = get_cleanup_param( - CleanupPolicy::CircularBuffer, - pt, - &mr, - 300, - WindowType::Tumbling, - 3600, - 30, - ) - .unwrap(); - assert_eq!(result, 130); - } - - #[test] - fn cleanup_param_read_based_spatial_instant_query() { - let (pt, mr) = match_query("sum(some_metric)"); - // lookback_buckets = ceil(300/300) = 1, num_steps = 1 → result = 1 - let result = get_cleanup_param( - CleanupPolicy::ReadBased, - pt, - &mr, - 300, - WindowType::Tumbling, - 0, - 0, - ) - .unwrap(); - assert_eq!(result, 1); - } - - #[test] - fn cleanup_param_read_based_spatial_range_query() { - let (pt, mr) = match_query("sum(some_metric)"); - // lookback_buckets = ceil(300/30) = 10, num_steps = 3600/30 + 1 = 121 - // result = 10 * 121 = 1210 - let result = get_cleanup_param( - CleanupPolicy::ReadBased, - pt, - &mr, - 300, - WindowType::Tumbling, - 3600, - 30, - ) - .unwrap(); - assert_eq!(result, 1210); - } - - #[test] - fn cleanup_param_circular_buffer_temporal_instant_query() { - let (pt, mr) = match_query("rate(some_metric[5m])"); - assert_eq!(pt, QueryPatternType::OnlyTemporal); - // t_lookback = 5m = 300s (from [5m] range vector), range_duration=0, step=0 - // effective_repeat = 60, ceil((300 + 0) / 60) = 5 - let result = get_cleanup_param( - CleanupPolicy::CircularBuffer, - pt, - &mr, - 60, - WindowType::Tumbling, - 0, - 0, - ) - .unwrap(); - assert_eq!(result, 5); - } - - #[test] - fn cleanup_param_no_cleanup_returns_error() { - let (pt, mr) = match_query("sum(some_metric)"); - let result = get_cleanup_param( - CleanupPolicy::NoCleanup, - pt, - &mr, - 300, - WindowType::Tumbling, - 0, - 0, - ); - assert!(result.is_err()); - } - - #[test] - fn cleanup_param_mismatched_range_and_step_returns_error() { - let (pt, mr) = match_query("sum(some_metric)"); - // range_duration > 0 but step == 0 is invalid - let result = get_cleanup_param( - CleanupPolicy::CircularBuffer, - pt, - &mr, - 300, - WindowType::Tumbling, - 3600, - 0, - ); - assert!(result.is_err()); - } -} diff --git a/asap-planner-rs/src/planner/mod.rs b/asap-planner-rs/src/planner/mod.rs deleted file mode 100644 index 44475bec5..000000000 --- a/asap-planner-rs/src/planner/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -pub mod logics; -pub mod patterns; -pub mod single_query; -pub mod sql_single_query; -pub use single_query::*; diff --git a/asap-planner-rs/src/planner/patterns.rs b/asap-planner-rs/src/planner/patterns.rs deleted file mode 100644 index d2de5afd8..000000000 --- a/asap-planner-rs/src/planner/patterns.rs +++ /dev/null @@ -1,119 +0,0 @@ -use promql_utilities::ast_matching::{PromQLPattern, PromQLPatternBuilder}; -use promql_utilities::query_logics::enums::{ - AggregationOperator, PromQLFunction, QueryPatternType, -}; - -/// Build all 5 patterns in priority order: ONLY_TEMPORAL (2), ONLY_SPATIAL (1), ONE_TEMPORAL_ONE_SPATIAL (2) -pub fn build_patterns() -> Vec<(QueryPatternType, PromQLPattern)> { - let metric_pattern = || PromQLPatternBuilder::metric(None, None, None, Some("metric")); - let range_vector_pattern = - || PromQLPatternBuilder::matrix_selector(metric_pattern(), None, Some("range_vector")); - - // Temporal functions that produce a single-value result (no quantile phi arg) - let temporal_funcs: Vec<&str> = [ - PromQLFunction::SumOverTime, - PromQLFunction::CountOverTime, - PromQLFunction::AvgOverTime, - PromQLFunction::MinOverTime, - PromQLFunction::MaxOverTime, - PromQLFunction::Increase, - PromQLFunction::Rate, - ] - .map(PromQLFunction::as_str) - .to_vec(); - - // Aggregation operators used in spatial and spatial-of-temporal patterns - let spatial_ops: Vec<&str> = [ - AggregationOperator::Sum, - AggregationOperator::Count, - AggregationOperator::Avg, - AggregationOperator::Quantile, - AggregationOperator::Min, - AggregationOperator::Max, - AggregationOperator::Topk, - ] - .map(AggregationOperator::as_str) - .to_vec(); - - // Spatial-of-temporal excludes topk (no topk(quantile_over_time(...)) pattern) - let spatial_ops_no_topk: Vec<&str> = [ - AggregationOperator::Sum, - AggregationOperator::Count, - AggregationOperator::Avg, - AggregationOperator::Quantile, - AggregationOperator::Min, - AggregationOperator::Max, - ] - .map(AggregationOperator::as_str) - .to_vec(); - - // ONLY_TEMPORAL pattern 1: quantile_over_time(phi, metric[range]) - let ot_quantile = PromQLPattern::new(PromQLPatternBuilder::function( - vec![PromQLFunction::QuantileOverTime.as_str()], - vec![ - PromQLPatternBuilder::number(None, None), - range_vector_pattern(), - ], - Some("function"), - Some("function_args"), - )); - - // ONLY_TEMPORAL pattern 2: sum_over_time/count_over_time/... (metric[range]) - let ot_temporal_funcs = PromQLPattern::new(PromQLPatternBuilder::function( - temporal_funcs.clone(), - vec![range_vector_pattern()], - Some("function"), - Some("function_args"), - )); - - // ONLY_SPATIAL pattern: agg_op(metric) - let os_spatial = PromQLPattern::new(PromQLPatternBuilder::aggregation( - spatial_ops, - metric_pattern(), - None, - None, - None, - Some("aggregation"), - )); - - // ONE_TEMPORAL_ONE_SPATIAL pattern 1: agg_op(quantile_over_time(phi, metric[range])) - let ottos_quantile = PromQLPattern::new(PromQLPatternBuilder::aggregation( - spatial_ops_no_topk.clone(), - PromQLPatternBuilder::function( - vec![PromQLFunction::QuantileOverTime.as_str()], - vec![ - PromQLPatternBuilder::number(None, None), - range_vector_pattern(), - ], - Some("function"), - Some("function_args"), - ), - None, - None, - None, - Some("aggregation"), - )); - - // ONE_TEMPORAL_ONE_SPATIAL pattern 2: agg_op(temporal_func(metric[range])) - let ottos_temporal = PromQLPattern::new(PromQLPatternBuilder::aggregation( - spatial_ops_no_topk, - PromQLPatternBuilder::function( - temporal_funcs, - vec![range_vector_pattern()], - Some("function"), - Some("function_args"), - ), - None, - None, - None, - Some("aggregation"), - )); - - vec![ - (QueryPatternType::OnlyTemporal, ot_quantile), - (QueryPatternType::OnlyTemporal, ot_temporal_funcs), - (QueryPatternType::OnlySpatial, os_spatial), - (QueryPatternType::OneTemporalOneSpatial, ottos_quantile), - (QueryPatternType::OneTemporalOneSpatial, ottos_temporal), - ] -} diff --git a/asap-planner-rs/src/planner/single_query.rs b/asap-planner-rs/src/planner/single_query.rs deleted file mode 100644 index 9bdb2255a..000000000 --- a/asap-planner-rs/src/planner/single_query.rs +++ /dev/null @@ -1,552 +0,0 @@ -use asap_types::enums::{CleanupPolicy, WindowType}; -use asap_types::PromQLSchema; -use promql_utilities::ast_matching::PromQLMatchResult; -use promql_utilities::data_model::KeyByLabelNames; -use promql_utilities::query_logics::enums::{ - AggregationOperator, AggregationType, PromQLFunction, QueryPatternType, QueryTreatmentType, - Statistic, -}; -use promql_utilities::query_logics::logics::{ - get_is_collapsable, map_statistic_to_precompute_operator, -}; -use promql_utilities::query_logics::parsing::{ - get_metric_and_spatial_filter, get_spatial_aggregation_output_labels, get_statistics_to_compute, -}; -use serde_json::Value; -use std::collections::HashMap; - -use crate::config::input::SketchParameterOverrides; -use crate::error::ControllerError; -use crate::planner::logics::{ - build_sketch_parameters_from_promql, get_cleanup_param, set_subpopulation_labels, - set_window_parameters, IntermediateWindowConfig, -}; -use crate::planner::patterns::build_patterns; -use crate::StreamingEngine; - -/// Represents one arm of a binary arithmetic expression in the planner. -#[derive(Debug, Clone)] -pub enum BinaryArm { - /// A PromQL query expression that may be acceleratable. - Query(String), - /// A scalar literal (e.g. `100` in `rate(x[5m]) * 100`). - Scalar(f64), -} - -/// Convert an AST expression to a `BinaryArm`. Scalar literals become -/// `BinaryArm::Scalar`; everything else is serialized to a query string. -/// Outer parentheses are stripped so nested binary arms can be re-parsed -/// as `Binary` expressions (not `Paren`). -fn expr_to_binary_arm(expr: &promql_parser::parser::Expr) -> BinaryArm { - let inner = strip_parens(expr); - if let promql_parser::parser::Expr::NumberLiteral(nl) = inner { - BinaryArm::Scalar(nl.val) - } else { - BinaryArm::Query(format!("{}", inner)) - } -} - -/// Recursively remove outer `Paren` wrappers from an expression. -fn strip_parens(expr: &promql_parser::parser::Expr) -> &promql_parser::parser::Expr { - if let promql_parser::parser::Expr::Paren(paren) = expr { - strip_parens(&paren.expr) - } else { - expr - } -} - -/// Internal representation of an aggregation config before IDs are assigned -#[derive(Debug, Clone)] -pub struct IntermediateAggConfig { - pub aggregation_type: AggregationType, - pub aggregation_sub_type: String, - pub window_type: WindowType, - pub window_size: u64, - pub slide_interval: u64, - pub spatial_filter: String, - pub metric: String, - pub table_name: Option, - pub value_column: Option, - pub parameters: HashMap, - pub rollup_labels: KeyByLabelNames, - pub grouping_labels: KeyByLabelNames, - pub aggregated_labels: KeyByLabelNames, -} - -impl IntermediateAggConfig { - /// Canonical deduplication key matching Python's get_identifying_key() - pub fn identifying_key(&self) -> String { - // Build a canonical string representation matching Python's tuple - let mut params_vec: Vec<(String, String)> = self - .parameters - .iter() - .map(|(k, v)| (k.clone(), v.to_string())) - .collect(); - params_vec.sort_by_key(|(k, _)| k.clone()); - - let mut label_parts = String::new(); - // sorted label keys: aggregated, grouping, rollup - let mut label_keys = vec!["aggregated", "grouping", "rollup"]; - label_keys.sort(); - for k in label_keys { - let labels = match k { - "aggregated" => &self.aggregated_labels, - "grouping" => &self.grouping_labels, - "rollup" => &self.rollup_labels, - _ => unreachable!(), - }; - label_parts.push_str(&format!("{}:{:?};", k, labels.labels)); - } - - format!( - "{}|{}|{}|{}|{}|{}|{}|{:?}|{:?}|{:?}|{}", - self.aggregation_type, - self.aggregation_sub_type, - self.window_type, - self.window_size, - self.slide_interval, - self.spatial_filter, - self.metric, - self.table_name, - self.value_column, - params_vec, - label_parts, - ) - } -} - -pub struct SingleQueryProcessor { - query: String, - t_repeat: u64, - prometheus_scrape_interval: u64, - metric_schema: PromQLSchema, - #[allow(dead_code)] - streaming_engine: StreamingEngine, - sketch_parameters: Option, - range_duration: u64, - step: u64, - cleanup_policy: CleanupPolicy, -} - -impl SingleQueryProcessor { - #[allow(clippy::too_many_arguments)] - pub fn new( - query: String, - t_repeat: u64, - prometheus_scrape_interval: u64, - metric_schema: PromQLSchema, - streaming_engine: StreamingEngine, - sketch_parameters: Option, - range_duration: u64, - step: u64, - cleanup_policy: CleanupPolicy, - ) -> Self { - Self { - query, - t_repeat, - prometheus_scrape_interval, - metric_schema, - streaming_engine, - sketch_parameters, - range_duration, - step, - cleanup_policy, - } - } - - /// Try to match query and return (pattern_type, match_result) or None - fn match_pattern( - &self, - ast: &promql_parser::parser::Expr, - ) -> Option<(QueryPatternType, PromQLMatchResult)> { - let patterns = build_patterns(); - for (pattern_type, pattern) in &patterns { - let result = pattern.matches(ast); - if result.matches { - return Some((*pattern_type, result)); - } - } - None - } - - /// Get treatment type (Exact vs Approximate) from pattern match - fn get_treatment_type( - pattern_type: QueryPatternType, - match_result: &PromQLMatchResult, - ) -> QueryTreatmentType { - match pattern_type { - QueryPatternType::OnlyTemporal | QueryPatternType::OneTemporalOneSpatial => { - let fn_name = match_result.get_function_name().unwrap_or_default(); - match fn_name.parse::() { - Ok(f) if f.is_approximate() => QueryTreatmentType::Approximate, - _ => QueryTreatmentType::Exact, - } - } - QueryPatternType::OnlySpatial => { - let op = match_result.get_aggregation_op().unwrap_or_default(); - match op.parse::() { - Ok(o) if o.is_approximate() => QueryTreatmentType::Approximate, - _ => QueryTreatmentType::Exact, - } - } - } - } - - /// Returns `Some((lhs, rhs))` if this query is a binary arithmetic expression. - /// Each arm is either a query string (`BinaryArm::Query`) or a scalar literal - /// (`BinaryArm::Scalar`). Returns `None` if the query is not a binary expression - /// or cannot be parsed. - pub fn get_binary_arm_queries(&self) -> Option<(BinaryArm, BinaryArm)> { - let ast = promql_parser::parser::parse(&self.query).ok()?; - if let promql_parser::parser::Expr::Binary(binary) = ast { - let lhs = expr_to_binary_arm(binary.lhs.as_ref()); - let rhs = expr_to_binary_arm(binary.rhs.as_ref()); - // Only handle arithmetic operators (not comparison or set operators) - if !binary.op.is_comparison_operator() && !binary.op.is_set_operator() { - return Some((lhs, rhs)); - } - } - None - } - - /// Create a new processor for an arm query, reusing all parameters from this processor. - pub fn make_arm_processor(&self, arm_query: String) -> Self { - SingleQueryProcessor::new( - arm_query, - self.t_repeat, - self.prometheus_scrape_interval, - self.metric_schema.clone(), - self.streaming_engine, - self.sketch_parameters.clone(), - self.range_duration, - self.step, - self.cleanup_policy, - ) - } - - /// Check if query should be processed (supported pattern) - pub fn is_supported(&self) -> bool { - if let Ok(ast) = promql_parser::parser::parse(&self.query) { - self.match_pattern(&ast).is_some() - } else { - false - } - } - - /// Check if query should be performant (enable_punting check) - pub fn should_be_performant(&self) -> bool { - let ast = match promql_parser::parser::parse(&self.query) { - Ok(a) => a, - Err(_) => return false, - }; - let (pattern_type, match_result) = match self.match_pattern(&ast) { - Some(x) => x, - None => return true, - }; - - if pattern_type == QueryPatternType::OnlyTemporal { - let fn_name = match_result.get_function_name().unwrap_or_default(); - let parsed_fn = fn_name.parse::(); - if matches!( - parsed_fn, - Ok(PromQLFunction::Rate - | PromQLFunction::Increase - | PromQLFunction::QuantileOverTime) - ) { - let num_data_points = self.t_repeat as f64 / self.prometheus_scrape_interval as f64; - if num_data_points < 60.0 { - return false; - } - if parsed_fn == Ok(PromQLFunction::QuantileOverTime) { - if let Some(range_dur) = match_result.get_range_duration() { - let range_secs = range_dur.num_seconds() as f64; - if range_secs / self.t_repeat as f64 > 15.0 { - return false; - } - } - } - } - } - true - } - - /// Generate streaming aggregation configs for this query - pub fn get_streaming_aggregation_configs( - &self, - ) -> Result<(Vec, Option), ControllerError> { - let ast = promql_parser::parser::parse(&self.query) - .map_err(|e| ControllerError::PromQLParse(e.to_string()))?; - - let (pattern_type, match_result) = self.match_pattern(&ast).ok_or_else(|| { - ControllerError::PlannerError(format!("Unsupported query: {}", self.query)) - })?; - - let treatment_type = Self::get_treatment_type(pattern_type, &match_result); - - let (metric, spatial_filter) = get_metric_and_spatial_filter(&match_result); - - let all_labels = self - .metric_schema - .get_labels(&metric) - .ok_or_else(|| ControllerError::UnknownMetric(metric.clone()))? - .clone(); - - let statistics = get_statistics_to_compute(pattern_type, &match_result); - - let mut window_cfg = IntermediateWindowConfig::default(); - set_window_parameters( - pattern_type, - self.t_repeat, - self.prometheus_scrape_interval, - "any", // aggregation_type doesn't matter (sliding always false) - self.step, - &mut window_cfg, - ); - - let (rollup, subpopulation_labels) = - get_label_routing(pattern_type, &match_result, &all_labels); - - let configs = build_agg_configs_for_statistics( - &statistics, - treatment_type, - &subpopulation_labels, - &rollup, - &window_cfg, - &metric, - None, - None, - &spatial_filter, - |agg_type: AggregationType, agg_sub_type: &str| { - build_sketch_parameters_from_promql( - agg_type, - agg_sub_type, - &match_result, - self.sketch_parameters.as_ref(), - ) - }, - ) - .map_err(ControllerError::PlannerError)?; - - // Calculate cleanup param - let cleanup_param = if self.cleanup_policy == CleanupPolicy::NoCleanup { - None - } else { - Some( - get_cleanup_param( - self.cleanup_policy, - pattern_type, - &match_result, - self.t_repeat, - window_cfg.window_type, - self.range_duration, - self.step, - ) - .map_err(ControllerError::PlannerError)?, - ) - }; - - Ok((configs, cleanup_param)) - } -} - -/// Returns `(rollup, subpopulation_labels)` for a given PromQL pattern type. -/// These are constant across all statistics in a query, so they are computed -/// once before the per-statistic loop. -fn get_label_routing( - pattern_type: QueryPatternType, - match_result: &PromQLMatchResult, - all_labels: &KeyByLabelNames, -) -> (KeyByLabelNames, KeyByLabelNames) { - match pattern_type { - QueryPatternType::OnlyTemporal => (KeyByLabelNames::empty(), all_labels.clone()), - QueryPatternType::OnlySpatial => { - // Match Python: if no by/without modifier, spatial_output = [] (rollup gets all labels). - // promql_utilities::get_spatial_aggregation_output_labels has a topk patch that returns - // all_labels when there is no modifier, but the Python planner returns [] in that case. - let has_modifier = match_result - .tokens - .get("aggregation") - .and_then(|t| t.aggregation.as_ref()) - .and_then(|a| a.modifier.as_ref()) - .is_some(); - let spatial_output = if has_modifier { - get_spatial_aggregation_output_labels(match_result, all_labels) - } else { - KeyByLabelNames::empty() - }; - (all_labels.difference(&spatial_output), spatial_output) - } - QueryPatternType::OneTemporalOneSpatial => { - let fn_name = match_result.get_function_name().unwrap_or_default(); - let agg_op = match_result.get_aggregation_op().unwrap_or_default(); - let collapsable = fn_name - .parse::() - .ok() - .zip(agg_op.parse::().ok()) - .is_some_and(|(f, o)| get_is_collapsable(f, o)); - if !collapsable { - (KeyByLabelNames::empty(), all_labels.clone()) - } else { - let spatial_output = - get_spatial_aggregation_output_labels(match_result, all_labels); - (all_labels.difference(&spatial_output), spatial_output) - } - } - } -} - -/// Shared per-statistic config builder used by both PromQL and SQL paths. -/// -/// `get_params(agg_type, agg_sub_type)` is a closure supplied by the caller -/// that resolves sketch parameters; it is the only thing that differs between -/// the two paths. -#[allow(clippy::too_many_arguments)] -pub fn build_agg_configs_for_statistics( - statistics: &[Statistic], - treatment_type: QueryTreatmentType, - subpopulation_labels: &KeyByLabelNames, - rollup: &KeyByLabelNames, - window_cfg: &IntermediateWindowConfig, - metric: &str, - table_name: Option<&str>, - value_column: Option<&str>, - spatial_filter: &str, - get_params: impl Fn(AggregationType, &str) -> Result, String>, -) -> Result, String> { - let mut configs = Vec::new(); - - for statistic in statistics.iter().copied() { - let (agg_type, agg_sub_type) = - map_statistic_to_precompute_operator(statistic, treatment_type)?; - - let mut grouping = KeyByLabelNames::empty(); - let mut aggregated = KeyByLabelNames::empty(); - set_subpopulation_labels( - statistic, - agg_type, - subpopulation_labels, - &mut rollup.clone(), - &mut grouping, - &mut aggregated, - ); - - if matches!( - agg_type, - AggregationType::CountMinSketch | AggregationType::HydraKLL - ) { - let delta_params = get_params(AggregationType::DeltaSetAggregator, "")?; - configs.push(IntermediateAggConfig { - aggregation_type: AggregationType::DeltaSetAggregator, - aggregation_sub_type: String::new(), - window_type: window_cfg.window_type, - window_size: window_cfg.window_size, - slide_interval: window_cfg.slide_interval, - spatial_filter: spatial_filter.to_string(), - metric: metric.to_string(), - table_name: table_name.map(str::to_string), - value_column: value_column.map(str::to_string), - parameters: delta_params, - rollup_labels: rollup.clone(), - grouping_labels: grouping.clone(), - aggregated_labels: aggregated.clone(), - }); - } - - let parameters = get_params(agg_type, &agg_sub_type)?; - configs.push(IntermediateAggConfig { - aggregation_type: agg_type, - aggregation_sub_type: agg_sub_type, - window_type: window_cfg.window_type, - window_size: window_cfg.window_size, - slide_interval: window_cfg.slide_interval, - spatial_filter: spatial_filter.to_string(), - metric: metric.to_string(), - table_name: table_name.map(str::to_string), - value_column: value_column.map(str::to_string), - parameters, - rollup_labels: rollup.clone(), - grouping_labels: grouping, - aggregated_labels: aggregated, - }); - } - - Ok(configs) -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::Value; - use std::collections::HashMap; - - fn base_config() -> IntermediateAggConfig { - IntermediateAggConfig { - aggregation_type: AggregationType::MultipleIncrease, - aggregation_sub_type: "rate".to_string(), - window_type: WindowType::Tumbling, - window_size: 300, - slide_interval: 300, - spatial_filter: String::new(), - metric: "http_requests_total".to_string(), - table_name: None, - value_column: None, - parameters: HashMap::new(), - rollup_labels: KeyByLabelNames::new(vec!["instance".to_string()]), - grouping_labels: KeyByLabelNames::empty(), - aggregated_labels: KeyByLabelNames::empty(), - } - } - - #[test] - fn identifying_key_is_stable() { - let cfg = base_config(); - assert_eq!(cfg.identifying_key(), cfg.identifying_key()); - } - - #[test] - fn identical_configs_have_same_key() { - assert_eq!( - base_config().identifying_key(), - base_config().identifying_key() - ); - } - - #[test] - fn different_aggregation_type_produces_different_key() { - let cfg1 = base_config(); - let mut cfg2 = base_config(); - cfg2.aggregation_type = AggregationType::DatasketchesKLL; - assert_ne!(cfg1.identifying_key(), cfg2.identifying_key()); - } - - #[test] - fn different_window_size_produces_different_key() { - let cfg1 = base_config(); - let mut cfg2 = base_config(); - cfg2.window_size = 60; - assert_ne!(cfg1.identifying_key(), cfg2.identifying_key()); - } - - #[test] - fn different_rollup_labels_produce_different_key() { - let cfg1 = base_config(); - let mut cfg2 = base_config(); - cfg2.rollup_labels = KeyByLabelNames::new(vec!["job".to_string()]); - assert_ne!(cfg1.identifying_key(), cfg2.identifying_key()); - } - - #[test] - fn parameter_insertion_order_does_not_affect_key() { - let mut cfg1 = base_config(); - let mut cfg2 = base_config(); - cfg1.parameters - .insert("depth".to_string(), Value::Number(3.into())); - cfg1.parameters - .insert("width".to_string(), Value::Number(1024.into())); - cfg2.parameters - .insert("width".to_string(), Value::Number(1024.into())); - cfg2.parameters - .insert("depth".to_string(), Value::Number(3.into())); - assert_eq!(cfg1.identifying_key(), cfg2.identifying_key()); - } -} diff --git a/asap-planner-rs/src/planner/sql_single_query.rs b/asap-planner-rs/src/planner/sql_single_query.rs deleted file mode 100644 index 057b3209e..000000000 --- a/asap-planner-rs/src/planner/sql_single_query.rs +++ /dev/null @@ -1,214 +0,0 @@ -use std::collections::HashSet; - -use asap_types::enums::{CleanupPolicy, WindowType}; -use promql_utilities::data_model::KeyByLabelNames; -use promql_utilities::query_logics::enums::{AggregationType, QueryTreatmentType, Statistic}; -use sql_utilities::ast_matching::sqlhelper::Table; -use sql_utilities::ast_matching::sqlpattern_matcher::{QueryType, SQLPatternMatcher}; -use sql_utilities::ast_matching::sqlpattern_parser::SQLPatternParser; -use sql_utilities::ast_matching::SQLSchema; -use sqlparser::dialect::ClickHouseDialect; -use sqlparser::parser::Parser as SqlParser; - -use crate::config::input::{SketchParameterOverrides, TableDefinition}; -use crate::error::ControllerError; -use crate::planner::logics::{ - build_sketch_parameters, get_sql_cleanup_param, IntermediateWindowConfig, -}; -use crate::planner::single_query::{build_agg_configs_for_statistics, IntermediateAggConfig}; -use crate::StreamingEngine; - -pub struct SQLSingleQueryProcessor { - query_string: String, - t_repeat: u64, - data_ingestion_interval: u64, - table_definitions: Vec, - #[allow(dead_code)] - streaming_engine: StreamingEngine, - sketch_parameters: Option, - cleanup_policy: CleanupPolicy, -} - -impl SQLSingleQueryProcessor { - #[allow(clippy::too_many_arguments)] - pub fn new( - query_string: String, - t_repeat: u64, - data_ingestion_interval: u64, - table_definitions: Vec, - streaming_engine: StreamingEngine, - sketch_parameters: Option, - cleanup_policy: CleanupPolicy, - ) -> Self { - Self { - query_string, - t_repeat, - data_ingestion_interval, - table_definitions, - streaming_engine, - sketch_parameters, - cleanup_policy, - } - } - - pub fn get_streaming_aggregation_configs( - &self, - query_evaluation_time: f64, - ) -> Result<(Vec, Option), ControllerError> { - let schema = build_sql_schema(&self.table_definitions); - - // Parse SQL - let stmts = SqlParser::parse_sql(&ClickHouseDialect {}, &self.query_string) - .map_err(|e| ControllerError::SqlParse(e.to_string()))?; - - // Parse query into SQLQueryData - let qdata = SQLPatternParser::new(&schema, query_evaluation_time) - .parse_query(&stmts) - .ok_or_else(|| { - ControllerError::SqlParse(format!( - "Failed to parse SQL query: {}", - self.query_string - )) - })?; - - // Match query to pattern - let sql_query = SQLPatternMatcher::new(schema.clone(), self.data_ingestion_interval as f64) - .query_info_to_pattern(&qdata); - - if !sql_query.is_valid() { - return Err(ControllerError::SqlParse(sql_query.msg.unwrap_or_default())); - } - - let n = sql_query.query_data.len(); - - if n != 1 { - return Err(ControllerError::SqlParse(format!( - "Nested SQL queries (n={}) are not supported", - n - ))); - } - - // Determine fields from query vecs - let agg_info = &sql_query.query_data[0].aggregation_info; - let query_type = &sql_query.query_type[0]; - let labels = &sql_query.query_data[0].labels; - let table_name = &sql_query.query_data[0].metric; - - let value_column = agg_info.get_value_column_name().to_string(); - - // Compute window - let window_cfg = - compute_sql_window(query_type, self.data_ingestion_interval, self.t_repeat); - - // Get all metadata columns for the table - let all_metadata = get_all_metadata_columns(&self.table_definitions, table_name)?; - - // Label routing - let spatial_output = KeyByLabelNames::new(labels.iter().cloned().collect::>()); - let rollup = all_metadata.difference(&spatial_output); - - let treatment_type = get_sql_treatment_type(agg_info.get_name()); - let statistics = get_sql_statistics(agg_info.get_name())?; - - let configs = build_agg_configs_for_statistics( - &statistics, - treatment_type, - &spatial_output, - &rollup, - &window_cfg, - table_name, - Some(table_name), - Some(&value_column), - "", - |agg_type: AggregationType, agg_sub_type: &str| { - build_sketch_parameters( - agg_type, - agg_sub_type, - None, - self.sketch_parameters.as_ref(), - ) - }, - ) - .map_err(ControllerError::SqlParse)?; - - let t_lookback = match query_type { - QueryType::Spatial => self.data_ingestion_interval, - _ => sql_query.query_data[0].time_info.get_duration() as u64, - }; - - let cleanup_param = if self.cleanup_policy == CleanupPolicy::NoCleanup { - None - } else { - Some( - get_sql_cleanup_param(self.cleanup_policy, t_lookback, self.t_repeat) - .map_err(ControllerError::PlannerError)?, - ) - }; - - Ok((configs, cleanup_param)) - } -} - -fn build_sql_schema(tables: &[TableDefinition]) -> SQLSchema { - let table_vec: Vec = tables - .iter() - .map(|t| { - Table::new( - t.name.clone(), - t.time_column.clone(), - t.value_columns.iter().cloned().collect::>(), - t.metadata_columns.iter().cloned().collect::>(), - ) - }) - .collect(); - SQLSchema::new(table_vec) -} - -fn get_sql_treatment_type(name: &str) -> QueryTreatmentType { - match name.to_uppercase().as_str() { - "MIN" | "MAX" => QueryTreatmentType::Exact, - _ => QueryTreatmentType::Approximate, - } -} - -fn get_sql_statistics(name: &str) -> Result, ControllerError> { - match name.to_uppercase().as_str() { - "QUANTILE" => Ok(vec![Statistic::Quantile]), - "SUM" => Ok(vec![Statistic::Sum]), - "COUNT" => Ok(vec![Statistic::Count]), - "AVG" => Ok(vec![Statistic::Sum, Statistic::Count]), - "MIN" => Ok(vec![Statistic::Min]), - "MAX" => Ok(vec![Statistic::Max]), - other => Err(ControllerError::SqlParse(format!( - "Unsupported aggregation: {}", - other - ))), - } -} - -fn compute_sql_window( - query_type: &QueryType, - data_ingestion_interval: u64, - t_repeat: u64, -) -> IntermediateWindowConfig { - let window_size = match query_type { - QueryType::Spatial => data_ingestion_interval, - _ => t_repeat, - }; - IntermediateWindowConfig { - window_size, - slide_interval: window_size, - window_type: WindowType::Tumbling, - } -} - -fn get_all_metadata_columns( - table_definitions: &[TableDefinition], - table_name: &str, -) -> Result { - let table = table_definitions - .iter() - .find(|t| t.name == table_name) - .ok_or_else(|| ControllerError::UnknownTable(table_name.to_string()))?; - Ok(KeyByLabelNames::new(table.metadata_columns.clone())) -} diff --git a/asap-planner-rs/src/prometheus_client.rs b/asap-planner-rs/src/prometheus_client.rs deleted file mode 100644 index 76958fd81..000000000 --- a/asap-planner-rs/src/prometheus_client.rs +++ /dev/null @@ -1,184 +0,0 @@ -use std::collections::HashSet; -use std::thread; -use std::time::Duration; - -use asap_types::PromQLSchema; -use promql_parser::parser::Expr; -use promql_utilities::data_model::KeyByLabelNames; -use tracing::{debug, warn}; - -use crate::error::ControllerError; - -/// Number of times to retry a Prometheus request on 503 (service not yet ready). -const MAX_RETRIES: u32 = 15; -/// Delay between retries. -const RETRY_DELAY: Duration = Duration::from_secs(2); - -/// Walk a PromQL AST and collect all metric names referenced by VectorSelectors. -fn collect_metric_names(expr: &Expr, names: &mut HashSet) { - match expr { - Expr::VectorSelector(vs) => { - if let Some(name) = &vs.name { - names.insert(name.clone()); - } - } - Expr::MatrixSelector(ms) => { - if let Some(name) = &ms.vs.name { - names.insert(name.clone()); - } - } - Expr::Call(call) => { - for arg in &call.args.args { - collect_metric_names(arg, names); - } - } - Expr::Aggregate(agg) => { - collect_metric_names(&agg.expr, names); - } - Expr::Binary(bin) => { - collect_metric_names(&bin.lhs, names); - collect_metric_names(&bin.rhs, names); - } - Expr::Subquery(sq) => { - collect_metric_names(&sq.expr, names); - } - _ => {} - } -} - -/// Extract all unique metric names referenced in a slice of PromQL query strings. -/// Queries that fail to parse are skipped with a warning. -pub fn extract_metric_names(queries: &[String]) -> HashSet { - let mut names = HashSet::new(); - for query in queries { - match promql_parser::parser::parse(query) { - Ok(expr) => collect_metric_names(&expr, &mut names), - Err(e) => warn!( - "Could not parse query {:?} for metric name extraction: {}", - query, e - ), - } - } - names -} - -/// Query Prometheus `GET /api/v1/series?match[]=` and return the set of label key names -/// for that metric, or `None` if no series were found. -/// -/// Internal `__*__` labels (e.g. `__name__`) are excluded from the result. -/// -/// TODO: This queries only the last 5 minutes of series data (Prometheus default when no -/// `start`/`end` parameters are provided). Expand to a configurable lookback window to capture -/// metrics that have not been seen recently. -fn fetch_labels_for_metric( - prometheus_url: &str, - metric_name: &str, -) -> Result>, ControllerError> { - let url = format!("{}/api/v1/series", prometheus_url.trim_end_matches('/')); - let client = reqwest::blocking::Client::new(); - - for attempt in 1..=MAX_RETRIES { - let response = client - .get(&url) - .query(&[("match[]", metric_name)]) - .send() - .map_err(|e| { - ControllerError::PrometheusClient(format!( - "HTTP request failed for metric '{}': {}", - metric_name, e - )) - })?; - - let status = response.status(); - - if status == reqwest::StatusCode::SERVICE_UNAVAILABLE { - warn!( - "Prometheus returned 503 for metric '{}' (attempt {}/{}); retrying in {}s", - metric_name, - attempt, - MAX_RETRIES, - RETRY_DELAY.as_secs(), - ); - thread::sleep(RETRY_DELAY); - continue; - } - - if !status.is_success() { - return Err(ControllerError::PrometheusClient(format!( - "Prometheus returned HTTP {} for metric '{}'", - status, metric_name - ))); - } - - let body: serde_json::Value = response.json().map_err(|e| { - ControllerError::PrometheusClient(format!( - "Failed to parse Prometheus response for metric '{}': {}", - metric_name, e - )) - })?; - - let data = match body.get("data").and_then(|d| d.as_array()) { - Some(arr) => arr, - None => { - warn!( - "Prometheus returned no 'data' array for metric '{}'; skipping", - metric_name - ); - return Ok(None); - } - }; - - if data.is_empty() { - warn!( - "Prometheus returned no series for metric '{}' in the last 5 minutes; skipping", - metric_name - ); - return Ok(None); - } - - // Collect all unique label key names across all returned series, - // filtering out internal __*__ labels. - let mut label_keys: HashSet = HashSet::new(); - for series in data { - if let Some(labels) = series.as_object() { - for key in labels.keys() { - if !key.starts_with("__") { - label_keys.insert(key.clone()); - } - } - } - } - - return Ok(Some(label_keys.into_iter().collect())); - } - - Err(ControllerError::PrometheusClient(format!( - "Prometheus returned 503 for metric '{}' after {} attempts; giving up", - metric_name, MAX_RETRIES - ))) -} - -/// Build a `PromQLSchema` by querying Prometheus for each metric name found in the given -/// PromQL queries. Metrics with no series in Prometheus are skipped with a warning. -pub fn build_schema_from_prometheus( - prometheus_url: &str, - queries: &[String], -) -> Result { - let metric_names = extract_metric_names(queries); - debug!("Inferred metric names from queries: {:?}", metric_names); - let mut schema = PromQLSchema::new(); - - for metric_name in &metric_names { - match fetch_labels_for_metric(prometheus_url, metric_name)? { - Some(labels) => { - debug!("Inferred labels for metric '{}': {:?}", metric_name, labels); - schema = schema.add_metric(metric_name.clone(), KeyByLabelNames::new(labels)); - } - None => { - // Warning already emitted inside fetch_labels_for_metric. - } - } - } - - Ok(schema) -} diff --git a/asap-planner-rs/src/query_log/converter.rs b/asap-planner-rs/src/query_log/converter.rs deleted file mode 100644 index 22babb868..000000000 --- a/asap-planner-rs/src/query_log/converter.rs +++ /dev/null @@ -1,44 +0,0 @@ -use crate::config::input::{AggregateCleanupConfig, ControllerConfig, QueryGroup}; - -use super::frequency::{InstantQueryInfo, RangeQueryInfo}; - -/// Build a `ControllerConfig` from extracted instant and range queries. -/// -/// Each query becomes its own `QueryGroup` (one query per group, no SLA fields needed). -pub fn to_controller_config( - instants: Vec, - ranges: Vec, -) -> ControllerConfig { - let mut query_groups: Vec = Vec::new(); - - for info in instants { - query_groups.push(QueryGroup { - id: None, - queries: vec![info.query], - repetition_delay: info.repetition_delay, - controller_options: Default::default(), - step: None, - range_duration: None, - }); - } - - for info in ranges { - query_groups.push(QueryGroup { - id: None, - queries: vec![info.query], - repetition_delay: info.repetition_delay, - controller_options: Default::default(), - step: Some(info.step), - range_duration: Some(info.range_duration), - }); - } - - ControllerConfig { - query_groups, - sketch_parameters: None, - aggregate_cleanup: Some(AggregateCleanupConfig { - policy: Some("read_based".to_string()), - }), - metrics: None, - } -} diff --git a/asap-planner-rs/src/query_log/frequency.rs b/asap-planner-rs/src/query_log/frequency.rs deleted file mode 100644 index 6cc3a6f10..000000000 --- a/asap-planner-rs/src/query_log/frequency.rs +++ /dev/null @@ -1,280 +0,0 @@ -use chrono::{DateTime, Utc}; -use std::collections::HashMap; - -use super::parser::LogEntry; - -/// A single (query_string, step) variant paired with all its log entries. -type QueryVariant<'a> = ((String, u64), Vec<&'a LogEntry>); - -#[derive(Debug, Clone, PartialEq)] -pub struct InstantQueryInfo { - pub query: String, - /// Median inter-arrival time rounded to nearest scrape interval (seconds). - pub repetition_delay: u64, -} - -#[derive(Debug, Clone, PartialEq)] -pub struct RangeQueryInfo { - pub query: String, - /// Median inter-arrival time rounded to nearest scrape interval (seconds). - pub repetition_delay: u64, - /// Step from params.step (seconds). - pub step: u64, - /// Median of (end − start) across occurrences (seconds). - pub range_duration: u64, -} - -/// Infer query repetition delays from a slice of parsed log entries. -/// -/// Returns `(instant_queries, range_queries)`. -/// -/// Rules: -/// - Entries with step == 0 are instant; others are range. -/// - Group by (query_string, step). -/// - If the same query string appears under multiple step values, keep the -/// variant with the most occurrences and warn about discarded variants. -/// - Groups with only 1 occurrence are skipped with a warning. -/// - `repetition_delay` = median inter-arrival time, rounded to nearest -/// `scrape_interval` second. -/// - If `|raw_median − rounded| / scrape_interval ≥ 0.1` a warning is emitted. -pub fn infer_queries( - entries: &[LogEntry], - scrape_interval: u64, -) -> (Vec, Vec) { - // Group by (query_string, step) - let mut groups: HashMap<(String, u64), Vec<&LogEntry>> = HashMap::new(); - for entry in entries { - groups - .entry((entry.query.clone(), entry.step)) - .or_default() - .push(entry); - } - - // Per query string, if it appears under multiple steps, keep the most frequent. - let mut by_query: HashMap> = HashMap::new(); - for ((query, step), entries) in groups { - by_query - .entry(query.clone()) - .or_default() - .push(((query, step), entries)); - } - - let mut instant_results: Vec = Vec::new(); - let mut range_results: Vec = Vec::new(); - - for (_query_str, mut variants) in by_query { - // Sort descending by count so [0] is the most frequent variant. - // `sort_by_key` with the negated length would overflow for i64; use - // `sort_by` reversed via Reverse to keep clippy happy under - // `clippy::unnecessary_sort_by` (rust 1.95+). - variants.sort_by_key(|v| std::cmp::Reverse(v.1.len())); - - if variants.len() > 1 { - tracing::warn!( - query = %variants[0].0.0, - kept_step = variants[0].0.1, - kept_count = variants[0].1.len(), - "query appears with multiple step values; keeping most-frequent variant" - ); - } - - let ((query, step), variant_entries) = variants.remove(0); - - if variant_entries.len() < 2 { - tracing::warn!(%query, "query appears only once in log; skipping"); - continue; - } - - let repetition_delay = infer_repetition_delay(&variant_entries, scrape_interval, &query); - - if step == 0 { - instant_results.push(InstantQueryInfo { - query, - repetition_delay, - }); - } else { - let range_duration = median_range_duration(&variant_entries); - range_results.push(RangeQueryInfo { - query, - repetition_delay, - step, - range_duration, - }); - } - } - - (instant_results, range_results) -} - -/// Compute median inter-arrival time from timestamps and round to nearest scrape interval. -fn infer_repetition_delay(entries: &[&LogEntry], scrape_interval: u64, query: &str) -> u64 { - let mut timestamps: Vec> = entries.iter().map(|e| e.ts).collect(); - timestamps.sort(); - - let deltas: Vec = timestamps - .windows(2) - .map(|w| (w[1] - w[0]).num_seconds() as f64) - .collect(); - - let raw_median = median_f64(&deltas); - let rounded = round_to_nearest(raw_median, scrape_interval); - - let misalignment = (raw_median - rounded as f64).abs() / scrape_interval as f64; - if misalignment >= 0.1 { - tracing::warn!( - %query, - raw_median_secs = raw_median, - rounded_secs = rounded, - misalignment_pct = misalignment * 100.0, - "inferred repetition_delay is poorly aligned with scrape_interval; result may be inaccurate" - ); - } - - rounded -} - -/// Median of (end − start) durations in seconds. -fn median_range_duration(entries: &[&LogEntry]) -> u64 { - let mut durations: Vec = entries - .iter() - .map(|e| (e.end - e.start).num_seconds() as f64) - .collect(); - durations.sort_by(|a, b| a.partial_cmp(b).unwrap()); - median_f64(&durations) as u64 -} - -/// Median of a sorted-or-unsorted slice of f64 values. -fn median_f64(values: &[f64]) -> f64 { - assert!(!values.is_empty()); - let mut sorted = values.to_vec(); - sorted.sort_by(|a, b| a.partial_cmp(b).unwrap()); - let mid = sorted.len() / 2; - if sorted.len().is_multiple_of(2) { - (sorted[mid - 1] + sorted[mid]) / 2.0 - } else { - sorted[mid] - } -} - -/// Round `value` to the nearest multiple of `interval`, minimum 1 interval. -fn round_to_nearest(value: f64, interval: u64) -> u64 { - let interval_f = interval as f64; - let rounded = (value / interval_f).round() as u64; - rounded.max(1) * interval -} - -// ── unit tests ──────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use chrono::TimeZone; - - fn make_instant_entries( - base_ts: DateTime, - interval_secs: i64, - count: u32, - ) -> Vec { - (0..count) - .map(|i| LogEntry { - query: "rate(http_requests_total[5m])".to_string(), - start: base_ts + chrono::Duration::seconds(interval_secs * i as i64), - end: base_ts + chrono::Duration::seconds(interval_secs * i as i64), - step: 0, - ts: base_ts + chrono::Duration::seconds(interval_secs * i as i64), - }) - .collect() - } - - fn make_range_entries( - base_ts: DateTime, - interval_secs: i64, - range_secs: i64, - step: u64, - count: u32, - ) -> Vec { - (0..count) - .map(|i| { - let start = base_ts + chrono::Duration::seconds(interval_secs * i as i64); - LogEntry { - query: "rate(http_requests_total[5m])".to_string(), - start, - end: start + chrono::Duration::seconds(range_secs), - step, - ts: base_ts + chrono::Duration::seconds(interval_secs * i as i64), - } - }) - .collect() - } - - fn base_ts() -> DateTime { - Utc.with_ymd_and_hms(2025, 12, 2, 18, 0, 0).unwrap() - } - - #[test] - fn single_occurrence_skipped() { - let entries = make_instant_entries(base_ts(), 60, 1); - let (instants, ranges) = infer_queries(&entries, 15); - assert!(instants.is_empty()); - assert!(ranges.is_empty()); - } - - #[test] - fn median_inter_arrival_odd_count() { - // 5 entries at exactly 60s apart → 4 deltas all 60s → median=60 → rounded=60 - let entries = make_instant_entries(base_ts(), 60, 5); - let (instants, _) = infer_queries(&entries, 15); - assert_eq!(instants.len(), 1); - assert_eq!(instants[0].repetition_delay, 60); - } - - #[test] - fn median_inter_arrival_even_count() { - // 4 entries at 60s apart → 3 deltas all 60s → median=60 → rounded=60 - let entries = make_instant_entries(base_ts(), 60, 4); - let (instants, _) = infer_queries(&entries, 15); - assert_eq!(instants.len(), 1); - assert_eq!(instants[0].repetition_delay, 60); - } - - #[test] - fn round_down_to_nearest_scrape() { - // raw=16s, scrape=15 → nearest=15 (|16-15|/15=6.7% < 10%, no warn) - assert_eq!(round_to_nearest(16.0, 15), 15); - } - - #[test] - fn round_up_to_nearest_scrape() { - // raw=23s, scrape=15 → nearest=30 (23 is closer to 30 than 15) - assert_eq!(round_to_nearest(23.0, 15), 30); - } - - #[test] - fn misaligned_still_returns_result() { - // raw=22s, scrape=15 → rounds to 15, but |22-15|/15=46.7% ≥ 10% → warn emitted - // We only verify the value is returned (warning is logged, not returned) - assert_eq!(round_to_nearest(22.0, 15), 15); - } - - #[test] - fn range_duration_from_start_end() { - let entries = make_range_entries(base_ts(), 60, 3600, 30, 5); - let (_, ranges) = infer_queries(&entries, 15); - assert_eq!(ranges.len(), 1); - assert_eq!(ranges[0].range_duration, 3600); - assert_eq!(ranges[0].step, 30); - } - - #[test] - fn same_query_multiple_steps_keeps_most_frequent() { - let mut entries = make_instant_entries(base_ts(), 60, 3); // step=0, 3 occurrences - let range_entries = make_range_entries(base_ts(), 60, 3600, 30, 2); // step=30, 2 occurrences - entries.extend(range_entries); - - let (instants, ranges) = infer_queries(&entries, 15); - // step=0 variant has more occurrences → kept as instant - assert_eq!(instants.len(), 1); - // step=30 variant discarded - assert!(ranges.is_empty()); - } -} diff --git a/asap-planner-rs/src/query_log/mod.rs b/asap-planner-rs/src/query_log/mod.rs deleted file mode 100644 index 135e3fe0c..000000000 --- a/asap-planner-rs/src/query_log/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub mod converter; -pub mod frequency; -pub mod parser; - -pub use converter::to_controller_config; -pub use frequency::{infer_queries, InstantQueryInfo, RangeQueryInfo}; -pub use parser::{parse_log_file, LogEntry}; diff --git a/asap-planner-rs/src/query_log/parser.rs b/asap-planner-rs/src/query_log/parser.rs deleted file mode 100644 index 7a14a0558..000000000 --- a/asap-planner-rs/src/query_log/parser.rs +++ /dev/null @@ -1,114 +0,0 @@ -use chrono::{DateTime, Utc}; -use serde::Deserialize; - -#[derive(Debug, Clone, PartialEq)] -pub struct LogEntry { - pub query: String, - pub start: DateTime, - pub end: DateTime, - /// Step in seconds; 0 means instant query. - pub step: u64, - pub ts: DateTime, -} - -// ── raw serde shapes ────────────────────────────────────────────────────────── - -#[derive(Deserialize)] -struct RawEntry { - params: RawParams, - ts: DateTime, -} - -#[derive(Deserialize)] -struct RawParams { - query: String, - start: DateTime, - end: DateTime, - step: u64, -} - -// ── public API ──────────────────────────────────────────────────────────────── - -/// Parse a Prometheus query log file (newline-delimited JSON). -/// Malformed or incomplete lines are skipped with a warning; they never cause a panic. -pub fn parse_log_file(path: &std::path::Path) -> Result, std::io::Error> { - let contents = std::fs::read_to_string(path)?; - Ok(parse_log_str(&contents)) -} - -/// Parse a Prometheus query log from an in-memory string. -pub fn parse_log_str(input: &str) -> Vec { - input - .lines() - .enumerate() - .filter_map(|(line_no, line)| { - let line = line.trim(); - if line.is_empty() { - return None; - } - match serde_json::from_str::(line) { - Ok(raw) => Some(LogEntry { - query: raw.params.query, - start: raw.params.start, - end: raw.params.end, - step: raw.params.step, - ts: raw.ts, - }), - Err(e) => { - tracing::warn!(line = line_no + 1, error = %e, "skipping malformed query log line"); - None - } - } - }) - .collect() -} - -// ── unit tests ──────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - - const INSTANT_LINE: &str = r#"{"params":{"end":"2025-12-02T18:00:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:00:00.000Z","step":0},"ts":"2025-12-02T18:00:00.001Z"}"#; - const RANGE_LINE: &str = r#"{"params":{"end":"2025-12-02T19:00:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:00:00.000Z","step":30},"ts":"2025-12-02T18:00:00.001Z"}"#; - - #[test] - fn parse_valid_instant_entry() { - let entries = parse_log_str(INSTANT_LINE); - assert_eq!(entries.len(), 1); - let e = &entries[0]; - assert_eq!(e.query, "rate(http_requests_total[5m])"); - assert_eq!(e.step, 0); - assert_eq!(e.start, e.end); - } - - #[test] - fn parse_valid_range_entry() { - let entries = parse_log_str(RANGE_LINE); - assert_eq!(entries.len(), 1); - let e = &entries[0]; - assert_eq!(e.step, 30); - assert_ne!(e.start, e.end); - let duration = (e.end - e.start).num_seconds(); - assert_eq!(duration, 3600); - } - - #[test] - fn malformed_json_returns_empty() { - let entries = parse_log_str("not valid json at all"); - assert!(entries.is_empty()); - } - - #[test] - fn missing_params_field_skipped() { - let entries = parse_log_str(r#"{"ts":"2025-12-02T18:00:00.000Z"}"#); - assert!(entries.is_empty()); - } - - #[test] - fn mixed_lines_skips_bad() { - let input = format!("{}\nnot json\n{}", INSTANT_LINE, INSTANT_LINE); - let entries = parse_log_str(&input); - assert_eq!(entries.len(), 2); - } -} diff --git a/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic.yaml b/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic.yaml deleted file mode 100644 index c6b810116..000000000 --- a/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic.yaml +++ /dev/null @@ -1,15 +0,0 @@ -query_groups: - - id: 1 - queries: - - "rate(errors_total[5m]) / rate(requests_total[5m])" - repetition_delay: 300 - controller_options: - accuracy_sla: 0.99 - latency_sla: 1.0 -metrics: - - metric: "errors_total" - labels: ["instance", "job"] - - metric: "requests_total" - labels: ["instance", "job"] -aggregate_cleanup: - policy: "read_based" diff --git a/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic_dedup.yaml b/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic_dedup.yaml deleted file mode 100644 index 4dc2eabe1..000000000 --- a/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic_dedup.yaml +++ /dev/null @@ -1,16 +0,0 @@ -query_groups: - - id: 1 - queries: - - "rate(errors_total[5m]) / rate(requests_total[5m])" - - "rate(errors_total[5m])" - repetition_delay: 300 - controller_options: - accuracy_sla: 0.99 - latency_sla: 1.0 -metrics: - - metric: "errors_total" - labels: ["instance", "job"] - - metric: "requests_total" - labels: ["instance", "job"] -aggregate_cleanup: - policy: "read_based" diff --git a/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic_nested.yaml b/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic_nested.yaml deleted file mode 100644 index f5abf8a38..000000000 --- a/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic_nested.yaml +++ /dev/null @@ -1,17 +0,0 @@ -query_groups: - - id: 1 - queries: - - "(rate(a_total[5m]) + rate(b_total[5m])) / rate(c_total[5m])" - repetition_delay: 300 - controller_options: - accuracy_sla: 0.99 - latency_sla: 1.0 -metrics: - - metric: "a_total" - labels: ["instance"] - - metric: "b_total" - labels: ["instance"] - - metric: "c_total" - labels: ["instance"] -aggregate_cleanup: - policy: "read_based" diff --git a/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic_non_acceleratable.yaml b/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic_non_acceleratable.yaml deleted file mode 100644 index 6f53e4cfa..000000000 --- a/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic_non_acceleratable.yaml +++ /dev/null @@ -1,15 +0,0 @@ -query_groups: - - id: 1 - queries: - - "foo(errors_total[5m]) / rate(requests_total[5m])" - repetition_delay: 300 - controller_options: - accuracy_sla: 0.99 - latency_sla: 1.0 -metrics: - - metric: "errors_total" - labels: ["instance"] - - metric: "requests_total" - labels: ["instance"] -aggregate_cleanup: - policy: "read_based" diff --git a/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic_scalar.yaml b/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic_scalar.yaml deleted file mode 100644 index 35e7dd32f..000000000 --- a/asap-planner-rs/tests/comparison/test_data/configs/binary_arithmetic_scalar.yaml +++ /dev/null @@ -1,13 +0,0 @@ -query_groups: - - id: 1 - queries: - - "rate(errors_total[5m]) * 100" - repetition_delay: 300 - controller_options: - accuracy_sla: 0.99 - latency_sla: 1.0 -metrics: - - metric: "errors_total" - labels: ["instance", "job"] -aggregate_cleanup: - policy: "read_based" diff --git a/asap-planner-rs/tests/comparison/test_data/configs/cleanup_circular.yaml b/asap-planner-rs/tests/comparison/test_data/configs/cleanup_circular.yaml deleted file mode 100644 index 40d6c1f01..000000000 --- a/asap-planner-rs/tests/comparison/test_data/configs/cleanup_circular.yaml +++ /dev/null @@ -1,10 +0,0 @@ -query_groups: - - id: 1 - queries: - - "rate(http_requests_total[5m])" - repetition_delay: 300 - controller_options: - accuracy_sla: 0.99 - latency_sla: 1.0 -aggregate_cleanup: - policy: "circular_buffer" diff --git a/asap-planner-rs/tests/comparison/test_data/configs/cleanup_read_based.yaml b/asap-planner-rs/tests/comparison/test_data/configs/cleanup_read_based.yaml deleted file mode 100644 index 2fcd79964..000000000 --- a/asap-planner-rs/tests/comparison/test_data/configs/cleanup_read_based.yaml +++ /dev/null @@ -1,10 +0,0 @@ -query_groups: - - id: 1 - queries: - - "rate(http_requests_total[5m])" - repetition_delay: 300 - controller_options: - accuracy_sla: 0.99 - latency_sla: 1.0 -aggregate_cleanup: - policy: "read_based" diff --git a/asap-planner-rs/tests/comparison/test_data/configs/deduplicated.yaml b/asap-planner-rs/tests/comparison/test_data/configs/deduplicated.yaml deleted file mode 100644 index 4fe889754..000000000 --- a/asap-planner-rs/tests/comparison/test_data/configs/deduplicated.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Two different query strings that produce the same aggregation config -# rate and increase both map to MultipleIncrease with same window params -query_groups: - - id: 1 - queries: - - "rate(http_requests_total[5m])" - repetition_delay: 300 - controller_options: - accuracy_sla: 0.99 - latency_sla: 1.0 - - id: 2 - queries: - - "increase(http_requests_total[5m])" - repetition_delay: 300 - controller_options: - accuracy_sla: 0.99 - latency_sla: 1.0 -aggregate_cleanup: - policy: "read_based" diff --git a/asap-planner-rs/tests/comparison/test_data/configs/increase.yaml b/asap-planner-rs/tests/comparison/test_data/configs/increase.yaml deleted file mode 100644 index d72355710..000000000 --- a/asap-planner-rs/tests/comparison/test_data/configs/increase.yaml +++ /dev/null @@ -1,10 +0,0 @@ -query_groups: - - id: 1 - queries: - - "increase(http_requests_total[5m])" - repetition_delay: 300 - controller_options: - accuracy_sla: 0.99 - latency_sla: 1.0 -aggregate_cleanup: - policy: "read_based" diff --git a/asap-planner-rs/tests/comparison/test_data/configs/mixed_workload.yaml b/asap-planner-rs/tests/comparison/test_data/configs/mixed_workload.yaml deleted file mode 100644 index c4cac069d..000000000 --- a/asap-planner-rs/tests/comparison/test_data/configs/mixed_workload.yaml +++ /dev/null @@ -1,11 +0,0 @@ -query_groups: - - id: 1 - queries: - - "rate(http_requests_total[5m])" - - "sum by (job) (rate(http_requests_total[5m]))" - repetition_delay: 300 - controller_options: - accuracy_sla: 0.99 - latency_sla: 1.0 -aggregate_cleanup: - policy: "read_based" diff --git a/asap-planner-rs/tests/comparison/test_data/configs/quantile_over_time.yaml b/asap-planner-rs/tests/comparison/test_data/configs/quantile_over_time.yaml deleted file mode 100644 index 0d37afb6f..000000000 --- a/asap-planner-rs/tests/comparison/test_data/configs/quantile_over_time.yaml +++ /dev/null @@ -1,10 +0,0 @@ -query_groups: - - id: 1 - queries: - - "quantile_over_time(0.99, http_requests_total[5m])" - repetition_delay: 300 - controller_options: - accuracy_sla: 0.99 - latency_sla: 1.0 -aggregate_cleanup: - policy: "read_based" diff --git a/asap-planner-rs/tests/comparison/test_data/configs/range_query.yaml b/asap-planner-rs/tests/comparison/test_data/configs/range_query.yaml deleted file mode 100644 index 2fcd79964..000000000 --- a/asap-planner-rs/tests/comparison/test_data/configs/range_query.yaml +++ /dev/null @@ -1,10 +0,0 @@ -query_groups: - - id: 1 - queries: - - "rate(http_requests_total[5m])" - repetition_delay: 300 - controller_options: - accuracy_sla: 0.99 - latency_sla: 1.0 -aggregate_cleanup: - policy: "read_based" diff --git a/asap-planner-rs/tests/comparison/test_data/configs/rate_increase.yaml b/asap-planner-rs/tests/comparison/test_data/configs/rate_increase.yaml deleted file mode 100644 index 2fcd79964..000000000 --- a/asap-planner-rs/tests/comparison/test_data/configs/rate_increase.yaml +++ /dev/null @@ -1,10 +0,0 @@ -query_groups: - - id: 1 - queries: - - "rate(http_requests_total[5m])" - repetition_delay: 300 - controller_options: - accuracy_sla: 0.99 - latency_sla: 1.0 -aggregate_cleanup: - policy: "read_based" diff --git a/asap-planner-rs/tests/comparison/test_data/configs/spatial_quantile.yaml b/asap-planner-rs/tests/comparison/test_data/configs/spatial_quantile.yaml deleted file mode 100644 index 91b9ca99f..000000000 --- a/asap-planner-rs/tests/comparison/test_data/configs/spatial_quantile.yaml +++ /dev/null @@ -1,10 +0,0 @@ -query_groups: - - id: 1 - queries: - - "quantile by (job) (0.99, http_requests_total)" - repetition_delay: 60 - controller_options: - accuracy_sla: 0.99 - latency_sla: 1.0 -aggregate_cleanup: - policy: "read_based" diff --git a/asap-planner-rs/tests/comparison/test_data/configs/sum_by.yaml b/asap-planner-rs/tests/comparison/test_data/configs/sum_by.yaml deleted file mode 100644 index 7c68d86e6..000000000 --- a/asap-planner-rs/tests/comparison/test_data/configs/sum_by.yaml +++ /dev/null @@ -1,10 +0,0 @@ -query_groups: - - id: 1 - queries: - - "sum by (job, method) (http_requests_total)" - repetition_delay: 300 - controller_options: - accuracy_sla: 0.99 - latency_sla: 1.0 -aggregate_cleanup: - policy: "read_based" diff --git a/asap-planner-rs/tests/comparison/test_data/configs/sum_by_overlapping.yaml b/asap-planner-rs/tests/comparison/test_data/configs/sum_by_overlapping.yaml deleted file mode 100644 index 1324c7813..000000000 --- a/asap-planner-rs/tests/comparison/test_data/configs/sum_by_overlapping.yaml +++ /dev/null @@ -1,10 +0,0 @@ -query_groups: - - id: 1 - queries: - - "sum by (job, method) (http_requests_total)" - repetition_delay: 60 - controller_options: - accuracy_sla: 0.99 - latency_sla: 1.0 -aggregate_cleanup: - policy: "read_based" diff --git a/asap-planner-rs/tests/comparison/test_data/configs/sum_over_time.yaml b/asap-planner-rs/tests/comparison/test_data/configs/sum_over_time.yaml deleted file mode 100644 index c560bddcb..000000000 --- a/asap-planner-rs/tests/comparison/test_data/configs/sum_over_time.yaml +++ /dev/null @@ -1,10 +0,0 @@ -query_groups: - - id: 1 - queries: - - "sum_over_time(http_requests_total[5m])" - repetition_delay: 300 - controller_options: - accuracy_sla: 0.99 - latency_sla: 1.0 -aggregate_cleanup: - policy: "read_based" diff --git a/asap-planner-rs/tests/comparison/test_data/configs/temporal_overlapping.yaml b/asap-planner-rs/tests/comparison/test_data/configs/temporal_overlapping.yaml deleted file mode 100644 index 4e74cb849..000000000 --- a/asap-planner-rs/tests/comparison/test_data/configs/temporal_overlapping.yaml +++ /dev/null @@ -1,13 +0,0 @@ -query_groups: - - id: 1 - queries: - - "rate(http_requests_total[5m])" - - "increase(http_requests_total[5m])" - - "sum_over_time(http_requests_total[5m])" - - "quantile_over_time(0.99, http_requests_total[5m])" - repetition_delay: 60 - controller_options: - accuracy_sla: 0.99 - latency_sla: 1.0 -aggregate_cleanup: - policy: "read_based" diff --git a/asap-planner-rs/tests/comparison/test_data/configs/topk.yaml b/asap-planner-rs/tests/comparison/test_data/configs/topk.yaml deleted file mode 100644 index 3d996d8c2..000000000 --- a/asap-planner-rs/tests/comparison/test_data/configs/topk.yaml +++ /dev/null @@ -1,10 +0,0 @@ -query_groups: - - id: 1 - queries: - - "topk(5, http_requests_total)" - repetition_delay: 60 - controller_options: - accuracy_sla: 0.99 - latency_sla: 1.0 -aggregate_cleanup: - policy: "read_based" diff --git a/asap-planner-rs/tests/comparison/test_data/metrics/http_requests.yaml b/asap-planner-rs/tests/comparison/test_data/metrics/http_requests.yaml deleted file mode 100644 index dc7ec91bd..000000000 --- a/asap-planner-rs/tests/comparison/test_data/metrics/http_requests.yaml +++ /dev/null @@ -1,3 +0,0 @@ -metrics: - - metric: http_requests_total - labels: [instance, job, method, status] diff --git a/asap-planner-rs/tests/comparison/test_data/query_logs/instant_only.log b/asap-planner-rs/tests/comparison/test_data/query_logs/instant_only.log deleted file mode 100644 index c94846d5a..000000000 --- a/asap-planner-rs/tests/comparison/test_data/query_logs/instant_only.log +++ /dev/null @@ -1,10 +0,0 @@ -{"params":{"end":"2025-12-02T18:00:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:00:00.000Z","step":0},"ts":"2025-12-02T18:00:00.001Z"} -{"params":{"end":"2025-12-02T18:01:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:01:00.000Z","step":0},"ts":"2025-12-02T18:01:00.001Z"} -{"params":{"end":"2025-12-02T18:02:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:02:00.000Z","step":0},"ts":"2025-12-02T18:02:00.001Z"} -{"params":{"end":"2025-12-02T18:03:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:03:00.000Z","step":0},"ts":"2025-12-02T18:03:00.001Z"} -{"params":{"end":"2025-12-02T18:04:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:04:00.000Z","step":0},"ts":"2025-12-02T18:04:00.001Z"} -{"params":{"end":"2025-12-02T18:00:00.000Z","query":"sum(http_requests_total)","start":"2025-12-02T18:00:00.000Z","step":0},"ts":"2025-12-02T18:00:00.002Z"} -{"params":{"end":"2025-12-02T18:01:00.000Z","query":"sum(http_requests_total)","start":"2025-12-02T18:01:00.000Z","step":0},"ts":"2025-12-02T18:01:00.002Z"} -{"params":{"end":"2025-12-02T18:02:00.000Z","query":"sum(http_requests_total)","start":"2025-12-02T18:02:00.000Z","step":0},"ts":"2025-12-02T18:02:00.002Z"} -{"params":{"end":"2025-12-02T18:03:00.000Z","query":"sum(http_requests_total)","start":"2025-12-02T18:03:00.000Z","step":0},"ts":"2025-12-02T18:03:00.002Z"} -{"params":{"end":"2025-12-02T18:04:00.000Z","query":"sum(http_requests_total)","start":"2025-12-02T18:04:00.000Z","step":0},"ts":"2025-12-02T18:04:00.002Z"} diff --git a/asap-planner-rs/tests/comparison/test_data/query_logs/range_only.log b/asap-planner-rs/tests/comparison/test_data/query_logs/range_only.log deleted file mode 100644 index bb1107b37..000000000 --- a/asap-planner-rs/tests/comparison/test_data/query_logs/range_only.log +++ /dev/null @@ -1,5 +0,0 @@ -{"params":{"end":"2025-12-02T19:00:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:00:00.000Z","step":30},"ts":"2025-12-02T18:00:00.001Z"} -{"params":{"end":"2025-12-02T19:01:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:01:00.000Z","step":30},"ts":"2025-12-02T18:01:00.001Z"} -{"params":{"end":"2025-12-02T19:02:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:02:00.000Z","step":30},"ts":"2025-12-02T18:02:00.001Z"} -{"params":{"end":"2025-12-02T19:03:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:03:00.000Z","step":30},"ts":"2025-12-02T18:03:00.001Z"} -{"params":{"end":"2025-12-02T19:04:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:04:00.000Z","step":30},"ts":"2025-12-02T18:04:00.001Z"} diff --git a/asap-planner-rs/tests/comparison/test_data/query_logs/single_occurrence.log b/asap-planner-rs/tests/comparison/test_data/query_logs/single_occurrence.log deleted file mode 100644 index 5857715ff..000000000 --- a/asap-planner-rs/tests/comparison/test_data/query_logs/single_occurrence.log +++ /dev/null @@ -1,2 +0,0 @@ -{"params":{"end":"2025-12-02T18:00:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:00:00.000Z","step":0},"ts":"2025-12-02T18:00:00.001Z"} -{"params":{"end":"2025-12-02T18:00:00.000Z","query":"sum(http_requests_total)","start":"2025-12-02T18:00:00.000Z","step":0},"ts":"2025-12-02T18:00:00.002Z"} diff --git a/asap-planner-rs/tests/comparison/test_data/query_logs/with_malformed.log b/asap-planner-rs/tests/comparison/test_data/query_logs/with_malformed.log deleted file mode 100644 index 1c5c2cff4..000000000 --- a/asap-planner-rs/tests/comparison/test_data/query_logs/with_malformed.log +++ /dev/null @@ -1,7 +0,0 @@ -{"params":{"end":"2025-12-02T18:00:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:00:00.000Z","step":0},"ts":"2025-12-02T18:00:00.001Z"} -not valid json at all -{"params":{"end":"2025-12-02T18:01:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:01:00.000Z","step":0},"ts":"2025-12-02T18:01:00.001Z"} -{"missing_params": true, "ts": "2025-12-02T18:01:30.000Z"} -{"params":{"end":"2025-12-02T18:02:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:02:00.000Z","step":0},"ts":"2025-12-02T18:02:00.001Z"} -{"params":{"end":"2025-12-02T18:03:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:03:00.000Z","step":0},"ts":"2025-12-02T18:03:00.001Z"} -{"params":{"end":"2025-12-02T18:04:00.000Z","query":"rate(http_requests_total[5m])","start":"2025-12-02T18:04:00.000Z","step":0},"ts":"2025-12-02T18:04:00.001Z"} diff --git a/asap-planner-rs/tests/integration.rs b/asap-planner-rs/tests/integration.rs deleted file mode 100644 index 3bae948ea..000000000 --- a/asap-planner-rs/tests/integration.rs +++ /dev/null @@ -1,588 +0,0 @@ -use asap_planner::{Controller, ControllerError, PromQLSchema, RuntimeOptions, StreamingEngine}; -use promql_utilities::data_model::KeyByLabelNames; -use std::path::Path; - -// ─── helpers ───────────────────────────────────────────────────────────────── - -fn arroyo_opts() -> RuntimeOptions { - RuntimeOptions { - prometheus_scrape_interval: 15, - streaming_engine: StreamingEngine::Arroyo, - enable_punting: false, - range_duration: 0, - step: 0, - } -} - -/// Standard test schema: http_requests_total with [instance, job, method, status]. -fn http_requests_schema() -> PromQLSchema { - PromQLSchema::new().add_metric( - "http_requests_total".to_string(), - KeyByLabelNames::new(vec![ - "instance".to_string(), - "job".to_string(), - "method".to_string(), - "status".to_string(), - ]), - ) -} - -/// Schema for binary arithmetic tests: errors_total and requests_total. -fn binary_arithmetic_schema() -> PromQLSchema { - PromQLSchema::new() - .add_metric( - "errors_total".to_string(), - KeyByLabelNames::new(vec!["instance".to_string(), "job".to_string()]), - ) - .add_metric( - "requests_total".to_string(), - KeyByLabelNames::new(vec!["instance".to_string(), "job".to_string()]), - ) -} - -/// Schema for nested binary arithmetic test: a_total, b_total, c_total. -fn nested_binary_arithmetic_schema() -> PromQLSchema { - PromQLSchema::new() - .add_metric( - "a_total".to_string(), - KeyByLabelNames::new(vec!["instance".to_string(), "job".to_string()]), - ) - .add_metric( - "b_total".to_string(), - KeyByLabelNames::new(vec!["instance".to_string(), "job".to_string()]), - ) - .add_metric( - "c_total".to_string(), - KeyByLabelNames::new(vec!["instance".to_string(), "job".to_string()]), - ) -} - -// ─── query_log integration tests ───────────────────────────────────────────── - -#[test] -fn query_log_instant_produces_valid_configs() { - let c = Controller::from_query_log_with_schema( - Path::new("tests/comparison/test_data/query_logs/instant_only.log"), - http_requests_schema(), - arroyo_opts(), - ) - .unwrap(); - let out = c.generate().unwrap(); - assert!(out.streaming_aggregation_count() > 0); - assert!(out.inference_query_count() > 0); -} - -#[test] -fn query_log_range_produces_valid_configs() { - let c = Controller::from_query_log_with_schema( - Path::new("tests/comparison/test_data/query_logs/range_only.log"), - http_requests_schema(), - arroyo_opts(), - ) - .unwrap(); - let out = c.generate().unwrap(); - // range_only.log has step=30, so effective window size must be ≤ 30 - assert!(out.all_tumbling_window_sizes_leq(30)); -} - -#[test] -fn query_log_single_occurrence_excluded() { - let c = Controller::from_query_log_with_schema( - Path::new("tests/comparison/test_data/query_logs/single_occurrence.log"), - http_requests_schema(), - arroyo_opts(), - ) - .unwrap(); - let out = c.generate().unwrap(); - assert_eq!(out.inference_query_count(), 0); -} - -#[test] -fn query_log_malformed_lines_skipped() { - // with_malformed.log has 5 valid entries for rate() interspersed with bad lines - let c = Controller::from_query_log_with_schema( - Path::new("tests/comparison/test_data/query_logs/with_malformed.log"), - http_requests_schema(), - arroyo_opts(), - ) - .unwrap(); - let out = c.generate().unwrap(); - assert!(out.inference_query_count() > 0); -} - -#[test] -fn query_log_output_files_written() { - let dir = tempfile::tempdir().unwrap(); - let c = Controller::from_query_log_with_schema( - Path::new("tests/comparison/test_data/query_logs/instant_only.log"), - http_requests_schema(), - arroyo_opts(), - ) - .unwrap(); - c.generate_to_dir(dir.path()).unwrap(); - assert!(dir.path().join("streaming_config.yaml").exists()); - assert!(dir.path().join("inference_config.yaml").exists()); -} - -#[test] -fn quantile_over_time_produces_kll() { - // quantile_over_time groups by all labels → 1 DatasketchesKLL config - // Arroyo/Flink maintains one sketch per unique label-value combination at runtime - let c = Controller::from_file_with_schema( - Path::new("tests/comparison/test_data/configs/quantile_over_time.yaml"), - http_requests_schema(), - arroyo_opts(), - ) - .unwrap(); - let out = c.generate().unwrap(); - assert_eq!(out.streaming_aggregation_count(), 1); - assert!(out.has_aggregation_type("DatasketchesKLL")); - assert!(!out.has_aggregation_type("DeltaSetAggregator")); -} - -#[test] -fn rate_produces_multiple_increase_only() { - let c = Controller::from_file_with_schema( - Path::new("tests/comparison/test_data/configs/rate_increase.yaml"), - http_requests_schema(), - arroyo_opts(), - ) - .unwrap(); - let out = c.generate().unwrap(); - assert_eq!(out.streaming_aggregation_count(), 1); - assert!(out.has_aggregation_type("MultipleIncrease")); -} - -#[test] -fn only_spatial_window_equals_scrape_interval() { - let c = Controller::from_file_with_schema( - Path::new("tests/comparison/test_data/configs/spatial_quantile.yaml"), - http_requests_schema(), - arroyo_opts(), - ) - .unwrap(); - let out = c.generate().unwrap(); - assert!(out.all_tumbling_window_sizes_eq(15)); -} - -#[test] -fn duplicate_aggregation_configs_are_deduped() { - let c = Controller::from_file_with_schema( - Path::new("tests/comparison/test_data/configs/deduplicated.yaml"), - http_requests_schema(), - arroyo_opts(), - ) - .unwrap(); - let out = c.generate().unwrap(); - assert_eq!(out.streaming_aggregation_count(), 1); - assert_eq!(out.inference_query_count(), 2); -} - -#[test] -fn topk_produces_count_min_sketch_with_heap() { - let c = Controller::from_file_with_schema( - Path::new("tests/comparison/test_data/configs/topk.yaml"), - http_requests_schema(), - arroyo_opts(), - ) - .unwrap(); - let out = c.generate().unwrap(); - assert!(out.has_aggregation_type("CountMinSketchWithHeap")); -} - -#[test] -fn range_query_uses_effective_repeat() { - let opts = RuntimeOptions { - range_duration: 3600, - step: 30, - ..arroyo_opts() - }; - let c = Controller::from_file_with_schema( - Path::new("tests/comparison/test_data/configs/range_query.yaml"), - http_requests_schema(), - opts, - ) - .unwrap(); - let out = c.generate().unwrap(); - assert!(out.all_tumbling_window_sizes_leq(30)); -} - -#[test] -fn output_files_written_to_dir() { - let dir = tempfile::tempdir().unwrap(); - let c = Controller::from_file_with_schema( - Path::new("tests/comparison/test_data/configs/mixed_workload.yaml"), - http_requests_schema(), - arroyo_opts(), - ) - .unwrap(); - c.generate_to_dir(dir.path()).unwrap(); - assert!(dir.path().join("streaming_config.yaml").exists()); - assert!(dir.path().join("inference_config.yaml").exists()); -} - -#[test] -fn rate_tumbling_window_size_equals_effective_repeat() { - // For range queries, effective_repeat = min(t_repeat=300, step=30) = 30 - // Tumbling window size must equal effective_repeat (sliding is always disabled) - let opts = RuntimeOptions { - range_duration: 3600, - step: 30, - ..arroyo_opts() - }; - let c = Controller::from_file_with_schema( - Path::new("tests/comparison/test_data/configs/rate_increase.yaml"), - http_requests_schema(), - opts, - ) - .unwrap(); - let out = c.generate().unwrap(); - assert!(out.all_tumbling_window_sizes_eq(30)); -} - -#[test] -fn increase_tumbling_window_size_equals_effective_repeat() { - // effective_repeat = min(t_repeat=300, step=30) = 30 - let opts = RuntimeOptions { - range_duration: 3600, - step: 30, - ..arroyo_opts() - }; - let c = Controller::from_file_with_schema( - Path::new("tests/comparison/test_data/configs/increase.yaml"), - http_requests_schema(), - opts, - ) - .unwrap(); - let out = c.generate().unwrap(); - assert!(out.has_aggregation_type("MultipleIncrease")); - assert!(out.all_tumbling_window_sizes_eq(30)); -} - -#[test] -fn quantile_over_time_tumbling_window_size_equals_effective_repeat() { - // effective_repeat = min(t_repeat=300, step=30) = 30 - let opts = RuntimeOptions { - range_duration: 3600, - step: 30, - ..arroyo_opts() - }; - let c = Controller::from_file_with_schema( - Path::new("tests/comparison/test_data/configs/quantile_over_time.yaml"), - http_requests_schema(), - opts, - ) - .unwrap(); - let out = c.generate().unwrap(); - assert!(out.has_aggregation_type("DatasketchesKLL")); - assert!(out.all_tumbling_window_sizes_eq(30)); -} - -#[test] -fn sum_over_time_produces_count_min_sketch_with_delta_set() { - // sum_over_time is Approximate → CountMinSketch + DeltaSetAggregator pairing - let c = Controller::from_file_with_schema( - Path::new("tests/comparison/test_data/configs/sum_over_time.yaml"), - http_requests_schema(), - arroyo_opts(), - ) - .unwrap(); - let out = c.generate().unwrap(); - assert_eq!(out.streaming_aggregation_count(), 2); - assert!(out.has_aggregation_type("CountMinSketch")); - assert!(out.has_aggregation_type("DeltaSetAggregator")); -} - -#[test] -fn sum_by_produces_count_min_sketch_with_delta_set() { - let c = Controller::from_file_with_schema( - Path::new("tests/comparison/test_data/configs/sum_by.yaml"), - http_requests_schema(), - arroyo_opts(), - ) - .unwrap(); - let out = c.generate().unwrap(); - assert_eq!(out.streaming_aggregation_count(), 2); - assert!(out.has_aggregation_type("CountMinSketch")); - assert!(out.has_aggregation_type("DeltaSetAggregator")); -} - -#[test] -fn sum_by_rollup_excludes_groupby_labels() { - // sum by (job, method) → rollup gets labels NOT in by-clause - let c = Controller::from_file_with_schema( - Path::new("tests/comparison/test_data/configs/sum_by.yaml"), - http_requests_schema(), - arroyo_opts(), - ) - .unwrap(); - let out = c.generate().unwrap(); - assert_eq!( - out.aggregation_labels("CountMinSketch", "rollup"), - vec!["instance", "status"] - ); -} - -// --- Error-path tests --- - -#[test] -fn unknown_cleanup_policy_returns_planner_error() { - let yaml = r#" -query_groups: - - id: 1 - queries: - - "rate(http_requests_total[5m])" - repetition_delay: 300 - controller_options: - accuracy_sla: 0.99 - latency_sla: 1.0 -aggregate_cleanup: - policy: "not_a_real_policy" -"#; - let c = Controller::from_yaml_with_schema(yaml, http_requests_schema(), arroyo_opts()).unwrap(); - assert!(matches!( - c.generate(), - Err(ControllerError::PlannerError(_)) - )); -} - -#[test] -fn duplicate_query_in_same_group_returns_error() { - let yaml = r#" -query_groups: - - id: 1 - queries: - - "rate(http_requests_total[5m])" - - "rate(http_requests_total[5m])" - repetition_delay: 300 - controller_options: - accuracy_sla: 0.99 - latency_sla: 1.0 -"#; - let c = Controller::from_yaml_with_schema(yaml, http_requests_schema(), arroyo_opts()).unwrap(); - assert!(matches!( - c.generate(), - Err(ControllerError::DuplicateQuery(_)) - )); -} - -#[test] -fn duplicate_query_across_groups_returns_error() { - let yaml = r#" -query_groups: - - id: 1 - queries: - - "rate(http_requests_total[5m])" - repetition_delay: 300 - controller_options: - accuracy_sla: 0.99 - latency_sla: 1.0 - - id: 2 - queries: - - "rate(http_requests_total[5m])" - repetition_delay: 60 - controller_options: - accuracy_sla: 0.99 - latency_sla: 1.0 -"#; - let c = Controller::from_yaml_with_schema(yaml, http_requests_schema(), arroyo_opts()).unwrap(); - assert!(matches!( - c.generate(), - Err(ControllerError::DuplicateQuery(_)) - )); -} - -#[test] -fn query_referencing_unknown_metric_is_skipped_with_warning() { - // Unknown metric no longer aborts the run; the query is silently skipped. - let yaml = r#" -query_groups: - - id: 1 - queries: - - "rate(unknown_metric[5m])" - repetition_delay: 300 - controller_options: - accuracy_sla: 0.99 - latency_sla: 1.0 -"#; - // Schema only knows about http_requests_total, not unknown_metric. - let c = Controller::from_yaml_with_schema(yaml, http_requests_schema(), arroyo_opts()).unwrap(); - let out = c.generate().unwrap(); - assert_eq!(out.inference_query_count(), 0); - assert_eq!(out.streaming_aggregation_count(), 0); -} - -#[test] -fn malformed_yaml_returns_parse_error() { - let result = - Controller::from_yaml_with_schema("{ invalid yaml :", PromQLSchema::new(), arroyo_opts()); - assert!(matches!(result, Err(ControllerError::YamlParse(_)))); -} - -#[test] -fn metrics_field_in_yaml_is_accepted_as_hint() { - // The `metrics` section is a backwards-compatible label hint used as a - // fallback when Prometheus has no series for a metric. It must parse cleanly. - let yaml = r#" -query_groups: - - id: 1 - queries: - - "rate(http_requests_total[5m])" - repetition_delay: 300 -metrics: - - metric: "http_requests_total" - labels: ["instance"] -"#; - let result = Controller::from_yaml_with_schema(yaml, http_requests_schema(), arroyo_opts()); - assert!(result.is_ok()); -} - -// --- Overlapping window tests --- -// Queries where range vector > t_repeat: e.g. [5m] repeated every 60s. -// Windows are always tumbling (sliding disabled); the planner emits windowSize=t_repeat -// and the cleanup param tells the query engine how many windows to retain to cover the range. - -#[test] -fn temporal_overlapping_window_size_equals_t_repeat() { - // [5m] range repeated every 60s → windowSize = 60, not 300 - let c = Controller::from_file_with_schema( - Path::new("tests/comparison/test_data/configs/temporal_overlapping.yaml"), - http_requests_schema(), - arroyo_opts(), - ) - .unwrap(); - let out = c.generate().unwrap(); - assert!(out.all_tumbling_window_sizes_eq(60)); -} - -#[test] -fn temporal_overlapping_all_function_types_present() { - // rate+increase → MultipleIncrease (deduped to 1), sum_over_time → CountMinSketch+DeltaSet, - // quantile_over_time → DatasketchesKLL; 4 unique streaming aggregation configs total - let c = Controller::from_file_with_schema( - Path::new("tests/comparison/test_data/configs/temporal_overlapping.yaml"), - http_requests_schema(), - arroyo_opts(), - ) - .unwrap(); - let out = c.generate().unwrap(); - assert_eq!(out.streaming_aggregation_count(), 4); - assert!(out.has_aggregation_type("MultipleIncrease")); - assert!(out.has_aggregation_type("CountMinSketch")); - assert!(out.has_aggregation_type("DatasketchesKLL")); -} - -#[test] -fn temporal_overlapping_cleanup_param_equals_range_over_repeat() { - // t_lookback = 5m = 300s, effective_repeat = 60s → ceil(300/60) = 5 - let c = Controller::from_file_with_schema( - Path::new("tests/comparison/test_data/configs/temporal_overlapping.yaml"), - http_requests_schema(), - arroyo_opts(), - ) - .unwrap(); - let out = c.generate().unwrap(); - assert_eq!( - out.inference_cleanup_param("rate(http_requests_total[5m])"), - Some(5) - ); - assert_eq!( - out.inference_cleanup_param("increase(http_requests_total[5m])"), - Some(5) - ); - assert_eq!( - out.inference_cleanup_param("sum_over_time(http_requests_total[5m])"), - Some(5) - ); - assert_eq!( - out.inference_cleanup_param("quantile_over_time(0.99, http_requests_total[5m])"), - Some(5) - ); -} - -// --- Binary arithmetic tests --- - -#[test] -fn binary_arithmetic_produces_two_leaf_configs() { - let c = Controller::from_file_with_schema( - Path::new("tests/comparison/test_data/configs/binary_arithmetic.yaml"), - binary_arithmetic_schema(), - arroyo_opts(), - ) - .unwrap(); - let out = c.generate().unwrap(); - // Two arms → two streaming aggregation configs - assert_eq!(out.streaming_aggregation_count(), 2); - // Two separate query_config entries (one per arm) - assert_eq!(out.inference_query_count(), 2); -} - -#[test] -fn binary_arithmetic_deduplicates_shared_arm() { - let c = Controller::from_file_with_schema( - Path::new("tests/comparison/test_data/configs/binary_arithmetic_dedup.yaml"), - binary_arithmetic_schema(), - arroyo_opts(), - ) - .unwrap(); - let out = c.generate().unwrap(); - // errors_total arm is shared — only 2 streaming configs total (not 3) - assert_eq!(out.streaming_aggregation_count(), 2); - // 2 query_config entries: rate(errors_total[5m]) and rate(requests_total[5m]) - assert_eq!(out.inference_query_count(), 2); -} - -#[test] -fn nested_binary_arithmetic_produces_three_leaf_configs() { - let c = Controller::from_file_with_schema( - Path::new("tests/comparison/test_data/configs/binary_arithmetic_nested.yaml"), - nested_binary_arithmetic_schema(), - arroyo_opts(), - ) - .unwrap(); - let out = c.generate().unwrap(); - assert_eq!(out.streaming_aggregation_count(), 3); - assert_eq!(out.inference_query_count(), 3); -} - -#[test] -fn binary_arithmetic_scalar_constant_produces_one_leaf_config() { - let c = Controller::from_file_with_schema( - Path::new("tests/comparison/test_data/configs/binary_arithmetic_scalar.yaml"), - binary_arithmetic_schema(), - arroyo_opts(), - ) - .unwrap(); - let out = c.generate().unwrap(); - // Only the vector arm needs a streaming config; 100 is a literal - assert_eq!(out.streaming_aggregation_count(), 1); - assert_eq!(out.inference_query_count(), 1); -} - -#[test] -fn binary_arithmetic_with_non_acceleratable_arm_produces_no_configs() { - let c = Controller::from_file_with_schema( - Path::new("tests/comparison/test_data/configs/binary_arithmetic_non_acceleratable.yaml"), - binary_arithmetic_schema(), - arroyo_opts(), - ) - .unwrap(); - let out = c.generate().unwrap(); - assert_eq!(out.streaming_aggregation_count(), 0); - assert_eq!(out.inference_query_count(), 0); -} - -#[test] -fn temporal_overlapping_rate_increase_deduped() { - // rate and increase produce identical MultipleIncrease configs → 1 streaming entry shared, - // but inference config still tracks 4 queries separately - let c = Controller::from_file_with_schema( - Path::new("tests/comparison/test_data/configs/temporal_overlapping.yaml"), - http_requests_schema(), - arroyo_opts(), - ) - .unwrap(); - let out = c.generate().unwrap(); - assert_eq!(out.inference_query_count(), 4); - assert_eq!(out.streaming_aggregation_count(), 4); // not 5 -} diff --git a/asap-planner-rs/tests/sql_integration.rs b/asap-planner-rs/tests/sql_integration.rs deleted file mode 100644 index 485a30790..000000000 --- a/asap-planner-rs/tests/sql_integration.rs +++ /dev/null @@ -1,765 +0,0 @@ -use asap_planner::{ControllerError, SQLController, SQLRuntimeOptions, StreamingEngine}; - -// ── helpers ────────────────────────────────────────────────────────────────── - -fn sql_opts() -> SQLRuntimeOptions { - SQLRuntimeOptions { - streaming_engine: StreamingEngine::Arroyo, - // Fixed evaluation time so NOW()-relative timestamps are deterministic. - query_evaluation_time: Some(1_000_000.0), - data_ingestion_interval: 15, - } -} - -/// Single-query config with a 3-column metadata schema. -/// -/// Schema: metrics_table -/// time_column : time -/// value_columns : [cpu_usage] -/// metadata_columns : [hostname, datacenter, region] -/// data_ingestion_interval = 15 s -fn one_query_config(query: &str, t_repeat: u64) -> String { - format!( - r#" -tables: - - name: metrics_table - time_column: time - value_columns: [cpu_usage] - metadata_columns: [hostname, datacenter, region] -query_groups: - - id: 1 - repetition_delay: {t_repeat} - controller_options: - accuracy_sla: 0.95 - latency_sla: 100.0 - queries: - - >- - {query} -aggregate_cleanup: - policy: read_based -"# - ) -} - -// ── single-query happy-path tests ───────────────────────────────────────────── -// -// All queries: SELECT (cpu_usage) FROM metrics_table -// WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() -// GROUP BY datacenter -// -// With 3 metadata columns [hostname, datacenter, region] and GROUP BY datacenter: -// rollup = [hostname, region] (the two NOT in GROUP BY) -// grouping = [datacenter] -// aggregated = [] (no label-level aggregation dimension in SQL) -// -// Each test checks ALL observable plan properties: -// - streaming_aggregation_count -// - inference_query_count (always 1) -// - aggregation types present -// - tumbling window size -// - table_name and value_column -// - label routing: rollup / grouping / aggregated -// - inference cleanup param - -// ── aggregate-type coverage (T = 300 s, range = 300 s) ─────────────────────── - -/// SUM is Approximate → CountMinSketch + DeltaSetAggregator. -/// window = range = 300 s, cleanup = ceil(300 / 300) = 1. -#[test] -fn temporal_sum() { - let q = "SELECT SUM(cpu_usage) FROM metrics_table WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY datacenter"; - let out = SQLController::from_yaml(&one_query_config(q, 300), sql_opts()) - .unwrap() - .generate() - .unwrap(); - - assert_eq!(out.streaming_aggregation_count(), 2); - assert_eq!(out.inference_query_count(), 1); - assert!(out.has_aggregation_type("CountMinSketch")); - assert!(out.has_aggregation_type("DeltaSetAggregator")); - assert!(out.all_tumbling_window_sizes_eq(300)); - assert_eq!( - out.aggregation_table_name("CountMinSketch"), - Some("metrics_table".to_string()) - ); - assert_eq!( - out.aggregation_value_column("CountMinSketch"), - Some("cpu_usage".to_string()) - ); - let mut rollup = out.aggregation_labels("CountMinSketch", "rollup"); - rollup.sort(); - assert_eq!(rollup, vec!["hostname".to_string(), "region".to_string()]); - assert_eq!( - out.aggregation_labels("CountMinSketch", "grouping"), - Vec::::new() - ); - assert_eq!( - out.aggregation_labels("CountMinSketch", "aggregated"), - vec!["datacenter".to_string()] - ); - assert_eq!(out.inference_cleanup_param(q), Some(1)); -} - -/// COUNT is Approximate → CountMinSketch + DeltaSetAggregator (identical plan to SUM). -/// window = 300 s, cleanup = 1. -#[test] -fn temporal_count() { - let q = "SELECT COUNT(cpu_usage) FROM metrics_table WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY datacenter"; - let out = SQLController::from_yaml(&one_query_config(q, 300), sql_opts()) - .unwrap() - .generate() - .unwrap(); - - assert_eq!(out.streaming_aggregation_count(), 2); - assert_eq!(out.inference_query_count(), 1); - assert!(out.has_aggregation_type("CountMinSketch")); - assert!(out.has_aggregation_type("DeltaSetAggregator")); - assert!(out.all_tumbling_window_sizes_eq(300)); - assert_eq!( - out.aggregation_table_name("CountMinSketch"), - Some("metrics_table".to_string()) - ); - assert_eq!( - out.aggregation_value_column("CountMinSketch"), - Some("cpu_usage".to_string()) - ); - let mut rollup = out.aggregation_labels("CountMinSketch", "rollup"); - rollup.sort(); - assert_eq!(rollup, vec!["hostname".to_string(), "region".to_string()]); - assert_eq!( - out.aggregation_labels("CountMinSketch", "grouping"), - Vec::::new() - ); - assert_eq!( - out.aggregation_labels("CountMinSketch", "aggregated"), - vec!["datacenter".to_string()] - ); - assert_eq!(out.inference_cleanup_param(q), Some(1)); -} - -/// MIN is Exact → MultipleMinMax only (no DeltaSetAggregator). -/// window = 300 s, cleanup = 1. -#[test] -fn temporal_min() { - let q = "SELECT MIN(cpu_usage) FROM metrics_table WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY datacenter"; - let out = SQLController::from_yaml(&one_query_config(q, 300), sql_opts()) - .unwrap() - .generate() - .unwrap(); - - assert_eq!(out.streaming_aggregation_count(), 1); - assert_eq!(out.inference_query_count(), 1); - assert!(out.has_aggregation_type("MultipleMinMax")); - assert!(!out.has_aggregation_type("DeltaSetAggregator")); - assert!(out.all_tumbling_window_sizes_eq(300)); - assert_eq!( - out.aggregation_table_name("MultipleMinMax"), - Some("metrics_table".to_string()) - ); - assert_eq!( - out.aggregation_value_column("MultipleMinMax"), - Some("cpu_usage".to_string()) - ); - let mut rollup = out.aggregation_labels("MultipleMinMax", "rollup"); - rollup.sort(); - assert_eq!(rollup, vec!["hostname".to_string(), "region".to_string()]); - assert_eq!( - out.aggregation_labels("MultipleMinMax", "grouping"), - Vec::::new() - ); - assert_eq!( - out.aggregation_labels("MultipleMinMax", "aggregated"), - vec!["datacenter".to_string()] - ); - assert_eq!(out.inference_cleanup_param(q), Some(1)); -} - -/// MAX is Exact → MultipleMinMax only (identical plan to MIN). -/// window = 300 s, cleanup = 1. -#[test] -fn temporal_max() { - let q = "SELECT MAX(cpu_usage) FROM metrics_table WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY datacenter"; - let out = SQLController::from_yaml(&one_query_config(q, 300), sql_opts()) - .unwrap() - .generate() - .unwrap(); - - assert_eq!(out.streaming_aggregation_count(), 1); - assert_eq!(out.inference_query_count(), 1); - assert!(out.has_aggregation_type("MultipleMinMax")); - assert!(!out.has_aggregation_type("DeltaSetAggregator")); - assert!(out.all_tumbling_window_sizes_eq(300)); - assert_eq!( - out.aggregation_table_name("MultipleMinMax"), - Some("metrics_table".to_string()) - ); - assert_eq!( - out.aggregation_value_column("MultipleMinMax"), - Some("cpu_usage".to_string()) - ); - let mut rollup = out.aggregation_labels("MultipleMinMax", "rollup"); - rollup.sort(); - assert_eq!(rollup, vec!["hostname".to_string(), "region".to_string()]); - assert_eq!( - out.aggregation_labels("MultipleMinMax", "grouping"), - Vec::::new() - ); - assert_eq!( - out.aggregation_labels("MultipleMinMax", "aggregated"), - vec!["datacenter".to_string()] - ); - assert_eq!(out.inference_cleanup_param(q), Some(1)); -} - -/// AVG decomposes into SUM + COUNT → 2 CountMinSketch + 1 DeltaSetAggregator (deduped) = 3 total. -/// window = 300 s, cleanup = 1. -#[test] -fn temporal_avg() { - let q = "SELECT AVG(cpu_usage) FROM metrics_table WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY datacenter"; - let out = SQLController::from_yaml(&one_query_config(q, 300), sql_opts()) - .unwrap() - .generate() - .unwrap(); - - assert_eq!(out.streaming_aggregation_count(), 3); - assert_eq!(out.inference_query_count(), 1); - assert!(out.has_aggregation_type("CountMinSketch")); - assert!(out.has_aggregation_type("DeltaSetAggregator")); - assert!(out.all_tumbling_window_sizes_eq(300)); - assert_eq!( - out.aggregation_table_name("CountMinSketch"), - Some("metrics_table".to_string()) - ); - assert_eq!( - out.aggregation_value_column("CountMinSketch"), - Some("cpu_usage".to_string()) - ); - let mut rollup = out.aggregation_labels("CountMinSketch", "rollup"); - rollup.sort(); - assert_eq!(rollup, vec!["hostname".to_string(), "region".to_string()]); - assert_eq!( - out.aggregation_labels("CountMinSketch", "grouping"), - Vec::::new() - ); - assert_eq!( - out.aggregation_labels("CountMinSketch", "aggregated"), - vec!["datacenter".to_string()] - ); - assert_eq!(out.inference_cleanup_param(q), Some(1)); -} - -/// QUANTILE (Clickhouse parametric syntax) → DatasketchesKLL only (no DeltaSetAggregator). -/// window = 300 s, cleanup = 1. -#[test] -fn temporal_quantile() { - let q = "SELECT quantile(0.95)(cpu_usage) FROM metrics_table WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY datacenter"; - let out = SQLController::from_yaml(&one_query_config(q, 300), sql_opts()) - .unwrap() - .generate() - .unwrap(); - - assert_eq!(out.streaming_aggregation_count(), 1); - assert_eq!(out.inference_query_count(), 1); - assert!(out.has_aggregation_type("DatasketchesKLL")); - assert!(!out.has_aggregation_type("DeltaSetAggregator")); - assert!(out.all_tumbling_window_sizes_eq(300)); - assert_eq!( - out.aggregation_table_name("DatasketchesKLL"), - Some("metrics_table".to_string()) - ); - assert_eq!( - out.aggregation_value_column("DatasketchesKLL"), - Some("cpu_usage".to_string()) - ); - let mut rollup = out.aggregation_labels("DatasketchesKLL", "rollup"); - rollup.sort(); - assert_eq!(rollup, vec!["hostname".to_string(), "region".to_string()]); - assert_eq!( - out.aggregation_labels("DatasketchesKLL", "grouping"), - vec!["datacenter".to_string()] - ); - assert_eq!( - out.aggregation_labels("DatasketchesKLL", "aggregated"), - Vec::::new() - ); - assert_eq!(out.inference_cleanup_param(q), Some(1)); -} - -// ── Elastic SQL syntax variants ─────────────────────────────────────────────── -// -// These three tests exercise syntactic forms used by Elastic: -// 1. PERCENTILE(col, integer_percentile) — integer 0–100, col-first arg order -// 2. DATEADD('s', …) — quoted unit string -// 3. CAST('…' AS DATETIME) — absolute timestamp bounds -// -// All produce the same plan as temporal_quantile: DatasketchesKLL, no -// DeltaSetAggregator, one streaming config, window = T = 300 s, cleanup = 1. - -/// PERCENTILE(col, 95) is the Elastic aggregation syntax. -/// The parser normalises it to QUANTILE internally, so the plan is identical -/// to temporal_quantile (DatasketchesKLL, grouping = [datacenter], rollup = [hostname, region]). -#[test] -fn temporal_quantile_percentile_syntax() { - let q = "SELECT PERCENTILE(cpu_usage, 95) FROM metrics_table WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY datacenter"; - let out = SQLController::from_yaml(&one_query_config(q, 300), sql_opts()) - .unwrap() - .generate() - .unwrap(); - - assert_eq!(out.streaming_aggregation_count(), 1); - assert_eq!(out.inference_query_count(), 1); - assert!(out.has_aggregation_type("DatasketchesKLL")); - assert!(!out.has_aggregation_type("DeltaSetAggregator")); - assert!(out.all_tumbling_window_sizes_eq(300)); - assert_eq!( - out.aggregation_table_name("DatasketchesKLL"), - Some("metrics_table".to_string()) - ); - assert_eq!( - out.aggregation_value_column("DatasketchesKLL"), - Some("cpu_usage".to_string()) - ); - let mut rollup = out.aggregation_labels("DatasketchesKLL", "rollup"); - rollup.sort(); - assert_eq!(rollup, vec!["hostname".to_string(), "region".to_string()]); - assert_eq!( - out.aggregation_labels("DatasketchesKLL", "grouping"), - vec!["datacenter".to_string()] - ); - assert_eq!( - out.aggregation_labels("DatasketchesKLL", "aggregated"), - Vec::::new() - ); - assert_eq!(out.inference_cleanup_param(q), Some(1)); -} - -/// DATEADD('s', …) — quoted unit string (Elastic style) vs unquoted `s`. -/// The parser accepts both forms; the plan must be identical to temporal_quantile. -#[test] -fn temporal_quantile_quoted_dateadd_unit() { - let q = "SELECT PERCENTILE(cpu_usage, 95) FROM metrics_table WHERE time BETWEEN DATEADD('s', -300, NOW()) AND NOW() GROUP BY datacenter"; - let out = SQLController::from_yaml(&one_query_config(q, 300), sql_opts()) - .unwrap() - .generate() - .unwrap(); - - assert_eq!(out.streaming_aggregation_count(), 1); - assert_eq!(out.inference_query_count(), 1); - assert!(out.has_aggregation_type("DatasketchesKLL")); - assert!(!out.has_aggregation_type("DeltaSetAggregator")); - assert!(out.all_tumbling_window_sizes_eq(300)); - assert_eq!( - out.aggregation_table_name("DatasketchesKLL"), - Some("metrics_table".to_string()) - ); - assert_eq!( - out.aggregation_value_column("DatasketchesKLL"), - Some("cpu_usage".to_string()) - ); - let mut rollup = out.aggregation_labels("DatasketchesKLL", "rollup"); - rollup.sort(); - assert_eq!(rollup, vec!["hostname".to_string(), "region".to_string()]); - assert_eq!( - out.aggregation_labels("DatasketchesKLL", "grouping"), - vec!["datacenter".to_string()] - ); - assert_eq!( - out.aggregation_labels("DatasketchesKLL", "aggregated"), - Vec::::new() - ); - assert_eq!(out.inference_cleanup_param(q), Some(1)); -} - -/// Full Elastic syntax: PERCENTILE(col, N) + DATEADD('s', …) + CAST('…' AS DATETIME). -/// Absolute timestamp bounds replace NOW(); the 300 s range still yields cleanup = 1. -#[test] -fn temporal_quantile_cast_datetime_bounds() { - let q = "SELECT PERCENTILE(cpu_usage, 95) FROM metrics_table WHERE time BETWEEN DATEADD('s', -300, CAST('2024-01-01T00:05:00Z' AS DATETIME)) AND CAST('2024-01-01T00:05:00Z' AS DATETIME) GROUP BY datacenter"; - let out = SQLController::from_yaml(&one_query_config(q, 300), sql_opts()) - .unwrap() - .generate() - .unwrap(); - - assert_eq!(out.streaming_aggregation_count(), 1); - assert_eq!(out.inference_query_count(), 1); - assert!(out.has_aggregation_type("DatasketchesKLL")); - assert!(!out.has_aggregation_type("DeltaSetAggregator")); - assert!(out.all_tumbling_window_sizes_eq(300)); - assert_eq!( - out.aggregation_table_name("DatasketchesKLL"), - Some("metrics_table".to_string()) - ); - assert_eq!( - out.aggregation_value_column("DatasketchesKLL"), - Some("cpu_usage".to_string()) - ); - let mut rollup = out.aggregation_labels("DatasketchesKLL", "rollup"); - rollup.sort(); - assert_eq!(rollup, vec!["hostname".to_string(), "region".to_string()]); - assert_eq!( - out.aggregation_labels("DatasketchesKLL", "grouping"), - vec!["datacenter".to_string()] - ); - assert_eq!( - out.aggregation_labels("DatasketchesKLL", "aggregated"), - Vec::::new() - ); - assert_eq!(out.inference_cleanup_param(q), Some(1)); -} - -// ── T-value variants for SUM (range = 300 s fixed) ─────────────────────────── -// -// These three tests use the same query and differ only in repetition_delay (T). -// Window size = T (the repetition delay), not the query range. -// They verify cleanup scales correctly with T and that T > range is valid. - -/// T = 30 s < range: window = 30, cleanup = ceil(300 / 30) = 10. -#[test] -fn temporal_sum_t30() { - let q = "SELECT SUM(cpu_usage) FROM metrics_table WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY datacenter"; - let out = SQLController::from_yaml(&one_query_config(q, 30), sql_opts()) - .unwrap() - .generate() - .unwrap(); - - assert_eq!(out.streaming_aggregation_count(), 2); - assert_eq!(out.inference_query_count(), 1); - assert!(out.has_aggregation_type("CountMinSketch")); - assert!(out.has_aggregation_type("DeltaSetAggregator")); - assert!(out.all_tumbling_window_sizes_eq(30)); - assert_eq!( - out.aggregation_table_name("CountMinSketch"), - Some("metrics_table".to_string()) - ); - assert_eq!( - out.aggregation_value_column("CountMinSketch"), - Some("cpu_usage".to_string()) - ); - let mut rollup = out.aggregation_labels("CountMinSketch", "rollup"); - rollup.sort(); - assert_eq!(rollup, vec!["hostname".to_string(), "region".to_string()]); - assert_eq!( - out.aggregation_labels("CountMinSketch", "grouping"), - Vec::::new() - ); - assert_eq!( - out.aggregation_labels("CountMinSketch", "aggregated"), - vec!["datacenter".to_string()] - ); - assert_eq!(out.inference_cleanup_param(q), Some(10)); -} - -/// T = 300 s == range: covered by temporal_sum above (cleanup = 1). -/// T = 600 s > range: window = 600, cleanup = ceil(300 / 600) = 1. -/// T larger than the query range is valid; the result is still 1 retained window. -#[test] -fn temporal_sum_t600() { - let q = "SELECT SUM(cpu_usage) FROM metrics_table WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY datacenter"; - let out = SQLController::from_yaml(&one_query_config(q, 600), sql_opts()) - .unwrap() - .generate() - .unwrap(); - - assert_eq!(out.streaming_aggregation_count(), 2); - assert_eq!(out.inference_query_count(), 1); - assert!(out.has_aggregation_type("CountMinSketch")); - assert!(out.has_aggregation_type("DeltaSetAggregator")); - assert!(out.all_tumbling_window_sizes_eq(600)); - assert_eq!( - out.aggregation_table_name("CountMinSketch"), - Some("metrics_table".to_string()) - ); - assert_eq!( - out.aggregation_value_column("CountMinSketch"), - Some("cpu_usage".to_string()) - ); - let mut rollup = out.aggregation_labels("CountMinSketch", "rollup"); - rollup.sort(); - assert_eq!(rollup, vec!["hostname".to_string(), "region".to_string()]); - assert_eq!( - out.aggregation_labels("CountMinSketch", "grouping"), - Vec::::new() - ); - assert_eq!( - out.aggregation_labels("CountMinSketch", "aggregated"), - vec!["datacenter".to_string()] - ); - assert_eq!(out.inference_cleanup_param(q), Some(1)); -} - -// ── multi-query tests ───────────────────────────────────────────────────────── - -/// MIN and MAX: distinct sub_types → 2 streaming configs (one "min", one "max"), 2 inference entries. -#[test] -fn two_queries_min_and_max_produce_separate_streaming_configs() { - let yaml = r#" -tables: - - name: metrics_table - time_column: time - value_columns: [cpu_usage] - metadata_columns: [hostname, datacenter, region] -query_groups: - - id: 1 - repetition_delay: 300 - controller_options: - accuracy_sla: 0.95 - latency_sla: 100.0 - queries: - - >- - SELECT MIN(cpu_usage) FROM metrics_table WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY datacenter - - id: 2 - repetition_delay: 300 - controller_options: - accuracy_sla: 0.95 - latency_sla: 100.0 - queries: - - >- - SELECT MAX(cpu_usage) FROM metrics_table WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY datacenter -aggregate_cleanup: - policy: read_based -"#; - let out = SQLController::from_yaml(yaml, sql_opts()) - .unwrap() - .generate() - .unwrap(); - - assert_eq!(out.streaming_aggregation_count(), 2); - assert_eq!(out.inference_query_count(), 2); - assert!(out.has_aggregation_type_and_sub_type("MultipleMinMax", "min")); - assert!(out.has_aggregation_type_and_sub_type("MultipleMinMax", "max")); -} - -/// Two SUM queries on different value columns: neither pair deduped (different value_column). -/// 2 × (CMS + DeltaSet) = 4 configs, 2 inference entries. -#[test] -fn two_queries_different_value_columns_produce_four_configs() { - let yaml = r#" -tables: - - name: metrics_table - time_column: time - value_columns: [cpu_usage, memory_usage] - metadata_columns: [hostname, datacenter, region] -query_groups: - - id: 1 - repetition_delay: 300 - controller_options: - accuracy_sla: 0.95 - latency_sla: 100.0 - queries: - - >- - SELECT SUM(cpu_usage) FROM metrics_table WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY datacenter - - >- - SELECT SUM(memory_usage) FROM metrics_table WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY datacenter -aggregate_cleanup: - policy: read_based -"#; - let out = SQLController::from_yaml(yaml, sql_opts()) - .unwrap() - .generate() - .unwrap(); - - assert_eq!(out.streaming_aggregation_count(), 4); - assert_eq!(out.inference_query_count(), 2); -} - -/// Identical SQL queries in separate groups are rejected as duplicates, same as PromQL. -#[test] -fn identical_queries_across_groups_return_duplicate_error() { - let q = "SELECT SUM(cpu_usage) FROM metrics_table WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY datacenter"; - let yaml = format!( - r#" -tables: - - name: metrics_table - time_column: time - value_columns: [cpu_usage] - metadata_columns: [hostname, datacenter, region] -query_groups: - - id: 1 - repetition_delay: 300 - controller_options: - accuracy_sla: 0.95 - latency_sla: 100.0 - queries: - - >- - {q} - - id: 2 - repetition_delay: 300 - controller_options: - accuracy_sla: 0.95 - latency_sla: 100.0 - queries: - - >- - {q} -aggregate_cleanup: - policy: read_based -"# - ); - let result = SQLController::from_yaml(&yaml, sql_opts()) - .unwrap() - .generate(); - assert!(matches!(result, Err(ControllerError::DuplicateQuery(_)))); -} - -/// SUM queries with same table/column but different window sizes -/// Since they will both produce tumnbling windows of 300, we will only get 2 configs total -#[test] -fn two_queries_different_windows_are_deduped() { - let yaml = r#" -tables: - - name: metrics_table - time_column: time - value_columns: [cpu_usage] - metadata_columns: [hostname, datacenter, region] -query_groups: - - id: 1 - repetition_delay: 300 - controller_options: - accuracy_sla: 0.95 - latency_sla: 100.0 - queries: - - >- - SELECT SUM(cpu_usage) FROM metrics_table WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY datacenter - - id: 2 - repetition_delay: 300 - controller_options: - accuracy_sla: 0.95 - latency_sla: 100.0 - queries: - - >- - SELECT SUM(cpu_usage) FROM metrics_table WHERE time BETWEEN DATEADD(s, -600, NOW()) AND NOW() GROUP BY datacenter -aggregate_cleanup: - policy: read_based -"#; - let out = SQLController::from_yaml(yaml, sql_opts()) - .unwrap() - .generate() - .unwrap(); - - assert_eq!(out.streaming_aggregation_count(), 2); - assert_eq!(out.inference_query_count(), 2); -} - -// ── output-file test ────────────────────────────────────────────────────────── - -/// generate_to_dir writes both streaming_config.yaml and inference_config.yaml. -#[test] -fn generate_to_dir_writes_both_yaml_files() { - let dir = tempfile::tempdir().unwrap(); - let q = "SELECT SUM(cpu_usage) FROM metrics_table WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY datacenter"; - SQLController::from_yaml(&one_query_config(q, 300), sql_opts()) - .unwrap() - .generate_to_dir(dir.path()) - .unwrap(); - assert!(dir.path().join("streaming_config.yaml").exists()); - assert!(dir.path().join("inference_config.yaml").exists()); -} - -// ── error-path tests ────────────────────────────────────────────────────────── - -#[test] -fn malformed_yaml_returns_parse_error() { - let result = SQLController::from_yaml("{ invalid yaml :", sql_opts()); - assert!(matches!(result, Err(ControllerError::YamlParse(_)))); -} - -#[test] -fn malformed_sql_returns_sql_parse_error() { - let q = "NOT VALID SQL AT ALL %%%"; - let result = SQLController::from_yaml(&one_query_config(q, 300), sql_opts()) - .unwrap() - .generate(); - assert!(matches!(result, Err(ControllerError::SqlParse(_)))); -} - -#[test] -fn query_referencing_unknown_table_returns_error() { - let q = "SELECT SUM(cpu_usage) FROM nonexistent_table WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY datacenter"; - let result = SQLController::from_yaml(&one_query_config(q, 300), sql_opts()) - .unwrap() - .generate(); - assert!(matches!( - result, - Err(ControllerError::UnknownTable(_)) | Err(ControllerError::SqlParse(_)) - )); -} - -#[test] -fn query_referencing_unknown_value_column_returns_sql_parse_error() { - let q = "SELECT SUM(nonexistent_col) FROM metrics_table WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY datacenter"; - let result = SQLController::from_yaml(&one_query_config(q, 300), sql_opts()) - .unwrap() - .generate(); - assert!(matches!(result, Err(ControllerError::SqlParse(_)))); -} - -#[test] -fn duplicate_queries_in_same_group_return_error() { - let q = "SELECT SUM(cpu_usage) FROM metrics_table WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY datacenter"; - let yaml = format!( - r#" -tables: - - name: metrics_table - time_column: time - value_columns: [cpu_usage] - metadata_columns: [hostname, datacenter, region] -query_groups: - - id: 1 - repetition_delay: 300 - controller_options: - accuracy_sla: 0.95 - latency_sla: 100.0 - queries: - - >- - {q} - - >- - {q} -aggregate_cleanup: - policy: read_based -"# - ); - let result = SQLController::from_yaml(&yaml, sql_opts()) - .unwrap() - .generate(); - assert!(matches!(result, Err(ControllerError::DuplicateQuery(_)))); -} - -#[test] -fn unknown_cleanup_policy_returns_planner_error() { - let q = "SELECT SUM(cpu_usage) FROM metrics_table WHERE time BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY datacenter"; - let yaml = format!( - r#" -tables: - - name: metrics_table - time_column: time - value_columns: [cpu_usage] - metadata_columns: [hostname, datacenter, region] -query_groups: - - id: 1 - repetition_delay: 300 - controller_options: - accuracy_sla: 0.95 - latency_sla: 100.0 - queries: - - >- - {q} -aggregate_cleanup: - policy: not_a_real_policy -"# - ); - let result = SQLController::from_yaml(&yaml, sql_opts()) - .unwrap() - .generate(); - assert!(matches!(result, Err(ControllerError::PlannerError(_)))); -} - -/// T that is not a multiple of data_ingestion_interval is invalid: sketch windows -/// must align with the ingestion cadence. -/// data_ingestion_interval = 15 s, T = 200 s → 200 mod 15 ≠ 0 → PlannerError. -#[test] -fn t_not_multiple_of_data_ingestion_interval_returns_planner_error() { - let q = "SELECT SUM(cpu_usage) FROM metrics_table WHERE time BETWEEN DATEADD(s, -200, NOW()) AND NOW() GROUP BY datacenter"; - let result = SQLController::from_yaml(&one_query_config(q, 200), sql_opts()) - .unwrap() - .generate(); - assert!(matches!(result, Err(ControllerError::PlannerError(_)))); -} diff --git a/asap-query-engine/Cargo.toml b/asap-query-engine/Cargo.toml index 672988df9..ab384665c 100644 --- a/asap-query-engine/Cargo.toml +++ b/asap-query-engine/Cargo.toml @@ -9,7 +9,6 @@ promql_utilities.workspace = true sql_utilities.workspace = true asap_types.workspace = true datafusion_summary_library.workspace = true -asap_planner.workspace = true # Shared external (workspace) serde.workspace = true diff --git a/asap-query-engine/Dockerfile b/asap-query-engine/Dockerfile index 2401dc4ee..e2305a3a2 100644 --- a/asap-query-engine/Dockerfile +++ b/asap-query-engine/Dockerfile @@ -19,7 +19,6 @@ COPY asap-common ./asap-common COPY Cargo.toml ./ COPY Cargo.lock ./ COPY asap-query-engine/Cargo.toml ./asap-query-engine/ -COPY asap-planner-rs/Cargo.toml ./asap-planner-rs/ # Create dummy source files so Cargo can resolve all workspace members # All explicit [[bin]] targets in Cargo.toml must have stubs here for the dependency cache layer @@ -29,9 +28,7 @@ RUN mkdir -p asap-query-engine/src/bin \ && echo "fn main() {}" > asap-query-engine/src/bin/test_e2e_precompute.rs \ && echo "fn main() {}" > asap-query-engine/src/bin/bench_precompute_sketch.rs \ && echo "fn main() {}" > asap-query-engine/src/bin/e2e_quickstart_resource_test.rs \ - && mkdir -p asap-query-engine/benches && echo "fn main() {}" > asap-query-engine/benches/simple_store_bench.rs \ - && mkdir -p asap-planner-rs/src && echo "fn main() {}" > asap-planner-rs/src/main.rs \ - && echo "pub fn placeholder() {}" >> asap-planner-rs/src/lib.rs + && mkdir -p asap-query-engine/benches && echo "fn main() {}" > asap-query-engine/benches/simple_store_bench.rs # Build dependencies (this layer will be cached) WORKDIR /code/asap-query-engine @@ -39,10 +36,9 @@ RUN cargo build --release && rm -rf src/ # Copy source code COPY asap-query-engine/src ./src -COPY asap-planner-rs/src /code/asap-planner-rs/src # Build the actual application -RUN touch src/main.rs && touch /code/asap-planner-rs/src/lib.rs && cargo build --release +RUN touch src/main.rs && cargo build --release # Runtime stage with Ubuntu 24.04 (has newer glibc/libstdc++) FROM ubuntu:24.04 diff --git a/asap-query-engine/src/bin/precompute_engine.rs b/asap-query-engine/src/bin/precompute_engine.rs index 8b35c3985..667f4c79a 100644 --- a/asap-query-engine/src/bin/precompute_engine.rs +++ b/asap-query-engine/src/bin/precompute_engine.rs @@ -280,7 +280,7 @@ async fn main() -> Result<(), Box> { handle_http_requests: true, adapter_config, }; - let mut http_server = HttpServer::new(http_config, query_engine, store.clone(), None); + let mut http_server = HttpServer::new(http_config, query_engine, store.clone()); // Per-metric storage-backend routing table (issue #46 // criterion ⑤). When provided, the HTTP handler consults this diff --git a/asap-query-engine/src/bin/test_e2e_precompute.rs b/asap-query-engine/src/bin/test_e2e_precompute.rs index 7b62f6586..8a4e96f70 100644 --- a/asap-query-engine/src/bin/test_e2e_precompute.rs +++ b/asap-query-engine/src/bin/test_e2e_precompute.rs @@ -132,7 +132,7 @@ async fn main() -> Result<(), Box> { fallback: None, }, }; - let http_server = HttpServer::new(http_config, query_engine, store.clone(), None); + let http_server = HttpServer::new(http_config, query_engine, store.clone()); tokio::spawn(async move { if let Err(e) = http_server.run().await { eprintln!("Query server error: {e}"); diff --git a/asap-query-engine/src/drivers/query/servers/http.rs b/asap-query-engine/src/drivers/query/servers/http.rs index 71b59e9a4..364a297d3 100644 --- a/asap-query-engine/src/drivers/query/servers/http.rs +++ b/asap-query-engine/src/drivers/query/servers/http.rs @@ -18,7 +18,6 @@ use crate::drivers::query::adapters::{create_http_adapter, AdapterConfig, HttpPr use crate::drivers::query::servers::metrics as srv_metrics; use crate::engines::SimpleEngine; use crate::routing::{EngineRouter, EngineRouterError, QueryEngine}; -use crate::query_tracker::QueryTracker; use crate::stores::Store; use asap_types::{AccuracyTarget, StorageBackend}; use promql_utilities::query_logics::enums::Statistic; @@ -65,7 +64,6 @@ pub struct HttpServer { /// map. See `docs/design-gorilla-s3-cold-engine.md` §8. query_router: Arc, store: Arc, - query_tracker: Option>, /// Hot-reloadable `StreamingConfig` source. `None` when hot-reload /// is not wired up by the caller (unit tests, legacy binaries). hot_reload_config: Option, @@ -116,7 +114,6 @@ struct AppState { /// See [`HttpServer::query_router`]. query_router: Arc, store: Arc, - query_tracker: Option>, adapter: Arc, fallback: Option>, hot_reload_config: Option, @@ -140,7 +137,6 @@ impl HttpServer { config: HttpServerConfig, query_engine: Arc, store: Arc, - query_tracker: Option>, ) -> Self { // Bootstrap the capability router with `SimpleEngine` registered // for the warm-tier (`sketch_warm`) `data_source_id`. Callers @@ -154,7 +150,6 @@ impl HttpServer { query_engine, query_router, store, - query_tracker, hot_reload_config: None, backend_storage_routing: None, schemas: None, @@ -312,7 +307,6 @@ impl HttpServer { query_engine: self.query_engine, query_router: self.query_router, store: self.store, - query_tracker: self.query_tracker, adapter: adapter.clone(), fallback: self.config.adapter_config.fallback.clone(), hot_reload_config: self.hot_reload_config.clone(), @@ -391,7 +385,6 @@ impl HttpServer { query_engine: self.query_engine.clone(), query_router: self.query_router.clone(), store: self.store.clone(), - query_tracker: self.query_tracker.clone(), adapter: adapter.clone(), fallback: self.config.adapter_config.fallback.clone(), hot_reload_config: self.hot_reload_config.clone(), @@ -497,10 +490,11 @@ async fn process_query_request( } } - // Record query for passive auto-discovery (if tracker is enabled) - if let Some(tracker) = &state.query_tracker { - tracker.record_instant(&parsed_request.query, parsed_request.time); - } + // (Phase γ: legacy in-backend query tracker removed — the + // ASAPCollector controller now observes queries via its own + // PromQL scrape side-channel and pushes plans / routing tables + // back to the backend. See README "post-consolidation backend + // scope" prose.) // Phase-6 Fix 1: per-query engine override. // @@ -1359,15 +1353,8 @@ async fn process_range_query_request( }; } - // Record query for passive auto-discovery (if tracker is enabled) - if let Some(tracker) = &state.query_tracker { - tracker.record_range( - &parsed_request.query, - parsed_request.start, - parsed_request.end, - parsed_request.step, - ); - } + // (Phase γ: legacy in-backend query tracker removed — see + // companion comment in `handle_instant_query`.) // Execute range query with engine let query_start_time = Instant::now(); @@ -1556,7 +1543,7 @@ mod tests { crate::data_model::QueryLanguage::promql, )); - let mut server = HttpServer::new(config, query_engine, store, None); + let mut server = HttpServer::new(config, query_engine, store); if let Some(handle) = hot_reload { server = server.with_hot_reload_config(handle); } @@ -1812,7 +1799,7 @@ aggregations: 15000, crate::data_model::QueryLanguage::promql, )); - let server = HttpServer::new(config, query_engine, store, None) + let server = HttpServer::new(config, query_engine, store) .with_hot_reload_config(hot_reload) .with_schemas(schemas); server @@ -2326,7 +2313,7 @@ aggregations: let sc = StreamingConfig::new(map); Arc::new(crate::stores::sketch_db::SchemaRegistry::from_streaming_config(&sc)) }; - let server = HttpServer::new(config, query_engine, store, None) + let server = HttpServer::new(config, query_engine, store) .with_backfill_registry(registry) .with_schemas(schemas); server @@ -2696,7 +2683,7 @@ aggregations: 15000, crate::data_model::QueryLanguage::promql, )); - let mut server = HttpServer::new(config, query_engine, store, None) + let mut server = HttpServer::new(config, query_engine, store) .with_hot_reload_config(hot_reload); for engine in extra_engines { server = server.with_query_engine(engine); @@ -2752,7 +2739,7 @@ aggregations: 15000, crate::data_model::QueryLanguage::promql, )); - let mut server = HttpServer::new(config, query_engine, store, None) + let mut server = HttpServer::new(config, query_engine, store) .with_hot_reload_config(hot_reload) .with_backend_storage_routing(Arc::new(routing)); for engine in extra_engines { @@ -3542,7 +3529,7 @@ aggregations: crate::data_model::QueryLanguage::promql, )); let routing_handle = HotReloadBackendStorageRouting::empty(); - let server = HttpServer::new(config, query_engine, store, None) + let server = HttpServer::new(config, query_engine, store) .with_hot_reload_config(hot_reload) .with_hot_reload_backend_storage_routing(routing_handle.clone()); let port = server.start_test_server().await.expect("start ok"); @@ -3915,7 +3902,7 @@ aggregations: 15000, crate::data_model::QueryLanguage::promql, )); - let mut server = HttpServer::new(config, query_engine, store, None) + let mut server = HttpServer::new(config, query_engine, store) .with_hot_reload_config(hot_reload); for (id, engine) in aliased_engines { server = server.with_query_engine_aliased(id, engine); diff --git a/asap-query-engine/src/lib.rs b/asap-query-engine/src/lib.rs index a353cbf19..47b203fd7 100644 --- a/asap-query-engine/src/lib.rs +++ b/asap-query-engine/src/lib.rs @@ -1,10 +1,8 @@ pub mod data_model; pub mod drivers; pub mod engines; -pub mod planner_client; pub mod precompute_engine; pub mod precompute_operators; -pub mod query_tracker; pub mod routing; pub mod stores; @@ -37,8 +35,6 @@ pub use precompute_engine::config::{LateDataPolicy, PrecomputeEngineConfig}; pub use precompute_engine::output_sink::StoreOutputSink; pub use precompute_engine::PrecomputeEngine; -pub use query_tracker::{QueryTracker, QueryTrackerConfig}; - pub use utils::{normalize_spatial_filter, read_inference_config, read_streaming_config}; pub type Result = std::result::Result>; diff --git a/asap-query-engine/src/main.rs b/asap-query-engine/src/main.rs index 96357fa0e..f0cc1a846 100644 --- a/asap-query-engine/src/main.rs +++ b/asap-query-engine/src/main.rs @@ -213,14 +213,6 @@ struct Args { #[arg(long)] schema_eviction_dry_run: bool, - /// Enable automatic query tracking and planning - #[arg(long)] - enable_query_tracker: bool, - - /// Query tracker: observation window in seconds before triggering planning - #[arg(long, default_value = "100")] - tracker_observation_window_secs: u64, - // ---- SimpleMapStore persistence ---- // // When --persistence-enabled is set, the store is constructed via @@ -624,36 +616,12 @@ async fn main() -> Result<()> { adapter_config, }; - let query_tracker = if args.enable_query_tracker { - use query_engine_rust::planner_client::LocalPlannerClient; - use query_engine_rust::QueryTrackerConfig; - - let tracker_config = QueryTrackerConfig { - observation_window_secs: args.tracker_observation_window_secs, - prometheus_scrape_interval: args.prometheus_scrape_interval, - }; - let runtime_options = asap_planner::RuntimeOptions { - prometheus_scrape_interval: args.prometheus_scrape_interval, - streaming_engine: asap_planner::StreamingEngine::Precompute, - enable_punting: false, - range_duration: 300, - step: args.prometheus_scrape_interval, - }; - let planner_client = Arc::new(LocalPlannerClient::new( - runtime_options, - args.query_language, - args.prometheus_server.clone(), - )); - let tracker = Arc::new(query_engine_rust::QueryTracker::new(tracker_config)); - let _tracker_handle = tracker.start_background_loop(planner_client); - info!( - "Query tracker enabled (observation window: {}s)", - args.tracker_observation_window_secs - ); - Some(tracker) - } else { - None - }; + // The legacy in-backend query tracker / LocalPlannerClient was + // removed in Phase γ (deletion of `asap-planner-rs`). The + // ASAPCollector controller is now the sole emitter of streaming + // configs / `BackendStorageRouting`; the backend is a pure + // executor that consumes plans pushed via + // `POST /api/v1/streaming-config` and `POST /api/v1/storage_routing`. // Forward the precompute engine's schema registry to the HTTP // server so `POST /api/v1/streaming-config` can drive schema @@ -661,7 +629,7 @@ async fn main() -> Result<()> { // design, §6). When precompute isn't enabled, the registry is // absent and the swap handler no-ops on schema reconciliation // (legacy per-batch reconcile in ingest still works). - let mut server = HttpServer::new(http_config, engine, store.clone(), query_tracker) + let mut server = HttpServer::new(http_config, engine, store.clone()) .with_hot_reload_config(hot_reload_config.clone()); // Per-metric storage-backend routing table (issue #46 diff --git a/asap-query-engine/src/planner_client.rs b/asap-query-engine/src/planner_client.rs deleted file mode 100644 index f1c5b3029..000000000 --- a/asap-query-engine/src/planner_client.rs +++ /dev/null @@ -1,180 +0,0 @@ -use anyhow::Result; -use asap_planner::{ - build_schema_from_prometheus, Controller, ControllerConfig, PlannerOutput, RuntimeOptions, -}; -use asap_types::enums::QueryLanguage; -use asap_types::inference_config::InferenceConfig; -use asap_types::streaming_config::StreamingConfig; -use tracing::warn; - -pub struct PlannerResult { - pub streaming_config: StreamingConfig, - pub inference_config: InferenceConfig, - pub punted_queries: Vec, -} - -#[async_trait::async_trait] -pub trait PlannerClient: Send + Sync { - async fn plan(&self, config: ControllerConfig) -> Result; -} - -pub struct LocalPlannerClient { - runtime_options: RuntimeOptions, - query_language: QueryLanguage, - prometheus_url: String, -} - -impl LocalPlannerClient { - pub fn new( - runtime_options: RuntimeOptions, - query_language: QueryLanguage, - prometheus_url: String, - ) -> Self { - Self { - runtime_options, - query_language, - prometheus_url, - } - } -} - -#[async_trait::async_trait] -impl PlannerClient for LocalPlannerClient { - async fn plan(&self, config: ControllerConfig) -> Result { - let opts = self.runtime_options.clone(); - let query_language = self.query_language; - let prometheus_url = self.prometheus_url.clone(); - - let output: PlannerOutput = tokio::task::spawn_blocking(move || { - let all_queries: Vec = config - .query_groups - .iter() - .flat_map(|qg| qg.queries.clone()) - .collect(); - let mut schema = match build_schema_from_prometheus(&prometheus_url, &all_queries) { - Ok(s) => s, - Err(e) => { - warn!( - "Prometheus metric discovery failed, falling back to config hints: {}", - e - ); - config.schema_from_hints() - } - }; - // Fall back to config-file hints for metrics not found in Prometheus. - if let Some(metric_hints) = &config.metrics { - for hint in metric_hints { - if !schema.config.contains_key(&hint.metric) { - schema = schema.add_metric( - hint.metric.clone(), - promql_utilities::data_model::KeyByLabelNames::new(hint.labels.clone()), - ); - } - } - } - let controller = Controller::new(config, schema, opts); - controller.generate() - }) - .await??; - - let inference_config = output.to_inference_config(query_language)?; - let streaming_config = output.to_streaming_config(query_language)?; - let punted_queries = output - .punted_queries - .iter() - .map(|p| p.query.clone()) - .collect(); - - Ok(PlannerResult { - streaming_config, - inference_config, - punted_queries, - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_planner::config::input::{ControllerOptions, MetricDefinition, QueryGroup}; - use asap_planner::ControllerConfig; - use asap_planner::{RuntimeOptions, StreamingEngine}; - - fn sample_controller_config() -> ControllerConfig { - ControllerConfig { - query_groups: vec![QueryGroup { - id: Some(1), - queries: vec!["sum(rate(http_requests_total[5m]))".to_string()], - repetition_delay: 60, - controller_options: ControllerOptions::default(), - step: None, - range_duration: None, - }], - metrics: Some(vec![MetricDefinition { - metric: "http_requests_total".to_string(), - labels: vec!["method".to_string(), "status".to_string()], - }]), - sketch_parameters: None, - aggregate_cleanup: None, - } - } - - fn sample_runtime_options() -> RuntimeOptions { - RuntimeOptions { - prometheus_scrape_interval: 15, - streaming_engine: StreamingEngine::Precompute, - enable_punting: false, - range_duration: 300, - step: 15, - } - } - - #[test] - fn test_controller_new_generate() { - let config = sample_controller_config(); - let schema = config.schema_from_hints(); - let opts = sample_runtime_options(); - let controller = Controller::new(config, schema, opts); - let output = controller.generate().expect("generate should succeed"); - - assert!(output.streaming_aggregation_count() > 0); - assert!(output.inference_query_count() > 0); - } - - #[test] - fn test_planner_output_struct_accessors() { - let config = sample_controller_config(); - let schema = config.schema_from_hints(); - let opts = sample_runtime_options(); - let controller = Controller::new(config, schema, opts); - let output = controller.generate().expect("generate should succeed"); - - let inference = output - .to_inference_config(QueryLanguage::promql) - .expect("to_inference_config should succeed"); - assert!(!inference.query_configs.is_empty()); - - let streaming = output - .to_streaming_config(QueryLanguage::promql) - .expect("to_streaming_config should succeed"); - assert!(!streaming.aggregation_configs.is_empty()); - } - - #[tokio::test] - async fn test_local_planner_client() { - // Use a dummy URL; the test config has metric hints so Prometheus discovery - // failures are tolerated via the fallback path. - let client = LocalPlannerClient::new( - sample_runtime_options(), - QueryLanguage::promql, - "http://localhost:9090".to_string(), - ); - let config = sample_controller_config(); - - let result = client.plan(config).await.expect("plan should succeed"); - - assert!(!result.streaming_config.aggregation_configs.is_empty()); - assert!(!result.inference_config.query_configs.is_empty()); - assert!(result.punted_queries.is_empty()); - } -} diff --git a/asap-query-engine/src/query_tracker/mod.rs b/asap-query-engine/src/query_tracker/mod.rs deleted file mode 100644 index f0043cd7f..000000000 --- a/asap-query-engine/src/query_tracker/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -mod tracker; - -pub use tracker::{QueryTracker, QueryTrackerConfig}; diff --git a/asap-query-engine/src/query_tracker/tracker.rs b/asap-query-engine/src/query_tracker/tracker.rs deleted file mode 100644 index f2323e7ef..000000000 --- a/asap-query-engine/src/query_tracker/tracker.rs +++ /dev/null @@ -1,241 +0,0 @@ -use std::sync::{Arc, Mutex}; -use std::time::Duration; - -use asap_planner::query_log::{infer_queries, to_controller_config, LogEntry}; -use chrono::{DateTime, Utc}; -use tokio::task::JoinHandle; -use tracing::{info, warn}; - -use crate::planner_client::PlannerClient; - -#[derive(Debug, Clone)] -pub struct QueryTrackerConfig { - /// How often to evaluate and trigger planning (default: 600s = 10 min). - pub observation_window_secs: u64, - /// Prometheus scrape interval, passed through to `infer_queries`. - pub prometheus_scrape_interval: u64, -} - -pub struct QueryTracker { - entries: Mutex>, - config: QueryTrackerConfig, -} - -impl QueryTracker { - pub fn new(config: QueryTrackerConfig) -> Self { - Self { - entries: Mutex::new(Vec::new()), - config, - } - } - - /// Record an instant query (step=0, start==end==time). - pub fn record_instant(&self, query: &str, time: f64) { - let ts = Utc::now(); - let query_time = epoch_secs_to_datetime(time); - let entry = LogEntry { - query: query.to_string(), - start: query_time, - end: query_time, - step: 0, - ts, - }; - self.entries.lock().unwrap().push(entry); - } - - /// Record a range query with start/end/step from the request. - pub fn record_range(&self, query: &str, start: f64, end: f64, step: f64) { - let ts = Utc::now(); - let entry = LogEntry { - query: query.to_string(), - start: epoch_secs_to_datetime(start), - end: epoch_secs_to_datetime(end), - step: step as u64, - ts, - }; - self.entries.lock().unwrap().push(entry); - } - - /// Spawn a background task that periodically evaluates collected queries and calls the planner. - pub fn start_background_loop( - self: &Arc, - planner_client: Arc, - ) -> JoinHandle<()> { - let tracker = Arc::clone(self); - tokio::spawn(async move { - let mut interval = - tokio::time::interval(Duration::from_secs(tracker.config.observation_window_secs)); - // The first tick completes immediately; skip it so we wait a full window first. - interval.tick().await; - - loop { - interval.tick().await; - Self::evaluate(&tracker, &planner_client).await; - } - }) - } - - async fn evaluate(tracker: &Arc, planner_client: &Arc) { - // Snapshot and drain collected entries. - let entries = { - let mut guard = tracker.entries.lock().unwrap(); - std::mem::take(&mut *guard) - }; - - if entries.is_empty() { - info!("query_tracker: no queries observed in this window, skipping"); - return; - } - - info!( - "query_tracker: evaluating {} observed query entries", - entries.len() - ); - - let scrape_interval = tracker.config.prometheus_scrape_interval; - let (instants, ranges) = infer_queries(&entries, scrape_interval); - - info!( - "query_tracker: inferred {} instant queries, {} range queries", - instants.len(), - ranges.len() - ); - - if instants.is_empty() && ranges.is_empty() { - info!("query_tracker: no queries met frequency threshold, skipping planner call"); - return; - } - - let controller_config = to_controller_config(instants, ranges); - - info!( - "query_tracker: calling planner with {} query groups", - controller_config.query_groups.len() - ); - - match planner_client.plan(controller_config).await { - Ok(result) => { - info!( - "query_tracker: planner succeeded — streaming aggregations: {}, inference queries: {}, punted: {}", - result.streaming_config.aggregation_configs.len(), - result.inference_config.query_configs.len(), - result.punted_queries.len(), - ); - } - Err(e) => { - warn!("query_tracker: planner failed: {}", e); - } - } - } -} - -fn epoch_secs_to_datetime(secs: f64) -> DateTime { - DateTime::from_timestamp(secs as i64, ((secs.fract()) * 1_000_000_000.0) as u32) - .unwrap_or_else(Utc::now) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::planner_client::PlannerResult; - use anyhow::Result; - use asap_types::inference_config::InferenceConfig; - use asap_types::streaming_config::StreamingConfig; - use std::collections::HashMap; - use std::sync::atomic::{AtomicUsize, Ordering}; - - struct MockPlannerClient { - call_count: AtomicUsize, - } - - impl MockPlannerClient { - fn new() -> Self { - Self { - call_count: AtomicUsize::new(0), - } - } - } - - #[async_trait::async_trait] - impl PlannerClient for MockPlannerClient { - async fn plan(&self, _config: asap_planner::ControllerConfig) -> Result { - self.call_count.fetch_add(1, Ordering::SeqCst); - Ok(PlannerResult { - streaming_config: StreamingConfig::new(HashMap::new()), - inference_config: InferenceConfig::new( - asap_types::enums::QueryLanguage::promql, - asap_types::enums::CleanupPolicy::NoCleanup, - ), - punted_queries: vec![], - }) - } - } - - #[test] - fn record_instant_appends_entry() { - let tracker = QueryTracker::new(QueryTrackerConfig { - observation_window_secs: 600, - prometheus_scrape_interval: 15, - }); - tracker.record_instant("rate(http_requests_total[5m])", 1700000000.0); - tracker.record_instant("rate(http_requests_total[5m])", 1700000060.0); - let entries = tracker.entries.lock().unwrap(); - assert_eq!(entries.len(), 2); - assert_eq!(entries[0].step, 0); - assert_eq!(entries[0].start, entries[0].end); - } - - #[test] - fn record_range_appends_entry() { - let tracker = QueryTracker::new(QueryTrackerConfig { - observation_window_secs: 600, - prometheus_scrape_interval: 15, - }); - tracker.record_range( - "rate(http_requests_total[5m])", - 1700000000.0, - 1700003600.0, - 30.0, - ); - let entries = tracker.entries.lock().unwrap(); - assert_eq!(entries.len(), 1); - assert_eq!(entries[0].step, 30); - assert_ne!(entries[0].start, entries[0].end); - } - - #[tokio::test] - async fn evaluate_calls_planner_with_entries() { - let tracker = Arc::new(QueryTracker::new(QueryTrackerConfig { - observation_window_secs: 600, - prometheus_scrape_interval: 15, - })); - - // Record enough entries for infer_queries to produce results (need >=2 per query). - for i in 0..5 { - tracker.record_instant( - "rate(http_requests_total[5m])", - 1700000000.0 + (i as f64 * 60.0), - ); - } - - let mock_client = Arc::new(MockPlannerClient::new()); - QueryTracker::evaluate(&tracker, &(mock_client.clone() as Arc)).await; - - assert_eq!(mock_client.call_count.load(Ordering::SeqCst), 1); - // Entries should be drained after evaluate. - assert!(tracker.entries.lock().unwrap().is_empty()); - } - - #[tokio::test] - async fn evaluate_skips_when_no_entries() { - let tracker = Arc::new(QueryTracker::new(QueryTrackerConfig { - observation_window_secs: 600, - prometheus_scrape_interval: 15, - })); - - let mock_client = Arc::new(MockPlannerClient::new()); - QueryTracker::evaluate(&tracker, &(mock_client.clone() as Arc)).await; - - assert_eq!(mock_client.call_count.load(Ordering::SeqCst), 0); - } -} diff --git a/asap-query-engine/src/tests/capability_miss_http_e2e_tests.rs b/asap-query-engine/src/tests/capability_miss_http_e2e_tests.rs index 8bbadbb09..2122e2fc7 100644 --- a/asap-query-engine/src/tests/capability_miss_http_e2e_tests.rs +++ b/asap-query-engine/src/tests/capability_miss_http_e2e_tests.rs @@ -183,7 +183,7 @@ async fn start_backend(controller_url: String, hot_reload: HotReloadStreamingCon adapter_config, }; let server = - HttpServer::new(config, engine, store, None).with_hot_reload_config(hot_reload.clone()); + HttpServer::new(config, engine, store).with_hot_reload_config(hot_reload.clone()); server .start_test_server() .await diff --git a/asap-query-engine/src/tests/clickhouse_forwarding_tests.rs b/asap-query-engine/src/tests/clickhouse_forwarding_tests.rs index 769974dfe..02de1e782 100644 --- a/asap-query-engine/src/tests/clickhouse_forwarding_tests.rs +++ b/asap-query-engine/src/tests/clickhouse_forwarding_tests.rs @@ -69,7 +69,7 @@ async fn setup_test_server(clickhouse_port: u16, database: &str) -> (HttpServer, QueryLanguage::sql, )); - let server = HttpServer::new(config, query_engine, store, None); + let server = HttpServer::new(config, query_engine, store); let actual_port = server .start_test_server() .await diff --git a/asap-query-engine/src/tests/elastic_forwarding_tests.rs b/asap-query-engine/src/tests/elastic_forwarding_tests.rs index 714f439a1..b7bb7f80c 100644 --- a/asap-query-engine/src/tests/elastic_forwarding_tests.rs +++ b/asap-query-engine/src/tests/elastic_forwarding_tests.rs @@ -137,7 +137,7 @@ async fn setup_test_server(elasticsearch_port: u16, index: &str) -> (HttpServer, QueryLanguage::elastic_querydsl, )); - let server = HttpServer::new(config, query_engine, store, None); + let server = HttpServer::new(config, query_engine, store); let actual_port = server .start_test_server() .await @@ -173,7 +173,7 @@ async fn setup_test_server_sql(elasticsearch_port: u16, index: &str) -> (HttpSer QueryLanguage::elastic_sql, )); - let server = HttpServer::new(config, query_engine, store, None); + let server = HttpServer::new(config, query_engine, store); let actual_port = server .start_test_server() .await @@ -408,7 +408,7 @@ async fn test_forwarding_disabled() { QueryLanguage::elastic_querydsl, )); - let server = HttpServer::new(config, query_engine, store, None); + let server = HttpServer::new(config, query_engine, store); let server_port = server .start_test_server() .await @@ -576,7 +576,7 @@ async fn test_sql_forwarding_disabled() { QueryLanguage::elastic_sql, )); - let server = HttpServer::new(config, query_engine, store, None); + let server = HttpServer::new(config, query_engine, store); let server_port = server .start_test_server() .await diff --git a/asap-query-engine/src/tests/prometheus_forwarding_tests.rs b/asap-query-engine/src/tests/prometheus_forwarding_tests.rs index e711357b8..1796d63da 100644 --- a/asap-query-engine/src/tests/prometheus_forwarding_tests.rs +++ b/asap-query-engine/src/tests/prometheus_forwarding_tests.rs @@ -88,7 +88,7 @@ async fn setup_test_server(prometheus_port: u16) -> (HttpServer, u16) { crate::data_model::QueryLanguage::promql, )); - let server = HttpServer::new(config, query_engine, store, None); + let server = HttpServer::new(config, query_engine, store); let actual_port = server .start_test_server() .await @@ -182,7 +182,7 @@ async fn test_forwarding_disabled() { crate::data_model::QueryLanguage::promql, )); - let server = HttpServer::new(config, query_engine, store, None); + let server = HttpServer::new(config, query_engine, store); let server_port = server .start_test_server() .await @@ -238,7 +238,7 @@ async fn test_prometheus_server_unreachable() { crate::data_model::QueryLanguage::promql, )); - let server = HttpServer::new(config, query_engine, store, None); + let server = HttpServer::new(config, query_engine, store); let server_port = server .start_test_server() .await diff --git a/asap-quickstart/Dockerfile.queryengine-local b/asap-quickstart/Dockerfile.queryengine-local index beb1f137a..03167152e 100644 --- a/asap-quickstart/Dockerfile.queryengine-local +++ b/asap-quickstart/Dockerfile.queryengine-local @@ -1,12 +1,12 @@ # Lightweight image that copies a pre-built query engine binary. # -# `reqwest` was switched to rustls-tls (see query_engine_rust/Cargo.toml -# and asap-planner-rs/Cargo.toml), so the binary no longer links against -# libssl at runtime. Any modern glibc base (Ubuntu 22.04+ or Debian 12+) -# works; we pin the LTS 24.04 for a stable default. `ca-certificates` -# is only needed if outbound TLS is made to hosts whose leaf certs -# aren't chained via webpki-roots — rustls-tls embeds Mozilla's bundle, -# so it's belt-and-suspenders for the common case. +# `reqwest` was switched to rustls-tls (see query_engine_rust/Cargo.toml), +# so the binary no longer links against libssl at runtime. Any modern +# glibc base (Ubuntu 22.04+ or Debian 12+) works; we pin the LTS 24.04 +# for a stable default. `ca-certificates` is only needed if outbound TLS +# is made to hosts whose leaf certs aren't chained via webpki-roots — +# rustls-tls embeds Mozilla's bundle, so it's belt-and-suspenders for +# the common case. FROM ubuntu:24.04 WORKDIR /app diff --git a/asap-quickstart/README.md b/asap-quickstart/README.md index e01424194..949bbb766 100644 --- a/asap-quickstart/README.md +++ b/asap-quickstart/README.md @@ -16,7 +16,15 @@ Then it adds ASAPQuery's components on top: - **Query Engine** - Prometheus-compatible API with sketch-based acceleration - **[Arroyo](https://github.com/ProjectASAP/arroyo) + asap-summary-ingest** - Streaming engine with pipelines configured for building sketches - **Kafka** - Message broker for streaming data from Arroyo to the Query Engine -- **asap-planner-rs** - Automatically configures sketches from PromQL queries +- **Plan emitter (init container)** - Automatically configures sketches from PromQL queries. + Today the demo uses the published `ghcr.io/projectasap/asap-planner-rs:v0.2.0` image + to emit `streaming_config.yaml` + `inference_config.yaml` into a shared volume + consumed by the Query Engine and asap-summary-ingest. The `asap-planner-rs` source + has been deleted from this repo (Phase γ); the planner now lives in + [`ASAPCollector/controller/`](https://github.com/ProjectASAP/ASAPCollector/tree/main/controller). + The Phase δ rewrite of this quickstart will replace the init container with a + controller HTTP push to `POST /api/v1/streaming-config` / + `POST /api/v1/storage_routing` on the Query Engine. Once you run the quickstart, you will see a pre-configured Grafana dashboard that compares Prometheues and ASAPQuery side-by-side. You will see **visually indistinguishable** results from Prometheus and ASAPQuery, with ASAPQuery being 100x faster @@ -88,9 +96,11 @@ Run `python3 set_data_cardinality.py ` where `M` is the number of labels To modify the queries in the Grafana dashboard and run ASAPQuery against those: -#### 1. Edit the asap-planner-rs Config +#### 1. Edit the Plan Emitter Config -Edit `config/controller-config.yaml`: +Edit `config/controller-config.yaml` (consumed by the published +`asap-planner-rs:v0.2.0` init container — see the component list above +for context on the post-Phase-γ migration): ```yaml query_groups: diff --git a/asap-quickstart/docker-compose.dev.yml b/asap-quickstart/docker-compose.dev.yml index 96b5fc0f3..c38ecd989 100644 --- a/asap-quickstart/docker-compose.dev.yml +++ b/asap-quickstart/docker-compose.dev.yml @@ -9,10 +9,10 @@ name: asapquery-quickstart # External users who just want the quickstart should use docker-compose.yml directly. services: - asap-planner-rs: - build: - context: .. - dockerfile: asap-planner-rs/Dockerfile + # asap-planner-rs source was deleted in Phase γ — the dev override + # falls back to the published `ghcr.io/projectasap/asap-planner-rs:v0.2.0` + # image referenced by docker-compose.yml. Phase δ replaces this init + # container with a controller HTTP push entirely. asap-summary-ingest: build: diff --git a/asap-summary-ingest/README.md b/asap-summary-ingest/README.md index c7f2c938f..04349ce6c 100644 --- a/asap-summary-ingest/README.md +++ b/asap-summary-ingest/README.md @@ -4,19 +4,19 @@ asap-summary-ingest is the pipeline configurator that creates Arroyo streaming p ## Purpose -Given `streaming_config.yaml` (generated by asap-planner-rs), asap-summary-ingest: +Given `streaming_config.yaml` (generated by the planner (in `ASAPCollector/controller/`)), asap-summary-ingest: 1. Renders SQL query templates using Jinja2 2. Creates Arroyo pipelines via REST API 3. Configures sketch-building UDFs with parameters 4. Sets up connections to Kafka for sketch output -This automation eliminates manual pipeline creation and ensures consistency with asap-planner-rs decisions. +This automation eliminates manual pipeline creation and ensures consistency with the planner (in `ASAPCollector/controller/`) decisions. ## How It Works ### Input: streaming_config.yaml -The asap-planner-rs generates this file describing which sketches to build: +The the planner (in `ASAPCollector/controller/`) generates this file describing which sketches to build: **TODO** diff --git a/docs/01-getting-started/architecture.md b/docs/01-getting-started/architecture.md index 1fb7f9dfe..821d2d291 100644 --- a/docs/01-getting-started/architecture.md +++ b/docs/01-getting-started/architecture.md @@ -186,7 +186,7 @@ graph LR | **asap-query-engine** | Answers PromQL queries using sketches | Rust | `asap-query-engine/` | | **Arroyo** | Stream processing for building sketches | Rust (forked) | [github.com/ProjectASAP/arroyo](https://github.com/ProjectASAP/arroyo) | | **asap-summary-ingest** | Configures Arroyo pipelines from config | Python | `asap-summary-ingest/` | -| **asap-planner-rs** | Auto-determines sketch parameters | Rust | `asap-planner-rs/` | +| **Planner** (Rust) | Auto-determines sketch parameters | Rust | [`ASAPCollector/controller/`](https://github.com/ProjectASAP/ASAPCollector/tree/main/controller) — moved out of this repo in Phase γ | | **Kafka** | Message broker for sketch distribution | Apache Kafka | (external) | | **Prometheus** | Time-series database (existing) | Go | (external) | | **Exporters** | Generate synthetic metrics for testing | Rust/Python | `asap-tools/data-sources/prometheus-exporters/` | @@ -264,10 +264,8 @@ ASAPQuery/ │ ├── templates/ # Jinja2 SQL templates │ └── utils/ # Arroyo API client │ -├── asap-planner-rs/ # Auto-configuration service -│ ├── main_controller.py # Entry point -│ ├── classes/ # Config data structures -│ └── utils/ # Decision logic +├── (Planner: lives in ASAPCollector/controller/, deleted from this +│ repo in Phase γ — see https://github.com/ProjectASAP/ASAPCollector) │ ├── asap-tools/ # Experiment framework & tooling │ ├── data-sources/ diff --git a/docs/02-components/README.md b/docs/02-components/README.md index dc3fd79f4..7cce644b8 100644 --- a/docs/02-components/README.md +++ b/docs/02-components/README.md @@ -9,7 +9,7 @@ This document provides an overview of all ASAP components and links to detailed | **asap-query-engine** | Answers PromQL queries using sketches | Rust | [Details](query-engine.md) · [Code](../../asap-query-engine/) · [Dev Docs](../../asap-query-engine/docs/README.md) | | **Arroyo** | Stream processing for building sketches | Rust (forked) | [Details](arroyo.md) · [Code](https://github.com/ProjectASAP/arroyo) | | **asap-summary-ingest** | Configures Arroyo pipelines from config | Python | [Details](arroyosketch.md) · [Code](../../asap-summary-ingest/) · [README](../../asap-summary-ingest/README.md) | -| **asap-planner-rs-rs** | Auto-determines sketch parameters | Rust | [Details](controller.md) · [Code](../../asap-planner-rs-rs/) | +| **Planner (in ASAPCollector)** | Auto-determines sketch parameters; pushes plans to backend via OpAMP / HTTP | Rust | [`ASAPCollector/controller/`](https://github.com/ProjectASAP/ASAPCollector/tree/main/controller) | | **Exporters** | Generate synthetic metrics for testing | Rust/Python | [Details](exporters.md) · [Code](../../asap-tools/data-sources/prometheus-exporters/) · [README](../../asap-tools/data-sources/prometheus-exporters/README.md) | | **asap-tools** | Experiment framework for CloudLab | Python | [Details](utilities.md) · [Code](../../asap-tools/) · [Docs](../../asap-tools/docs/architecture.md) | @@ -19,7 +19,7 @@ This document provides an overview of all ASAP components and links to detailed graph TB subgraph "Configuration (Offline)" U[User] -->|edits| CC[controller-config.yaml] - CC --> C[asap-planner-rs] + CC --> C[Planner in ASAPCollector] C -->|streaming_config.yaml| AS[asap-summary-ingest] C -->|inference_config.yaml| Q AS -->|create pipelines| A @@ -73,10 +73,13 @@ These run continuously to serve queries: These run once to set up the system: -- **[asap-planner-rs](controller.md)** - Determines optimal sketch parameters +- **Planner** (in [ASAPCollector/controller](https://github.com/ProjectASAP/ASAPCollector/tree/main/controller)) + - Determines optimal sketch parameters - Analyzes query workload - Selects sketch algorithms - - Generates configs for Arroyo and QueryEngine + - Pushes streaming + inference configs to Arroyo and QueryEngine + via OpAMP / HTTP (replaces the deleted `asap-planner-rs` library + + CLI; see Phase γ) - **[asap-summary-ingest](arroyosketch.md)** - Creates Arroyo pipelines - Reads streaming_config.yaml @@ -111,18 +114,20 @@ Performance-critical components written in Rust: Configuration and orchestration in Python: -- **asap-planner-rs** - Query analysis and config generation - **asap-summary-ingest** - Pipeline configuration - **asap-tools** - Experiment framework - **Python Exporters** - Simpler metric generators +(Planner config-generation moved to the `ASAPCollector/controller/` +Rust crate after Phase γ deletion of `asap-planner-rs/`.) + ## Component Dependencies ``` asap-query-engine ├── Kafka (runtime) - Consumes sketches ├── Prometheus (runtime, optional) - Fallback queries -└── inference_config.yaml (config) - From asap-planner-rs +└── inference_config.yaml (config) - From ASAPCollector controller Arroyo ├── Prometheus (runtime) - Remote write source @@ -131,9 +136,9 @@ Arroyo asap-summary-ingest ├── Arroyo (runtime) - Creates pipelines via API -└── streaming_config.yaml (config) - From asap-planner-rs +└── streaming_config.yaml (config) - From ASAPCollector controller -asap-planner-rs +Planner (lives in ASAPCollector/controller/, not this repo) ├── controller-config.yaml (input) - User-provided ├── streaming_config.yaml (output) - For asap-summary-ingest └── inference_config.yaml (output) - For asap-query-engine @@ -153,7 +158,8 @@ asap-tools - [asap-query-engine](query-engine.md) - Query processor deep dive - [Arroyo](arroyo.md) - Streaming engine + ASAP customizations - [asap-summary-ingest](arroyosketch.md) - Pipeline configurator -- [asap-planner-rs](controller.md) - Auto-configuration service +- Planner — auto-configuration service. See + [`ASAPCollector/controller/`](https://github.com/ProjectASAP/ASAPCollector/tree/main/controller). - [Exporters](exporters.md) - Metric generators - [asap-tools](utilities.md) - Experiment framework @@ -162,7 +168,6 @@ asap-tools For implementation details, see READMEs co-located with code: - [asap-query-engine/docs/](../../asap-query-engine/docs/README.md) - Extensibility guides -- [asap-planner-rs/README.md](../../asap-planner-rs/README.md) - asap-planner-rs internals - [asap-summary-ingest/README.md](../../asap-summary-ingest/README.md) - Pipeline config internals - [asap-tools/data-sources/prometheus-exporters/README.md](../../asap-tools/data-sources/prometheus-exporters/README.md) - Exporter implementations - [asap-tools/docs/](../../asap-tools/docs/architecture.md) - Experiment framework architecture diff --git a/docs/03-how-to-guides/development/add-new-sketch.md b/docs/03-how-to-guides/development/add-new-sketch.md index 75cf698ec..dd614c151 100644 --- a/docs/03-how-to-guides/development/add-new-sketch.md +++ b/docs/03-how-to-guides/development/add-new-sketch.md @@ -75,9 +75,9 @@ pub use your_sketch_accumulator::*; --- -## Step 4: asap-planner-rs - Sketch Parameters (Optional) +## Step 4: Planner - Sketch Parameters (Optional) -**Usually**: asap-planner-rs picks up sketch automatically from asap-common mapping. Custom sketch parameters (size, epsilon, etc.) can be added in the Rust source under `asap-planner-rs/src/`. +**Usually**: the planner picks up the sketch automatically from the asap-common mapping. Custom sketch parameters (size, epsilon, etc.) can be added in the Rust source under [`ASAPCollector/controller/`](https://github.com/ProjectASAP/ASAPCollector/tree/main/controller). The legacy `asap-planner-rs/src/` location was deleted in Phase γ. --- @@ -86,7 +86,7 @@ pub use your_sketch_accumulator::*; - [ ] `validate_udfs.py` passes (ArroyoSketch) - [ ] `cargo build --release` succeeds (asap-query-engine) - [ ] `cargo test` passes (asap-query-engine) -- [ ] End-to-end: asap-planner-rs → asap-summary-ingest → Arroyo → Kafka → QueryEngine → Query result +- [ ] End-to-end: ASAPCollector controller (planner) → asap-summary-ingest → Arroyo → Kafka → QueryEngine → Query result --- diff --git a/docs/03-how-to-guides/operations/bootstrap-config-from-query-log.md b/docs/03-how-to-guides/operations/bootstrap-config-from-query-log.md index 0bc3dfbee..bae2fbd3a 100644 --- a/docs/03-how-to-guides/operations/bootstrap-config-from-query-log.md +++ b/docs/03-how-to-guides/operations/bootstrap-config-from-query-log.md @@ -30,16 +30,12 @@ metrics: ### 3. Run the planner -```bash -asap-planner \ - --query-log /var/log/prometheus/query.log \ - --metrics-config metrics.yaml \ - --output_dir ./configs \ - --prometheus_scrape_interval 15 \ - --streaming_engine arroyo -``` - -This writes `streaming_config.yaml` and `inference_config.yaml` to `./configs/`. +The standalone `asap-planner` CLI was removed in Phase γ; the planner +now lives inside the [ASAPCollector controller](https://github.com/ProjectASAP/ASAPCollector/tree/main/controller). +Drive it via the controller's CLI / HTTP surface (see the controller +README for the equivalent invocation). The controller writes +`streaming_config.yaml` + `inference_config.yaml` and pushes the +resulting `BackendStorageRouting` to the backend over HTTP. ## Notes diff --git a/docs/03-how-to-guides/operations/publishing-docker-images.md b/docs/03-how-to-guides/operations/publishing-docker-images.md index c0bc316f9..37ad31b81 100644 --- a/docs/03-how-to-guides/operations/publishing-docker-images.md +++ b/docs/03-how-to-guides/operations/publishing-docker-images.md @@ -16,7 +16,6 @@ That's it. GitHub Actions will build and push all images tagged as both `v0.2.0` | Image | Source | |---|---| | `ghcr.io/projectasap/asap-base` | `asap-common/installation/` | -| `ghcr.io/projectasap/asap-planner-rs` | `asap-planner-rs/` | | `ghcr.io/projectasap/asap-summary-ingest` | `asap-summary-ingest/` | | `ghcr.io/projectasap/asap-query-engine` | `asap-query-engine/` | | `ghcr.io/projectasap/asap-prometheus-client` | `asap-tools/queriers/prometheus-client/` | diff --git a/docs/README.md b/docs/README.md index 4e424d596..bb00f143d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -14,7 +14,9 @@ Deep dives into each component: - [Query Engine](02-components/query-engine.md) - Rust query processor - [Arroyo](02-components/arroyo.md) - Streaming engine (fork + customizations) - [asap-summary-ingest](02-components/arroyosketch.md) - Pipeline configurator -- [asap-planner-rs](02-components/controller.md) - Auto-configuration service +- Planner — auto-configuration service. Lives in + [`ASAPCollector/controller/`](https://github.com/ProjectASAP/ASAPCollector/tree/main/controller) + after the Phase γ deletion of `asap-planner-rs/`. - [Exporters](02-components/exporters.md) - Metric generators - [asap-tools](02-components/utilities.md) - Experiment framework @@ -55,6 +57,5 @@ Developer practices and infrastructure: Technical details co-located with code: - [asap-query-engine](../asap-query-engine/docs/README.md) - Extensibility guides - [asap-tools/Experiments](../asap-tools/docs/architecture.md) - Experiment framework architecture -- [asap-planner-rs](../asap-planner-rs/README.md) - Controller internals - [asap-summary-ingest](../asap-summary-ingest/README.md) - Pipeline configuration - [asap-tools/data-sources/prometheus-exporters](../asap-tools/data-sources/prometheus-exporters/README.md) - Exporter implementations