diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index bc8950db..20fe2945 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -1,22 +1,26 @@ name: Rust CI +# TEMPORARILY DISABLED: auto-triggers are off because CI can't fetch the private +# promql-parser dependency until the CARGO_PRIVATE_GIT_TOKEN secret is added (see +# THIRD_PARTY.md). The auth scaffolding below is ready — re-enable the push / +# pull_request triggers once the credential is configured. Manual runs still work. on: - push: - branches: [ main ] - paths: - - 'crates/**' - - 'Cargo.toml' - - 'Cargo.lock' - - '.github/workflows/rust.yml' - pull_request: - types: [opened, synchronize, reopened, ready_for_review] - branches: [ main ] - paths: - - 'crates/**' - - 'Cargo.toml' - - 'Cargo.lock' - - '.github/workflows/rust.yml' workflow_dispatch: + # push: + # branches: [ main ] + # paths: + # - 'crates/**' + # - 'Cargo.toml' + # - 'Cargo.lock' + # - '.github/workflows/rust.yml' + # pull_request: + # types: [opened, synchronize, reopened, ready_for_review] + # branches: [ main ] + # paths: + # - 'crates/**' + # - 'Cargo.toml' + # - 'Cargo.lock' + # - '.github/workflows/rust.yml' concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -24,6 +28,10 @@ concurrency: env: CARGO_TERM_COLOR: always + # The promql-parser dependency is a private git repo (ProjectASAP/promql-parser). + # Cargo must fetch it via the git CLI so it picks up the credential helper + # configured by the "Authenticate private git deps" step below. + CARGO_NET_GIT_FETCH_WITH_CLI: "true" jobs: format-and-lint: @@ -32,6 +40,13 @@ jobs: steps: - uses: actions/checkout@v4 + # Grant read access to the private promql-parser mirror. Requires an org/repo + # secret CARGO_PRIVATE_GIT_TOKEN (fine-grained PAT or GitHub App token with + # read access to ProjectASAP/promql-parser). + - name: Authenticate private git deps + run: | + git config --global url."https://x-access-token:${{ secrets.CARGO_PRIVATE_GIT_TOKEN }}@github.com/".insteadOf "/" + - name: Install Rust uses: dtolnay/rust-toolchain@stable with: @@ -64,6 +79,13 @@ jobs: steps: - uses: actions/checkout@v4 + # Grant read access to the private promql-parser mirror. Requires an org/repo + # secret CARGO_PRIVATE_GIT_TOKEN (fine-grained PAT or GitHub App token with + # read access to ProjectASAP/promql-parser). + - name: Authenticate private git deps + run: | + git config --global url."https://x-access-token:${{ secrets.CARGO_PRIVATE_GIT_TOKEN }}@github.com/".insteadOf "/" + - name: Install Rust uses: dtolnay/rust-toolchain@stable with: diff --git a/Cargo.lock b/Cargo.lock index 755e871e..e922417e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -305,6 +305,11 @@ dependencies = [ [[package]] name = "asap-control-core" version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "thiserror", +] [[package]] name = "asap-control-lower" @@ -312,6 +317,7 @@ version = "0.1.0" dependencies = [ "asap-control-core", "datafusion", + "promql-parser", "tokio", ] @@ -365,6 +371,15 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -477,6 +492,12 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "cactus" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbc26382d871df4b7442e3df10a9402bf3cf5e55cbd66f12be38861425f0564" + [[package]] name = "cc" version = "1.2.62" @@ -495,6 +516,20 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfgrammar" +version = "0.13.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe45e18904af7af10e4312df7c97251e98af98c70f42f1f2587aecfcbee56bf" +dependencies = [ + "indexmap", + "lazy_static", + "num-traits", + "regex", + "serde", + "vob", +] + [[package]] name = "chrono" version = "0.4.39" @@ -503,7 +538,9 @@ checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ "android-tzdata", "iana-time-zone", + "js-sys", "num-traits", + "wasm-bindgen", "windows-targets", ] @@ -631,9 +668,9 @@ dependencies = [ [[package]] name = "dashmap" -version = "6.1.0" +version = "6.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" dependencies = [ "cfg-if", "crossbeam-utils", @@ -1048,6 +1085,15 @@ dependencies = [ "strum", ] +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", +] + [[package]] name = "digest" version = "0.10.7" @@ -1072,9 +1118,9 @@ dependencies = [ [[package]] name = "either" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "equivalent" @@ -1098,6 +1144,16 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -1130,6 +1186,12 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" @@ -1243,6 +1305,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "getopts" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" +dependencies = [ + "unicode-width", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -1654,6 +1725,60 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "lrlex" +version = "0.13.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c71364e868116ee891b0f93559eb9eca5675bec28b22d33c58481e66c3951d7e" +dependencies = [ + "cfgrammar", + "getopts", + "lazy_static", + "lrpar", + "num-traits", + "quote", + "regex", + "regex-syntax", + "serde", + "vergen", +] + +[[package]] +name = "lrpar" +version = "0.13.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b265a81193d94c92d1c9c715498d6fa505bce3f789ceecb24ab5d6fa2dbc71" +dependencies = [ + "bincode", + "cactus", + "cfgrammar", + "filetime", + "indexmap", + "lazy_static", + "lrtable", + "num-traits", + "packedvec", + "regex", + "serde", + "static_assertions", + "vergen", + "vob", +] + +[[package]] +name = "lrtable" +version = "0.13.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc36d15214ca997a5097845be1f932b7ee6125c36f5c5e55f6c49e027ddeb6de" +dependencies = [ + "cfgrammar", + "fnv", + "num-traits", + "serde", + "sparsevec", + "vob", +] + [[package]] name = "lz4_flex" version = "0.11.6" @@ -1733,6 +1858,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "num-integer" version = "0.1.46" @@ -1784,6 +1915,15 @@ dependencies = [ "libc", ] +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + [[package]] name = "object_store" version = "0.11.2" @@ -1820,6 +1960,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "packedvec" +version = "1.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69e0a534dd2e6aefce319af62a0aa0066a76bdfcec0201dfe02df226bc9ec70" +dependencies = [ + "num-traits", + "serde", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -1940,6 +2090,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1968,6 +2124,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "promql-parser" +version = "0.9.0" +source = "git+https://github.com/ProjectASAP/promql-parser?branch=asap#c51beafb361af4cc95ed62ae377862c660ceb757" +dependencies = [ + "cfgrammar", + "chrono", + "lazy_static", + "lrlex", + "lrpar", + "regex", +] + [[package]] name = "quote" version = "1.0.45" @@ -2125,6 +2294,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ "serde_core", + "serde_derive", ] [[package]] @@ -2228,6 +2398,18 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" +[[package]] +name = "sparsevec" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68b4a8ce3045f0fe173fb5ae3c6b7dcfbec02bfa650bb8618b2301f52af0134d" +dependencies = [ + "num-traits", + "packedvec", + "serde", + "vob", +] + [[package]] name = "sqlparser" version = "0.51.0" @@ -2324,6 +2506,26 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "thrift" version = "0.17.0" @@ -2335,6 +2537,39 @@ dependencies = [ "ordered-float", ] +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "libc", + "num-conv", + "num_threads", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tiny-keccak" version = "2.0.2" @@ -2495,12 +2730,33 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "vergen" +version = "8.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2990d9ea5967266ea0ccf413a4aa5c42a93dbcfda9cb49a97de6931726b12566" +dependencies = [ + "anyhow", + "rustversion", + "time", +] + [[package]] name = "version_check" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vob" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc936b5a7202a703aeaf7ce05e7931db2e0c8126813f97db3e9e06d867b0bb38" +dependencies = [ + "num-traits", + "serde", +] + [[package]] name = "walkdir" version = "2.5.0" diff --git a/THIRD_PARTY.md b/THIRD_PARTY.md new file mode 100644 index 00000000..dce20c37 --- /dev/null +++ b/THIRD_PARTY.md @@ -0,0 +1,83 @@ +# Third-party software + +ASAPController bundles third-party Rust crates. This file records the notable +direct dependencies, their licenses, and any attribution obligations. It is a +convenience summary, **not legal advice** — for an exhaustive, machine-generated +list (including transitive dependencies) run e.g. `cargo tree` or +[`cargo about`](https://github.com/EmbarkStudios/cargo-about). + +All licenses below are OSI-approved and **permissive** (MIT / Apache-2.0); none +are copyleft. ASAPController itself is therefore not obligated to be open-sourced +on account of these dependencies. + +## Direct dependencies + +| Crate | License | Source | Notes | +|---|---|---|---| +| `promql-parser` | Apache-2.0 | **Private mirror** `ProjectASAP/promql-parser` of [`GreptimeTeam/promql-parser`](https://github.com/GreptimeTeam/promql-parser) | L1 PromQL parsing. See below. | +| `serde` (+ derive) | MIT OR Apache-2.0 | crates.io | Serialization of the intent-algebra IR. | +| `serde_json` | MIT OR Apache-2.0 | crates.io | JSON (de)serialization in tests/IR. | +| `thiserror` | MIT OR Apache-2.0 | crates.io | Error types. | + +Transitive dependencies pulled in by the above (notably the `lrpar` / `lrlex` / +`cfgrammar` parser-toolkit stack behind `promql-parser`, and `regex`) carry their +own licenses — overwhelmingly MIT and/or Apache-2.0. Regenerate the full set with +`cargo about generate` if a complete NOTICE bundle is needed for a release. + +## `promql-parser` — vendored Apache-2.0 mirror + +`promql-parser` is consumed as a **git dependency on a private mirror** +(`ProjectASAP/promql-parser`) of the upstream Apache-2.0 project +`GreptimeTeam/promql-parser`, so we can carry local PromQL grammar/function +additions ahead of upstream releases. Branch layout: + +- **`main`** — an untouched mirror of upstream `GreptimeTeam/promql-parser`. +- **`asap`** — our working branch; ASAPController's `crates/lower` manifest pins + this branch. It currently adds the experimental functions present in + Prometheus `promql/parser/functions.go` but missing upstream (`mad_over_time`, + `first_over_time`, `ts_of_{first,last,max,min}_over_time`, `histogram_quantiles`, + `info`, `max_of`, `min_of`, `step`, `range`). Modified files are marked per + Apache-2.0 §4(b). + +Apache-2.0 explicitly permits copying, modifying, **keeping modifications +private**, and commercial use, and it is **not copyleft**. The conditions in +§4 ("Redistribution") apply only when the software is **distributed outside the +organization**. If/when ASAPController (with this parser compiled in) is +distributed externally, retain the following: + +- a copy of the **Apache-2.0 license** text (kept in the mirror as `LICENSE`); +- the upstream **`NOTICE`** file's attribution content, if present; +- original copyright / patent / attribution notices in the source; and +- a prominent note in **each file we modify** stating that it was changed + (Apache-2.0 §4(b)). + +Purely internal use (private mirror, internal builds, no external distribution) +carries essentially none of these obligations beyond keeping `LICENSE`/`NOTICE` +in the mirror, which the mirror already does. + +### Keeping the mirror in sync with upstream + +The mirror was created as a one-way copy (not a GitHub fork). `main` stays a +pristine upstream mirror; local edits live on `asap`. To pull future upstream +changes, refresh `main` then rebase `asap`: + +```sh +# refresh the pristine mirror branch +git clone --bare https://github.com/GreptimeTeam/promql-parser.git +cd promql-parser.git +git push https://github.com/ProjectASAP/promql-parser.git +refs/heads/main:refs/heads/main +# then, in a normal clone: git checkout asap && git rebase main && git push --force-with-lease +``` + +### CI / build access + +Because `promql-parser` is a private git dependency, any build needs read access +to `ProjectASAP/promql-parser`: + +- **Local dev:** `gh auth setup-git` (uses your GitHub credentials). +- **CI:** the `rust.yml` workflow sets `CARGO_NET_GIT_FETCH_WITH_CLI=true` and + configures a git credential helper from the secret **`CARGO_PRIVATE_GIT_TOKEN`** + — add that org/repo secret (a fine-grained PAT or GitHub App token with read + access to the mirror) or CI will fail to fetch the dependency. Until the secret + exists, `rust.yml`'s auto-triggers are **disabled** (manual `workflow_dispatch` + only); re-enable the commented `push` / `pull_request` triggers afterward. diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 99972465..beadf2e0 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -2,3 +2,10 @@ name = "asap-control-core" version = "0.1.0" edition = "2021" + +[dependencies] +serde = { version = "1", features = ["derive"] } +thiserror = "2" + +[dev-dependencies] +serde_json = "1" diff --git a/crates/core/src/intent_algebra/agg_intent.rs b/crates/core/src/intent_algebra/agg_intent.rs new file mode 100644 index 00000000..73922f41 --- /dev/null +++ b/crates/core/src/intent_algebra/agg_intent.rs @@ -0,0 +1,310 @@ +//! Layer 3 aggregation-intent vocabulary — "what to compute, not how". +//! +//! L3 carries intent ("compute a quantile to ε=0.01 accuracy"); the choice +//! between `HashAgg` / `SortAgg` / `SketchAgg(KLL{k=200})` is an L4 cost-aware +//! decision, not encoded here. +//! +//! `AggIntent::TopK` is a first-class *intent* — a dedicated heavy-hitter +//! sketch (SpaceSaving, CMS-with-heap) computes it in one pass. Generic +//! `ORDER BY value LIMIT k` stays as the `QueryExpr::Sort + Limit` operator +//! pair. L1→L2→L3 lowering picks one or the other deterministically. + +use std::time::Duration; + +use serde::{Deserialize, Serialize}; + +use crate::intent_algebra::query_expr::DataModel; +use crate::intent_algebra::schema::{Column, ColumnId, DataType}; +use crate::types::AccuracyTarget; + +/// "What to compute" at L3 — the vocabulary the planner pivots on. +/// +/// Grouping for `TopK` rides on the enclosing `QueryExpr::Aggregate.by` +/// (positional `ColumnId`s), like every other aggregate; the intent itself +/// carries only `k` + the accuracy target. +/// +/// The single-column reducers (`Sum` / `Min` / `Max` / `Avg` / `StdDev` / +/// `Variance`) carry `col: Option` — the positional input column +/// they reduce. `None` is the PromQL convention "the time-series sample +/// value"; SQL `SUM(bytes), AVG(latency)` sets distinct `Some(id)`s so a +/// multi-aggregate node binds each reducer to the right column. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum AggIntent { + // ── Data-model-agnostic ────────────────────────────────────────────── + Count { + accuracy: AccuracyTarget, + }, + Sum { + #[serde(default)] + col: Option, + }, + Min { + #[serde(default)] + col: Option, + }, + Max { + #[serde(default)] + col: Option, + }, + Avg { + #[serde(default)] + col: Option, + }, + /// Sample standard deviation when `population == false`; population stddev + /// otherwise. PromQL `stddev` / `stddev_over_time`; SQL `STDDEV(col)`. + StdDev { + #[serde(default)] + col: Option, + population: bool, + }, + /// Variance — PromQL `stdvar` / `stdvar_over_time`; SQL `VARIANCE(col)`. + Variance { + #[serde(default)] + col: Option, + population: bool, + }, + Quantile { + q: f64, + accuracy: AccuracyTarget, + }, + /// Heavy-hitter top-k — served by a dedicated sketch in one pass. The + /// group-by keys live on the enclosing `Aggregate.by`. + TopK { + k: usize, + accuracy: AccuracyTarget, + }, + Cardinality { + accuracy: AccuracyTarget, + }, + + // ── Time-series streaming derivatives ──────────────────────────────── + // Carry PromQL's counter-reset adjustment; not equivalent to Sum/Count + // over a Window. Kept distinct so delta-set aggregators bind directly. + Rate { + window: Duration, + }, + Increase { + window: Duration, + }, +} + +impl AggIntent { + /// Which data model this intent semantically requires. L4 rules consult + /// this to skip non-applicable intents (e.g. `Rate` over a tabular source). + pub fn requires(&self) -> DataModel { + match self { + Self::Rate { .. } | Self::Increase { .. } => DataModel::TimeSeries, + _ => DataModel::Any, + } + } + + /// Whether this is a *per-series* reduction — it reduces a single series' + /// samples over its range window (one value out per series), so it does + /// **not** collapse across series and every label column is preserved. + /// `rate`/`increase` carry their window in the intent. (Cross-series + /// reductions like `sum`/`avg` over a series set return `false`.) + pub fn is_per_series(&self) -> bool { + matches!(self, Self::Rate { .. } | Self::Increase { .. }) + } + + /// The positional input column this intent reduces, if it carries one. + /// `None` = the synthetic time-series sample value (PromQL) or an + /// argument-less aggregate (`Count` / `Cardinality` / `TopK`). Used by + /// schema derivation to resolve each reducer's input column. + pub fn input_col(&self) -> Option { + match self { + AggIntent::Sum { col } + | AggIntent::Min { col } + | AggIntent::Max { col } + | AggIntent::Avg { col } + | AggIntent::StdDev { col, .. } + | AggIntent::Variance { col, .. } => *col, + _ => None, + } + } + + /// Output column name + type produced by this intent over `input`. + /// Used by `QueryExpr::Aggregate`'s schema-derivation rule. The PromQL + /// convention names the column after the intent kind so consumers can + /// locate it without an alias lookup. + pub fn output_column(&self, input: &Column) -> Column { + match self { + AggIntent::Count { .. } => col("count", DataType::Int64, false), + AggIntent::Sum { .. } => col("sum", input.dtype.clone(), false), + AggIntent::Min { .. } => col("min", input.dtype.clone(), input.nullable), + AggIntent::Max { .. } => col("max", input.dtype.clone(), input.nullable), + AggIntent::Avg { .. } => col("avg", DataType::Float64, false), + AggIntent::StdDev { .. } => col("stddev", DataType::Float64, false), + AggIntent::Variance { .. } => col("variance", DataType::Float64, false), + AggIntent::Quantile { q, .. } => col( + &format!("quantile_{}", quantile_suffix(*q)), + DataType::Float64, + false, + ), + // TopK output is a per-row struct/list; modeled as Utf8 at L3 + // (the L4 sketch-bound IR upgrades the dtype). + AggIntent::TopK { k, .. } => col(&format!("topk_{k}"), DataType::Utf8, false), + AggIntent::Cardinality { .. } => col("cardinality", DataType::Int64, false), + AggIntent::Rate { .. } => col("rate", DataType::Float64, false), + AggIntent::Increase { .. } => col("increase", DataType::Float64, false), + } + } +} + +fn col(name: &str, dtype: DataType, nullable: bool) -> Column { + Column::new(name, dtype, nullable) +} + +/// `0.99` → `"0_99"`, `0.5` → `"0_5"`. Used by `Quantile` output naming so +/// `quantile_0_99` is a valid identifier downstream. +fn quantile_suffix(q: f64) -> String { + let mut s = format!("{q}"); + if let Some(stripped) = s.strip_prefix('-') { + s = format!("neg_{stripped}"); + } + s.replace('.', "_") +} + +// ── AggIntent helpers ──────────────────────────────────────────────────────── + +/// Two instances of this aggregation can be merged +/// (`agg(A ∪ B) = combine(agg(A), agg(B))`). `Avg` / `StdDev` / `Variance` +/// need richer partial state than a single value, so they are not mergeable. +pub fn agg_is_mergeable(op: &AggIntent) -> bool { + !matches!( + op, + AggIntent::Avg { .. } | AggIntent::StdDev { .. } | AggIntent::Variance { .. } + ) +} + +/// Whether this op implies `exact_required` — no sketch benefit. The exact +/// intents are `Sum / Count / Avg / Min / Max`. +pub fn agg_is_exact(op: &AggIntent) -> bool { + matches!( + op, + AggIntent::Sum { .. } + | AggIntent::Count { .. } + | AggIntent::Avg { .. } + | AggIntent::Min { .. } + | AggIntent::Max { .. } + ) +} + +/// Accuracy parameter as a fractional ε (`0.0` for exact ops), unpacked from +/// the typed `AccuracyTarget` on Quantile / Cardinality / Count / TopK. +pub fn agg_accuracy(op: &AggIntent) -> f64 { + match op { + AggIntent::Quantile { accuracy, .. } + | AggIntent::Cardinality { accuracy } + | AggIntent::Count { accuracy } + | AggIntent::TopK { accuracy, .. } => accuracy_target_to_f64(accuracy), + _ => 0.0, + } +} + +fn accuracy_target_to_f64(t: &AccuracyTarget) -> f64 { + match t { + AccuracyTarget::Exact => 0.0, + AccuracyTarget::Epsilon(eps) => *eps, + AccuracyTarget::EpsilonDelta { epsilon, .. } => *epsilon, + } +} + +/// Default `Cardinality` intent — HLL standard error at precision p=14. +pub fn default_cardinality() -> AggIntent { + AggIntent::Cardinality { + accuracy: AccuracyTarget::Epsilon(1.04 / ((1u64 << 14) as f64).sqrt()), + } +} + +/// Default `Quantile` intent at φ = `q`, `accuracy = ε 0.01`. +pub fn default_quantile(q: f64) -> AggIntent { + AggIntent::Quantile { + q, + accuracy: AccuracyTarget::Epsilon(0.01), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::intent_algebra::schema::{Column, DataType}; + + fn c(name: &str, dtype: DataType) -> Column { + Column::new(name, dtype, false) + } + + #[test] + fn output_column_names_are_intent_keyed() { + let v = c("value", DataType::Float64); + assert_eq!( + AggIntent::Count { + accuracy: AccuracyTarget::Exact + } + .output_column(&v) + .name, + "count" + ); + assert_eq!(AggIntent::Sum { col: None }.output_column(&v).name, "sum"); + assert_eq!( + AggIntent::Quantile { + q: 0.99, + accuracy: AccuracyTarget::Epsilon(0.01) + } + .output_column(&v) + .name, + "quantile_0_99" + ); + } + + #[test] + fn sum_preserves_input_dtype() { + assert!(matches!( + AggIntent::Sum { col: None } + .output_column(&c("c", DataType::Int64)) + .dtype, + DataType::Int64 + )); + } + + #[test] + fn mergeability_and_exactness() { + assert!(agg_is_mergeable(&AggIntent::Sum { col: None })); + assert!(!agg_is_mergeable(&AggIntent::Avg { col: None })); + assert!(!agg_is_mergeable(&AggIntent::StdDev { + col: None, + population: false + })); + assert!(agg_is_exact(&AggIntent::Min { col: None })); + assert!(!agg_is_exact(&default_cardinality())); + } + + #[test] + fn input_col_tracks_only_reducers() { + assert_eq!(AggIntent::Sum { col: Some(3) }.input_col(), Some(3)); + assert_eq!( + AggIntent::Avg { col: None }.input_col(), + None, + "None = PromQL sample value" + ); + assert_eq!( + AggIntent::Count { + accuracy: AccuracyTarget::Exact + } + .input_col(), + None + ); + } + + #[test] + fn agg_intent_serde_roundtrip() { + let v = AggIntent::Quantile { + q: 0.99, + accuracy: AccuracyTarget::Epsilon(0.01), + }; + let json = serde_json::to_string(&v).unwrap(); + let back: AggIntent = serde_json::from_str(&json).unwrap(); + assert_eq!(v, back); + } +} diff --git a/crates/core/src/intent_algebra/binder.rs b/crates/core/src/intent_algebra/binder.rs new file mode 100644 index 00000000..0004e327 --- /dev/null +++ b/crates/core/src/intent_algebra/binder.rs @@ -0,0 +1,203 @@ +//! The L3 **Binder** — name resolution as an explicit pass. +//! +//! [`Binder::bind`] produces the complete, self-contained [`Schema`] every +//! `ColumnId` in the converted canonical tree indexes into. The converter +//! ([`super::lower::convert`]) then becomes purely structural: it threads the +//! Binder's schema and positional resolution downstream is **total**. +//! +//! The default [`UsageDerivedCatalog`] knows nothing — every schema is derived +//! purely from the query's own usage. That is the honest state for the +//! observability domain (metric label sets are open-ended). A registry-backed +//! `SchemaCatalog` is future work; the `Binder` pass does not change when it +//! lands, only the catalog impl swaps. + +use crate::intent_algebra::expr_ir::ColumnRef; +use crate::intent_algebra::expr_ir::L2Expr; +use crate::intent_algebra::relational::QueryExpr as LQueryExpr; +use crate::intent_algebra::schema::{Column, DataType, Schema}; + +/// The DB / source-schema metadata source — resolves a source (metric / +/// table) name to its known columns. +pub trait SchemaCatalog { + /// Columns known for `source`. `None` when unknown — the [`Binder`] then + /// falls back to a usage-derived column set. + fn columns_for(&self, source: &str) -> Option>; +} + +/// The default catalog: knows nothing. Every schema the [`Binder`] produces +/// is derived purely from the query's own usage. +pub struct UsageDerivedCatalog; + +impl SchemaCatalog for UsageDerivedCatalog { + fn columns_for(&self, _source: &str) -> Option> { + None + } +} + +/// The L3 Binder — the explicit name-resolution pass. +pub struct Binder { + catalog: C, +} + +impl Default for Binder { + fn default() -> Self { + Self::new() + } +} + +impl Binder { + pub fn new() -> Self { + Self { + catalog: UsageDerivedCatalog, + } + } +} + +impl Binder { + pub fn with_catalog(catalog: C) -> Self { + Self { catalog } + } + + /// Resolve the complete [`Schema`] in scope for a query rooted at `tree`. + /// + /// Contains the time axis, the synthetic `value` column, and one column + /// per distinct name referenced anywhere in the tree — so positional + /// `ColumnId` resolution downstream is total. + pub fn bind(&self, tree: &LQueryExpr) -> Schema { + let mut columns: Vec = tree + .source_name() + .and_then(|name| self.catalog.columns_for(name)) + .unwrap_or_else(default_leaf_columns); + + // Ensure the (ts, value) floor is present. + for floor in default_leaf_columns() { + if !columns.iter().any(|c| c.name == floor.name) { + columns.push(floor); + } + } + + // Append one column per referenced-but-unknown name (group keys etc.). + for name in collect_referenced_columns(tree) { + if !columns.iter().any(|c| c.name == name) { + columns.push(Column::new(name, DataType::Utf8, true)); + } + } + + let time_index = columns.iter().position(|c| c.name == "ts"); + Schema { + columns, + time_index, + unique_keys: Vec::new(), + } + } +} + +/// The conventional PromQL leaf shape: `(ts: Timestamp, value: Float64)`. +fn default_leaf_columns() -> Vec { + vec![ + Column::new("ts", DataType::Timestamp, false), + Column::new("value", DataType::Float64, false), + ] +} + +/// Collect every distinct column name the converter resolves positionally: +/// group keys (`Aggregate.keys`, `TopK.by`, `Partition.keys`) **and** the +/// columns referenced by name in filter / having / project / sort / join +/// expressions (e.g. a PromQL label matcher `m{env="prod"}` references `env`). +/// The Binder seeds these into the usage-derived leaf so positional resolution +/// downstream is total. +/// Push a `ColumnRef`'s bare name (the schema-seedable identifier). `Qualified` +/// collapses to its `name`; `SampleValue`/`Wildcard` carry no name. +fn push_ref_name(c: &ColumnRef, out: &mut Vec) { + match c { + ColumnRef::Named(n) => out.push(n.clone()), + ColumnRef::Qualified { name, .. } => out.push(name.clone()), + ColumnRef::SampleValue | ColumnRef::Wildcard => {} + } +} + +fn collect_referenced_columns(tree: &LQueryExpr) -> Vec { + fn named(expr: &L2Expr, out: &mut Vec) { + for c in expr.columns_referenced() { + push_ref_name(c, out); + } + } + let mut out: Vec = Vec::new(); + tree.walk(&mut |node| match node { + LQueryExpr::Aggregate { keys, having, .. } => { + keys.iter().for_each(|k| push_ref_name(k, &mut out)); + if let Some(h) = having { + named(h, &mut out); + } + } + LQueryExpr::TopK { by, .. } => by.iter().for_each(|k| push_ref_name(k, &mut out)), + LQueryExpr::Partition { keys, .. } => out.extend(keys.keys().iter().cloned()), + LQueryExpr::Filter { pred, .. } => named(pred, &mut out), + LQueryExpr::Project { cols, .. } => { + for item in cols { + named(&item.expr, &mut out); + } + } + LQueryExpr::Sort { keys, .. } => { + for k in keys { + named(&k.expr, &mut out); + } + } + LQueryExpr::Join { pred: Some(p), .. } => named(p, &mut out), + _ => {} + }); + out.sort(); + out.dedup(); + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::intent_algebra::query_expr::PartitionKeys; + use crate::intent_algebra::relational::{QueryExpr as LQueryExpr, SourceSpec}; + + fn src(name: &str) -> LQueryExpr { + LQueryExpr::Source(SourceSpec::new(name)) + } + + #[test] + fn bare_source_yields_ts_value_floor() { + let schema = Binder::new().bind(&src("m")); + assert_eq!(schema.columns.len(), 2); + assert_eq!(schema.columns[0].name, "ts"); + assert_eq!(schema.columns[1].name, "value"); + assert_eq!(schema.time_index, Some(0)); + } + + #[test] + fn partition_keys_land_in_schema() { + let tree = LQueryExpr::Partition { + keys: PartitionKeys::By(vec!["host".into()]), + input: Box::new(src("hits")), + }; + let schema = Binder::new().bind(&tree); + assert!(schema.column_id("host").is_some()); + } + + #[test] + fn custom_catalog_supplies_base_columns() { + struct FixedCatalog; + impl SchemaCatalog for FixedCatalog { + fn columns_for(&self, source: &str) -> Option> { + (source == "known").then(|| { + vec![ + Column::new("ts", DataType::Timestamp, false), + Column::new("value", DataType::Float64, false), + Column::new("datacenter", DataType::Utf8, false), + ] + }) + } + } + let schema = Binder::with_catalog(FixedCatalog).bind(&src("known")); + let dc = schema + .column_id("datacenter") + .and_then(|id| schema.columns.get(id)); + assert!(matches!(dc, Some(c) if !c.nullable)); + } +} diff --git a/crates/core/src/intent_algebra/column_resolution.rs b/crates/core/src/intent_algebra/column_resolution.rs new file mode 100644 index 00000000..7fc87fa5 --- /dev/null +++ b/crates/core/src/intent_algebra/column_resolution.rs @@ -0,0 +1,239 @@ +//! Schema-driven column resolution for the Layer-2 `relational` IR. +//! +//! The Layer-2 IR uses `ColumnRef` (name-based, optionally table-qualified); +//! the canonical IR uses positional [`ColumnId`] resolved against a per-node +//! [`Schema`]. These helpers bridge the two — the [`Binder`](super::binder) +//! builds the schema, and [`resolve_column_refs`] turns the L2 refs (group +//! keys, dedup columns) into positional ids, qualifier-aware. + +use thiserror::Error; + +use crate::intent_algebra::agg_intent::AggIntent; +use crate::intent_algebra::expr_ir::ColumnRef; +use crate::intent_algebra::expr_ir::{L2Expr, L3Expr}; +use crate::intent_algebra::relational::QueryExpr; +use crate::intent_algebra::schema::{Column, ColumnId, DataType, Schema}; + +/// Errors returned by the resolution helpers. +#[derive(Debug, Error, PartialEq, Eq)] +pub enum ResolveError { + #[error("column `{name}` not found in schema (have: {available:?})")] + NotFound { + name: String, + available: Vec, + }, + #[error("ColumnRef::SampleValue has no `value` column in schema (have: {available:?})")] + NoSampleValue { available: Vec }, + #[error("ColumnRef::Wildcard cannot be resolved to a single ColumnId")] + WildcardNotPositional, +} + +/// Synthesize the conventional PromQL leaf schema `(ts, value)` for a metric. +pub fn infer_source_schema(_metric_or_table: &str) -> Schema { + Schema::with_time_index( + vec![ + Column::new("ts", DataType::Timestamp, false), + Column::new("value", DataType::Float64, false), + ], + 0, + Vec::new(), + ) +} + +/// Synthesise the root schema by walking to the outermost `Source` leaf. +pub fn infer_schema_for_root(expr: &QueryExpr) -> Schema { + match expr.source_name() { + Some(name) => infer_source_schema(name), + None => Schema::default(), + } +} + +/// Resolve a single [`ColumnRef`] to a positional [`ColumnId`]. +pub fn resolve_column_ref(col: &ColumnRef, schema: &Schema) -> Result { + match col { + ColumnRef::Named(name) => schema + .column_id(name) + .ok_or_else(|| ResolveError::NotFound { + name: name.clone(), + available: schema.columns.iter().map(|c| c.name.clone()).collect(), + }), + // Prefer the (table, name) match; fall back to the bare name for + // schemas whose columns carry no qualifier. + ColumnRef::Qualified { table, name } => schema + .column_id_qualified(table, name) + .or_else(|| schema.column_id(name)) + .ok_or_else(|| ResolveError::NotFound { + name: format!("{table}.{name}"), + available: schema.columns.iter().map(|c| c.name.clone()).collect(), + }), + ColumnRef::SampleValue => schema + .column_id("value") + .or_else(|| { + // After an aggregate the sample value is renamed (e.g. "avg"); + // fall back to the sole non-timestamp column when unambiguous. + let non_ts: Vec = (0..schema.columns.len()) + .filter(|&i| Some(i) != schema.time_index) + .collect(); + (non_ts.len() == 1).then(|| non_ts[0]) + }) + .ok_or_else(|| ResolveError::NoSampleValue { + available: schema.columns.iter().map(|c| c.name.clone()).collect(), + }), + ColumnRef::Wildcard => Err(ResolveError::WildcardNotPositional), + } +} + +/// Resolve every entry, short-circuiting on the first error. +pub fn resolve_column_refs( + cols: &[ColumnRef], + schema: &Schema, +) -> Result, ResolveError> { + cols.iter().map(|c| resolve_column_ref(c, schema)).collect() +} + +/// Resolve a Layer-2 [`L2Expr`] (name-based) into a positional [`L3Expr`] by +/// resolving every column reference against `schema`. Structural otherwise. +pub fn resolve_expr(expr: &L2Expr, schema: &Schema) -> Result { + let boxed = |e: &L2Expr| -> Result, ResolveError> { + Ok(Box::new(resolve_expr(e, schema)?)) + }; + let each = |es: &[L2Expr]| -> Result, ResolveError> { + es.iter().map(|e| resolve_expr(e, schema)).collect() + }; + Ok(match expr { + L2Expr::Column(c) => L3Expr::Column(resolve_column_ref(c, schema)?), + L2Expr::Literal(s) => L3Expr::Literal(s.clone()), + L2Expr::Compare { left, op, right } => L3Expr::Compare { + left: boxed(left)?, + op: op.clone(), + right: boxed(right)?, + }, + L2Expr::BoolAnd(v) => L3Expr::BoolAnd(each(v)?), + L2Expr::BoolOr(v) => L3Expr::BoolOr(each(v)?), + L2Expr::Not(e) => L3Expr::Not(boxed(e)?), + L2Expr::IsNull(e) => L3Expr::IsNull(boxed(e)?), + L2Expr::IsNotNull(e) => L3Expr::IsNotNull(boxed(e)?), + L2Expr::Cast { expr, to, try_cast } => L3Expr::Cast { + expr: boxed(expr)?, + to: to.clone(), + try_cast: *try_cast, + }, + L2Expr::InList { + expr, + list, + negated, + } => L3Expr::InList { + expr: boxed(expr)?, + list: each(list)?, + negated: *negated, + }, + L2Expr::FunctionCall { name, args } => L3Expr::FunctionCall { + name: name.clone(), + args: each(args)?, + }, + L2Expr::Arith { op, left, right } => L3Expr::Arith { + op: op.clone(), + left: boxed(left)?, + right: boxed(right)?, + }, + L2Expr::Case { + operand, + branches, + else_expr, + } => L3Expr::Case { + operand: operand.as_deref().map(&boxed).transpose()?, + branches: branches + .iter() + .map(|(w, t)| Ok((resolve_expr(w, schema)?, resolve_expr(t, schema)?))) + .collect::, ResolveError>>()?, + else_expr: else_expr.as_deref().map(&boxed).transpose()?, + }, + }) +} + +/// Output schema produced by an `Aggregate { by, aggs }` over `input`. +/// Mirrors `QueryExpr::output_schema_in`'s `Aggregate` arm; out-of-range `by` +/// ids are silently dropped (callers needing the strict check resolve `by` +/// via [`resolve_column_refs`], which surfaces `NotFound`). +pub fn output_schema_for_aggregate( + input: &Schema, + by: &[ColumnId], + aggs: &[AggIntent], + output_names: &[String], +) -> Schema { + let mut out_cols: Vec = Vec::with_capacity(by.len() + aggs.len()); + for &id in by { + if let Some(c) = input.columns.get(id) { + out_cols.push(c.clone()); + } + } + let value_col_idx = input + .column_id("value") + .or_else(|| (0..input.columns.len()).find(|i| !by.contains(i))); + let probe = value_col_idx + .and_then(|i| input.columns.get(i)) + .cloned() + .unwrap_or_else(|| Column::new("value", DataType::Float64, false)); + for (i, intent) in aggs.iter().enumerate() { + let in_col = intent + .input_col() + .and_then(|id| input.columns.get(id)) + .unwrap_or(&probe); + let mut out = intent.output_column(in_col); + if let Some(name) = output_names.get(i).filter(|s| !s.is_empty()) { + out.name = name.clone(); + } + out_cols.push(out); + } + let unique_keys = if by.is_empty() { + Vec::new() + } else { + vec![(0..by.len()).collect()] + }; + Schema { + columns: out_cols, + time_index: None, + unique_keys, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn source_schema_has_ts_and_value() { + let s = infer_source_schema("m"); + assert_eq!(s.columns.len(), 2); + assert_eq!(s.time_index, Some(0)); + assert!(!s.has_unique_key()); + } + + #[test] + fn resolve_sample_value() { + let s = infer_source_schema("m"); + assert_eq!(resolve_column_ref(&ColumnRef::SampleValue, &s), Ok(1)); + } + + #[test] + fn resolve_unknown_name_errors() { + let s = infer_source_schema("m"); + let err = resolve_column_ref(&ColumnRef::Named("host".into()), &s).unwrap_err(); + assert!(matches!(err, ResolveError::NotFound { .. })); + } + + #[test] + fn aggregate_strips_time_and_keeps_unique_keys() { + let mut input = infer_source_schema("m"); + input + .columns + .push(Column::new("host", DataType::Utf8, false)); + let out = + output_schema_for_aggregate(&input, &[2usize], &[AggIntent::Sum { col: None }], &[]); + assert_eq!(out.columns.len(), 2); // host, sum + assert_eq!(out.columns[0].name, "host"); + assert_eq!(out.columns[1].name, "sum"); + assert!(out.time_index.is_none()); + assert_eq!(out.unique_keys, vec![vec![0]]); + } +} diff --git a/crates/core/src/intent_algebra/cse.rs b/crates/core/src/intent_algebra/cse.rs new file mode 100644 index 00000000..9d5401e1 --- /dev/null +++ b/crates/core/src/intent_algebra/cse.rs @@ -0,0 +1,229 @@ +//! Workload-level Common Sub-Expression Elimination. +//! +//! Multi-root planning hoists shared sub-DAGs into `LetBinding`s so the cost +//! model can credit the producer once. Legality is gated by +//! [`cse_reuse_is_legal`](super::schema::cse_reuse_is_legal): a candidate +//! sub-DAG only becomes a binding when its output schema has at least one +//! `unique_keys` set — the load-bearing field for this pass. +//! +//! Scope: the basic "≥2 roots with identical `Aggregate`-child sub-trees" +//! case. The fully-general algorithm (alpha-equivalence, schema-merge, +//! nested CSE) is a downstream optimisation, not part of the IR contract. + +use crate::intent_algebra::names::{BindingName, QueryId}; +use crate::intent_algebra::query_expr::QueryExpr; +use crate::intent_algebra::schema::cse_reuse_is_legal; + +/// Multi-root container produced by the CSE pass. +#[derive(Debug, Clone, PartialEq)] +pub struct CseWorkloadPlan { + /// Named shared producers, each referenced by ≥2 roots via `QueryExpr::Ref`. + pub bindings: Vec<(BindingName, QueryExpr)>, + /// One root per input query, in input order. + pub roots: Vec<(QueryId, QueryExpr)>, +} + +/// Hoist sub-expressions structurally identical across ≥2 roots into shared +/// `LetBinding`s, leaving each root with a `Ref` where the duplicate lived. +/// +/// A candidate is hoisted only when +/// `cse_reuse_is_legal(&candidate.output_schema(), consumers)` returns `Ok`. +pub fn dedupe_subtrees(roots: Vec<(QueryId, QueryExpr)>) -> CseWorkloadPlan { + if roots.len() < 2 { + return CseWorkloadPlan { + bindings: vec![], + roots, + }; + } + + // Count `Aggregate`-child sub-trees that appear in ≥2 roots. Group by + // structural equality (`QueryExpr: PartialEq`) rather than `Debug` + // output: `{:?}` is not a guaranteed-injective, stable identity contract. + // The candidate set is one entry per distinct root child, so this linear + // scan is bounded by the number of distinct queries. + let mut candidate_counts: Vec<(QueryExpr, usize)> = Vec::new(); + for (_, root) in &roots { + if let QueryExpr::Aggregate { child, .. } = root { + if matches!(**child, QueryExpr::Ref { .. }) { + continue; + } + match candidate_counts + .iter_mut() + .find(|(e, _)| e == child.as_ref()) + { + Some(entry) => entry.1 += 1, + None => candidate_counts.push(((**child).clone(), 1)), + } + } + } + + // Pick the most-shared legal candidate (biggest reuse first). + let mut chosen: Option<(QueryExpr, usize)> = None; + for (expr, count) in candidate_counts.into_iter() { + if count < 2 { + continue; + } + let Ok(out_schema) = expr.output_schema() else { + continue; + }; + if cse_reuse_is_legal(&out_schema, count).is_err() { + continue; + } + match &chosen { + Some((_, best)) if *best >= count => {} + _ => chosen = Some((expr, count)), + } + } + + let Some((shared_expr, _)) = chosen else { + return CseWorkloadPlan { + bindings: vec![], + roots, + }; + }; + + let binding_name = BindingName::new("shared_0"); + let mut rewritten: Vec<(QueryId, QueryExpr)> = Vec::with_capacity(roots.len()); + for (qid, root) in roots { + let new_root = match root { + QueryExpr::Aggregate { + by, + aggs, + output_names, + having, + child, + } if *child == shared_expr => QueryExpr::Aggregate { + by, + aggs, + output_names, + having, + child: Box::new(QueryExpr::Ref { + name: binding_name.clone(), + }), + }, + other => other, + }; + rewritten.push((qid, new_root)); + } + + CseWorkloadPlan { + bindings: vec![(binding_name, shared_expr)], + roots: rewritten, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::intent_algebra::agg_intent::AggIntent; + use crate::intent_algebra::query_expr::{Source, WindowKind}; + use crate::intent_algebra::schema::{Column, DataType, Schema}; + use crate::types::AccuracyTarget; + use std::time::Duration; + + fn col(name: &str, dtype: DataType) -> Column { + Column::new(name, dtype, false) + } + + fn ts_scan() -> QueryExpr { + QueryExpr::Scan { + source: Source::TimeSeries { + metric: "http_request_duration_seconds".into(), + }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + col("ts", DataType::Timestamp), + col("service", DataType::Utf8), + col("value", DataType::Float64), + ], + 0, + vec![vec![0, 1]], + ), + } + } + + fn windowed_scan() -> QueryExpr { + QueryExpr::Window { + kind: WindowKind::Sliding, + size: Duration::from_secs(300), + slide: None, + child: Box::new(ts_scan()), + } + } + + #[test] + fn dedupe_subtrees_single_root_passthrough() { + let q = QueryExpr::Aggregate { + by: vec![1], + aggs: vec![AggIntent::Quantile { + q: 0.99, + accuracy: AccuracyTarget::Epsilon(0.01), + }], + output_names: vec![], + having: None, + child: Box::new(windowed_scan()), + }; + let out = dedupe_subtrees(vec![(QueryId::new("q1"), q.clone())]); + assert!(out.bindings.is_empty()); + assert_eq!(out.roots[0].1, q); + } + + #[test] + fn dedupe_subtrees_basic() { + let mk = |q: f64| QueryExpr::Aggregate { + by: vec![1], + aggs: vec![AggIntent::Quantile { + q, + accuracy: AccuracyTarget::Epsilon(0.01), + }], + output_names: vec![], + having: None, + child: Box::new(windowed_scan()), + }; + let out = dedupe_subtrees(vec![ + (QueryId::new("q1"), mk(0.99)), + (QueryId::new("q2"), mk(0.95)), + ]); + assert_eq!(out.bindings.len(), 1); + assert_eq!(out.bindings[0].0, BindingName::new("shared_0")); + assert_eq!(out.bindings[0].1, windowed_scan()); + for (_, root) in &out.roots { + match root { + QueryExpr::Aggregate { child, .. } => assert_eq!( + **child, + QueryExpr::Ref { + name: BindingName::new("shared_0") + } + ), + other => panic!("expected Aggregate root, got {other:?}"), + } + } + } + + #[test] + fn dedupe_subtrees_no_shared_subexpr_when_unique_keys_absent() { + // Schema without unique_keys → CSE refuses to share even if identical. + let scan_no_uk = QueryExpr::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + col("ts", DataType::Timestamp), + col("value", DataType::Float64), + ], + 0, + vec![], + ), + }; + let mk = || QueryExpr::Aggregate { + by: vec![], + aggs: vec![AggIntent::Sum { col: None }], + output_names: vec![], + having: None, + child: Box::new(scan_no_uk.clone()), + }; + let out = dedupe_subtrees(vec![(QueryId::new("q1"), mk()), (QueryId::new("q2"), mk())]); + assert!(out.bindings.is_empty(), "no unique_keys → no hoisting"); + } +} diff --git a/crates/core/src/intent_algebra/expr.rs b/crates/core/src/intent_algebra/expr.rs deleted file mode 100644 index 90444f3a..00000000 --- a/crates/core/src/intent_algebra/expr.rs +++ /dev/null @@ -1,721 +0,0 @@ -use std::sync::Arc; -use std::time::Duration; - -use super::expr_ir::L3Expr; -use super::schema::{HasSchema, L3Schema, SchemaCatalog}; -use crate::types::AccuracyTarget; - -// ── Leaf / supporting types ─────────────────────────────────────────────────── - -/// A row-level filter predicate (WHERE clause / PromQL label matcher). -#[derive(Debug, Clone)] -pub struct Predicate(pub L3Expr); - -/// One item in a SELECT projection list. -#[derive(Debug, Clone)] -pub struct ProjectItem { - pub expr: L3Expr, - pub alias: Option, -} - -/// A GROUP BY key reference (column name). -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct GroupKey(pub String); -/// A reference to a column by name. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct ColumnRef(pub String); - -/// A set of partitioning keys (sharding hint for L5 stage allocator). -#[derive(Debug, Clone)] -pub struct PartitionKeys; - -/// One key in an ORDER BY or window OVER clause. -#[derive(Debug, Clone)] -pub struct SortKey { - pub expr: L3Expr, - pub ascending: bool, - pub nulls_first: bool, -} -/// An analytic window frame (ROWS / RANGE BETWEEN …). -#[derive(Debug, Clone)] -pub struct WindowFrame; -/// PromQL vector-match modifiers (`on`/`ignoring` + `group_left`/`group_right`). -#[derive(Debug, Clone)] -pub struct VectorMatch; -/// Reference to a metric by name (PromQL / OTLP). -#[derive(Debug, Clone)] -pub struct MetricRef; -/// Closed time interval [start_ms, end_ms] in milliseconds since Unix epoch. -/// Either bound may be `None`, meaning unbounded on that side. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct TimeRange { - pub start_ms: Option, - pub end_ms: Option, -} -/// Label matchers applied to a time-series scan. -#[derive(Debug, Clone)] -pub struct LabelFilter; -/// Reference to a relational table by name. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct TableRef(pub String); -/// Join key specification (USING / ON column reference). -#[derive(Debug, Clone)] -pub struct JoinKey; - -// ── Enum supporting types ───────────────────────────────────────────────────── - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum JoinKind { - Inner, - Left, - Right, - Full, - Cross, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum SetOpKind { - Union, - Intersect, - Except, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum BinaryOpKind { - // Arithmetic - Add, - Sub, - Mul, - Div, - Mod, - // Comparison - Eq, - NotEq, - Lt, - LtEq, - Gt, - GtEq, - // Boolean / PromQL set operators - And, - Or, - Unless, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum WindowFuncKind { - RowNumber, - Rank, - DenseRank, - Lag, - Lead, - FirstValue, - LastValue, - NthValue(Option), - Sum, - Avg, - Count, - Min, - Max, -} - -/// Which data model a `Source` or `AggIntent` operates over. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum DataModel { - TimeSeries, - Tabular, - /// Agnostic — works over either data model. - Any, -} - -// ── Time window kind ────────────────────────────────────────────────────────── - -/// The lifecycle / flush semantics of a streaming time window. -/// Used by `QueryExpr::TimeWindow`; distinct from SQL analytic frames -/// (`QueryExpr::WindowFunc`). -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum TimeWindowKind { - /// Non-overlapping fixed-size windows. - Tumbling, - /// Overlapping windows advancing by `slide` interval. - Sliding, - /// Windows that open on activity and close after a gap of inactivity. - Session, -} - -// ── Leaf data source ────────────────────────────────────────────────────────── - -/// The leaf data source of a query. Carried by `QueryExpr::Scan` to keep -/// L3 data-model-agnostic: everything above `Scan` (`Filter`, `Aggregate`, -/// etc.) works identically regardless of `Source` variant. -#[derive(Debug, Clone)] -pub enum Source { - /// Time-series input — deployment-model-asapquery / asaplifecycle shape. - TimeSeries { - metric: MetricRef, - time: TimeRange, - labels: LabelFilter, - }, - /// Tabular input — deployment-model-asapfusion / future-OLAP shape. - Table { - table_ref: TableRef, - columns: Vec, - /// Time range extracted from a WHERE predicate on the table's designated - /// time column. `None` means no time bound (full-history scan). - time_range: Option, - }, - /// Recursive join over sources (multi-table tabular queries). - Join { - left: Box, - right: Box, - on: JoinKey, - }, -} - -impl Source { - pub fn data_model(&self) -> DataModel { - match self { - Source::TimeSeries { .. } => DataModel::TimeSeries, - Source::Table { .. } | Source::Join { .. } => DataModel::Tabular, - } - } -} - -// ── Aggregation intent ──────────────────────────────────────────────────────── - -/// What to compute, not how. Sketch type and parameters are chosen by L4 -/// rules; `AggIntent` is the L3 statement of intent only. -/// -/// Heavy-hitter top-k (`TopK`) is a first-class intent because dedicated -/// sketch primitives (SpaceSaving, CMS-with-heap) compute it in one pass. -/// Generic ordering+limit stays as `QueryExpr::Sort + QueryExpr::Limit`. -#[derive(Debug, Clone)] -pub enum AggIntent { - // ── Data-model-agnostic ─────────────────────────────────────────────────── - Count { - accuracy: AccuracyTarget, - }, - Sum { - col: Option, - }, - Min { - col: Option, - }, - Max { - col: Option, - }, - Avg { - col: Option, - }, - /// Sample stddev when `population == false`; population stddev otherwise. - Stddev { - col: Option, - population: bool, - }, - Quantile { - q: f64, - accuracy: AccuracyTarget, - }, - /// Heavy-hitter top-k. Distinct from generic `Sort + Limit` — a - /// dedicated sketch (SpaceSaving, CMS-with-heap) computes it as a single - /// primitive. Recognised by L1→L2→L3 lowering on `ORDER BY count DESC - /// LIMIT k` / PromQL `topk(k, …)`. - TopK { - k: usize, - by: Vec, - accuracy: AccuracyTarget, - }, - Cardinality { - accuracy: AccuracyTarget, - }, - - // ── Time-series streaming derivatives ──────────────────────────────────── - // Include PromQL counter-reset adjustment; not equivalent to Sum/Count - // over a Window. Kept distinct so delta-set aggregators bind directly. - Rate { - window: Duration, - }, - Increase { - window: Duration, - }, -} - -impl AggIntent { - /// Which data model this intent semantically requires. L4 rules consult - /// this to skip non-applicable intents (e.g. `Rate` over a `Source::Table`). - pub fn requires(&self) -> DataModel { - match self { - Self::Rate { .. } | Self::Increase { .. } => DataModel::TimeSeries, - _ => DataModel::Any, - } - } - - /// The column this intent aggregates, if tracked. Used by schema derivation - /// to resolve the actual field type (e.g. so `MIN(ts: Int64)` → `Int64`). - /// Returns `None` for intents with no specific column (`Count`, `Cardinality`, - /// `TopK`, ...) and for the `"*"` wildcard sentinel produced by `agg_col` - /// when the aggregate arg is not a named column. - pub fn col(&self) -> Option<&ColumnRef> { - match self { - Self::Sum { col } - | Self::Min { col } - | Self::Max { col } - | Self::Avg { col } - | Self::Stddev { col, .. } => col.as_ref(), - _ => None, - } - } - - /// Output column type for a single-column aggregate result. - /// - /// Returns `None` for `TopK`, which produces multiple output columns. - /// Use `QueryExpr::output_schema` for TopK schema derivation instead. - pub fn output_type(&self, input: &super::schema::L3Field) -> Option { - use super::schema::L3DataType; - match self { - Self::Count { .. } | Self::Cardinality { .. } => Some(L3DataType::Int64), - Self::Min { .. } | Self::Max { .. } => Some(input.dtype.clone()), - Self::Sum { .. } - | Self::Avg { .. } - | Self::Stddev { .. } - | Self::Quantile { .. } - | Self::Rate { .. } - | Self::Increase { .. } => Some(L3DataType::Float64), - Self::TopK { .. } => None, - } - } -} - -// ── L3 DAG node ─────────────────────────────────────────────────────────────── - -/// A node in the L3 DAG. Wraps the expression and its derived output schema -/// so that every edge implicitly carries a typed schema: holding an -/// `Arc` gives you both the child expression and the schema of the -/// data flowing on that edge. -#[derive(Debug, Clone)] -pub struct L3Node { - pub expr: QueryExpr, - /// Output schema of `expr` — the schema of the data flowing on the edge - /// leading *from* this node to its parent(s). - pub schema: L3Schema, -} - -// ── L3 intent algebra IR ────────────────────────────────────────────────────── - -/// Language- and deployment-independent intent-only IR. No sketch types, -/// no sketch parameters, no language-specific operators. Traversing from -/// the root node yields a DAG; shared sub-expressions appear as multiple -/// `Rc` references to the same `L3Node`. -#[derive(Debug, Clone)] -pub enum QueryExpr { - // ── Base relations ──────────────────────────────────────────────────────── - /// Outermost leaf. `source` carries the data-model-specific leaf shape. - Scan { - source: Source, - predicates: Vec, - }, - /// Reference to a named `LetBinding` sub-expression; resolved at plan time. - Ref(String), - - // ── Filtering & projection ──────────────────────────────────────────────── - /// σ — row-level filter. Output schema = child schema (unchanged). - Filter { - child: Arc, - pred: Predicate, - }, - /// π — column projection. Output schema = child schema projected to `cols`. - Project { - child: Arc, - cols: Vec, - }, - - // ── Aggregation ─────────────────────────────────────────────────────────── - /// γ + α — GROUP BY + aggregate intents. Concrete operator (HashAgg / - /// SortAgg / SketchAgg) chosen by L4; `aggs` carry intent only. - Aggregate { - child: Arc, - by: Vec, - aggs: Vec, - having: Option, - /// Output column names parallel to `aggs`. When non-empty (populated by - /// the SQL lowerer from DataFusion's aggregate schema), `output_schema` - /// uses these names so that an enclosing `Project` can resolve aggregate - /// outputs by the names DataFusion assigned them (e.g. `"MIN(metrics.ts)"`). - /// Empty = fall back to synthetic `"agg_{i}"` names. - output_names: Vec, - }, - - // ── Time / streaming windows ────────────────────────────────────────────── - /// ψ — tumbling / sliding / session window over the time axis. Defines - /// the flush / reset lifecycle for aggregates in its sub-DAG. SQL analytic - /// frames are a different node (`WindowFunc`). - TimeWindow { - child: Arc, - kind: TimeWindowKind, - size: Duration, - slide: Option, - }, - - // ── Distributed-execution structure ─────────────────────────────────────── - /// Logical-only partitioning marker. Output schema = child schema. - /// Carries a sharding hint for the L5 stage allocator. - Partition { - child: Arc, - keys: PartitionKeys, - }, - /// δ — SQL `DISTINCT` / row deduplication. - Distinct { - child: Arc, - cols: Vec, - }, - /// ⊕ — exact union of sub-results from independent stages or shards. - /// Sketch unions are a separate node in `SummaryExpr` because they carry - /// sketch-family / params type constraints. - Merge { - children: Vec>, - }, - - // ── Joins ───────────────────────────────────────────────────────────────── - /// Logical join. L4 picks the physical alternative (HashJoin / - /// SortMergeJoin / SketchJoin) based on selectivity, memory budget, and - /// accuracy target. - Join { - kind: JoinKind, - left: Arc, - right: Arc, - pred: Option, - }, - - // ── Set operators ───────────────────────────────────────────────────────── - SetOp { - kind: SetOpKind, - all: bool, - left: Arc, - right: Arc, - }, - - // ── Ordering & limiting ─────────────────────────────────────────────────── - /// Generic order-by for non-heavy-hitter cases (`ORDER BY name LIMIT 10`). - /// Heavy-hitter shapes lower to `AggIntent::TopK` instead. - Sort { - child: Arc, - keys: Vec, - }, - Limit { - child: Arc, - /// `None` means no upper bound (only an offset applies). - n: Option, - offset: u64, - }, - - // ── Subquery / CTE ──────────────────────────────────────────────────────── - Subquery { - child: Arc, - alias: String, - }, - /// SQL `WITH name AS (expr) … body`; lowering target for PromQL - /// recording-rule bindings. The `expr` sub-DAG may be referenced N times - /// via `Ref(name)` in `body`, giving the DAG its fan-in. - LetBinding { - name: String, - expr: Arc, - body: Arc, - }, - - // ── Analytic window functions ───────────────────────────────────────────── - /// SQL `OVER (PARTITION BY … ORDER BY … ROWS BETWEEN …)`. - /// Distinct from `TimeWindow` — that is a streaming window over the time - /// axis; this is an analytic frame over already-grouped rows. - WindowFunc { - child: Arc, - func: WindowFuncKind, - /// Expressions the function operates on (e.g. `LAG(value)` → `[Column("value")]`). - /// Empty for rank-only funcs (`ROW_NUMBER`, `RANK`, `DENSE_RANK`). - args: Vec, - partition_by: Vec, - order_by: Vec, - frame: Option, - }, - - // ── Binary composition ──────────────────────────────────────────────────── - /// Arithmetic / comparison / boolean composition (PromQL binary ops - /// including `and` / `or` / `unless`, SQL boolean composition). - BinaryOp { - op: BinaryOpKind, - lhs: Arc, - rhs: Arc, - vector_match: Option, - }, -} - -impl HasSchema for QueryExpr { - fn output_schema(&self, input_schemas: &[&L3Schema], catalog: &SchemaCatalog) -> L3Schema { - use super::schema::{L3Field, L3Schema}; - - // Shorthand: first child's schema (most nodes have exactly one child). - let child = || input_schemas[0]; - - match self { - // ── Leaf: Table scan — schema comes from the catalog ────────────── - QueryExpr::Scan { source, .. } => match source { - Source::Table { table_ref, .. } => { - let table = catalog - .tables - .get(&table_ref.0) - .unwrap_or_else(|| panic!("table '{}' not in catalog", table_ref.0)); - let fields: Vec = table - .columns - .iter() - .map(|c| L3Field { - name: c.name.clone(), - dtype: c.data_type.clone(), - nullable: c.nullable, - }) - .collect(); - let time_index = table - .time_column - .as_ref() - .and_then(|tc| fields.iter().position(|f| &f.name == tc)); - L3Schema { fields, time_index } - } - Source::TimeSeries { .. } | Source::Join { .. } => { - todo!("schema derivation for TimeSeries and Join sources (PromQL path)") - } - }, - - // ── Project: one output field per ProjectItem ───────────────────── - QueryExpr::Project { cols, .. } => { - use super::expr_ir::L3Scalar; - use super::schema::L3DataType; - - let cs = child(); - let time_col_src = cs.time_index.map(|ti| cs.fields[ti].name.clone()); - - let pairs: Vec<(L3Field, bool)> = cols - .iter() - .map(|item| match &item.expr { - L3Expr::Column(col_ref) => { - // TODO: DataFusion may qualify column names with the table name - // (e.g. "metrics.value") while child schema stores bare names - // ("value"). Strip the qualifier before lookup so the fallback - // Float64 default is not silently applied to real columns. - let child_f = cs.fields.iter().find(|f| f.name == col_ref.0); - let (dtype, nullable) = child_f - .map(|f| (f.dtype.clone(), f.nullable)) - .unwrap_or((L3DataType::Float64, true)); - let out_name = item.alias.as_deref().unwrap_or(&col_ref.0).to_string(); - let is_time = time_col_src.as_deref() == Some(col_ref.0.as_str()); - ( - L3Field { - name: out_name, - dtype, - nullable, - }, - is_time, - ) - } - // CAST: output type is the cast target. - L3Expr::Cast { to, .. } => { - let name = item.alias.as_deref().unwrap_or("cast").to_string(); - ( - L3Field { - name, - dtype: to.clone(), - nullable: true, - }, - false, - ) - } - // Literal: infer type from the scalar variant. - L3Expr::Literal(scalar) => { - let (dtype, nullable) = match scalar { - L3Scalar::Int64(_) => (L3DataType::Int64, false), - L3Scalar::Float64(_) => (L3DataType::Float64, false), - L3Scalar::Utf8(_) => (L3DataType::Utf8, false), - L3Scalar::Boolean(_) => (L3DataType::Boolean, false), - L3Scalar::Null => (L3DataType::Float64, true), - }; - let name = item.alias.as_deref().unwrap_or("literal").to_string(); - ( - L3Field { - name, - dtype, - nullable, - }, - false, - ) - } - // Arithmetic, CASE, function calls, boolean exprs: - // default to Float64 (full type inference is future work). - _ => { - let name = item.alias.as_deref().unwrap_or("expr").to_string(); - ( - L3Field { - name, - dtype: L3DataType::Float64, - nullable: true, - }, - false, - ) - } - }) - .collect(); - - let time_index = pairs.iter().position(|(_, is_time)| *is_time); - let fields = pairs.into_iter().map(|(f, _)| f).collect(); - L3Schema { fields, time_index } - } - - // ── Pass-through: output schema == child schema ─────────────────── - QueryExpr::Filter { .. } - | QueryExpr::Sort { .. } - | QueryExpr::Limit { .. } - | QueryExpr::Distinct { .. } - | QueryExpr::Partition { .. } - | QueryExpr::TimeWindow { .. } => child().clone(), - - // ── Aggregate: GROUP BY cols + one output col per AggIntent ─────── - QueryExpr::Aggregate { - by, - aggs, - output_names, - .. - } => { - let cs = child(); - - // TopK is the only multi-column AggIntent: produces the TopK - // by-columns looked up from the child schema, followed by a - // synthetic "count" Int64 column. - if let [AggIntent::TopK { by: topk_by, .. }] = aggs.as_slice() { - let mut fields: Vec = topk_by - .iter() - .filter_map(|col| cs.fields.iter().find(|f| f.name == col.0).cloned()) - .collect(); - fields.push(L3Field { - name: "count".to_string(), - dtype: super::schema::L3DataType::Int64, - nullable: false, - }); - return L3Schema { - fields, - time_index: None, - }; - } - - // General case: GROUP BY fields (preserving child type) followed - // by one output field per AggIntent. Use the tracked col to look - // up the real field type; fall back to Float64 for untracked intents. - let float64_dummy = L3Field { - name: String::new(), - dtype: super::schema::L3DataType::Float64, - nullable: true, - }; - let by_fields: Vec = by - .iter() - .filter_map(|key| cs.fields.iter().find(|f| f.name == key.0).cloned()) - .collect(); - let agg_fields: Vec = aggs - .iter() - .enumerate() - .map(|(i, agg)| { - let col_field = agg - .col() - .and_then(|c| cs.fields.iter().find(|f| f.name == c.0)) - .cloned() - .unwrap_or_else(|| float64_dummy.clone()); - let name = output_names - .get(i) - .cloned() - .unwrap_or_else(|| format!("agg_{i}")); - L3Field { - name, - dtype: agg - .output_type(&col_field) - .unwrap_or(super::schema::L3DataType::Float64), - nullable: true, - } - }) - .collect(); - let time_col_name = cs.time_index.map(|ti| cs.fields[ti].name.clone()); - let all_fields: Vec = by_fields.into_iter().chain(agg_fields).collect(); - // Propagate time_index if a GROUP BY key preserved the time column in - // the output. Agg output columns are never the time axis. - let time_index = time_col_name.and_then(|tc| { - if by.iter().any(|key| key.0 == tc) { - all_fields.iter().position(|f| f.name == tc) - } else { - None - } - }); - L3Schema { - fields: all_fields, - time_index, - } - } - - // ── WindowFunc: child schema + one new column ───────────────────── - QueryExpr::WindowFunc { func, args, .. } => { - use super::schema::{L3DataType, L3Field}; - let cs = child(); - - // Resolve the first arg's type from the child schema. - // Falls back to Float64 for non-column exprs or unknown columns. - let arg_field = args.first().and_then(|a| match a { - L3Expr::Column(col_ref) => cs.fields.iter().find(|f| f.name == col_ref.0), - _ => None, - }); - let arg_dtype = || arg_field.map_or(L3DataType::Float64, |f| f.dtype.clone()); - - // TODO: the output column name should come from DataFusion's Window plan - // node schema (the actual name the enclosing Projection was built against, - // e.g. "row_number_1"), not a hardcoded string. An enclosing Project that - // references DataFusion's real name won't resolve it from this schema. - // Fix: thread the Window node's output field name through lower_window and - // store it on WindowFunc, then use it here instead of the hardcoded literals. - let (win_name, win_dtype, win_nullable) = match func { - WindowFuncKind::RowNumber => ("row_number", L3DataType::Int64, false), - WindowFuncKind::Rank => ("rank", L3DataType::Int64, false), - WindowFuncKind::DenseRank => ("dense_rank", L3DataType::Int64, false), - WindowFuncKind::Count => ("count", L3DataType::Int64, false), - WindowFuncKind::Sum => ("sum", L3DataType::Float64, true), - WindowFuncKind::Avg => ("avg", L3DataType::Float64, true), - // Navigation funcs: same type as arg, always nullable (boundary rows) - WindowFuncKind::Lag => ("lag", arg_dtype(), true), - WindowFuncKind::Lead => ("lead", arg_dtype(), true), - WindowFuncKind::FirstValue => ("first_value", arg_dtype(), true), - WindowFuncKind::LastValue => ("last_value", arg_dtype(), true), - WindowFuncKind::NthValue(_) => ("nth_value", arg_dtype(), true), - // Min/Max: preserve input type and nullability - WindowFuncKind::Min => { - ("min", arg_dtype(), arg_field.is_none_or(|f| f.nullable)) - } - WindowFuncKind::Max => { - ("max", arg_dtype(), arg_field.is_none_or(|f| f.nullable)) - } - }; - - let mut fields = cs.fields.clone(); - fields.push(L3Field { - name: win_name.to_string(), - dtype: win_dtype, - nullable: win_nullable, - }); - L3Schema { - fields, - time_index: cs.time_index, - } - } - - // ── Merge: all shards have identical schemas; use first ──────────── - QueryExpr::Merge { .. } => input_schemas[0].clone(), - - // ── SetOp: output is left-shaped (SQL semantics) ────────────────── - QueryExpr::SetOp { .. } => input_schemas[0].clone(), - - // ── Everything else: not yet implemented ────────────────────────── - _ => todo!( - "output_schema not yet implemented for {:?}", - std::mem::discriminant(self) - ), - } - } -} diff --git a/crates/core/src/intent_algebra/expr_ir.rs b/crates/core/src/intent_algebra/expr_ir.rs index 4005b8fd..97a1d990 100644 --- a/crates/core/src/intent_algebra/expr_ir.rs +++ b/crates/core/src/intent_algebra/expr_ir.rs @@ -1,10 +1,48 @@ -use super::expr::ColumnRef; -use super::schema::L3DataType; +//! Language-independent scalar expression IR. +//! +//! One generic [`Expr`] spans the lowering boundary; the two layers are +//! aliases that differ only in the column-reference type `C`: +//! +//! - [`L2Expr`] = `Expr` — name-based. The per-language front ends +//! emit it (PromQL label matchers, SQL `WHERE` / projection / sort-key +//! expressions) on the Layer-2 `relational` tree. +//! - [`L3Expr`] = `Expr` — **positional**. The canonical L3 +//! `query_expr` tree carries it; the converter resolves every `ColumnRef` +//! against the in-scope schema to produce it, so L3 column identity is +//! unambiguous (no name collisions across a join). +//! +//! `Expr` shares the scalar/operator vocabulary +//! ([`L3Scalar`], [`CompareOp`], [`ArithOp`]) — the **union** of what the two +//! front ends need: PromQL contributes `Regex` / `NotRegex` (`=~` / `!~`); SQL +//! contributes arithmetic, `CASE`, `IN`, `CAST`, `IS [NOT] NULL`, scalar +//! function calls, and the `LIKE` / `ILIKE` comparison family. -// ── Scalar literals ─────────────────────────────────────────────────────────── +use serde::{Deserialize, Serialize}; -/// A typed scalar constant. Used in `L3Expr::Literal`. -#[derive(Debug, Clone, PartialEq)] +use super::schema::{ColumnId, DataType}; + +/// A name-based column reference. This is an L2 / front-end concept — the +/// converter resolves every `ColumnRef` into a positional [`ColumnId`], so it +/// does not appear in the L3 [`QueryExpr`](super::query_expr::QueryExpr). It +/// includes the two PromQL-conventional synthetic columns. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum ColumnRef { + Named(String), + /// Table-qualified reference (`t.col` / `alias.col`). Resolved by + /// `(table, name)` so a column name shared across a join (`a.k` vs `b.k`) + /// binds to the correct side. + Qualified { + table: String, + name: String, + }, + /// The implicit metric sample value (PromQL — always the series value). + SampleValue, + /// All rows / COUNT(*). + Wildcard, +} + +/// A typed scalar constant. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum L3Scalar { Int64(i64), Float64(f64), @@ -13,10 +51,12 @@ pub enum L3Scalar { Null, } -// ── Comparison operators ────────────────────────────────────────────────────── - -/// Binary comparison operators for `L3Expr::Compare`. -#[derive(Debug, Clone, PartialEq, Eq)] +/// Binary comparison operators. +/// +/// `Regex` / `NotRegex` carry PromQL/RE2 regex-match semantics (`=~` / `!~`): +/// the right-hand side is a regular-expression pattern, not a literal value. +/// `Like` / `ILike` (+ negations) are the SQL pattern-match analogues. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum CompareOp { Eq, Ne, @@ -24,16 +64,41 @@ pub enum CompareOp { Le, Gt, Ge, + /// SQL `LIKE` — RHS is a `%`/`_` glob pattern. Like, + /// SQL `NOT LIKE`. NotLike, + /// SQL `ILIKE` — case-insensitive `LIKE`. ILike, + /// SQL `NOT ILIKE`. NotILike, + /// RHS is a regular-expression pattern; matches PromQL `=~`. + Regex, + /// RHS is a regular-expression pattern; matches PromQL `!~`. + NotRegex, } -// ── Arithmetic operators ────────────────────────────────────────────────────── +impl std::fmt::Display for CompareOp { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + CompareOp::Eq => "==", + CompareOp::Ne => "!=", + CompareOp::Lt => "<", + CompareOp::Le => "<=", + CompareOp::Gt => ">", + CompareOp::Ge => ">=", + CompareOp::Like => "LIKE", + CompareOp::NotLike => "NOT LIKE", + CompareOp::ILike => "ILIKE", + CompareOp::NotILike => "NOT ILIKE", + CompareOp::Regex => "=~", + CompareOp::NotRegex => "!~", + }) + } +} -/// Binary arithmetic operators for `L3Expr::Arith`. -#[derive(Debug, Clone, PartialEq, Eq)] +/// Binary arithmetic operators. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum ArithOp { Add, Sub, @@ -42,115 +107,134 @@ pub enum ArithOp { Mod, } -// ── Expression IR ───────────────────────────────────────────────────────────── +impl std::fmt::Display for ArithOp { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + ArithOp::Add => "+", + ArithOp::Sub => "-", + ArithOp::Mul => "*", + ArithOp::Div => "/", + ArithOp::Mod => "%", + }) + } +} -/// A scalar expression used in filter predicates, projection lists, and sort -/// keys. Flat conjunctions (`BoolAnd`) and disjunctions (`BoolOr`) make -/// per-conjunct selectivity estimation and PromQL label-matcher lowering -/// straightforward without recursive descent. -#[derive(Debug, Clone, PartialEq)] -pub enum L3Expr { - /// Reference to a named column. - Column(ColumnRef), +/// Scalar expression, generic over its column-reference type `C`. The two +/// lowering layers are aliases over the *same* shape — only the column +/// reference differs — so there is one definition (and one set of helpers) to +/// maintain, and the converter is a structural map that swaps `C`: +/// +/// - [`L2Expr`] = `Expr` — name-based, front-end-emitted. +/// - [`L3Expr`] = `Expr` — positional, resolved against the schema. +/// +/// Flat conjunctions (`BoolAnd`) / disjunctions (`BoolOr`) make per-conjunct +/// selectivity estimation and label-matcher lowering straightforward without +/// recursive descent. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum Expr { + /// A column reference — `ColumnRef` (L2) or positional `ColumnId` (L3). + Column(C), /// A constant literal value. Literal(L3Scalar), /// `left op right` — binary comparison. Compare { - left: Box, + left: Box>, op: CompareOp, - right: Box, + right: Box>, }, /// Flat conjunction (logical AND). An empty list is vacuously true. - BoolAnd(Vec), + BoolAnd(Vec>), /// Flat disjunction (logical OR). An empty list is vacuously false. - BoolOr(Vec), + BoolOr(Vec>), /// Logical NOT. - Not(Box), + Not(Box>), /// `expr IS NULL`. - IsNull(Box), + IsNull(Box>), /// `expr IS NOT NULL`. - IsNotNull(Box), - /// `CAST(expr AS to)`. `try_cast` is `true` for SQL `TRY_CAST`, which - /// returns `NULL` on conversion failure instead of raising an error. + IsNotNull(Box>), + /// `CAST(expr AS to)`; `try_cast` for SQL `TRY_CAST` (NULL on failure). Cast { - expr: Box, - to: L3DataType, + expr: Box>, + to: DataType, try_cast: bool, }, /// `expr [NOT] IN (v1, v2, …)`. InList { - expr: Box, - list: Vec, + expr: Box>, + list: Vec>, negated: bool, }, /// Scalar function call, e.g. `LOWER(col)`, `ABS(x)`. - FunctionCall { name: String, args: Vec }, + FunctionCall { name: String, args: Vec> }, /// Binary arithmetic: `left op right`. Arith { op: ArithOp, - left: Box, - right: Box, + left: Box>, + right: Box>, }, - /// SQL `CASE` expression (both searched and simple forms). - /// `operand` is present for simple CASE (`CASE expr WHEN ...`), - /// absent for searched CASE (`CASE WHEN condition THEN ...`). + /// SQL `CASE` (both searched and simple forms). `operand` present for the + /// simple form (`CASE expr WHEN …`), absent for searched. Case { - operand: Option>, - branches: Vec<(L3Expr, L3Expr)>, - else_expr: Option>, + operand: Option>>, + branches: Vec<(Expr, Expr)>, + else_expr: Option>>, }, } -impl L3Expr { - /// If this expression is a `BoolAnd`, return its elements. - /// Otherwise return a single-element slice containing `self`. - /// Lets callers iterate all top-level conjuncts without cloning. - pub fn conjuncts(&self) -> &[L3Expr] { +/// Layer-2 (name-based) scalar expression. Front ends emit this; the converter +/// resolves it into a positional [`L3Expr`]. +pub type L2Expr = Expr; + +/// Canonical L3 (positional) scalar expression — column references are +/// [`ColumnId`]s resolved against the in-scope schema, so identity is +/// unambiguous across joins / duplicate names. +pub type L3Expr = Expr; + +impl Expr { + /// If this expression is a `BoolAnd`, return its elements; otherwise a + /// single-element slice containing `self`. + pub fn conjuncts(&self) -> &[Expr] { match self { - L3Expr::BoolAnd(v) => v.as_slice(), + Expr::BoolAnd(v) => v.as_slice(), _ => std::slice::from_ref(self), } } - /// If this expression is a `BoolOr`, return its elements. - /// Otherwise return a single-element slice containing `self`. - pub fn disjuncts(&self) -> &[L3Expr] { + /// If this expression is a `BoolOr`, return its elements; otherwise a + /// single-element slice containing `self`. + pub fn disjuncts(&self) -> &[Expr] { match self { - L3Expr::BoolOr(v) => v.as_slice(), + Expr::BoolOr(v) => v.as_slice(), _ => std::slice::from_ref(self), } } - /// Recursively collect all `ColumnRef`s referenced anywhere in this - /// expression. Used by L4 for column-lineage and selectivity estimation. - pub fn columns_referenced(&self) -> Vec<&ColumnRef> { + /// Recursively collect every column reference anywhere in this expression. + /// Used by the Binder (L2) to seed usage-derived leaf schemas, and available + /// to L4 (L3) for column-lineage / selectivity. + pub fn columns_referenced(&self) -> Vec<&C> { match self { - L3Expr::Column(c) => vec![c], - L3Expr::Literal(_) => vec![], - L3Expr::Compare { left, right, .. } => { + Expr::Column(c) => vec![c], + Expr::Literal(_) => vec![], + Expr::Compare { left, right, .. } | Expr::Arith { left, right, .. } => { let mut v = left.columns_referenced(); v.extend(right.columns_referenced()); v } - L3Expr::BoolAnd(parts) | L3Expr::BoolOr(parts) => { + Expr::BoolAnd(parts) | Expr::BoolOr(parts) => { parts.iter().flat_map(|e| e.columns_referenced()).collect() } - L3Expr::Not(e) | L3Expr::IsNull(e) | L3Expr::IsNotNull(e) => e.columns_referenced(), - L3Expr::Cast { expr, .. } => expr.columns_referenced(), - L3Expr::InList { expr, list, .. } => { + Expr::Not(e) | Expr::IsNull(e) | Expr::IsNotNull(e) => e.columns_referenced(), + Expr::Cast { expr, .. } => expr.columns_referenced(), + Expr::InList { expr, list, .. } => { let mut v = expr.columns_referenced(); v.extend(list.iter().flat_map(|e| e.columns_referenced())); v } - L3Expr::FunctionCall { args, .. } => { + Expr::FunctionCall { args, .. } => { args.iter().flat_map(|e| e.columns_referenced()).collect() } - L3Expr::Arith { left, right, .. } => { - let mut v = left.columns_referenced(); - v.extend(right.columns_referenced()); - v - } - L3Expr::Case { + Expr::Case { operand, branches, else_expr, diff --git a/crates/core/src/intent_algebra/lower.rs b/crates/core/src/intent_algebra/lower.rs new file mode 100644 index 00000000..2238788d --- /dev/null +++ b/crates/core/src/intent_algebra/lower.rs @@ -0,0 +1,729 @@ +//! Layer-2 → canonical L3 IR converter. +//! +//! Recursively converts a whole [`relational::QueryExpr`] tree into a whole +//! [`query_expr::QueryExpr`] tree. The single-statistic sketchable `Aggregate` +//! fuses directly in canonical terms (window-swap, `Partition` wrap); see the +//! `Aggregate` arm. +//! +//! Name resolution is an explicit pass: [`convert_root`] runs the +//! [`Binder`](super::binder) first to build the complete, self-contained +//! schema every `ColumnId` indexes into, so positional resolution downstream +//! is total. + +use thiserror::Error; + +use crate::intent_algebra::agg_intent::AggIntent; +use crate::intent_algebra::binder::Binder; +use crate::intent_algebra::column_resolution::{ + output_schema_for_aggregate, resolve_column_refs, resolve_expr, ResolveError, +}; +use crate::intent_algebra::expr_ir::{ColumnRef, L2Expr, L3Expr, L3Scalar}; +use crate::intent_algebra::names::BindingName; +use crate::intent_algebra::query_expr::{ + PartitionKeys as CPartitionKeys, Predicate, ProjectItem, QueryExpr as CQueryExpr, SortKey, + Source, WindowKind, +}; +use crate::intent_algebra::relational::{AggFunc, QueryExpr as LQueryExpr, SourceSpec}; +use crate::intent_algebra::schema::{ColumnId, Schema}; +use crate::types::AccuracyTarget; + +/// Errors produced while converting a Layer-2 tree to canonical. +#[derive(Debug, Error)] +pub enum ConvertError { + /// A column reference (`Aggregate` key, `TopK` key, aggregate input column) + /// did not resolve against the child's derived schema. + #[error("column resolution failed: {0}")] + Resolve(#[from] ResolveError), + /// Deriving the schema of an already-converted child failed (needed to + /// resolve positional column references against it). + #[error("schema derivation failed: {0}")] + Schema(#[from] crate::intent_algebra::query_expr::QueryExprError), +} + +/// Lower a Layer-2 tree to canonical L3, threading `accuracy` onto every +/// approximate intent (`Count`, `Quantile`, `Cardinality`, `TopK`). +pub fn convert_root( + legacy: &LQueryExpr, + accuracy: &AccuracyTarget, +) -> Result { + let fallback = Binder::new().bind(legacy); + convert(legacy, &fallback, accuracy) +} + +/// Convert a Layer-2 tree to canonical L3. +/// +/// `fallback` is the leaf schema used for schema-less (PromQL) `Source`s — +/// the Binder's usage-derived `(ts, value)` floor + referenced labels. SQL +/// leaves carry their own resolved schema on [`SourceSpec::schema`], so the +/// fallback is unused for them. Positional column references (`Aggregate` +/// keys + input columns, `TopK` keys) resolve against the **converted child's +/// derived output schema**, so a `JOIN`'s concatenated schema and a table's +/// real columns bind to the right positions. +pub fn convert( + legacy: &LQueryExpr, + fallback: &Schema, + acc: &AccuracyTarget, +) -> Result { + Ok(match legacy { + LQueryExpr::Source(spec) => scan(spec, fallback, &[])?, + + LQueryExpr::Ref(name) => CQueryExpr::Ref { + name: BindingName::new(name.clone()), + }, + + // Fold label matchers / pushed-down predicates directly onto the Scan + // when the immediate child is a `Source`; otherwise emit a `Filter`. + // Predicate column refs resolve positionally against the input schema. + LQueryExpr::Filter { pred, input } => match input.as_ref() { + LQueryExpr::Source(spec) => scan(spec, fallback, pred.conjuncts())?, + other => { + let child = convert(other, fallback, acc)?; + let child_schema = child.output_schema()?; + CQueryExpr::Filter { + pred: Predicate(resolve_expr(pred, &child_schema)?), + child: Box::new(child), + } + } + }, + + LQueryExpr::Aggregate { + keys, + aggs, + having, + input, + } => { + // Single-statistic aggregate (no HAVING) over a *time-series* leaf + // fuses: a `Window` input becomes `Window { Aggregate { by: [] } }`; + // GROUP BY keys wrap the result in a `Partition` (the streaming + // sketch canonical shape). Tabular (SQL) GROUP BY instead falls + // through to the positional `Aggregate.by` path below, so the group + // keys land in the output schema (a SELECT projects them). The + // reducer's input column resolves against the aggregate's *direct* + // input (the scan under any window). + if aggs.len() == 1 && having.is_none() && !input.leaf_is_tabular() { + let (agg_input_l2, window): (&LQueryExpr, Option<(_, _)>) = match input.as_ref() { + LQueryExpr::Window { + duration, + slide, + input: win_input, + } => (win_input, Some((*duration, *slide))), + other => (other, None), + }; + let agg_child = convert(agg_input_l2, fallback, acc)?; + let agg_in_schema = agg_child.output_schema()?; + let intent = agg_func_to_intent( + &aggs[0].func, + acc, + resolve_agg_col(&aggs[0].col, &agg_in_schema)?, + ); + // Resolve the group keys positionally against the aggregate's + // input so the grouping lives in `Aggregate.by` — the *same* + // shape SQL produces. Only when this is a *non-windowed* + // reduction: an instant aggregate (`sum by (job) (m)`) or a + // cross-series reduction over a label-preserving `rate`/ + // `increase` (`sum by (job) (rate(m[w]))`), where the key is in + // scope. A *windowed* reduction here is per-series (e.g. + // `avg_over_time`) — its keys belong to an enclosing level, so + // keep the legacy name-based `Partition` marker instead of + // folding them into a per-series `by`. + let by = if window.is_none() { + resolve_column_refs(keys, &agg_in_schema).unwrap_or_default() + } else { + Vec::new() + }; + let grouped_positionally = by.len() == keys.len(); + let aggregate = CQueryExpr::Aggregate { + by, + aggs: vec![intent], + output_names: vec![aggs[0].alias.clone().unwrap_or_default()], + having: None, + child: Box::new(agg_child), + }; + let sketch = match window { + Some((duration, slide)) => CQueryExpr::Window { + kind: if slide.is_some() { + WindowKind::Sliding + } else { + WindowKind::Tumbling + }, + size: duration, + slide, + child: Box::new(aggregate), + }, + None => aggregate, + }; + return Ok(if grouped_positionally { + sketch + } else { + // Fallback (windowed per-series reduction): keep the legacy + // name-based `Partition`. These keys are PromQL labels + // (unqualified), so the bare names suffice. + CQueryExpr::Partition { + keys: CPartitionKeys::By(ref_names(keys)), + child: Box::new(sketch), + } + }); + } + + // Plain canonical `Aggregate`: multi-agg or HAVING-bearing. Keys + + // per-reducer input columns resolve against the child's (input) + // schema; HAVING references the aggregate's *output* columns, so it + // resolves against the derived output schema instead. + let child = convert(input, fallback, acc)?; + let child_schema = child.output_schema()?; + let by = resolve_column_refs(keys, &child_schema)?; + let intents: Vec = aggs + .iter() + .map(|item| -> Result { + let col = resolve_agg_col(&item.col, &child_schema)?; + Ok(agg_func_to_intent(&item.func, acc, col)) + }) + .collect::, _>>()?; + let output_names: Vec = aggs + .iter() + .map(|item| item.alias.clone().unwrap_or_default()) + .collect(); + let having = having + .as_ref() + .map(|h| -> Result { + let out_schema = + output_schema_for_aggregate(&child_schema, &by, &intents, &output_names); + Ok(Predicate(resolve_expr(h, &out_schema)?)) + }) + .transpose()?; + CQueryExpr::Aggregate { + by, + aggs: intents, + output_names, + having, + child: Box::new(child), + } + } + + LQueryExpr::Window { + duration, + slide, + input, + } => CQueryExpr::Window { + kind: if slide.is_some() { + WindowKind::Sliding + } else { + WindowKind::Tumbling + }, + size: *duration, + slide: *slide, + child: Box::new(convert(input, fallback, acc)?), + }, + + // π — resolve each project item's expression to positional against the + // child's schema. + LQueryExpr::Project { cols, input } => { + let child = convert(input, fallback, acc)?; + let child_schema = child.output_schema()?; + let cols = cols + .iter() + .map(|item| -> Result { + Ok(ProjectItem { + alias: item.alias.clone(), + expr: resolve_expr(&item.expr, &child_schema)?, + }) + }) + .collect::, _>>()?; + CQueryExpr::Project { + cols, + child: Box::new(child), + } + } + + LQueryExpr::Partition { keys, input } => CQueryExpr::Partition { + keys: keys.clone(), + child: Box::new(convert(input, fallback, acc)?), + }, + + LQueryExpr::Distinct { cols, input } => { + // Resolve the L2 (name-based) dedup keys to positional ids against + // the converted child's schema, like every other L3 column ref. + let child = convert(input, fallback, acc)?; + let cols = resolve_column_refs(cols, &child.output_schema()?)?; + CQueryExpr::Distinct { + cols, + child: Box::new(child), + } + } + + LQueryExpr::TopK { k, by, input } => { + let child = convert(input, fallback, acc)?; + let child_schema = child.output_schema()?; + let by = resolve_column_refs(by, &child_schema)?; + CQueryExpr::Aggregate { + by, + aggs: vec![AggIntent::TopK { + k: *k as usize, + accuracy: acc.clone(), + }], + output_names: vec![], + having: None, + child: Box::new(child), + } + } + + LQueryExpr::Merge { inputs } => CQueryExpr::Merge { + children: inputs + .iter() + .map(|i| convert(i, fallback, acc)) + .collect::, _>>()?, + }, + + LQueryExpr::Join { + kind, + pred, + left, + right, + } => { + // Each branch is bound independently (different leaves / label sets). + let left = convert_root(left, acc)?; + let right = convert_root(right, acc)?; + // The join predicate resolves against the concatenated left++right + // schema (the Join's own output shape), so left refs land at + // 0..left_len and right refs at left_len.. . + let mut concat = left.output_schema()?; + concat.columns.extend(right.output_schema()?.columns); + let pred = match pred { + Some(p) => Predicate(resolve_expr(p, &concat)?), + None => Predicate(L3Expr::Literal(L3Scalar::Boolean(true))), + }; + CQueryExpr::Join { + kind: kind.clone(), + pred, + left: Box::new(left), + right: Box::new(right), + } + } + + LQueryExpr::SetOp { + kind, + all, + left, + right, + } => CQueryExpr::SetOp { + kind: kind.clone(), + all: *all, + left: Box::new(convert_root(left, acc)?), + right: Box::new(convert_root(right, acc)?), + }, + + LQueryExpr::Sort { keys, input } => { + let child = convert(input, fallback, acc)?; + let child_schema = child.output_schema()?; + let keys = keys + .iter() + .map(|k| -> Result { + Ok(SortKey { + expr: resolve_expr(&k.expr, &child_schema)?, + ascending: k.ascending, + nulls_first: k.nulls_first, + }) + }) + .collect::, _>>()?; + CQueryExpr::Sort { + keys, + child: Box::new(child), + } + } + + LQueryExpr::Limit { n, offset, input } => CQueryExpr::Limit { + n: *n as usize, + offset: *offset as usize, + child: Box::new(convert(input, fallback, acc)?), + }, + + LQueryExpr::LetBinding { name, expr, body } => CQueryExpr::LetBinding { + name: BindingName::new(name.clone()), + expr: Box::new(convert(expr, fallback, acc)?), + child: Box::new(convert(body, fallback, acc)?), + }, + + LQueryExpr::PromQLSubquery { + range, + resolution, + input, + } => CQueryExpr::Subquery { + range: *range, + resolution: *resolution, + child: Box::new(convert(input, fallback, acc)?), + }, + + // Analytic window: args / partition-by / order-by resolve positionally + // against the child's output schema. + LQueryExpr::WindowFunc { + func, + args, + partition_by, + order_by, + output_name, + input, + } => { + let child = convert(input, fallback, acc)?; + let child_schema = child.output_schema()?; + let args = args + .iter() + .map(|a| resolve_expr(a, &child_schema)) + .collect::, _>>()?; + let partition_by = resolve_column_refs(partition_by, &child_schema)?; + let order_by = order_by + .iter() + .map(|k| -> Result { + Ok(SortKey { + expr: resolve_expr(&k.expr, &child_schema)?, + ascending: k.ascending, + nulls_first: k.nulls_first, + }) + }) + .collect::, _>>()?; + CQueryExpr::WindowFunc { + func: func.clone(), + args, + partition_by, + order_by, + output_name: output_name.clone(), + child: Box::new(child), + } + } + + LQueryExpr::BinaryOp { + op, + lhs, + rhs, + vector_match, + } => CQueryExpr::BinaryOp { + op: op.clone(), + // A binary op's two sides may scan different metrics with different + // label sets, so each branch must resolve against its OWN bound + // schema. `convert_root` re-runs the Binder per sub-tree; threading + // the parent `schema` (derived from the left leaf only) would bind + // the right side's columns to the wrong positions. + lhs: Box::new(convert_root(lhs, acc)?), + rhs: Box::new(convert_root(rhs, acc)?), + vector_match: vector_match.clone(), + }, + }) +} + +/// Build a canonical `Scan`. A schema-bearing [`SourceSpec`] (SQL table) emits +/// a `Source::Table` carrying that resolved schema; a schema-less one (PromQL) +/// emits a `Source::TimeSeries` carrying the Binder's usage-derived `fallback`. +/// The L2 predicate conjuncts are resolved positionally against the leaf schema. +fn scan( + spec: &SourceSpec, + fallback: &Schema, + pred_conjuncts: &[L2Expr], +) -> Result { + let (source, schema) = match &spec.schema { + Some(s) => ( + Source::Table { + table_ref: spec.name.clone(), + }, + s.clone(), + ), + None => ( + Source::TimeSeries { + metric: spec.name.clone(), + }, + fallback.clone(), + ), + }; + let predicates = pred_conjuncts + .iter() + .map(|e| -> Result { Ok(Predicate(resolve_expr(e, &schema)?)) }) + .collect::, _>>()?; + Ok(CQueryExpr::Scan { + source, + predicates, + schema, + }) +} + +/// Bare names of a `ColumnRef` list, dropping `SampleValue`/`Wildcard`. Used +/// only for the legacy name-based `Partition` fallback (PromQL labels, which +/// are unqualified — `Qualified` collapses to its bare `name`). +fn ref_names(refs: &[ColumnRef]) -> Vec { + refs.iter() + .filter_map(|c| match c { + ColumnRef::Named(n) => Some(n.clone()), + ColumnRef::Qualified { name, .. } => Some(name.clone()), + ColumnRef::SampleValue | ColumnRef::Wildcard => None, + }) + .collect() +} + +/// Resolve a Layer-2 aggregate-input [`ColumnRef`] to a positional input +/// column. `SampleValue` / `Wildcard` carry no specific column → `Ok(None)` +/// (the PromQL sample-value / `COUNT(*)` convention); a `Named` column +/// (`SUM(bytes)`) must resolve to its position, else it is an error — silently +/// dropping it to `None` would reduce the wrong column (the schema probe). +fn resolve_agg_col(col: &ColumnRef, schema: &Schema) -> Result, ResolveError> { + match col { + ColumnRef::Named(name) => { + schema + .column_id(name) + .map(Some) + .ok_or_else(|| ResolveError::NotFound { + name: name.clone(), + available: schema.columns.iter().map(|c| c.name.clone()).collect(), + }) + } + ColumnRef::Qualified { table, name } => schema + .column_id_qualified(table, name) + .or_else(|| schema.column_id(name)) + .map(Some) + .ok_or_else(|| ResolveError::NotFound { + name: format!("{table}.{name}"), + available: schema.columns.iter().map(|c| c.name.clone()).collect(), + }), + ColumnRef::SampleValue | ColumnRef::Wildcard => Ok(None), + } +} + +/// Map a Layer-2 [`AggFunc`] to its canonical [`AggIntent`], threading the +/// workload's accuracy target onto the approximate intents and the resolved +/// input column (`col`) onto the single-column reducers. `col = None` is the +/// PromQL sample-value convention. +fn agg_func_to_intent(func: &AggFunc, acc: &AccuracyTarget, col: Option) -> AggIntent { + match func { + AggFunc::Count => AggIntent::Count { + accuracy: acc.clone(), + }, + AggFunc::Sum => AggIntent::Sum { col }, + AggFunc::Avg => AggIntent::Avg { col }, + AggFunc::Min => AggIntent::Min { col }, + AggFunc::Max => AggIntent::Max { col }, + AggFunc::StdDev { population } => AggIntent::StdDev { + col, + population: *population, + }, + AggFunc::Variance { population } => AggIntent::Variance { + col, + population: *population, + }, + AggFunc::Quantile(q) => AggIntent::Quantile { + q: *q, + accuracy: acc.clone(), + }, + AggFunc::CountDistinct => AggIntent::Cardinality { + accuracy: acc.clone(), + }, + AggFunc::HeavyHitters { k } => AggIntent::TopK { + k: *k as usize, + accuracy: acc.clone(), + }, + AggFunc::Rate { window } => AggIntent::Rate { window: *window }, + AggFunc::Increase { window } => AggIntent::Increase { window: *window }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::intent_algebra::agg_intent::AggIntent; + use crate::intent_algebra::expr_ir::{CompareOp, L2Expr, L3Expr, L3Scalar}; + use crate::intent_algebra::query_expr::{JoinKind, QueryExpr as CQueryExpr}; + use crate::intent_algebra::relational::{ + AggFunc, AggItem, QueryExpr as LQueryExpr, SourceSpec, + }; + use crate::intent_algebra::schema::{Column, DataType, Schema}; + + fn col(name: &str, dtype: DataType) -> Column { + Column::new(name, dtype, false) + } + + /// A SQL-shaped `SELECT SUM(bytes), AVG(latency) FROM t` lowers each + /// reducer onto its own input column (positional), and the derived output + /// schema types each result off that column (`SUM(bytes:Int64)→Int64`). + #[test] + fn multi_column_aggregate_threads_per_agg_col() { + let schema = Schema::with_time_index( + vec![ + col("ts", DataType::Timestamp), + col("bytes", DataType::Int64), + col("latency", DataType::Float64), + col("value", DataType::Float64), + ], + 0, + vec![], + ); + let tree = LQueryExpr::Aggregate { + keys: vec![], + aggs: vec![ + AggItem { + alias: Some("total_bytes".into()), + func: AggFunc::Sum, + col: ColumnRef::Named("bytes".into()), + }, + AggItem { + alias: Some("avg_latency".into()), + func: AggFunc::Avg, + col: ColumnRef::Named("latency".into()), + }, + ], + having: None, + input: Box::new(LQueryExpr::Source(SourceSpec::new("t"))), + }; + + let l3 = convert(&tree, &schema, &AccuracyTarget::Exact).unwrap(); + let CQueryExpr::Aggregate { by, aggs, .. } = &l3 else { + panic!("expected Aggregate, got {l3:?}"); + }; + assert!(by.is_empty()); + // bytes is column 1, latency is column 2 in the input schema. + assert_eq!( + aggs, + &vec![ + AggIntent::Sum { col: Some(1) }, + AggIntent::Avg { col: Some(2) }, + ] + ); + + // Output schema types each reducer off its own input column, and names + // it from the AggItem alias (threaded via Aggregate.output_names). + let out = l3.output_schema().unwrap(); + assert_eq!(out.columns[0], col("total_bytes", DataType::Int64)); // SUM(bytes:Int64) + assert_eq!(out.columns[1], col("avg_latency", DataType::Float64)); // AVG(latency)→Float64 + } + + /// PromQL's single sample-value reducer stays `col: None`. + #[test] + fn promql_sample_value_agg_stays_col_none() { + let schema = Schema::with_time_index( + vec![ + col("ts", DataType::Timestamp), + col("value", DataType::Float64), + ], + 0, + vec![], + ); + let tree = LQueryExpr::Aggregate { + keys: vec![], + aggs: vec![AggItem { + alias: Some("value".into()), + func: AggFunc::Sum, + col: ColumnRef::SampleValue, + }], + having: None, + input: Box::new(LQueryExpr::Source(SourceSpec::new("m"))), + }; + let l3 = convert(&tree, &schema, &AccuracyTarget::Exact).unwrap(); + // single-agg fused path → bare Aggregate (no keys → no Partition) + let CQueryExpr::Aggregate { aggs, .. } = &l3 else { + panic!("expected Aggregate, got {l3:?}"); + }; + assert_eq!(aggs, &vec![AggIntent::Sum { col: None }]); + } + + /// `SELECT region, SUM(bytes), COUNT(*) FROM logs JOIN meta … GROUP BY region` + /// — keys and per-agg input columns resolve against the JOIN's *concatenated* + /// schema, not either leaf's. logs=[id,bytes] meta=[id,region] → + /// concat [id(0), bytes(1), id(2), region(3)]. + #[test] + fn aggregate_over_join_resolves_against_concatenated_schema() { + let logs = LQueryExpr::Source(SourceSpec::with_schema( + "logs", + Schema::new(vec![ + col("id", DataType::Int64), + col("bytes", DataType::Int64), + ]), + )); + let meta = LQueryExpr::Source(SourceSpec::with_schema( + "meta", + Schema::new(vec![ + col("id", DataType::Int64), + col("region", DataType::Utf8), + ]), + )); + let join = LQueryExpr::Join { + kind: JoinKind::Inner, + pred: None, + left: Box::new(logs), + right: Box::new(meta), + }; + let tree = LQueryExpr::Aggregate { + keys: vec![ColumnRef::Named("region".into())], + aggs: vec![ + AggItem { + alias: Some("tot".into()), + func: AggFunc::Sum, + col: ColumnRef::Named("bytes".into()), + }, + AggItem { + alias: Some("n".into()), + func: AggFunc::Count, + col: ColumnRef::Wildcard, + }, + ], + having: None, + input: Box::new(join), + }; + let l3 = convert_root(&tree, &AccuracyTarget::Exact).unwrap(); + let CQueryExpr::Aggregate { + by, aggs, child, .. + } = &l3 + else { + panic!("expected multi-agg Aggregate, got {l3:?}"); + }; + assert_eq!(by, &vec![3], "region is column 3 of the joined schema"); + assert_eq!( + aggs[0], + AggIntent::Sum { col: Some(1) }, + "bytes is column 1" + ); + assert!(matches!(aggs[1], AggIntent::Count { .. })); + assert!(matches!(child.as_ref(), CQueryExpr::Join { .. })); + } + + /// `GROUP BY region HAVING > 5` — HAVING references the aggregate + /// OUTPUT column (`n`), absent from the input schema, so it must resolve + /// against the derived output schema `[region(0), tot(1), n(2)]`. + #[test] + fn having_resolves_against_aggregate_output_schema() { + let schema = Schema::new(vec![ + col("region", DataType::Utf8), + col("bytes", DataType::Int64), + ]); + let tree = LQueryExpr::Aggregate { + keys: vec![ColumnRef::Named("region".into())], + aggs: vec![ + AggItem { + alias: Some("tot".into()), + func: AggFunc::Sum, + col: ColumnRef::Named("bytes".into()), + }, + AggItem { + alias: Some("n".into()), + func: AggFunc::Count, + col: ColumnRef::Wildcard, + }, + ], + having: Some(L2Expr::Compare { + left: Box::new(L2Expr::Column(ColumnRef::Named("n".into()))), + op: CompareOp::Gt, + right: Box::new(L2Expr::Literal(L3Scalar::Int64(5))), + }), + input: Box::new(LQueryExpr::Source(SourceSpec::with_schema("t", schema))), + }; + let l3 = convert(&tree, &Schema::default(), &AccuracyTarget::Exact).unwrap(); + let CQueryExpr::Aggregate { + having: Some(having), + .. + } = &l3 + else { + panic!("expected Aggregate with HAVING, got {l3:?}"); + }; + let L3Expr::Compare { left, .. } = &having.0 else { + panic!("expected Compare HAVING, got {:?}", having.0); + }; + assert_eq!( + **left, + L3Expr::Column(2), + "HAVING `n` resolves to the count output column (index 2), not the input schema" + ); + } +} diff --git a/crates/core/src/intent_algebra/mod.rs b/crates/core/src/intent_algebra/mod.rs index 16a53799..9eeb9568 100644 --- a/crates/core/src/intent_algebra/mod.rs +++ b/crates/core/src/intent_algebra/mod.rs @@ -1,11 +1,41 @@ -pub mod expr; +//! Layers 2–3 of the controller pipeline. +//! +//! - [`relational`] — the Layer-2 per-language algebra tree the parser front +//! ends emit (PromQL / SQL). +//! - [`lower`] — the L2→L3 converter ([`convert_root`]), which runs the +//! [`Binder`] for name resolution and folds single-statistic sketchable +//! aggregates into canonical shapes. +//! - [`query_expr`] — the canonical, language- and deployment-independent L3 +//! intent algebra ([`QueryExpr`] + [`AggIntent`]), with positional +//! [`ColumnId`] schema flow. +//! - [`cse`] — workload-level common-sub-expression elimination over L3. + +pub mod agg_intent; +pub mod binder; +pub mod column_resolution; +pub mod cse; pub mod expr_ir; +pub mod lower; +pub mod names; +pub mod query_expr; +pub mod relational; pub mod schema; -pub use expr::{ - AggIntent, BinaryOpKind, ColumnRef, DataModel, GroupKey, JoinKey, JoinKind, L3Node, - LabelFilter, MetricRef, PartitionKeys, Predicate, ProjectItem, QueryExpr, SetOpKind, SortKey, - Source, TableRef, TimeRange, TimeWindowKind, VectorMatch, WindowFrame, WindowFuncKind, +pub use agg_intent::{ + agg_accuracy, agg_is_exact, agg_is_mergeable, default_cardinality, default_quantile, AggIntent, +}; +pub use binder::{Binder, SchemaCatalog, UsageDerivedCatalog}; +pub use column_resolution::{ + infer_schema_for_root, infer_source_schema, output_schema_for_aggregate, resolve_column_ref, + resolve_column_refs, resolve_expr, ResolveError, +}; +pub use cse::{dedupe_subtrees, CseWorkloadPlan}; +pub use expr_ir::{ArithOp, ColumnRef, CompareOp, Expr, L2Expr, L3Expr, L3Scalar}; +pub use lower::{convert, convert_root, ConvertError}; +pub use names::{BindingName, QueryId}; +pub use query_expr::{ + BinaryOpKind, BindingScope, DataModel, GroupSide, JoinKind, PartitionKeys, Predicate, + ProjectItem, QueryExpr, QueryExprError, SetOpKind, SortKey, Source, VectorGrouping, + VectorMatch, VectorMatchKind, WindowFuncKind, WindowKind, }; -pub use expr_ir::{ArithOp, CompareOp, L3Expr, L3Scalar}; -pub use schema::{ColumnDef, HasSchema, L3DataType, L3Field, L3Schema, SchemaCatalog, TableSchema}; +pub use schema::{cse_reuse_is_legal, Column, ColumnId, CseError, DataType, Schema}; diff --git a/crates/core/src/intent_algebra/names.rs b/crates/core/src/intent_algebra/names.rs new file mode 100644 index 00000000..071bf905 --- /dev/null +++ b/crates/core/src/intent_algebra/names.rs @@ -0,0 +1,27 @@ +use serde::{Deserialize, Serialize}; + +/// Name of a `LetBinding` / `Ref` sub-expression (CSE producer alias). +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct BindingName(pub String); + +impl BindingName { + pub fn new(s: impl Into) -> Self { + Self(s.into()) + } + pub fn as_str(&self) -> &str { + &self.0 + } +} + +/// Stable identifier for one query within a workload (CSE root key). +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct QueryId(pub String); + +impl QueryId { + pub fn new(s: impl Into) -> Self { + Self(s.into()) + } + pub fn as_str(&self) -> &str { + &self.0 + } +} diff --git a/crates/core/src/intent_algebra/query_expr.rs b/crates/core/src/intent_algebra/query_expr.rs new file mode 100644 index 00000000..d8604028 --- /dev/null +++ b/crates/core/src/intent_algebra/query_expr.rs @@ -0,0 +1,870 @@ +//! The canonical Layer-3 intent algebra IR. +//! +//! Language- and deployment-independent. Box-owned tree (DAG fan-in is +//! expressed via `LetBinding` / `Ref`); column identity is **positional** +//! (`Aggregate.by: Vec`), resolved by the [`Binder`](super::binder) +//! against the self-contained [`Schema`] carried on each `Scan`. + +use std::collections::HashMap; +use std::time::Duration; + +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +use super::agg_intent::AggIntent; +use super::expr_ir::{ArithOp, CompareOp, L3Expr, L3Scalar}; +use super::names::BindingName; +use super::schema::{Column, ColumnId, DataType, Schema}; + +/// Errors from schema derivation over a canonical tree. +#[derive(Debug, Error)] +pub enum QueryExprError { + #[error("unresolved ref: {0}")] + UnresolvedRef(String), + #[error("by-column id {0} out of range (input has {1} columns)")] + InvalidGroupByColumn(ColumnId, usize), + #[error("Merge requires at least one child")] + EmptyMerge, +} + +// ── Leaf / supporting types ─────────────────────────────────────────────────── + +/// Lifecycle / flush semantics of a streaming time window. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum WindowKind { + Tumbling, + Sliding, + Session, +} + +/// Which data model a `Source` / `AggIntent` operates over. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum DataModel { + TimeSeries, + Tabular, + Any, +} + +/// The leaf data source of a `Scan`. The schema itself rides on the +/// `Scan.schema` field (Binder-built); `Source` carries only the leaf's +/// identity. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum Source { + /// Time-series leaf — PromQL / DC lifecycle. Produces `(ts, value, *labels)`. + TimeSeries { metric: String }, + /// Tabular leaf — asap-fusion / future OLAP. Columns ride on `Scan.schema`. + Table { table_ref: String }, +} + +impl Source { + pub fn data_model(&self) -> DataModel { + match self { + Source::TimeSeries { .. } => DataModel::TimeSeries, + Source::Table { .. } => DataModel::Tabular, + } + } +} + +/// Grouping key set (`by (...)` / `without (...)`). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum PartitionKeys { + By(Vec), + /// **Reserved**: PromQL `without(...)` is rejected up front (the + /// usage-derived schema can't enumerate the label complement), so no front + /// end produces this variant yet. + Without(Vec), +} + +impl PartitionKeys { + pub fn keys(&self) -> &[String] { + match self { + PartitionKeys::By(k) | PartitionKeys::Without(k) => k, + } + } + pub fn is_empty(&self) -> bool { + self.keys().is_empty() + } +} + +/// Operator on the query-level `BinaryOp` node. Reuses the scalar IR's +/// [`ArithOp`] / [`CompareOp`] so every arithmetic/comparison operator has +/// exactly one representation (and one `Display`) across the IR; the remaining +/// variants are PromQL vector-set / power ops with no scalar-IR counterpart. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum BinaryOpKind { + /// Arithmetic — `Add/Sub/Mul/Div/Mod` (shared with `L3Expr::Arith`). + Arith(ArithOp), + /// Comparison — `Eq/Ne/Lt/Le/Gt/Ge` + `Like/ILike/Regex` family (shared + /// with `L3Expr::Compare`). + Compare(CompareOp), + /// PromQL logical-set intersection (`and`). + And, + /// PromQL logical-set union (`or`). + Or, + /// PromQL logical-set complement (`unless`). + Unless, + /// Exponentiation (`^`) — PromQL vector op, no scalar-IR counterpart. + Pow, + /// `atan2` — PromQL vector op, no scalar-IR counterpart. + Atan2, +} + +impl std::fmt::Display for BinaryOpKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + BinaryOpKind::Arith(op) => write!(f, "{op}"), + BinaryOpKind::Compare(op) => write!(f, "{op}"), + BinaryOpKind::And => f.write_str("AND"), + BinaryOpKind::Or => f.write_str("OR"), + BinaryOpKind::Unless => f.write_str("unless"), + BinaryOpKind::Pow => f.write_str("^"), + BinaryOpKind::Atan2 => f.write_str("atan2"), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum JoinKind { + Inner, + Left, + Right, + Full, + Cross, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum SetOpKind { + Union, + Intersect, + Except, +} + +/// SQL analytic window function (`fn(...) OVER (…)`). Distinct from a streaming +/// time `Window`: this is an analytic frame over already-materialised rows. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WindowFuncKind { + RowNumber, + Rank, + DenseRank, + Lag, + Lead, + FirstValue, + LastValue, + /// `NTH_VALUE(expr, n)` — `n` is resolved from the (literal) 2nd argument. + NthValue(Option), + Sum, + Avg, + Count, + Min, + Max, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct SortKey { + pub expr: L3Expr, + pub ascending: bool, + pub nulls_first: bool, +} + +/// PromQL vector-match modifier (`on`/`ignoring` + `group_left`/`group_right`). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct VectorMatch { + pub kind: VectorMatchKind, + pub labels: Vec, + pub grouping: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum VectorMatchKind { + On, + Ignoring, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct VectorGrouping { + pub side: GroupSide, + pub labels: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum GroupSide { + Left, + Right, +} + +/// A row-level filter predicate (WHERE clause / PromQL label matcher). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Predicate(pub L3Expr); + +/// One item in a SELECT projection list. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ProjectItem { + pub alias: Option, + pub expr: L3Expr, +} + +// ── L3 intent algebra IR ────────────────────────────────────────────────────── + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum QueryExpr { + /// Outermost leaf. `schema` is the authoritative, self-contained output + /// schema (Binder-built); `predicates` are leaf-level row filters + /// (PromQL label matchers, pushed-down `WHERE` conjuncts). + Scan { + source: Source, + #[serde(default)] + predicates: Vec, + schema: Schema, + }, + /// Reference to a `LetBinding` by name; resolved at plan time. + Ref { name: BindingName }, + + /// σ — row-level filter. Output schema = child schema. + Filter { + pred: Predicate, + child: Box, + }, + /// π — column projection. + Project { + cols: Vec, + child: Box, + }, + + /// γ + α — GROUP BY (positional) + aggregate intents. + Aggregate { + by: Vec, + aggs: Vec, + /// Output column names parallel to `aggs`. A non-empty entry overrides + /// the synthetic intent-keyed name — SQL threads DataFusion's generated + /// name (e.g. `"sum(metrics.bytes)"`) here so an enclosing `Project` + /// resolves the aggregate output by the name it references. An empty + /// entry (or empty vec) falls back to `AggIntent::output_column`'s name + /// (PromQL's convention). + #[serde(default)] + output_names: Vec, + #[serde(default)] + having: Option, + child: Box, + }, + + /// ψ — tumbling / sliding / session window over the time axis. Window + /// over Aggregate is the canonical windowed-aggregate shape. + Window { + kind: WindowKind, + size: Duration, + #[serde(default)] + slide: Option, + child: Box, + }, + + /// Logical-only partitioning marker (sharding hint for L5). + Partition { + keys: PartitionKeys, + child: Box, + }, + /// δ — SQL `DISTINCT` / row deduplication. Positional like every other L3 + /// column reference; empty = dedup on all columns (`SELECT DISTINCT *`). + Distinct { + cols: Vec, + child: Box, + }, + /// ⊕ — exact union of sub-results from independent stages / shards. + Merge { children: Vec }, + + /// Logical join. L4 picks the physical alternative. + Join { + kind: JoinKind, + pred: Predicate, + left: Box, + right: Box, + }, + SetOp { + kind: SetOpKind, + all: bool, + left: Box, + right: Box, + }, + + /// Generic order-by for non-heavy-hitter cases. + Sort { + keys: Vec, + child: Box, + }, + Limit { + n: usize, + offset: usize, + child: Box, + }, + + /// SQL `WITH name AS (expr) … child`; PromQL recording-rule binding. + LetBinding { + name: BindingName, + expr: Box, + child: Box, + }, + + /// PromQL sub-query (`[range:resolution]`). Logical pass-through. + Subquery { + range: Duration, + #[serde(default)] + resolution: Option, + child: Box, + }, + + /// SQL analytic window function: `func(args) OVER (PARTITION BY … ORDER BY …)`. + /// Output schema = child schema + one column named `output_name` (the name + /// the enclosing `Project` references). Window frames are not modelled yet. + WindowFunc { + func: WindowFuncKind, + /// Operand expressions (`LAG(value)` → `[Column(value_id)]`); empty for + /// the rank-only functions (`ROW_NUMBER`/`RANK`/`DENSE_RANK`). + args: Vec, + partition_by: Vec, + order_by: Vec, + /// The output column's name — DataFusion's window-expr field name, so a + /// `Project` above resolves it (cf. `Aggregate.output_names`). + output_name: String, + child: Box, + }, + + /// Arithmetic / comparison / boolean composition (PromQL binary ops). + BinaryOp { + op: BinaryOpKind, + lhs: Box, + rhs: Box, + #[serde(default)] + vector_match: Option, + }, +} + +impl QueryExpr { + /// Output schema of the root of a single query (empty binding scope). + pub fn output_schema(&self) -> Result { + self.output_schema_in(&BindingScope::default()) + } + + /// Output schema given an explicit `LetBinding` scope. + pub fn output_schema_in(&self, scope: &BindingScope) -> Result { + match self { + QueryExpr::Scan { schema, .. } => Ok(schema.clone()), + + // ψ — a window reshapes the time axis but not the column set. Over a + // time-indexed Scan it preserves the time_index; over an Aggregate + // (the canonical Window-over-Aggregate fused shape) the child has + // already consumed the time axis, so the child schema passes through. + QueryExpr::Window { child, .. } => child.output_schema_in(scope), + + QueryExpr::Aggregate { + by, + aggs, + output_names, + child, + .. + } => { + let in_schema = child.output_schema_in(scope)?; + + // Per-series range reduction (`rate`/`increase`): one value out + // per series, so it is *label-preserving* — every label column + // survives and only the sample value is replaced (kept named + // `value`, the PromQL convention). This is what lets an outer + // cross-series `Aggregate.by` resolve its group keys positionally + // over `sum by (job) (rate(...))`. + if by.is_empty() && !aggs.is_empty() && aggs.iter().all(|a| a.is_per_series()) { + let value_idx = in_schema.column_id("value").or_else(|| { + (0..in_schema.columns.len()).find(|&i| Some(i) != in_schema.time_index) + }); + let mut columns = in_schema.columns.clone(); + if let Some(vi) = value_idx { + let mut out = aggs[0].output_column(&columns[vi]); + out.name = "value".into(); + columns[vi] = out; + } + return Ok(Schema { + columns, + time_index: in_schema.time_index, + unique_keys: in_schema.unique_keys.clone(), + }); + } + + let mut out_cols: Vec = Vec::with_capacity(by.len() + aggs.len()); + for &id in by { + let c = + in_schema + .columns + .get(id) + .ok_or(QueryExprError::InvalidGroupByColumn( + id, + in_schema.columns.len(), + ))?; + out_cols.push(c.clone()); + } + let value_col_idx = in_schema + .column_id("value") + .or_else(|| (0..in_schema.columns.len()).find(|i| !by.contains(i))); + let probe = value_col_idx + .and_then(|i| in_schema.columns.get(i)) + .cloned() + .unwrap_or_else(|| Column::new("value", DataType::Float64, false)); + // Each reducer types off its own input column (`SUM(bytes)` vs + // `AVG(latency)` in one node); `None` falls back to the sample- + // value probe (PromQL's single-column convention). A non-empty + // `output_names[i]` overrides the synthetic output column name. + for (i, intent) in aggs.iter().enumerate() { + let in_col = intent + .input_col() + .and_then(|id| in_schema.columns.get(id)) + .unwrap_or(&probe); + let mut out = intent.output_column(in_col); + if let Some(name) = output_names.get(i).filter(|s| !s.is_empty()) { + out.name = name.clone(); + } + out_cols.push(out); + } + let unique_keys = if by.is_empty() { + Vec::new() + } else { + vec![(0..by.len()).collect()] + }; + Ok(Schema { + columns: out_cols, + time_index: None, + unique_keys, + }) + } + + QueryExpr::LetBinding { name, expr, child } => { + let bound = expr.output_schema_in(scope)?; + let extended = scope.with(name.clone(), bound); + child.output_schema_in(&extended) + } + QueryExpr::Ref { name } => scope + .lookup(name) + .cloned() + .ok_or_else(|| QueryExprError::UnresolvedRef(name.as_str().into())), + + QueryExpr::Filter { child, .. } + | QueryExpr::Partition { child, .. } + | QueryExpr::Sort { child, .. } + | QueryExpr::Limit { child, .. } + | QueryExpr::Subquery { child, .. } => child.output_schema_in(scope), + + // π — one output column per projection item. Each item's type is + // inferred from its expression against the child schema; the name + // is the explicit alias or a derived default. Projection may drop + // the grouping/time columns, so unique_keys reset and time_index + // is re-found by name. + QueryExpr::Project { cols, child } => { + let in_schema = child.output_schema_in(scope)?; + let columns: Vec = cols + .iter() + .enumerate() + .map(|(i, item)| { + let (dtype, nullable) = infer_expr_type(&item.expr, &in_schema); + let name = item + .alias + .clone() + .unwrap_or_else(|| default_proj_name(&item.expr, i, &in_schema)); + Column::new(name, dtype, nullable) + }) + .collect(); + let time_index = columns.iter().position(|c| c.name == "ts"); + Ok(Schema { + columns, + time_index, + unique_keys: Vec::new(), + }) + } + + QueryExpr::Distinct { cols, child } => { + let mut out = child.output_schema_in(scope)?; + // Deduplicating on `cols` makes them a unique key of the result. + if !cols.is_empty() { + out.add_unique_key(cols.clone()); + } + Ok(out) + } + + QueryExpr::Merge { children } => children + .first() + .ok_or(QueryExprError::EmptyMerge) + .and_then(|c| c.output_schema_in(scope)), + // Set operations are union-compatible: both sides share the left's + // column shape, so the output schema is the left's. (Row identity + // is not preserved across a UNION, so unique_keys are dropped.) + QueryExpr::SetOp { left, .. } => { + let mut s = left.output_schema_in(scope)?; + s.unique_keys.clear(); + Ok(s) + } + // ⋈ — output is the concatenation of both inputs' columns. Outer + // joins make the non-preserved side nullable. Post-join row + // identity isn't provable in general, so unique_keys reset. + QueryExpr::Join { + kind, left, right, .. + } => { + let l = left.output_schema_in(scope)?; + let r = right.output_schema_in(scope)?; + let (left_null, right_null) = match kind { + JoinKind::Left => (false, true), + JoinKind::Right => (true, false), + JoinKind::Full => (true, true), + JoinKind::Inner | JoinKind::Cross => (false, false), + }; + let l_len = l.columns.len(); + let mut columns = Vec::with_capacity(l_len + r.columns.len()); + columns.extend(l.columns.iter().cloned().map(|mut c| { + c.nullable |= left_null; + c + })); + columns.extend(r.columns.iter().cloned().map(|mut c| { + c.nullable |= right_null; + c + })); + let time_index = l.time_index.or(r.time_index.map(|i| i + l_len)); + Ok(Schema { + columns, + time_index, + unique_keys: Vec::new(), + }) + } + // ψ-analytic — child schema + one appended window-output column. + QueryExpr::WindowFunc { + func, + args, + output_name, + child, + .. + } => { + let mut out = child.output_schema_in(scope)?; + // First operand's (dtype, nullable) from the child schema, owned + // so the borrow ends before we append. + let arg = args.first().and_then(|a| match a { + L3Expr::Column(id) => out.columns.get(*id), + _ => None, + }); + let arg_dtype = || arg.map_or(DataType::Float64, |c| c.dtype.clone()); + let (dtype, nullable) = match func { + WindowFuncKind::RowNumber + | WindowFuncKind::Rank + | WindowFuncKind::DenseRank + | WindowFuncKind::Count => (DataType::Int64, false), + WindowFuncKind::Sum | WindowFuncKind::Avg => (DataType::Float64, true), + // Navigation funcs: arg type, nullable (boundary rows are NULL). + WindowFuncKind::Lag + | WindowFuncKind::Lead + | WindowFuncKind::FirstValue + | WindowFuncKind::LastValue + | WindowFuncKind::NthValue(_) => (arg_dtype(), true), + WindowFuncKind::Min | WindowFuncKind::Max => { + (arg_dtype(), arg.is_none_or(|c| c.nullable)) + } + }; + out.columns + .push(Column::new(output_name.clone(), dtype, nullable)); + Ok(out) + } + + QueryExpr::BinaryOp { lhs, .. } => lhs.output_schema_in(scope), + } + } +} + +/// Infer the `(DataType, nullable)` a scalar [`L3Expr`] produces against an +/// input [`Schema`]. Used by `Project` schema derivation. Approximate at L3: +/// unknown columns and bare `FunctionCall`s fall back to a permissive default +/// (the L4/emit layer refines with a real function/type registry). +fn infer_expr_type(expr: &L3Expr, schema: &Schema) -> (DataType, bool) { + match expr { + L3Expr::Column(id) => schema + .columns + .get(*id) + .map(|c| (c.dtype.clone(), c.nullable)) + .unwrap_or((DataType::Float64, true)), + L3Expr::Literal(s) => match s { + L3Scalar::Int64(_) => (DataType::Int64, false), + L3Scalar::Float64(_) => (DataType::Float64, false), + L3Scalar::Utf8(_) => (DataType::Utf8, false), + L3Scalar::Boolean(_) => (DataType::Bool, false), + L3Scalar::Null => (DataType::Float64, true), + }, + // Boolean-valued expressions (SQL three-valued logic → nullable). + L3Expr::Compare { .. } + | L3Expr::BoolAnd(_) + | L3Expr::BoolOr(_) + | L3Expr::Not(_) + | L3Expr::IsNull(_) + | L3Expr::IsNotNull(_) + | L3Expr::InList { .. } => (DataType::Bool, true), + L3Expr::Arith { left, right, .. } => { + let (lt, ln) = infer_expr_type(left, schema); + let (rt, rn) = infer_expr_type(right, schema); + let dtype = if matches!(lt, DataType::Int64) && matches!(rt, DataType::Int64) { + DataType::Int64 + } else { + DataType::Float64 + }; + (dtype, ln || rn) + } + L3Expr::Cast { to, try_cast, expr } => { + let (_, nullable) = infer_expr_type(expr, schema); + (to.clone(), *try_cast || nullable) + } + // No function/type registry at L3 — default permissive. + L3Expr::FunctionCall { .. } => (DataType::Float64, true), + L3Expr::Case { + branches, + else_expr, + .. + } => branches + .first() + .map(|(_, then)| (infer_expr_type(then, schema).0, true)) + .or_else(|| else_expr.as_ref().map(|e| infer_expr_type(e, schema))) + .unwrap_or((DataType::Float64, true)), + } +} + +/// Default output-column name for a projection item with no explicit alias: +/// a bare column keeps its (schema) name; anything else gets `col_{i}`. +fn default_proj_name(expr: &L3Expr, idx: usize, schema: &Schema) -> String { + match expr { + L3Expr::Column(id) => schema + .columns + .get(*id) + .map(|c| c.name.clone()) + .unwrap_or_else(|| format!("col_{idx}")), + _ => format!("col_{idx}"), + } +} + +/// Lexical scope for `LetBinding` / `Ref` resolution. +#[derive(Debug, Default, Clone)] +pub struct BindingScope { + bindings: HashMap, +} + +impl BindingScope { + pub fn new() -> Self { + Self::default() + } + pub fn with(&self, name: BindingName, schema: Schema) -> Self { + let mut bindings = self.bindings.clone(); + bindings.insert(name.as_str().into(), schema); + Self { bindings } + } + pub fn lookup(&self, name: &BindingName) -> Option<&Schema> { + self.bindings.get(name.as_str()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::intent_algebra::expr_ir::{ArithOp, CompareOp}; + + fn col(name: &str, dtype: DataType, nullable: bool) -> Column { + Column::new(name, dtype, nullable) + } + + fn scan( + columns: Vec, + time_index: Option, + uk: Vec>, + ) -> QueryExpr { + QueryExpr::Scan { + source: Source::Table { + table_ref: "t".into(), + }, + predicates: vec![], + schema: Schema { + columns, + time_index, + unique_keys: uk, + }, + } + } + + #[test] + fn project_retypes_and_renames_per_item() { + let child = scan( + vec![ + col("ts", DataType::Timestamp, false), + col("host", DataType::Utf8, false), + col("value", DataType::Float64, false), + ], + Some(0), + vec![vec![0, 1]], + ); + let q = QueryExpr::Project { + cols: vec![ + // bare column passthrough keeps its (schema) name + type: host=col 1 + ProjectItem { + alias: None, + expr: L3Expr::Column(1), + }, + // arithmetic over value (col 2) → Float64 + ProjectItem { + alias: Some("dbl".into()), + expr: L3Expr::Arith { + op: ArithOp::Add, + left: Box::new(L3Expr::Column(2)), + right: Box::new(L3Expr::Column(2)), + }, + }, + // comparison → Bool (nullable under 3-valued logic) + ProjectItem { + alias: Some("flag".into()), + expr: L3Expr::Compare { + left: Box::new(L3Expr::Column(2)), + op: CompareOp::Gt, + right: Box::new(L3Expr::Literal(L3Scalar::Float64(0.0))), + }, + }, + ], + child: Box::new(child), + }; + let s = q.output_schema().unwrap(); + assert_eq!(s.columns.len(), 3); + assert_eq!(s.columns[0], col("host", DataType::Utf8, false)); + assert_eq!(s.columns[1], col("dbl", DataType::Float64, false)); + assert_eq!(s.columns[2], col("flag", DataType::Bool, true)); + // projection drops the time axis + unique keys (ts not retained) + assert!(s.time_index.is_none()); + assert!(s.unique_keys.is_empty()); + } + + #[test] + fn per_series_rate_preserves_labels() { + // A per-series range reduction (`rate`) is label-preserving: it produces + // one value per series, so every label survives and only the sample + // value is replaced (kept named `value`). This is what lets an outer + // cross-series `Aggregate.by` group on those labels positionally. + let child = scan( + vec![ + col("ts", DataType::Timestamp, false), + col("value", DataType::Float64, false), + col("job", DataType::Utf8, true), + ], + Some(0), + vec![], + ); + let rate = QueryExpr::Aggregate { + by: vec![], + aggs: vec![AggIntent::Rate { + window: Duration::from_secs(300), + }], + output_names: vec![], + having: None, + child: Box::new(child), + }; + let s = rate.output_schema().unwrap(); + assert_eq!( + s.columns + .iter() + .map(|c| c.name.as_str()) + .collect::>(), + vec!["ts", "value", "job"], + "rate preserves all labels; only the sample value is replaced" + ); + assert_eq!(s.time_index, Some(0)); + assert!(s.column_id("job").is_some(), "label survives the reduction"); + } + + #[test] + fn project_keeps_time_index_when_ts_passed_through() { + let child = scan( + vec![ + col("ts", DataType::Timestamp, false), + col("value", DataType::Float64, false), + ], + Some(0), + vec![], + ); + let q = QueryExpr::Project { + cols: vec![ + // value=col 1, ts=col 0 + ProjectItem { + alias: None, + expr: L3Expr::Column(1), + }, + ProjectItem { + alias: None, + expr: L3Expr::Column(0), + }, + ], + child: Box::new(child), + }; + let s = q.output_schema().unwrap(); + assert_eq!(s.columns[0].name, "value"); + assert_eq!(s.columns[1].name, "ts"); + assert_eq!(s.time_index, Some(1)); + } + + fn join(kind: JoinKind) -> QueryExpr { + let left = scan(vec![col("a", DataType::Int64, false)], None, vec![vec![0]]); + let right = scan(vec![col("b", DataType::Utf8, false)], None, vec![]); + QueryExpr::Join { + kind, + pred: Predicate(L3Expr::Literal(L3Scalar::Boolean(true))), + left: Box::new(left), + right: Box::new(right), + } + } + + #[test] + fn inner_join_concatenates_both_sides() { + let s = join(JoinKind::Inner).output_schema().unwrap(); + assert_eq!(s.columns.len(), 2); + assert_eq!(s.columns[0], col("a", DataType::Int64, false)); + assert_eq!(s.columns[1], col("b", DataType::Utf8, false)); + // post-join row identity not provable → no unique keys + assert!(s.unique_keys.is_empty()); + } + + #[test] + fn left_join_makes_right_side_nullable() { + let s = join(JoinKind::Left).output_schema().unwrap(); + assert!(!s.columns[0].nullable, "preserved left side stays non-null"); + assert!(s.columns[1].nullable, "right side nullable under LEFT JOIN"); + } + + #[test] + fn full_join_makes_both_sides_nullable() { + let s = join(JoinKind::Full).output_schema().unwrap(); + assert!(s.columns[0].nullable); + assert!(s.columns[1].nullable); + } + + #[test] + fn setop_takes_left_shape_and_drops_unique_keys() { + let left = scan( + vec![ + col("k", DataType::Utf8, false), + col("v", DataType::Int64, false), + ], + None, + vec![vec![0]], + ); + let right = scan( + vec![ + col("k", DataType::Utf8, false), + col("v", DataType::Int64, false), + ], + None, + vec![vec![0]], + ); + let q = QueryExpr::SetOp { + kind: SetOpKind::Union, + all: false, + left: Box::new(left), + right: Box::new(right), + }; + let s = q.output_schema().unwrap(); + assert_eq!(s.columns.len(), 2); + assert_eq!(s.columns[0].name, "k"); + assert!( + s.unique_keys.is_empty(), + "UNION does not preserve row identity" + ); + } +} diff --git a/crates/core/src/intent_algebra/relational.rs b/crates/core/src/intent_algebra/relational.rs new file mode 100644 index 00000000..9fbcdc50 --- /dev/null +++ b/crates/core/src/intent_algebra/relational.rs @@ -0,0 +1,299 @@ +//! The Layer-2 relational IR — the per-language query algebra the parser +//! front ends emit, before [`convert_root`](super::lower::convert_root) lowers +//! it to the canonical L3 [`query_expr::QueryExpr`](super::query_expr::QueryExpr). +//! +//! Leaf / scalar types (`ColumnRef`, `PartitionKeys`, `SortKey`, +//! `BinaryOpKind`, `VectorMatch`) are owned by `query_expr` and re-used here so +//! there is one canonical spelling. Filter / having / project expressions use +//! the shared language-independent [`L3Expr`](super::expr_ir::L3Expr). + +use std::time::Duration; + +pub use super::expr_ir::{ColumnRef, L2Expr}; +pub use super::query_expr::{BinaryOpKind, PartitionKeys, VectorMatch, WindowFuncKind}; +use super::schema::Schema; + +/// SELECT-list item at Layer 2 — a name-based [`L2Expr`] + optional alias. +/// (`query_expr::ProjectItem` is the positional L3 sibling.) +#[derive(Debug, Clone, PartialEq)] +pub struct L2ProjectItem { + pub alias: Option, + pub expr: L2Expr, +} + +/// ORDER BY key at Layer 2 — a name-based [`L2Expr`] + direction. +#[derive(Debug, Clone, PartialEq)] +pub struct L2SortKey { + pub expr: L2Expr, + pub ascending: bool, + pub nulls_first: bool, +} + +/// Base relation / metric stream source. +#[derive(Debug, Clone, PartialEq)] +pub struct SourceSpec { + /// Metric name (PromQL) or table name (SQL). + pub name: String, + /// Front-end-resolved leaf schema. `Some` for SQL tables (DataFusion knows + /// the columns); `None` for PromQL, where the [`Binder`](super::binder) + /// synthesises a usage-derived schema (the `(ts, value)` floor + referenced + /// labels). The presence of a schema also selects the L3 `Source` variant: + /// `Some` → `Source::Table`, `None` → `Source::TimeSeries`. + pub schema: Option, +} + +impl SourceSpec { + /// A PromQL-style leaf whose schema the Binder synthesises. + pub fn new(name: impl Into) -> Self { + Self { + name: name.into(), + schema: None, + } + } + + /// A SQL-style leaf carrying its front-end-resolved schema. + pub fn with_schema(name: impl Into, schema: Schema) -> Self { + Self { + name: name.into(), + schema: Some(schema), + } + } +} + +/// One aggregate function in a GROUP BY / AGGREGATE node. +#[derive(Debug, Clone, PartialEq)] +pub struct AggItem { + /// Output alias (`None` = use the intent's conventional name). Matches + /// `L2ProjectItem.alias`'s convention — no `""` sentinel. + pub alias: Option, + pub func: AggFunc, + pub col: ColumnRef, +} + +/// Layer-2 aggregate functions. Mapped to canonical [`AggIntent`] by +/// [`super::lower::convert`]. +#[derive(Debug, Clone, PartialEq)] +pub enum AggFunc { + Count, + Sum, + Avg, + Min, + Max, + StdDev { + population: bool, + }, + Variance { + population: bool, + }, + Quantile(f64), + /// COUNT DISTINCT — maps to `Cardinality`. + CountDistinct, + /// Heavy-hitter top-k — maps to `AggIntent::TopK`. + HeavyHitters { + k: u64, + }, + /// PromQL `rate()` / `irate()` — carries the range-vector window so the + /// canonical `Rate` intent owns it (no separate `Window` node). + Rate { + window: Duration, + }, + /// PromQL `increase()` — see `Rate`. + Increase { + window: Duration, + }, +} + +/// The Layer-2 relational query IR. +#[derive(Debug, Clone, PartialEq)] +pub enum QueryExpr { + /// A named metric stream or table — the outermost leaf. + Source(SourceSpec), + /// Reference to a CTE / let-binding by name. **Reserved**: no front end + /// emits `Ref`/`LetBinding` yet (CSE runs on L3); the converter arm exists + /// for forward-compatibility (e.g. PromQL recording rules). + Ref(String), + + /// σ — row-level filter (WHERE / PromQL label matchers). + Filter { pred: L2Expr, input: Box }, + + /// π — projection / SELECT list (SQL). Column refs in `cols` resolve by + /// name against the child schema during conversion. + Project { + cols: Vec, + input: Box, + }, + + /// γ + α — GROUP BY (`keys`) followed by aggregate functions. Keys are + /// `ColumnRef` (not bare strings) so a table-qualified key (`b.k`) resolves + /// to the correct join side, matching the scalar-predicate path. + Aggregate { + keys: Vec, + aggs: Vec, + having: Option, + input: Box, + }, + + /// ψ — time window (PromQL `[5m]`). + Window { + duration: Duration, + slide: Option, + input: Box, + }, + + /// Partition the stream by key-tuple (`by (dims)` / `without (dims)`). + /// **Reserved**: PromQL `by(...)` emits `Aggregate.keys` (the converter + /// synthesizes any L3 `Partition`); no front end emits an L2 `Partition`. + Partition { + keys: PartitionKeys, + input: Box, + }, + /// δ — deduplicate on `cols`. + Distinct { + cols: Vec, + input: Box, + }, + /// τ — heavy-hitter top-k. `by` are the grouping keys (qualified-capable). + TopK { + k: u64, + by: Vec, + input: Box, + }, + /// ⊕ — merge sub-results from independent branches. **Reserved**: SQL + /// `UNION` lowers to `SetOp`; no front end emits `Merge` yet (reserved for + /// sharded / fan-in plans). + Merge { inputs: Vec }, + + Join { + kind: super::query_expr::JoinKind, + pred: Option, + left: Box, + right: Box, + }, + SetOp { + kind: super::query_expr::SetOpKind, + all: bool, + left: Box, + right: Box, + }, + + Sort { + keys: Vec, + input: Box, + }, + Limit { + n: u64, + offset: u64, + input: Box, + }, + + /// **Reserved**: see [`Ref`](Self::Ref) — no front end emits `LetBinding` + /// yet (DAG fan-in / CSE is expressed on L3). + LetBinding { + name: String, + expr: Box, + body: Box, + }, + + /// PromQL sub-query syntax: `[range:resolution]`. + PromQLSubquery { + range: Duration, + resolution: Option, + input: Box, + }, + + /// SQL analytic window function `func(args) OVER (PARTITION BY … ORDER BY …)`. + WindowFunc { + func: WindowFuncKind, + args: Vec, + partition_by: Vec, + order_by: Vec, + output_name: String, + input: Box, + }, + + /// Binary op between two instant-vector expressions (PromQL `+`, `/`, …). + BinaryOp { + op: BinaryOpKind, + lhs: Box, + rhs: Box, + vector_match: Option, + }, +} + +impl QueryExpr { + /// Walk the tree depth-first, calling `f` on every node. + pub fn walk(&self, f: &mut F) { + f(self); + match self { + QueryExpr::Source(_) | QueryExpr::Ref(_) => {} + QueryExpr::Filter { input, .. } + | QueryExpr::Project { input, .. } + | QueryExpr::Aggregate { input, .. } + | QueryExpr::Window { input, .. } + | QueryExpr::Partition { input, .. } + | QueryExpr::Distinct { input, .. } + | QueryExpr::TopK { input, .. } + | QueryExpr::Sort { input, .. } + | QueryExpr::Limit { input, .. } + | QueryExpr::WindowFunc { input, .. } + | QueryExpr::PromQLSubquery { input, .. } => input.walk(f), + QueryExpr::Merge { inputs } => { + for i in inputs { + i.walk(f); + } + } + QueryExpr::Join { left, right, .. } + | QueryExpr::SetOp { left, right, .. } + | QueryExpr::BinaryOp { + lhs: left, + rhs: right, + .. + } => { + left.walk(f); + right.walk(f); + } + QueryExpr::LetBinding { expr, body, .. } => { + expr.walk(f); + body.walk(f); + } + } + } + + /// The leftmost `Source` leaf. + pub fn leaf_source(&self) -> Option<&SourceSpec> { + match self { + QueryExpr::Source(s) => Some(s), + QueryExpr::Filter { input, .. } + | QueryExpr::Project { input, .. } + | QueryExpr::Aggregate { input, .. } + | QueryExpr::Window { input, .. } + | QueryExpr::Partition { input, .. } + | QueryExpr::Distinct { input, .. } + | QueryExpr::TopK { input, .. } + | QueryExpr::Sort { input, .. } + | QueryExpr::Limit { input, .. } + | QueryExpr::WindowFunc { input, .. } + | QueryExpr::PromQLSubquery { input, .. } => input.leaf_source(), + QueryExpr::Merge { inputs } => inputs.first()?.leaf_source(), + QueryExpr::Join { left, .. } + | QueryExpr::SetOp { left, .. } + | QueryExpr::BinaryOp { lhs: left, .. } => left.leaf_source(), + QueryExpr::LetBinding { body, .. } => body.leaf_source(), + QueryExpr::Ref(_) => None, + } + } + + /// Outermost metric/table name from the first `Source` leaf. + pub fn source_name(&self) -> Option<&str> { + self.leaf_source().map(|s| s.name.as_str()) + } + + /// Whether the leftmost `Source` leaf carries a resolved schema — i.e. it is + /// a SQL table (`Source::Table`). Time-series (PromQL) leaves return + /// `false`. The converter uses this to keep the time-series fused-Partition + /// canonical shape for PromQL while routing tabular GROUP BY through a + /// positional `Aggregate.by` (so group keys land in the output schema). + pub fn leaf_is_tabular(&self) -> bool { + self.leaf_source().is_some_and(|s| s.schema.is_some()) + } +} diff --git a/crates/core/src/intent_algebra/schema.rs b/crates/core/src/intent_algebra/schema.rs index 69034018..235afb72 100644 --- a/crates/core/src/intent_algebra/schema.rs +++ b/crates/core/src/intent_algebra/schema.rs @@ -1,95 +1,381 @@ -use std::collections::HashMap; - -/// Catalog of known relational tables and their column definitions. -/// Passed to `lower_batch` so the SQL lowerer can resolve table and column -/// types and identify each table's designated time column. -#[derive(Debug, Clone, Default)] -pub struct SchemaCatalog { - pub tables: HashMap, -} +//! Layer 3 schema flow — every L3 edge carries a typed `Schema`. +//! +//! Per `control_plane/docs/design.md` §6 "Schema flow — every L3 edge carries +//! a typed schema". The DAG is type-checked: a node's output schema is a +//! function of its inputs and parameters and is verifiable independently +//! of the surrounding context. +//! +//! `Schema::unique_keys` is the load-bearing field for the workload-level +//! CSE pass (`design.md` §6 "DAG, not tree" + the batched-queries example +//! around line ~1284). Two `QueryExpr::Ref` consumers can share a producer +//! only when its output schema is provably stable across reads — the +//! unique-key metadata is what lets the deduper assert that. +//! +//! Single-query plans don't read this field; it lives here so the metadata +//! is available the moment workload-aware planning lands without requiring +//! an L3-wide schema change. -/// Schema for a single relational table. -#[derive(Debug, Clone)] -pub struct TableSchema { - pub columns: Vec, - /// Name of the column that holds the row timestamp. When set, WHERE - /// predicates on this column are extracted into `Source::Table.time_range` - /// rather than left as opaque `Filter` predicates. - pub time_column: Option, -} +#![allow(dead_code)] -impl TableSchema { - /// Returns `Err` if `time_column` names a column that does not exist in `columns`. - pub fn validate(&self) -> Result<(), String> { - if let Some(tc) = &self.time_column { - if !self.columns.iter().any(|c| &c.name == tc) { - return Err(format!( - "time_column '{tc}' not found in table columns {:?}", - self.columns.iter().map(|c| &c.name).collect::>() - )); - } - } - Ok(()) - } -} +use serde::{Deserialize, Serialize}; + +/// Index into [`Schema::columns`] used everywhere a column position is +/// referenced (group-by keys, unique-key sets, the time axis index). +/// +/// Aliased to `usize` to match `design.md`'s `Vec>` for +/// `unique_keys`. Kept as a named type so downstream code can pattern on +/// the intent ("this is a column position, not just any number"). +pub type ColumnId = usize; -/// One column in a `TableSchema`. -#[derive(Debug, Clone)] -pub struct ColumnDef { +/// One column in a [`Schema`]. Mirrors `design.md` §6 `Field` — +/// `name + dtype + nullable`. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct Column { + /// Column name as it appears in the producer's output. PromQL leaves + /// produce label-name + the synthetic `value` / `timestamp` columns; + /// SQL leaves carry their `information_schema` names. pub name: String, - pub data_type: L3DataType, + /// Column data type. Kept narrow at L3 (`Int64` / `Float64` / `Utf8` + /// / `Bool` / `Timestamp`); `Sketch(...)` is an L4-only addition per + /// design.md §6.4 and is intentionally absent here. + pub dtype: DataType, + /// Whether NULL values are allowed in this column. PromQL value + /// columns are non-nullable; SQL columns inherit their DDL nullability. pub nullable: bool, + /// Optional table/alias qualifier (SQL `t.col` / `t AS a` → `a`). Travels + /// with the column through joins so a `ColumnRef::Qualified` can pick the + /// right side when both carry the same `name`. `None` for PromQL labels and + /// unqualified columns. + #[serde(default)] + pub table: Option, } -// ── Data types ──────────────────────────────────────────────────────────────── +impl Column { + /// An unqualified column (`table = None`). + pub fn new(name: impl Into, dtype: DataType, nullable: bool) -> Self { + Self { + name: name.into(), + dtype, + nullable, + table: None, + } + } -/// Column types that may appear on an L3 DAG edge. -/// L4 extends this set with `L4DataType::Sketch`; L3 edges never carry -/// sketch-state columns. -#[derive(Debug, Clone, PartialEq)] -pub enum L3DataType { + /// This column re-qualified under `table` (e.g. by a `SubqueryAlias`). + pub fn with_table(mut self, table: impl Into) -> Self { + self.table = Some(table.into()); + self + } +} + +/// L3 column data types. Deliberately narrow: no sketch state at this +/// layer (see `design.md` §6.4 for the L4 `DataType::Sketch(...)` +/// extension). +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DataType { + /// 64-bit signed integer. Counter columns, group-cardinality outputs. Int64, + /// 64-bit IEEE-754 float. Quantile / Avg / Sum-over-floats output. Float64, + /// UTF-8 string. PromQL label values, SQL `VARCHAR` / `TEXT`. Utf8, - Boolean, + /// Boolean — predicate output, `unless` / `and` / `or` PromQL ops. + Bool, + /// Wall-clock timestamp. PromQL leaves carry exactly one of these + /// (the `time_index` column); SQL leaves may or may not. Timestamp, - Duration, - /// Key→Value map (e.g. PromQL label set encoded as a column). - Map(Box, Box), - List(Box), } -// ── Schema ──────────────────────────────────────────────────────────────────── +/// Per-edge L3 schema. Flowing between any two L3 operators, on every +/// node's input and output. +/// +/// `unique_keys` is metadata for reuse-aware planning: each inner `Vec` +/// is a set of column indices that together uniquely identify rows. The +/// outer `Vec` allows multiple unique-key sets (primary key + another +/// unique constraint). Populated by per-node input/output spec — +/// `Aggregate { by, .. }` emits `unique_keys = [by]`; `Distinct { cols }` +/// adds `cols`; most other nodes pass through. +/// +/// **Consumed by**: workload-level CSE (`CostModel::workload_cost` in the +/// design, not yet shipped). The single-query path, the `Bind*` rules, +/// push-down, and L5 emitters do not read this field. +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct Schema { + /// Columns flowing on this edge, in positional order. + pub columns: Vec, + /// Index into `columns` for the time axis, if any. PromQL leaves + /// always carry one; SQL leaves may or may not. + #[serde(default)] + pub time_index: Option, + /// Unique-key sets — each inner vec is a tuple of column indices + /// that together uniquely identifies a row. Empty `Vec` means + /// "no provable unique constraint" (the conservative default). + #[serde(default)] + pub unique_keys: Vec>, +} -#[derive(Debug, Clone, PartialEq)] -pub struct L3Field { - pub name: String, - pub dtype: L3DataType, - pub nullable: bool, +impl Schema { + /// Construct a `Schema` from columns alone — no time index, no + /// unique-key constraint. Used by `Scan` over a tabular source + /// when the catalog supplies no primary-key metadata. + pub fn new(columns: Vec) -> Self { + Self { + columns, + time_index: None, + unique_keys: Vec::new(), + } + } + + /// Construct a `Scan`-style schema with explicit `time_index` + + /// inferred unique keys (e.g. PromQL leaves: `[time_index, label_set]`). + pub fn with_time_index( + columns: Vec, + time_index: ColumnId, + unique_keys: Vec>, + ) -> Self { + Self { + columns, + time_index: Some(time_index), + unique_keys, + } + } + + /// Look up a column by name (first match). `None` if not present. + pub fn column_id(&self, name: &str) -> Option { + self.columns.iter().position(|c| c.name == name) + } + + /// Look up a column by `(table, name)` qualifier — disambiguates columns + /// that share a `name` across a join (`a.k` vs `b.k`). `None` if no column + /// has both that qualifier and name. + pub fn column_id_qualified(&self, table: &str, name: &str) -> Option { + self.columns + .iter() + .position(|c| c.name == name && c.table.as_deref() == Some(table)) + } + + /// Whether this schema has *any* provable unique key. The CSE pass + /// reads this to decide whether two `Ref` consumers can safely share + /// a producer (see `design.md` §6 line ~1284 + the unit test in + /// `tests::cse_substitution_legal_only_with_unique_keys`). + pub fn has_unique_key(&self) -> bool { + !self.unique_keys.is_empty() + } + + /// Append `cols` as an additional unique-key set if not already present. + /// Used by `Distinct { cols }` per design.md §6 schema-flow table: + /// "the input schema with `unique_keys` tightened to include `cols`". + pub fn add_unique_key(&mut self, cols: Vec) { + if !self.unique_keys.contains(&cols) { + self.unique_keys.push(cols); + } + } } -/// Schema carried on every edge of the L3 DAG. Describes the columns -/// flowing between two operators. Type-checked at plan construction time: -/// a node whose predicate references a column absent from its child's -/// `L3Schema` is a plan-time error. -#[derive(Debug, Clone, PartialEq)] -pub struct L3Schema { - pub fields: Vec, - /// Index into `fields` for the time axis, if any. - /// PromQL `Scan` leaves always carry one; SQL leaves may or may not. - pub time_index: Option, +// ── CSE legality (the load-bearing consumer of `unique_keys`) ──────────────── +// +// Phase F per `control_plane/docs/design.md` §6 Schema flow + the batched- +// queries example (§6 line ~1320): +// +// "CSE legality leans on `Schema::unique_keys` (§6 Schema flow): two +// `QueryExpr::Ref` consumers can share a producer only when its +// output schema is provably stable across reads — the unique-key +// metadata is what lets the deduper assert that without re-running +// the producer's logic." +// +// `cse_reuse_is_legal` is the gatekeeper. The workload-level CSE pass +// (`intent_algebra::cse::dedupe_subtrees`) consults it before emitting +// a `LetBinding` to share a producer between ≥2 `Ref` consumers. + +use thiserror::Error; + +/// Errors returned by [`cse_reuse_is_legal`] when shared-producer reuse +/// would violate the design's stability invariant. +#[derive(Debug, Error, PartialEq, Eq)] +pub enum CseError { + /// Producer schema lacks any `unique_keys` set — row identity is + /// not provably stable across reads, so two `Ref` consumers cannot + /// safely share it. The deduper falls back to per-consumer + /// recomputation. Per design.md §6 line ~1356. + #[error( + "shared-producer CSE refused: producer schema has no unique_keys \ + (design.md §6 schema-flow — without a provable unique key the \ + deduper cannot assert row identity across reads)" + )] + NoUniqueKeys, + /// Trivially-callable case: only one consumer means no reuse to + /// gate. Returned so the caller can short-circuit instead of + /// emitting a degenerate `LetBinding`. + #[error("CSE not applicable: {0} consumer(s) — need ≥ 2 for shared-producer reuse")] + InsufficientConsumers(usize), } -// ── Schema derivation trait ─────────────────────────────────────────────────── - -/// Implemented by `QueryExpr` to compute the output schema of a node given -/// its children's output schemas. The `L3Node` wrapper stores the derived -/// schema so derivation runs once at construction, not on every traversal. -pub trait HasSchema { - /// # Panics - /// Panics if a `Scan` node references a table that is not present in - /// `catalog`. Callers must ensure every table referenced by the expression - /// tree is registered in the catalog before calling this method. - /// `lower_batch` enforces this invariant via upfront catalog validation. - fn output_schema(&self, input_schemas: &[&L3Schema], catalog: &SchemaCatalog) -> L3Schema; +/// Two `QueryExpr::Ref` nodes can share a producer (same `LetBinding`) +/// only when the producer's output schema has stable per-row identity — +/// i.e. `Schema::unique_keys` is non-empty. This is the gatekeeper: +/// returns `Ok(())` if shared-producer reuse is legal, otherwise `Err`. +/// +/// Per design.md §6 line ~1356 — `unique_keys` is what makes CSE +/// provably correct. The deduper consults this before emitting a +/// `LetBinding`, and `CostModel::workload_cost` only credits a shared +/// binding when this gate has fired green. +/// +/// `consumer_count` is the number of `QueryExpr::Ref { name }` sites the +/// deduper has identified for the candidate binding. Single-consumer +/// cases short-circuit with `InsufficientConsumers` — a `LetBinding` +/// with one `Ref` is just a no-op alias and shouldn't be hoisted. +pub fn cse_reuse_is_legal(producer_schema: &Schema, consumer_count: usize) -> Result<(), CseError> { + if consumer_count < 2 { + return Err(CseError::InsufficientConsumers(consumer_count)); + } + if !producer_schema.has_unique_key() { + return Err(CseError::NoUniqueKeys); + } + Ok(()) +} + +// ── Tests ───────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + fn col(name: &str, dtype: DataType) -> Column { + Column::new(name, dtype, false) + } + + /// `cse_reuse_is_legal` accepts a producer schema with at least one + /// `unique_keys` set + ≥2 consumers. This is the design.md §6 + /// "load-bearing" green path. + #[test] + fn cse_reuse_legal_when_unique_keys_set() { + let producer = Schema::with_time_index( + vec![ + col("ts", DataType::Timestamp), + col("service", DataType::Utf8), + col("value", DataType::Float64), + ], + 0, + vec![vec![0, 1]], + ); + assert_eq!(cse_reuse_is_legal(&producer, 2), Ok(())); + assert_eq!(cse_reuse_is_legal(&producer, 5), Ok(())); + } + + /// Schema without `unique_keys` is the conservative-default case — + /// the deduper must refuse to share it. Pins design.md §6 line + /// ~1356 ("Without it, the deduper has to be conservative and reuse + /// drops on the floor"). + #[test] + fn cse_reuse_illegal_when_unique_keys_empty() { + let producer = Schema::new(vec![col("a", DataType::Int64), col("b", DataType::Float64)]); + assert_eq!( + cse_reuse_is_legal(&producer, 2), + Err(CseError::NoUniqueKeys) + ); + } + + /// Single-consumer case is short-circuited — no `LetBinding` should + /// be emitted for one `Ref` because there's no reuse to credit. + #[test] + fn cse_reuse_rejects_single_consumer() { + let producer = Schema::with_time_index( + vec![col("ts", DataType::Timestamp), col("v", DataType::Float64)], + 0, + vec![vec![0]], + ); + assert_eq!( + cse_reuse_is_legal(&producer, 1), + Err(CseError::InsufficientConsumers(1)) + ); + assert_eq!( + cse_reuse_is_legal(&producer, 0), + Err(CseError::InsufficientConsumers(0)) + ); + } + + /// Empty `unique_keys` rejection takes precedence over the consumer + /// count check only when both pass — but here we verify the + /// insufficient-consumers branch fires first (a defensive ordering + /// so callers see the clearer error when they get the call wrong). + #[test] + fn cse_reuse_consumer_check_precedes_unique_key_check() { + let producer = Schema::new(vec![col("a", DataType::Int64)]); + // Both conditions fail; consumer check is reported. + assert_eq!( + cse_reuse_is_legal(&producer, 1), + Err(CseError::InsufficientConsumers(1)) + ); + } + + #[test] + fn schema_new_has_no_time_or_unique_key() { + let s = Schema::new(vec![col("k", DataType::Utf8), col("v", DataType::Float64)]); + assert!(s.time_index.is_none()); + assert!(!s.has_unique_key()); + assert_eq!(s.column_id("k"), Some(0)); + assert_eq!(s.column_id("v"), Some(1)); + assert_eq!(s.column_id("missing"), None); + } + + #[test] + fn schema_with_time_index_populates_metadata() { + let s = Schema::with_time_index( + vec![ + col("ts", DataType::Timestamp), + col("service", DataType::Utf8), + col("value", DataType::Float64), + ], + 0, + vec![vec![0, 1]], + ); + assert_eq!(s.time_index, Some(0)); + assert!(s.has_unique_key()); + assert_eq!(s.unique_keys, vec![vec![0, 1]]); + } + + #[test] + fn add_unique_key_dedupes() { + let mut s = Schema::new(vec![col("a", DataType::Utf8), col("b", DataType::Utf8)]); + s.add_unique_key(vec![0]); + s.add_unique_key(vec![0]); + s.add_unique_key(vec![0, 1]); + assert_eq!(s.unique_keys, vec![vec![0], vec![0, 1]]); + } + + #[test] + fn schema_serde_roundtrip() { + let s = Schema::with_time_index( + vec![ + col("ts", DataType::Timestamp), + col("value", DataType::Float64), + ], + 0, + vec![vec![0]], + ); + let json = serde_json::to_string(&s).unwrap(); + let back: Schema = serde_json::from_str(&json).unwrap(); + assert_eq!(s, back); + } + + #[test] + fn column_table_defaults_to_none_when_absent() { + // `Column.table` is `#[serde(default)]` so schemas serialized before the + // qualifier field existed still deserialize (to `table: None`) instead + // of erroring. Drop the key from a serialized column to simulate that. + let mut v = serde_json::to_value(col("svc", DataType::Utf8)).unwrap(); + assert!(v.as_object_mut().unwrap().remove("table").is_some()); + let back: Column = serde_json::from_value(v).unwrap(); + assert_eq!(back, col("svc", DataType::Utf8)); + assert!(back.table.is_none()); + } + + #[test] + fn qualified_column_serde_roundtrip() { + let c = col("service", DataType::Utf8).with_table("hosts"); + let back: Column = serde_json::from_str(&serde_json::to_string(&c).unwrap()).unwrap(); + assert_eq!(back, c); + assert_eq!(back.table.as_deref(), Some("hosts")); + } } diff --git a/crates/core/src/sketch_algebra/expr.rs b/crates/core/src/sketch_algebra/expr.rs index bcfc25c4..116ade73 100644 --- a/crates/core/src/sketch_algebra/expr.rs +++ b/crates/core/src/sketch_algebra/expr.rs @@ -1,9 +1,8 @@ use std::rc::Rc; -use std::sync::Arc; use super::schema::L4Schema; use super::sketch::{SketchQuery, SummaryKind, SummaryParams}; -use crate::intent_algebra::{ColumnRef, GroupKey, L3Node}; +use crate::intent_algebra::{ColumnId, ColumnRef, QueryExpr}; // ── L4 DAG node ─────────────────────────────────────────────────────────────── @@ -23,16 +22,17 @@ pub struct L4Node { /// Sketch-bound IR produced by L4 optimizer rules. L4 rules selectively /// replace logical aggregates and joins in the L3 `QueryExpr` with their /// sketch-bound counterparts; everything not rewritten passes through as -/// `Logical(Rc)`. +/// `Logical(Box)`. /// /// Traversing from the root node yields a DAG; shared sub-expressions appear -/// as multiple `Rc` references to the same `L4Node` or `L3Node`. +/// as multiple `Rc` references to the same `L4Node` (L3 fan-in is expressed +/// via `QueryExpr`'s own `LetBinding`/`Ref`). #[derive(Debug, Clone)] pub enum SummaryExpr { /// Any L3 node that no L4 rule rewrote (e.g. `Filter`, `Project`, `Sort`). /// Output schema is the inner L3 node's schema, lifted to `L4Schema` /// with all fields as `L4DataType::Primitive`. - Logical(Arc), + Logical(Box), /// Sketch aggregation. L4 chose `sketch` + `params` from the catalog /// for `AggIntent` under `DeploymentConstraints`. @@ -44,8 +44,8 @@ pub enum SummaryExpr { params: SummaryParams, /// The column being summarised (fed into the sketch). col: ColumnRef, - /// GROUP BY keys carried through to the output schema. - by: Vec, + /// GROUP BY keys (positional) carried through to the output schema. + by: Vec, }, /// Sketch-aware join (KMV / theta for join-cardinality; join-sample for diff --git a/crates/core/src/sketch_algebra/schema.rs b/crates/core/src/sketch_algebra/schema.rs index 1ee536e7..c2fdcaac 100644 --- a/crates/core/src/sketch_algebra/schema.rs +++ b/crates/core/src/sketch_algebra/schema.rs @@ -1,5 +1,5 @@ use super::sketch::{SummaryKind, SummaryParams}; -use crate::intent_algebra::L3DataType; +use crate::intent_algebra::DataType; // ── L4 data types ───────────────────────────────────────────────────────────── @@ -13,7 +13,7 @@ use crate::intent_algebra::L3DataType; #[derive(Debug, Clone, PartialEq)] pub enum L4DataType { /// Any base L3 column type — passed through unchanged from L3 edges. - Primitive(L3DataType), + Primitive(DataType), /// Opaque summary state (exact accumulator or approximate sketch). /// The `(kind, params)` pair is the type identity: two summary columns /// are compatible only if both match exactly. diff --git a/crates/core/src/types.rs b/crates/core/src/types.rs index 4dfad52c..089ea9fa 100644 --- a/crates/core/src/types.rs +++ b/crates/core/src/types.rs @@ -1,5 +1,7 @@ +use serde::{Deserialize, Serialize}; + /// Accuracy requirement that a query result must satisfy. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum AccuracyTarget { /// Additive error bound ε: |estimate − true| ≤ ε · (domain size). Epsilon(f64), diff --git a/crates/core/tests/expr_ir.rs b/crates/core/tests/expr_ir.rs deleted file mode 100644 index 68e49f7a..00000000 --- a/crates/core/tests/expr_ir.rs +++ /dev/null @@ -1,308 +0,0 @@ -use asap_control_core::intent_algebra::{ArithOp, ColumnRef, CompareOp, L3Expr, L3Scalar}; - -// ── L3Scalar ────────────────────────────────────────────────────────────────── - -#[test] -fn l3scalar_int64_eq() { - assert_eq!(L3Scalar::Int64(42), L3Scalar::Int64(42)); - assert_ne!(L3Scalar::Int64(1), L3Scalar::Int64(2)); -} - -#[test] -fn l3scalar_float64_eq() { - assert_eq!(L3Scalar::Float64(1.5), L3Scalar::Float64(1.5)); - assert_ne!(L3Scalar::Float64(1.5), L3Scalar::Float64(2.5)); -} - -#[test] -fn l3scalar_utf8_eq() { - assert_eq!( - L3Scalar::Utf8("hello".into()), - L3Scalar::Utf8("hello".into()) - ); - assert_ne!(L3Scalar::Utf8("a".into()), L3Scalar::Utf8("b".into())); -} - -#[test] -fn l3scalar_null_eq() { - assert_eq!(L3Scalar::Null, L3Scalar::Null); -} - -// ── L3Expr construction ─────────────────────────────────────────────────────── - -#[test] -fn l3expr_column_eq() { - let a = L3Expr::Column(ColumnRef("ts".into())); - let b = L3Expr::Column(ColumnRef("ts".into())); - assert_eq!(a, b); - assert_ne!( - L3Expr::Column(ColumnRef("ts".into())), - L3Expr::Column(ColumnRef("x".into())) - ); -} - -#[test] -fn l3expr_literal_eq() { - assert_eq!( - L3Expr::Literal(L3Scalar::Int64(5)), - L3Expr::Literal(L3Scalar::Int64(5)) - ); -} - -#[test] -fn l3expr_compare_eq() { - let make = || L3Expr::Compare { - left: Box::new(L3Expr::Column(ColumnRef("v".into()))), - op: CompareOp::Gt, - right: Box::new(L3Expr::Literal(L3Scalar::Float64(0.0))), - }; - assert_eq!(make(), make()); -} - -// ── conjuncts() ─────────────────────────────────────────────────────────────── - -#[test] -fn bool_and_conjuncts_returns_all_elements() { - let a = L3Expr::Column(ColumnRef("a".into())); - let b = L3Expr::Column(ColumnRef("b".into())); - let expr = L3Expr::BoolAnd(vec![a.clone(), b.clone()]); - let c = expr.conjuncts(); - assert_eq!(c.len(), 2); - assert_eq!(c[0], a); - assert_eq!(c[1], b); -} - -#[test] -fn non_and_conjuncts_returns_self_as_slice() { - let col = L3Expr::Column(ColumnRef("x".into())); - let c = col.conjuncts(); - assert_eq!(c.len(), 1); - assert_eq!(c[0], col); -} - -#[test] -fn literal_conjuncts_returns_self() { - let lit = L3Expr::Literal(L3Scalar::Boolean(true)); - assert_eq!(lit.conjuncts().len(), 1); -} - -// ── disjuncts() ─────────────────────────────────────────────────────────────── - -#[test] -fn bool_or_disjuncts_returns_all_elements() { - let a = L3Expr::Literal(L3Scalar::Int64(1)); - let b = L3Expr::Literal(L3Scalar::Int64(2)); - let expr = L3Expr::BoolOr(vec![a.clone(), b.clone()]); - let d = expr.disjuncts(); - assert_eq!(d.len(), 2); - assert_eq!(d[0], a); - assert_eq!(d[1], b); -} - -#[test] -fn non_or_disjuncts_returns_self() { - let lit = L3Expr::Literal(L3Scalar::Null); - assert_eq!(lit.disjuncts().len(), 1); -} - -// ── columns_referenced() ────────────────────────────────────────────────────── - -#[test] -fn columns_referenced_from_column_node() { - let expr = L3Expr::Column(ColumnRef("ts".into())); - let refs = expr.columns_referenced(); - assert_eq!(refs.len(), 1); - assert_eq!(refs[0].0, "ts"); -} - -#[test] -fn columns_referenced_from_literal_is_empty() { - let expr = L3Expr::Literal(L3Scalar::Int64(99)); - assert!(expr.columns_referenced().is_empty()); -} - -#[test] -fn columns_referenced_from_compare() { - let expr = L3Expr::Compare { - left: Box::new(L3Expr::Column(ColumnRef("a".into()))), - op: CompareOp::Gt, - right: Box::new(L3Expr::Column(ColumnRef("b".into()))), - }; - let refs = expr.columns_referenced(); - assert_eq!(refs.len(), 2); - assert!(refs.iter().any(|r| r.0 == "a")); - assert!(refs.iter().any(|r| r.0 == "b")); -} - -#[test] -fn columns_referenced_from_compare_with_literal() { - let expr = L3Expr::Compare { - left: Box::new(L3Expr::Column(ColumnRef("value".into()))), - op: CompareOp::Ge, - right: Box::new(L3Expr::Literal(L3Scalar::Float64(0.0))), - }; - let refs = expr.columns_referenced(); - assert_eq!(refs.len(), 1); - assert_eq!(refs[0].0, "value"); -} - -#[test] -fn columns_referenced_from_bool_and() { - let expr = L3Expr::BoolAnd(vec![ - L3Expr::Compare { - left: Box::new(L3Expr::Column(ColumnRef("region".into()))), - op: CompareOp::Eq, - right: Box::new(L3Expr::Literal(L3Scalar::Utf8("us".into()))), - }, - L3Expr::Compare { - left: Box::new(L3Expr::Column(ColumnRef("value".into()))), - op: CompareOp::Gt, - right: Box::new(L3Expr::Literal(L3Scalar::Float64(0.0))), - }, - ]); - let refs = expr.columns_referenced(); - assert_eq!(refs.len(), 2); - assert!(refs.iter().any(|r| r.0 == "region")); - assert!(refs.iter().any(|r| r.0 == "value")); -} - -#[test] -fn columns_referenced_from_not() { - let expr = L3Expr::Not(Box::new(L3Expr::Column(ColumnRef("flag".into())))); - let refs = expr.columns_referenced(); - assert_eq!(refs.len(), 1); - assert_eq!(refs[0].0, "flag"); -} - -#[test] -fn columns_referenced_from_is_null() { - let expr = L3Expr::IsNull(Box::new(L3Expr::Column(ColumnRef("x".into())))); - let refs = expr.columns_referenced(); - assert_eq!(refs.len(), 1); - assert_eq!(refs[0].0, "x"); -} - -#[test] -fn columns_referenced_from_is_not_null() { - let expr = L3Expr::IsNotNull(Box::new(L3Expr::Column(ColumnRef("y".into())))); - let refs = expr.columns_referenced(); - assert_eq!(refs.len(), 1); - assert_eq!(refs[0].0, "y"); -} - -#[test] -fn columns_referenced_from_in_list() { - let expr = L3Expr::InList { - expr: Box::new(L3Expr::Column(ColumnRef("region".into()))), - list: vec![ - L3Expr::Literal(L3Scalar::Utf8("us".into())), - L3Expr::Literal(L3Scalar::Utf8("eu".into())), - ], - negated: false, - }; - let refs = expr.columns_referenced(); - assert_eq!(refs.len(), 1); - assert_eq!(refs[0].0, "region"); -} - -#[test] -fn columns_referenced_from_function_call() { - let expr = L3Expr::FunctionCall { - name: "lower".into(), - args: vec![L3Expr::Column(ColumnRef("host".into()))], - }; - let refs = expr.columns_referenced(); - assert_eq!(refs.len(), 1); - assert_eq!(refs[0].0, "host"); -} - -// ── ArithOp / L3Expr::Arith ─────────────────────────────────────────────────── - -#[test] -fn arith_eq() { - let make = || L3Expr::Arith { - op: ArithOp::Mul, - left: Box::new(L3Expr::Column(ColumnRef("value".into()))), - right: Box::new(L3Expr::Literal(L3Scalar::Float64(2.0))), - }; - assert_eq!(make(), make()); -} - -#[test] -fn arith_columns_referenced_from_both_sides() { - let expr = L3Expr::Arith { - op: ArithOp::Add, - left: Box::new(L3Expr::Column(ColumnRef("a".into()))), - right: Box::new(L3Expr::Column(ColumnRef("b".into()))), - }; - let refs = expr.columns_referenced(); - assert_eq!(refs.len(), 2); - assert!(refs.iter().any(|r| r.0 == "a")); - assert!(refs.iter().any(|r| r.0 == "b")); -} - -#[test] -fn arith_columns_referenced_literal_side_is_empty() { - let expr = L3Expr::Arith { - op: ArithOp::Mul, - left: Box::new(L3Expr::Column(ColumnRef("value".into()))), - right: Box::new(L3Expr::Literal(L3Scalar::Float64(2.0))), - }; - let refs = expr.columns_referenced(); - assert_eq!(refs.len(), 1); - assert_eq!(refs[0].0, "value"); -} - -// ── L3Expr::Case ───────────────────────────────────────────────────────────── - -#[test] -fn case_searched_columns_referenced_from_all_branches() { - // CASE WHEN a > 0 THEN b ELSE c END - let expr = L3Expr::Case { - operand: None, - branches: vec![( - L3Expr::Compare { - left: Box::new(L3Expr::Column(ColumnRef("a".into()))), - op: CompareOp::Gt, - right: Box::new(L3Expr::Literal(L3Scalar::Int64(0))), - }, - L3Expr::Column(ColumnRef("b".into())), - )], - else_expr: Some(Box::new(L3Expr::Column(ColumnRef("c".into())))), - }; - let refs = expr.columns_referenced(); - assert_eq!(refs.len(), 3); - assert!(refs.iter().any(|r| r.0 == "a")); - assert!(refs.iter().any(|r| r.0 == "b")); - assert!(refs.iter().any(|r| r.0 == "c")); -} - -#[test] -fn case_simple_operand_included_in_refs() { - // CASE value WHEN 1 THEN x END - let expr = L3Expr::Case { - operand: Some(Box::new(L3Expr::Column(ColumnRef("value".into())))), - branches: vec![( - L3Expr::Literal(L3Scalar::Int64(1)), - L3Expr::Column(ColumnRef("x".into())), - )], - else_expr: None, - }; - let refs = expr.columns_referenced(); - assert!(refs.iter().any(|r| r.0 == "value")); - assert!(refs.iter().any(|r| r.0 == "x")); -} - -// ── CompareOp::ILike / NotILike ─────────────────────────────────────────────── - -#[test] -fn compare_op_ilike_eq() { - assert_eq!(CompareOp::ILike, CompareOp::ILike); - assert_ne!(CompareOp::ILike, CompareOp::Like); -} - -#[test] -fn compare_op_not_ilike_eq() { - assert_eq!(CompareOp::NotILike, CompareOp::NotILike); - assert_ne!(CompareOp::NotILike, CompareOp::NotLike); -} diff --git a/crates/core/tests/schema_derivation.rs b/crates/core/tests/schema_derivation.rs deleted file mode 100644 index 84f91e16..00000000 --- a/crates/core/tests/schema_derivation.rs +++ /dev/null @@ -1,1202 +0,0 @@ -use std::collections::HashMap; -use std::sync::Arc; - -use asap_control_core::intent_algebra::{ - AggIntent, ArithOp, ColumnDef, ColumnRef, DataModel, GroupKey, HasSchema, L3DataType, L3Expr, - L3Field, L3Node, L3Scalar, L3Schema, Predicate, ProjectItem, QueryExpr, SchemaCatalog, - SetOpKind, SortKey, Source, TableRef, TableSchema, WindowFuncKind, -}; -use asap_control_core::types::AccuracyTarget; - -// ── Helpers ─────────────────────────────────────────────────────────────────── - -fn field(name: &str, dtype: L3DataType) -> L3Field { - L3Field { - name: name.to_string(), - dtype, - nullable: false, - } -} - -fn nullable_field(name: &str, dtype: L3DataType) -> L3Field { - L3Field { - name: name.to_string(), - dtype, - nullable: true, - } -} - -fn schema(fields: Vec) -> L3Schema { - L3Schema { - fields, - time_index: None, - } -} - -fn schema_with_time(fields: Vec, time_index: usize) -> L3Schema { - L3Schema { - fields, - time_index: Some(time_index), - } -} - -fn make_node(expr: QueryExpr, s: L3Schema) -> Arc { - Arc::new(L3Node { expr, schema: s }) -} - -fn empty_catalog() -> SchemaCatalog { - SchemaCatalog { - tables: HashMap::new(), - } -} - -fn metrics_catalog() -> SchemaCatalog { - let mut tables = HashMap::new(); - tables.insert( - "metrics".to_string(), - TableSchema { - columns: vec![ - ColumnDef { - name: "ts".to_string(), - data_type: L3DataType::Int64, - nullable: false, - }, - ColumnDef { - name: "value".to_string(), - data_type: L3DataType::Float64, - nullable: true, - }, - ColumnDef { - name: "region".to_string(), - data_type: L3DataType::Utf8, - nullable: true, - }, - ], - time_column: Some("ts".to_string()), - }, - ); - tables.insert( - "events".to_string(), - TableSchema { - columns: vec![ - ColumnDef { - name: "id".to_string(), - data_type: L3DataType::Int64, - nullable: false, - }, - ColumnDef { - name: "name".to_string(), - data_type: L3DataType::Utf8, - nullable: true, - }, - ], - time_column: None, - }, - ); - SchemaCatalog { tables } -} - -fn exact() -> AccuracyTarget { - AccuracyTarget::Exact -} - -fn eps(e: f64) -> AccuracyTarget { - AccuracyTarget::Epsilon(e) -} - -fn child_schema() -> L3Schema { - schema(vec![ - field("ts", L3DataType::Int64), - nullable_field("value", L3DataType::Float64), - ]) -} -fn dummy_scan(s: L3Schema) -> Arc { - make_node( - QueryExpr::Scan { - source: Source::Table { - table_ref: TableRef("metrics".into()), - columns: vec![], - time_range: None, - }, - predicates: vec![], - }, - s, - ) -} -fn metrics_child_schema() -> L3Schema { - schema(vec![ - field("ts", L3DataType::Int64), - nullable_field("value", L3DataType::Float64), - nullable_field("region", L3DataType::Utf8), - nullable_field("host", L3DataType::Utf8), - ]) -} - -fn timed_two_col_schema() -> L3Schema { - schema_with_time( - vec![ - field("ts", L3DataType::Int64), - nullable_field("value", L3DataType::Float64), - ], - 0, - ) -} - -mod agg_intent_requires { - use super::*; - - // ── AggIntent::requires() ───────────────────────────────────────────────────── - - #[test] - fn requires_count_is_any() { - assert_eq!( - AggIntent::Count { accuracy: exact() }.requires(), - DataModel::Any - ); - } - - #[test] - fn requires_sum_is_any() { - assert_eq!( - AggIntent::Sum { - col: Some(ColumnRef("x".into())) - } - .requires(), - DataModel::Any - ); - } - - #[test] - fn requires_min_is_any() { - assert_eq!( - AggIntent::Min { - col: Some(ColumnRef("x".into())) - } - .requires(), - DataModel::Any - ); - } - - #[test] - fn requires_max_is_any() { - assert_eq!( - AggIntent::Max { - col: Some(ColumnRef("x".into())) - } - .requires(), - DataModel::Any - ); - } - - #[test] - fn requires_avg_is_any() { - assert_eq!( - AggIntent::Avg { - col: Some(ColumnRef("x".into())) - } - .requires(), - DataModel::Any - ); - } - - #[test] - fn requires_stddev_sample_is_any() { - assert_eq!( - AggIntent::Stddev { - col: Some(ColumnRef("x".into())), - population: false - } - .requires(), - DataModel::Any - ); - } - - #[test] - fn requires_stddev_population_is_any() { - assert_eq!( - AggIntent::Stddev { - col: Some(ColumnRef("x".into())), - population: true - } - .requires(), - DataModel::Any - ); - } - - #[test] - fn requires_quantile_is_any() { - assert_eq!( - AggIntent::Quantile { - q: 0.99, - accuracy: exact() - } - .requires(), - DataModel::Any - ); - } - - #[test] - fn requires_cardinality_is_any() { - assert_eq!( - AggIntent::Cardinality { accuracy: exact() }.requires(), - DataModel::Any - ); - } - - #[test] - fn requires_topk_is_any() { - assert_eq!( - AggIntent::TopK { - k: 10, - by: vec![], - accuracy: exact() - } - .requires(), - DataModel::Any - ); - } - - #[test] - fn requires_rate_is_timeseries() { - assert_eq!( - AggIntent::Rate { - window: std::time::Duration::from_secs(60) - } - .requires(), - DataModel::TimeSeries - ); - } - - #[test] - fn requires_increase_is_timeseries() { - assert_eq!( - AggIntent::Increase { - window: std::time::Duration::from_secs(300) - } - .requires(), - DataModel::TimeSeries - ); - } -} - -mod agg_intent_output_type { - use super::*; - - // ── AggIntent::output_type() ────────────────────────────────────────────────── - - #[test] - fn output_type_count_is_int64() { - let f = field("x", L3DataType::Float64); - assert_eq!( - AggIntent::Count { accuracy: exact() }.output_type(&f), - Some(L3DataType::Int64) - ); - } - - #[test] - fn output_type_count_ignores_input_type() { - // Count is always Int64 regardless of the aggregated column's type. - let f = field("x", L3DataType::Utf8); - assert_eq!( - AggIntent::Count { - accuracy: eps(0.01) - } - .output_type(&f), - Some(L3DataType::Int64) - ); - } - - #[test] - fn output_type_cardinality_is_int64() { - let f = field("host", L3DataType::Utf8); - assert_eq!( - AggIntent::Cardinality { accuracy: exact() }.output_type(&f), - Some(L3DataType::Int64) - ); - } - - #[test] - fn output_type_sum_is_float64() { - let f = field("value", L3DataType::Float64); - assert_eq!( - AggIntent::Sum { - col: Some(ColumnRef("value".into())) - } - .output_type(&f), - Some(L3DataType::Float64) - ); - } - - #[test] - fn output_type_avg_is_float64() { - let f = field("value", L3DataType::Int64); - assert_eq!( - AggIntent::Avg { - col: Some(ColumnRef("value".into())) - } - .output_type(&f), - Some(L3DataType::Float64) - ); - } - - #[test] - fn output_type_stddev_sample_is_float64() { - let f = field("value", L3DataType::Float64); - assert_eq!( - AggIntent::Stddev { - col: Some(ColumnRef("value".into())), - population: false - } - .output_type(&f), - Some(L3DataType::Float64) - ); - } - - #[test] - fn output_type_stddev_population_is_float64() { - let f = field("value", L3DataType::Float64); - assert_eq!( - AggIntent::Stddev { - col: Some(ColumnRef("value".into())), - population: true - } - .output_type(&f), - Some(L3DataType::Float64) - ); - } - - #[test] - fn output_type_quantile_is_float64() { - let f = field("latency", L3DataType::Float64); - assert_eq!( - AggIntent::Quantile { - q: 0.5, - accuracy: exact() - } - .output_type(&f), - Some(L3DataType::Float64) - ); - } - - #[test] - fn output_type_rate_is_float64() { - let f = field("bytes", L3DataType::Float64); - assert_eq!( - AggIntent::Rate { - window: std::time::Duration::from_secs(60) - } - .output_type(&f), - Some(L3DataType::Float64) - ); - } - - #[test] - fn output_type_increase_is_float64() { - let f = field("counter", L3DataType::Float64); - assert_eq!( - AggIntent::Increase { - window: std::time::Duration::from_secs(60) - } - .output_type(&f), - Some(L3DataType::Float64) - ); - } - - #[test] - fn output_type_min_preserves_int64_input() { - let f = field("count", L3DataType::Int64); - assert_eq!( - AggIntent::Min { - col: Some(ColumnRef("count".into())) - } - .output_type(&f), - Some(L3DataType::Int64) - ); - } - - #[test] - fn output_type_min_preserves_float64_input() { - let f = field("value", L3DataType::Float64); - assert_eq!( - AggIntent::Min { - col: Some(ColumnRef("value".into())) - } - .output_type(&f), - Some(L3DataType::Float64) - ); - } - - #[test] - fn output_type_max_preserves_utf8_input() { - let f = field("name", L3DataType::Utf8); - assert_eq!( - AggIntent::Max { - col: Some(ColumnRef("name".into())) - } - .output_type(&f), - Some(L3DataType::Utf8) - ); - } - - #[test] - fn output_type_topk_is_none() { - let f = field("region", L3DataType::Utf8); - assert_eq!( - AggIntent::TopK { - k: 10, - by: vec![ColumnRef("region".into())], - accuracy: exact() - } - .output_type(&f), - None - ); - } -} - -mod scan_schema { - use super::*; - - // ── HasSchema::output_schema() — Scan ───────────────────────────────────────── - - #[test] - fn scan_schema_columns_from_catalog() { - let catalog = metrics_catalog(); - let scan = QueryExpr::Scan { - source: Source::Table { - table_ref: TableRef("metrics".into()), - columns: vec![], - time_range: None, - }, - predicates: vec![], - }; - let s = scan.output_schema(&[], &catalog); - assert_eq!(s.fields.len(), 3); - assert_eq!(s.fields[0].name, "ts"); - assert_eq!(s.fields[0].dtype, L3DataType::Int64); - assert_eq!(s.fields[1].name, "value"); - assert_eq!(s.fields[2].name, "region"); - } - - #[test] - fn scan_schema_time_index_set_for_time_column() { - let catalog = metrics_catalog(); - let scan = QueryExpr::Scan { - source: Source::Table { - table_ref: TableRef("metrics".into()), - columns: vec![], - time_range: None, - }, - predicates: vec![], - }; - let s = scan.output_schema(&[], &catalog); - // "ts" is at index 0 and is the time_column - assert_eq!(s.time_index, Some(0)); - } - - #[test] - fn scan_schema_no_time_index_when_no_time_column() { - let catalog = metrics_catalog(); - let scan = QueryExpr::Scan { - source: Source::Table { - table_ref: TableRef("events".into()), - columns: vec![], - time_range: None, - }, - predicates: vec![], - }; - let s = scan.output_schema(&[], &catalog); - assert_eq!(s.time_index, None); - assert_eq!(s.fields.len(), 2); - } -} - -mod passthrough_schemas { - use super::*; - - // ── HasSchema::output_schema() — pass-through nodes ────────────────────────── - - #[test] - fn filter_passes_through_child_schema() { - let cs = child_schema(); - let node = QueryExpr::Filter { - child: dummy_scan(cs.clone()), - pred: Predicate(L3Expr::Literal(L3Scalar::Boolean(true))), - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - assert_eq!(out.fields, cs.fields); - assert_eq!(out.time_index, cs.time_index); - } - - #[test] - fn sort_passes_through_child_schema() { - let cs = child_schema(); - let node = QueryExpr::Sort { - child: dummy_scan(cs.clone()), - keys: vec![SortKey { - expr: L3Expr::Column(ColumnRef("ts".into())), - ascending: true, - nulls_first: false, - }], - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - assert_eq!(out.fields, cs.fields); - } - - #[test] - fn limit_passes_through_child_schema() { - let cs = child_schema(); - let node = QueryExpr::Limit { - child: dummy_scan(cs.clone()), - n: Some(10), - offset: 0, - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - assert_eq!(out.fields, cs.fields); - } - - #[test] - fn distinct_passes_through_child_schema() { - let cs = child_schema(); - let node = QueryExpr::Distinct { - child: dummy_scan(cs.clone()), - cols: vec![], - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - assert_eq!(out.fields, cs.fields); - } -} - -mod aggregate_schema { - use super::*; - - // ── HasSchema::output_schema() — Aggregate ──────────────────────────────────── - - #[test] - fn aggregate_count_star_no_group_by() { - // SELECT COUNT(*) FROM metrics - let cs = metrics_child_schema(); - let node = QueryExpr::Aggregate { - child: dummy_scan(cs.clone()), - by: vec![], - aggs: vec![AggIntent::Count { accuracy: exact() }], - having: None, - output_names: vec![], - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - assert_eq!(out.fields.len(), 1); - assert_eq!(out.fields[0].dtype, L3DataType::Int64); - } - - #[test] - fn aggregate_group_by_adds_by_cols_first() { - // SELECT region, COUNT(*) FROM metrics GROUP BY region - let cs = metrics_child_schema(); - let node = QueryExpr::Aggregate { - child: dummy_scan(cs.clone()), - by: vec![GroupKey("region".into())], - aggs: vec![AggIntent::Count { accuracy: exact() }], - having: None, - output_names: vec![], - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - // region col + count col - assert_eq!(out.fields.len(), 2); - assert_eq!(out.fields[0].name, "region"); - assert_eq!(out.fields[0].dtype, L3DataType::Utf8); - assert_eq!(out.fields[1].dtype, L3DataType::Int64); - } - - #[test] - fn aggregate_multiple_aggs() { - // SELECT COUNT(*), SUM(value), MIN(value) FROM metrics - let cs = metrics_child_schema(); - let node = QueryExpr::Aggregate { - child: dummy_scan(cs.clone()), - by: vec![], - aggs: vec![ - AggIntent::Count { accuracy: exact() }, - AggIntent::Sum { - col: Some(ColumnRef("value".into())), - }, - AggIntent::Min { - col: Some(ColumnRef("value".into())), - }, - ], - having: None, - output_names: vec![], - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - assert_eq!(out.fields.len(), 3); - assert_eq!(out.fields[0].dtype, L3DataType::Int64); // Count - assert_eq!(out.fields[1].dtype, L3DataType::Float64); // Sum(value: Float64) - assert_eq!(out.fields[2].dtype, L3DataType::Float64); // Min(value: Float64) - } - - #[test] - fn aggregate_topk_produces_by_cols_plus_count() { - // TopK { k: 5, by: [host] } → [host(Utf8), count(Int64)] - let cs = metrics_child_schema(); - let node = QueryExpr::Aggregate { - child: dummy_scan(cs.clone()), - by: vec![], - aggs: vec![AggIntent::TopK { - k: 5, - by: vec![ColumnRef("host".into())], - accuracy: exact(), - }], - having: None, - output_names: vec![], - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - assert_eq!(out.fields.len(), 2); - assert_eq!(out.fields[0].name, "host"); - assert_eq!(out.fields[0].dtype, L3DataType::Utf8); - assert_eq!(out.fields[1].name, "count"); - assert_eq!(out.fields[1].dtype, L3DataType::Int64); - } - - #[test] - fn aggregate_topk_multi_key() { - // TopK { k: 10, by: [region, host] } → [region(Utf8), host(Utf8), count(Int64)] - let cs = metrics_child_schema(); - let node = QueryExpr::Aggregate { - child: dummy_scan(cs.clone()), - by: vec![], - aggs: vec![AggIntent::TopK { - k: 10, - by: vec![ColumnRef("region".into()), ColumnRef("host".into())], - accuracy: exact(), - }], - having: None, - output_names: vec![], - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - assert_eq!(out.fields.len(), 3); - assert_eq!(out.fields[0].name, "region"); - assert_eq!(out.fields[1].name, "host"); - assert_eq!(out.fields[2].name, "count"); - assert_eq!(out.fields[2].dtype, L3DataType::Int64); - } - - #[test] - fn aggregate_drops_time_index_when_time_col_not_in_group_by() { - // GROUP BY value (not ts): time axis is consumed by the aggregation. - let cs = schema_with_time( - vec![ - field("ts", L3DataType::Int64), - nullable_field("value", L3DataType::Float64), - ], - 0, - ); - let node = QueryExpr::Aggregate { - child: dummy_scan(cs.clone()), - by: vec![], - aggs: vec![AggIntent::Count { accuracy: exact() }], - having: None, - output_names: vec![], - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - assert_eq!(out.time_index, None); - } - - #[test] - fn aggregate_propagates_time_index_when_time_col_in_group_by() { - // GROUP BY ts — the time column survives into the output; time_index must be set. - let cs = schema_with_time( - vec![ - field("ts", L3DataType::Int64), - nullable_field("value", L3DataType::Float64), - ], - 0, - ); - let node = QueryExpr::Aggregate { - child: dummy_scan(cs.clone()), - by: vec![GroupKey("ts".into())], - aggs: vec![AggIntent::Sum { - col: Some(ColumnRef("value".into())), - }], - having: None, - output_names: vec!["sum_value".into()], - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - // Output fields: [ts (by), sum_value (agg)]; ts is at index 0. - assert_eq!(out.time_index, Some(0)); - assert_eq!(out.fields[0].name, "ts"); - } -} - -mod project_schema { - use super::*; - - // ── HasSchema::output_schema() — Project ────────────────────────────────────── - - #[test] - fn project_column_items_derive_schema_from_child() { - // SELECT ts, value — both columns exist in the child schema. - let cs = child_schema(); // [ts(Int64), value(Float64)] - let node = QueryExpr::Project { - child: dummy_scan(cs.clone()), - cols: vec![ - ProjectItem { - expr: L3Expr::Column(ColumnRef("ts".into())), - alias: None, - }, - ProjectItem { - expr: L3Expr::Column(ColumnRef("value".into())), - alias: None, - }, - ], - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - assert_eq!(out.fields.len(), 2); - assert_eq!(out.fields[0].name, "ts"); - assert_eq!(out.fields[0].dtype, L3DataType::Int64); - assert_eq!(out.fields[1].name, "value"); - assert_eq!(out.fields[1].dtype, L3DataType::Float64); - } - - #[test] - fn project_alias_renames_output_field() { - // SELECT value AS v — output field is named "v", type preserved. - let cs = child_schema(); - let node = QueryExpr::Project { - child: dummy_scan(cs.clone()), - cols: vec![ProjectItem { - expr: L3Expr::Column(ColumnRef("value".into())), - alias: Some("v".into()), - }], - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - assert_eq!(out.fields.len(), 1); - assert_eq!(out.fields[0].name, "v"); - assert_eq!(out.fields[0].dtype, L3DataType::Float64); - } - - #[test] - fn project_subsets_columns() { - // SELECT value — only one of two child columns projected. - let cs = child_schema(); - let node = QueryExpr::Project { - child: dummy_scan(cs.clone()), - cols: vec![ProjectItem { - expr: L3Expr::Column(ColumnRef("value".into())), - alias: None, - }], - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - assert_eq!(out.fields.len(), 1); - assert_eq!(out.fields[0].name, "value"); - } - - #[test] - fn project_preserves_time_index_when_time_col_included() { - // SELECT ts, value — ts is at index 0 in child (time_index=0); should be preserved. - let cs = schema_with_time( - vec![ - field("ts", L3DataType::Int64), - nullable_field("value", L3DataType::Float64), - ], - 0, - ); - let node = QueryExpr::Project { - child: dummy_scan(cs.clone()), - cols: vec![ - ProjectItem { - expr: L3Expr::Column(ColumnRef("ts".into())), - alias: None, - }, - ProjectItem { - expr: L3Expr::Column(ColumnRef("value".into())), - alias: None, - }, - ], - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - assert_eq!(out.time_index, Some(0)); - } - - #[test] - fn project_preserves_time_index_when_col_reordered() { - // SELECT value, ts — ts moves to index 1; time_index should update. - let cs = schema_with_time( - vec![ - field("ts", L3DataType::Int64), - nullable_field("value", L3DataType::Float64), - ], - 0, - ); - let node = QueryExpr::Project { - child: dummy_scan(cs.clone()), - cols: vec![ - ProjectItem { - expr: L3Expr::Column(ColumnRef("value".into())), - alias: None, - }, - ProjectItem { - expr: L3Expr::Column(ColumnRef("ts".into())), - alias: None, - }, - ], - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - assert_eq!(out.time_index, Some(1)); - } - - #[test] - fn project_drops_time_index_when_time_col_excluded() { - // SELECT value — ts not projected; time_index should be None. - let cs = schema_with_time( - vec![ - field("ts", L3DataType::Int64), - nullable_field("value", L3DataType::Float64), - ], - 0, - ); - let node = QueryExpr::Project { - child: dummy_scan(cs.clone()), - cols: vec![ProjectItem { - expr: L3Expr::Column(ColumnRef("value".into())), - alias: None, - }], - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - assert_eq!(out.time_index, None); - } -} - -mod window_schema { - use super::*; - - // ── HasSchema::output_schema() — WindowFunc ────────────────────────────────── - - #[test] - fn window_func_row_number_appends_int64_column() { - let cs = timed_two_col_schema(); - let node = QueryExpr::WindowFunc { - child: dummy_scan(cs.clone()), - func: WindowFuncKind::RowNumber, - args: vec![], - partition_by: vec![], - order_by: vec![], - frame: None, - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - assert_eq!(out.fields.len(), 3); - let win = out.fields.last().unwrap(); - assert_eq!(win.dtype, L3DataType::Int64); - assert!(!win.nullable); - } - - #[test] - fn window_func_rank_appends_int64_column() { - let cs = timed_two_col_schema(); - let node = QueryExpr::WindowFunc { - child: dummy_scan(cs.clone()), - func: WindowFuncKind::Rank, - args: vec![], - partition_by: vec![], - order_by: vec![], - frame: None, - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - let win = out.fields.last().unwrap(); - assert_eq!(win.dtype, L3DataType::Int64); - assert!(!win.nullable); - } - - #[test] - fn window_func_lag_uses_arg_column_type() { - // LAG(value) → output type matches value: Float64, nullable - let cs = timed_two_col_schema(); - let node = QueryExpr::WindowFunc { - child: dummy_scan(cs.clone()), - func: WindowFuncKind::Lag, - args: vec![L3Expr::Column(ColumnRef("value".into()))], - partition_by: vec![], - order_by: vec![], - frame: None, - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - let win = out.fields.last().unwrap(); - assert_eq!(win.dtype, L3DataType::Float64); - assert!(win.nullable); - } - - #[test] - fn window_func_lag_int_col_preserves_type() { - // LAG(ts) → Int64, nullable - let cs = timed_two_col_schema(); - let node = QueryExpr::WindowFunc { - child: dummy_scan(cs.clone()), - func: WindowFuncKind::Lag, - args: vec![L3Expr::Column(ColumnRef("ts".into()))], - partition_by: vec![], - order_by: vec![], - frame: None, - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - let win = out.fields.last().unwrap(); - assert_eq!(win.dtype, L3DataType::Int64); - assert!(win.nullable); - } - - #[test] - fn window_func_min_preserves_arg_type() { - // MIN(ts) OVER (...) → Int64 (same as ts) - let cs = timed_two_col_schema(); - let node = QueryExpr::WindowFunc { - child: dummy_scan(cs.clone()), - func: WindowFuncKind::Min, - args: vec![L3Expr::Column(ColumnRef("ts".into()))], - partition_by: vec![], - order_by: vec![], - frame: None, - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - let win = out.fields.last().unwrap(); - assert_eq!(win.dtype, L3DataType::Int64); - } - - #[test] - fn window_func_count_appends_int64_not_nullable() { - let cs = timed_two_col_schema(); - let node = QueryExpr::WindowFunc { - child: dummy_scan(cs.clone()), - func: WindowFuncKind::Count, - args: vec![L3Expr::Column(ColumnRef("value".into()))], - partition_by: vec![], - order_by: vec![], - frame: None, - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - let win = out.fields.last().unwrap(); - assert_eq!(win.dtype, L3DataType::Int64); - assert!(!win.nullable); - } - - #[test] - fn window_func_preserves_child_fields_and_time_index() { - let cs = timed_two_col_schema(); - let node = QueryExpr::WindowFunc { - child: dummy_scan(cs.clone()), - func: WindowFuncKind::RowNumber, - args: vec![], - partition_by: vec![], - order_by: vec![], - frame: None, - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - assert_eq!(out.fields[0].name, "ts"); - assert_eq!(out.fields[1].name, "value"); - assert_eq!(out.time_index, Some(0)); - } -} - -mod merge_and_set_op_schema { - use super::*; - - // ── HasSchema::output_schema() — Merge ─────────────────────────────────────── - - #[test] - fn merge_uses_first_child_schema() { - let cs = timed_two_col_schema(); - let node = QueryExpr::Merge { - children: vec![dummy_scan(cs.clone()), dummy_scan(cs.clone())], - }; - let out = node.output_schema(&[&cs, &cs], &empty_catalog()); - assert_eq!(out.fields, cs.fields); - assert_eq!(out.time_index, cs.time_index); - } - - // ── HasSchema::output_schema() — SetOp ─────────────────────────────────────── - - fn right_schema() -> L3Schema { - schema(vec![ - field("a", L3DataType::Int64), - field("b", L3DataType::Utf8), - ]) - } - - #[test] - fn set_op_union_uses_left_schema() { - let left = timed_two_col_schema(); - let right = right_schema(); - let node = QueryExpr::SetOp { - kind: SetOpKind::Union, - all: false, - left: dummy_scan(left.clone()), - right: dummy_scan(right.clone()), - }; - let out = node.output_schema(&[&left, &right], &empty_catalog()); - assert_eq!(out.fields[0].name, "ts"); - assert_eq!(out.fields[1].name, "value"); - } - - #[test] - fn set_op_intersect_uses_left_schema() { - let left = timed_two_col_schema(); - let right = right_schema(); - let node = QueryExpr::SetOp { - kind: SetOpKind::Intersect, - all: false, - left: dummy_scan(left.clone()), - right: dummy_scan(right.clone()), - }; - let out = node.output_schema(&[&left, &right], &empty_catalog()); - assert_eq!(out.fields, left.fields); - } - - #[test] - fn set_op_except_uses_left_schema() { - let left = timed_two_col_schema(); - let right = right_schema(); - let node = QueryExpr::SetOp { - kind: SetOpKind::Except, - all: false, - left: dummy_scan(left.clone()), - right: dummy_scan(right.clone()), - }; - let out = node.output_schema(&[&left, &right], &empty_catalog()); - assert_eq!(out.fields, left.fields); - } - - #[test] - fn set_op_preserves_time_index_from_left() { - let left = timed_two_col_schema(); // time_index = Some(0) - let right = right_schema(); // time_index = None - let node = QueryExpr::SetOp { - kind: SetOpKind::Union, - all: true, - left: dummy_scan(left.clone()), - right: dummy_scan(right.clone()), - }; - let out = node.output_schema(&[&left, &right], &empty_catalog()); - assert_eq!(out.time_index, Some(0)); - } -} - -mod project_non_column { - use super::*; - - // ── HasSchema::output_schema() — Project, non-column items ─────────────────── - - #[test] - fn project_cast_item_uses_target_type() { - // SELECT CAST(ts AS FLOAT64) AS ts_f — output type is the cast target. - let cs = child_schema(); // [ts: Int64, value: Float64] - let node = QueryExpr::Project { - child: dummy_scan(cs.clone()), - cols: vec![ProjectItem { - expr: L3Expr::Cast { - expr: Box::new(L3Expr::Column(ColumnRef("ts".into()))), - to: L3DataType::Float64, - try_cast: false, - }, - alias: Some("ts_f".into()), - }], - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - assert_eq!(out.fields.len(), 1); - assert_eq!(out.fields[0].name, "ts_f"); - assert_eq!(out.fields[0].dtype, L3DataType::Float64); - } - - #[test] - fn project_int_literal_item_uses_int64_type() { - // SELECT 42 AS n — output type is Int64. - let cs = child_schema(); - let node = QueryExpr::Project { - child: dummy_scan(cs.clone()), - cols: vec![ProjectItem { - expr: L3Expr::Literal(L3Scalar::Int64(42)), - alias: Some("n".into()), - }], - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - assert_eq!(out.fields[0].dtype, L3DataType::Int64); - assert_eq!(out.fields[0].name, "n"); - } - - #[test] - fn project_float_literal_item_uses_float64_type() { - let cs = child_schema(); - let node = QueryExpr::Project { - child: dummy_scan(cs.clone()), - cols: vec![ProjectItem { - expr: L3Expr::Literal(L3Scalar::Float64(std::f64::consts::PI)), - alias: Some("pi".into()), - }], - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - assert_eq!(out.fields[0].dtype, L3DataType::Float64); - } - - #[test] - fn project_arith_item_defaults_to_float64() { - // SELECT value * 2 AS doubled — arithmetic defaults to Float64. - let cs = child_schema(); - let node = QueryExpr::Project { - child: dummy_scan(cs.clone()), - cols: vec![ProjectItem { - expr: L3Expr::Arith { - op: ArithOp::Mul, - left: Box::new(L3Expr::Column(ColumnRef("value".into()))), - right: Box::new(L3Expr::Literal(L3Scalar::Int64(2))), - }, - alias: Some("doubled".into()), - }], - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - assert_eq!(out.fields[0].dtype, L3DataType::Float64); - assert_eq!(out.fields[0].name, "doubled"); - } - - #[test] - fn project_case_item_defaults_to_float64() { - // CASE WHEN value > 0 THEN 1 ELSE 0 END — defaults to Float64. - let cs = child_schema(); - let node = QueryExpr::Project { - child: dummy_scan(cs.clone()), - cols: vec![ProjectItem { - expr: L3Expr::Case { - operand: None, - branches: vec![( - L3Expr::Compare { - left: Box::new(L3Expr::Column(ColumnRef("value".into()))), - op: asap_control_core::intent_algebra::CompareOp::Gt, - right: Box::new(L3Expr::Literal(L3Scalar::Float64(0.0))), - }, - L3Expr::Literal(L3Scalar::Int64(1)), - )], - else_expr: Some(Box::new(L3Expr::Literal(L3Scalar::Int64(0)))), - }, - alias: Some("tier".into()), - }], - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - assert_eq!(out.fields[0].dtype, L3DataType::Float64); - assert_eq!(out.fields[0].name, "tier"); - } - - #[test] - fn project_time_index_tracks_aliased_time_col() { - // SELECT ts AS t — aliased; time_index should still point at the right output position. - let cs = schema_with_time( - vec![ - field("ts", L3DataType::Int64), - nullable_field("value", L3DataType::Float64), - ], - 0, - ); - let node = QueryExpr::Project { - child: dummy_scan(cs.clone()), - cols: vec![ProjectItem { - expr: L3Expr::Column(ColumnRef("ts".into())), - alias: Some("t".into()), - }], - }; - let out = node.output_schema(&[&cs], &empty_catalog()); - assert_eq!(out.time_index, Some(0)); - assert_eq!(out.fields[0].name, "t"); - } -} diff --git a/crates/lower/Cargo.toml b/crates/lower/Cargo.toml index 68593ff1..710d8bb6 100644 --- a/crates/lower/Cargo.toml +++ b/crates/lower/Cargo.toml @@ -5,6 +5,11 @@ edition = "2021" [dependencies] asap-control-core = { path = "../core" } +# Private mirror of GreptimeTeam/promql-parser (Apache-2.0). `main` tracks +# upstream untouched; the `asap` branch carries our local grammar/function +# additions (see THIRD_PARTY.md). +promql-parser = { git = "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/ProjectASAP/promql-parser", branch = "asap" } +# SQL front end: parse + plan via DataFusion, then lower its LogicalPlan to L2. datafusion = "43" [dev-dependencies] diff --git a/crates/lower/src/error.rs b/crates/lower/src/error.rs index 113ef582..e22002f5 100644 --- a/crates/lower/src/error.rs +++ b/crates/lower/src/error.rs @@ -2,14 +2,33 @@ use std::fmt; #[derive(Debug)] pub enum LoweringError { + /// The `promql-parser` crate rejected the query string (L1 parse failure). + Parse(String), + /// A PromQL function (`rate`, `*_over_time`, …) not supported in this version. + UnsupportedFunction(String), + /// A PromQL aggregation operator (`sum`, `topk`, …) not supported. + UnsupportedAggregateOp(String), + /// A structural feature (PromQL offset/`@`/`without`; SQL JOIN/subquery/…) + /// not supported in this version. + UnsupportedFeature(String), + /// A required function / aggregator argument was missing. + MissingArgument(String), + /// An argument had the wrong shape (e.g. a non-numeric `topk` parameter). + InvalidParameter(String), + /// The workload's query language is not handled by this lowerer. + WrongLanguage(String), + /// The L2→L3 converter failed (name resolution against the bound schema). + Convert(asap_control_core::intent_algebra::ConvertError), + + // ── SQL front end (DataFusion) ─────────────────────────────────────────── + /// DataFusion failed to parse / plan the SQL query. DataFusion(datafusion::error::DataFusionError), + /// A table referenced by the query is absent from the catalog. TableNotFound(String), - /// A SQL feature (JOIN, subquery, etc.) not supported in this version. - UnsupportedFeature(String), + /// A SQL aggregate function not supported in this version. UnsupportedAggregate(String), + /// A SQL scalar expression that could not be lowered. InvalidExpression(String), - /// The workload's query language is not handled by this lowerer. - WrongLanguage(String), /// The SQL dialect is not supported (only DataFusionSQL is implemented). UnsupportedDialect(String), } @@ -17,12 +36,21 @@ pub enum LoweringError { impl fmt::Display for LoweringError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + Self::Parse(e) => write!(f, "PromQL parse error: {e}"), + Self::UnsupportedFunction(n) => write!(f, "unsupported PromQL function: {n}"), + Self::UnsupportedAggregateOp(n) => write!(f, "unsupported PromQL aggregate op: {n}"), + // Raised by both front ends (PromQL offset/`@`/`without`/negation; + // SQL join type/subquery/derived table), so keep the label neutral — + // the message string carries the specifics. + Self::UnsupportedFeature(m) => write!(f, "unsupported feature: {m}"), + Self::MissingArgument(m) => write!(f, "missing argument: {m}"), + Self::InvalidParameter(m) => write!(f, "invalid parameter: {m}"), + Self::WrongLanguage(l) => write!(f, "unsupported query language: {l}"), + Self::Convert(e) => write!(f, "L2→L3 conversion failed: {e}"), Self::DataFusion(e) => write!(f, "DataFusion error: {e}"), Self::TableNotFound(t) => write!(f, "table not found in catalog: {t}"), - Self::UnsupportedFeature(msg) => write!(f, "unsupported SQL feature: {msg}"), - Self::UnsupportedAggregate(name) => write!(f, "unsupported aggregate: {name}"), - Self::InvalidExpression(msg) => write!(f, "invalid expression: {msg}"), - Self::WrongLanguage(lang) => write!(f, "unsupported query language: {lang}"), + Self::UnsupportedAggregate(n) => write!(f, "unsupported aggregate: {n}"), + Self::InvalidExpression(m) => write!(f, "invalid expression: {m}"), Self::UnsupportedDialect(d) => write!(f, "unsupported SQL dialect: {d}"), } } @@ -30,8 +58,29 @@ impl fmt::Display for LoweringError { impl std::error::Error for LoweringError {} +impl From for LoweringError { + fn from(e: asap_control_core::intent_algebra::ConvertError) -> Self { + Self::Convert(e) + } +} + impl From for LoweringError { fn from(e: datafusion::error::DataFusionError) -> Self { Self::DataFusion(e) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unsupported_feature_label_is_language_neutral() { + // `UnsupportedFeature` is raised by both front ends, so its Display must + // not hardcode "PromQL" — a SQL user rejecting a subquery shouldn't see + // "unsupported PromQL feature: subquery". + let msg = LoweringError::UnsupportedFeature("subquery".into()).to_string(); + assert_eq!(msg, "unsupported feature: subquery"); + assert!(!msg.contains("PromQL"), "got: {msg}"); + } +} diff --git a/crates/lower/src/lib.rs b/crates/lower/src/lib.rs index 3fdb6e97..c38dcb65 100644 --- a/crates/lower/src/lib.rs +++ b/crates/lower/src/lib.rs @@ -1,39 +1,98 @@ +//! L1→L3 lowering passes for the ASAP controller core. +//! +//! Both front ends end at the canonical intent algebra via the same L2→L3 +//! [`convert_root`]: PromQL parses with `promql-parser`, SQL parses + plans with +//! DataFusion. Each emits the per-language +//! [`relational::QueryExpr`](asap_control_core::intent_algebra::relational); the +//! shared converter runs the [`Binder`](asap_control_core::intent_algebra::Binder) +//! for positional name resolution and folds single-statistic sketchable +//! aggregates into canonical shapes. + pub mod error; -pub mod schema_pass; +pub mod promql; pub mod sql; -use asap_control_core::intent_algebra::expr::QueryExpr; -use asap_control_core::intent_algebra::schema::SchemaCatalog; +use asap_control_core::intent_algebra::{convert_root, QueryExpr}; use asap_control_core::types::AccuracyTarget; use asap_control_core::workload::{QueryLanguage, QueryWorkload, SqlDialect}; pub use error::LoweringError; -pub use schema_pass::populate_schemas; -pub use sql::SqlLowerer; +pub use promql::PromqlLowerer; +pub use sql::{SqlCatalog, SqlLowerer}; -/// Lower every SQL batch entry in `workload` to a `QueryExpr`. +/// Lower a single PromQL query string to the canonical L3 `QueryExpr`. +/// +/// `accuracy` is threaded onto every approximate intent (`Count`, `Quantile`, +/// `Cardinality`, `TopK`). The returned tree carries a self-contained `Schema` +/// on its `Scan`; call [`QueryExpr::output_schema`] for any node's schema. +pub fn lower_promql(query: &str, accuracy: AccuracyTarget) -> Result { + let l2 = PromqlLowerer::lower(query)?; + let l3 = convert_root(&l2, &accuracy)?; + Ok(l3) +} + +/// Lower every PromQL batch entry in `workload` to a `QueryExpr`. /// /// One `Result` per entry — errors are per-query, not fatal for the batch. -/// Returns an empty `Vec` if `workload.query_batch` is absent or empty. -/// Returns `WrongLanguage` for every entry if the workload language is not SQL. +/// Returns an empty `Vec` if `workload.query_batch` is absent or empty, and a +/// `WrongLanguage` error for every entry if the workload language is not PromQL. +pub fn lower_promql_batch(workload: &QueryWorkload) -> Vec> { + let entries = match &workload.query_batch { + Some(e) if !e.is_empty() => e, + _ => return vec![], + }; + + if !matches!(workload.language, QueryLanguage::PromQL) { + let lang = format!("{:?}", workload.language); + return entries + .iter() + .map(|_| Err(LoweringError::WrongLanguage(lang.clone()))) + .collect(); + } + + entries + .iter() + .map(|entry| { + let accuracy = entry + .requirements + .as_ref() + .and_then(|r| r.accuracy.clone()) + .unwrap_or(AccuracyTarget::Exact); + lower_promql(&entry.query.0, accuracy) + }) + .collect() +} + +/// Lower a single SQL query string to the canonical L3 `QueryExpr`. /// -/// **The returned trees have empty schemas on every node** (`L3Schema::fields` -/// is `vec![]`). Call [`populate_schemas`] on each successful result before -/// inspecting node schemas or passing the trees to schema-aware stages. -pub async fn lower_batch( +/// The `catalog` supplies table schemas (used both to plan the SQL with +/// DataFusion and to carry positional column identity into L3). `accuracy` is +/// threaded onto every approximate intent by the shared converter. +pub async fn lower_sql( + query: &str, + catalog: &SqlCatalog, + accuracy: AccuracyTarget, +) -> Result { + let l2 = SqlLowerer::new(catalog).lower(query).await?; + let l3 = convert_root(&l2, &accuracy)?; + Ok(l3) +} + +/// Lower every SQL batch entry in `workload` to a `QueryExpr`. +/// +/// One `Result` per entry — errors are per-query, not fatal for the batch. +/// Returns `WrongLanguage` for every entry if the workload is not SQL, and +/// `UnsupportedDialect` for non-DataFusion SQL dialects. +pub async fn lower_sql_batch( workload: &QueryWorkload, - catalog: &SchemaCatalog, + catalog: &SqlCatalog, ) -> Vec> { let entries = match &workload.query_batch { Some(e) if !e.is_empty() => e, _ => return vec![], }; - // Guard: only SQL languages are handled by this lowerer. - // `QueryLanguage::DataFusion` is an alias for `QueryLanguage::SQL(SqlDialect::DataFusionSQL)`: - // both were kept for backward compatibility with callers that pre-date the `SQL(dialect)` - // variant. New code should prefer `SQL(DataFusionSQL)`; `DataFusion` may be removed in a - // future version once all callers have migrated. + // `DataFusion` is a legacy alias for `SQL(DataFusionSQL)`; accept both. if !matches!( workload.language, QueryLanguage::SQL(_) | QueryLanguage::DataFusion @@ -44,9 +103,6 @@ pub async fn lower_batch( .map(|_| Err(LoweringError::WrongLanguage(lang.clone()))) .collect(); } - - // Guard: only DataFusionSQL is implemented; other SQL dialects need their - // own parser and plan-conversion layer before reaching SqlLowerer. if let QueryLanguage::SQL(dialect) = &workload.language { if !matches!(dialect, SqlDialect::DataFusionSQL) { let d = format!("{dialect:?}"); @@ -64,8 +120,7 @@ pub async fn lower_batch( .as_ref() .and_then(|r| r.accuracy.clone()) .unwrap_or(AccuracyTarget::Exact); - let lowerer = SqlLowerer::new(catalog, accuracy); - results.push(lowerer.lower(&entry.query.0).await); + results.push(lower_sql(&entry.query.0, catalog, accuracy).await); } results } diff --git a/crates/lower/src/promql.rs b/crates/lower/src/promql.rs new file mode 100644 index 00000000..654149c4 --- /dev/null +++ b/crates/lower/src/promql.rs @@ -0,0 +1,727 @@ +//! Layers 1→2 lowering: PromQL string → Layer-2 `relational::QueryExpr`. +//! +//! - **L1 (parse)** is delegated to `promql-parser` 0.8. +//! - **L2 (per-language tree)** is built here: the walk interprets PromQL +//! semantics (range vectors, aggregate operators, label matchers) and emits +//! the language-flavored [`relational::QueryExpr`] the controller's L2→L3 +//! converter ([`convert_root`](asap_control_core::intent_algebra::convert_root)) +//! consumes. Canonicalisation (window-over-aggregate fold, GROUP-BY → +//! `Partition`, positional name binding) happens in that converter, not here. +//! +//! # PromQL → L2 mapping (summary) +//! +//! | PromQL | L2 shape (→ canonical via `convert_root`) | +//! |---|---| +//! | `quantile_over_time(φ, m{f}[w])` | `Aggregate{[Quantile(φ)], Window{w, Filter(Source)}}` | +//! | `histogram_quantile(φ, )` | `Aggregate{[Quantile(φ)]}` over the fully-lowered `` (preserves any `sum by (le)`/`rate`) | +//! | `OUTER_op(inner_func(m[w]))` (e.g. `sum(rate(m[w]))`) | `Aggregate{[OUTER_op]}` over `Aggregate{[inner_func]}` — two levels | +//! | `avg/min/max/sum_over_time(m[w])` | `Aggregate{[Avg/Min/Max/Sum], Window{w}}` | +//! | `stddev/stdvar_over_time(m[w])` | `Aggregate{[StdDev/Variance], Window{w}}` | +//! | `count_over_time(m[w])` | `Aggregate{[Count], Window{w}}` | +//! | `rate/irate(m[w])` | `Aggregate{[Rate{w}]}` (no Window) — `irate` shares the `rate` *intent*; the avg-vs-last-two-samples difference is an L4 estimation method | +//! | `increase(m[w])` | `Aggregate{[Increase{w}]}` (no Window) | +//! | `changes` / `resets` / `group` / `offset` / `@` | **rejected** — distinct semantics with no intent-algebra representation yet | +//! | `OUTER by (dims) (…)` | `Aggregate.keys = dims` (→ `Partition` in L3) | +//! | `count by (d) (…)` | `Aggregate{[CountDistinct], …}` (→ `Cardinality`) | +//! | `topk(k, count_over_time(…))` | `TopK{k, by}` (heavy-hitter, one pass) | +//! | `topk(k, )` / `bottomk(k, …)` | `Sort{value} → Limit{k}` | +//! | `m{f}` | `Filter(Source)` | +//! | `a OP b` | `BinaryOp{vector_match}` | +//! | `expr[r:res]` | `PromQLSubquery{r, res}` | + +use std::time::Duration; + +use promql_parser::label::{MatchOp, Matcher}; +use promql_parser::parser::{ + self, token, AggregateExpr, BinaryExpr, Call, Expr, LabelModifier, VectorMatchCardinality, + VectorSelector, +}; + +use asap_control_core::intent_algebra::query_expr::{ + BinaryOpKind, GroupSide, VectorGrouping, VectorMatch, VectorMatchKind, +}; +use asap_control_core::intent_algebra::relational::{ + AggFunc, AggItem, L2SortKey, QueryExpr as L2, SourceSpec, +}; +use asap_control_core::intent_algebra::{ArithOp, ColumnRef, CompareOp, L2Expr, L3Scalar}; + +use crate::error::LoweringError; + +type Result = std::result::Result; + +/// Parses (L1) and lowers (→ L2 relational) a PromQL query string. +pub struct PromqlLowerer; + +#[derive(Debug, Clone)] +enum Outer { + None, + Plain(OuterIntent), + Count, + TopK { k: u64, descending: bool }, +} + +#[derive(Debug, Clone)] +enum OuterIntent { + Sum, + Avg, + Min, + Max, + StdDev, + Variance, + Quantile(f64), +} + +#[derive(Debug, Clone)] +enum InnerFunc { + Quantile(f64), + Avg, + Min, + Max, + Sum, + StdDev, + Variance, + Count, + Rate(Duration), + Increase(Duration), +} + +struct Inner { + metric: String, + matchers: Vec, + window: Option, + func: Option, +} + +/// Maximum PromQL expression nesting depth the walker accepts. Real queries +/// nest only a handful deep; this bounds the recursive descent (`walk` and the +/// mutually-recursive helpers) so a pathologically nested query is rejected +/// rather than overflowing the stack. +const MAX_DEPTH: usize = 256; + +impl PromqlLowerer { + pub fn lower(query: &str) -> Result { + let ast = parser::parse(query).map_err(LoweringError::Parse)?; + // Reject over-deep nesting up front, so the (mutually-recursive) walk + // below cannot blow the stack. The check itself recurses at most + // `MAX_DEPTH` frames before erroring, so it is bounded too. + check_depth(&ast, MAX_DEPTH)?; + walk(&ast) + } +} + +/// Bounded depth check over the parser AST: errors once nesting would exceed +/// `budget` frames, descending into every child expression. +fn check_depth(expr: &Expr, budget: usize) -> Result<()> { + let Some(budget) = budget.checked_sub(1) else { + return Err(LoweringError::UnsupportedFeature(format!( + "query nesting exceeds the {MAX_DEPTH}-level limit" + ))); + }; + match expr { + Expr::Aggregate(a) => { + check_depth(&a.expr, budget)?; + if let Some(p) = &a.param { + check_depth(p, budget)?; + } + } + Expr::Unary(u) => check_depth(&u.expr, budget)?, + Expr::Binary(b) => { + check_depth(&b.lhs, budget)?; + check_depth(&b.rhs, budget)?; + } + Expr::Paren(p) => check_depth(&p.expr, budget)?, + Expr::Subquery(s) => check_depth(&s.expr, budget)?, + Expr::Call(c) => { + for arg in &c.args.args { + check_depth(arg, budget)?; + } + } + Expr::MatrixSelector(_) + | Expr::VectorSelector(_) + | Expr::NumberLiteral(_) + | Expr::StringLiteral(_) + | Expr::Extension(_) => {} + } + Ok(()) +} + +fn walk(expr: &Expr) -> Result { + match expr { + Expr::Aggregate(agg) => walk_aggregate(agg), + Expr::Call(call) if call.func.name == "histogram_quantile" => walk_histogram_quantile(call), + Expr::Call(call) => build(lower_inner_call(call)?, vec![], Outer::None), + Expr::Binary(bin) => walk_binary(bin), + Expr::Paren(p) => walk(&p.expr), + // `UnaryExpr` is built only by negation (`Neg`); unary `+` is folded to + // identity and `-` to a negated `NumberLiteral`, so this always + // wraps a vector expression whose samples must be sign-flipped. The L2 + // PromQL path has no scalar/negate node to express that (there's no + // `-1 * x`, since `walk` rejects bare scalar operands), so reject it + // rather than silently dropping the sign and computing `+expr`. + Expr::Unary(_) => Err(LoweringError::UnsupportedFeature( + "unary negation (`-expr`): no negate/scalar node in the L2 PromQL path".into(), + )), + Expr::Subquery(sq) => Ok(L2::PromQLSubquery { + range: sq.range, + resolution: sq.step, + input: Box::new(walk(&sq.expr)?), + }), + Expr::VectorSelector(vs) => { + let (metric, matchers) = vs_parts(vs)?; + Ok(filtered_source(metric, matchers)) + } + Expr::MatrixSelector(ms) => { + let (metric, matchers) = vs_parts(&ms.vs)?; + Ok(L2::Window { + duration: ms.range, + slide: None, + input: Box::new(filtered_source(metric, matchers)), + }) + } + Expr::NumberLiteral(_) | Expr::StringLiteral(_) => Err(LoweringError::UnsupportedFeature( + "bare scalar/string at top level".into(), + )), + Expr::Extension(_) => Err(LoweringError::UnsupportedFeature( + "extension expression".into(), + )), + } +} + +fn walk_aggregate(agg: &AggregateExpr) -> Result { + let keys = resolve_group(agg)?; + let inner = lower_inner(&agg.expr)?; + let op = agg.op.id(); + + let outer = if op == token::T_TOPK { + Outer::TopK { + k: count_param(agg)?, + descending: true, + } + } else if op == token::T_BOTTOMK { + Outer::TopK { + k: count_param(agg)?, + descending: false, + } + } else if op == token::T_COUNT { + Outer::Count + } else if op == token::T_SUM { + Outer::Plain(OuterIntent::Sum) + } else if op == token::T_GROUP { + // `group(v)` yields a constant 1 per group (presence), not a sum of + // values. Folding it onto `Sum` changed the result; reject until a + // distinct group-presence intent exists. + return Err(LoweringError::UnsupportedAggregateOp( + "`group` (constant-1 presence) is not `sum`; no distinct intent yet".into(), + )); + } else if op == token::T_AVG { + Outer::Plain(OuterIntent::Avg) + } else if op == token::T_MIN { + Outer::Plain(OuterIntent::Min) + } else if op == token::T_MAX { + Outer::Plain(OuterIntent::Max) + } else if op == token::T_STDDEV { + Outer::Plain(OuterIntent::StdDev) + } else if op == token::T_STDVAR { + Outer::Plain(OuterIntent::Variance) + } else if op == token::T_QUANTILE { + Outer::Plain(OuterIntent::Quantile(quantile_param(num_param(agg)?)?)) + } else { + return Err(LoweringError::UnsupportedAggregateOp(format!( + "aggregate token {op}" + ))); + }; + + build(inner, keys, outer) +} + +/// `histogram_quantile(φ, )` lowers `` in full — preserving any +/// `sum by (le)` / `rate` structure inside it — and wraps the result in an +/// `Aggregate{[Quantile(φ)]}`. The φ-quantile reduces across the `le` buckets, +/// so the wrapper carries no grouping keys: the usage-derived schema can't +/// enumerate the non-`le` labels to group by (the same limitation that rejects +/// `without`). This handles the canonical +/// `histogram_quantile(φ, sum by (le) (rate(m_bucket[w])))` pattern, which the +/// old "extract the matrix and substitute a bare Quantile" path could not. +fn walk_histogram_quantile(call: &Call) -> Result { + let phi = quantile_param(num_arg(call, 0)?)?; + let inner = walk(arg(call, 1)?)?; + Ok(outer_aggregate(vec![], AggFunc::Quantile(phi), inner)) +} + +fn walk_binary(bin: &BinaryExpr) -> Result { + let lhs = walk(&bin.lhs)?; + let rhs = walk(&bin.rhs)?; + let op = binop(bin.op.id())?; + let vector_match = bin.modifier.as_ref().map(|m| { + let (kind, labels) = match &m.matching { + Some(LabelModifier::Include(ls)) => (VectorMatchKind::On, ls.labels.clone()), + Some(LabelModifier::Exclude(ls)) => (VectorMatchKind::Ignoring, ls.labels.clone()), + None => (VectorMatchKind::On, vec![]), + }; + let grouping = match &m.card { + VectorMatchCardinality::ManyToOne(ls) => Some(VectorGrouping { + side: GroupSide::Left, + labels: ls.labels.clone(), + }), + VectorMatchCardinality::OneToMany(ls) => Some(VectorGrouping { + side: GroupSide::Right, + labels: ls.labels.clone(), + }), + _ => None, + }; + VectorMatch { + kind, + labels, + grouping, + } + }); + Ok(L2::BinaryOp { + op, + lhs: Box::new(lhs), + rhs: Box::new(rhs), + vector_match, + }) +} + +fn lower_inner(expr: &Expr) -> Result { + match expr { + Expr::VectorSelector(vs) => { + let (metric, matchers) = vs_parts(vs)?; + Ok(Inner { + metric, + matchers, + window: None, + func: None, + }) + } + Expr::MatrixSelector(ms) => { + let (metric, matchers) = vs_parts(&ms.vs)?; + Ok(Inner { + metric, + matchers, + window: Some(ms.range), + func: None, + }) + } + Expr::Paren(p) => lower_inner(&p.expr), + Expr::Call(call) => lower_inner_call(call), + other => Err(LoweringError::UnsupportedFeature(format!( + "aggregate argument: {:?}", + std::mem::discriminant(other) + ))), + } +} + +fn lower_inner_call(call: &Call) -> Result { + let name = call.func.name; + let at0 = |func: InnerFunc| -> Result { + let (metric, matchers, window) = extract_matrix(arg(call, 0)?)?; + Ok(Inner { + metric, + matchers, + window: Some(window), + func: Some(func), + }) + }; + match name { + "rate" | "irate" => { + let (metric, matchers, window) = extract_matrix(arg(call, 0)?)?; + Ok(Inner { + metric, + matchers, + window: Some(window), + func: Some(InnerFunc::Rate(window)), + }) + } + "increase" => { + let (metric, matchers, window) = extract_matrix(arg(call, 0)?)?; + Ok(Inner { + metric, + matchers, + window: Some(window), + func: Some(InnerFunc::Increase(window)), + }) + } + "quantile_over_time" => { + let phi = quantile_param(num_arg(call, 0)?)?; + let (metric, matchers, window) = extract_matrix(arg(call, 1)?)?; + Ok(Inner { + metric, + matchers, + window: Some(window), + func: Some(InnerFunc::Quantile(phi)), + }) + } + "avg_over_time" => at0(InnerFunc::Avg), + "min_over_time" => at0(InnerFunc::Min), + "max_over_time" => at0(InnerFunc::Max), + "sum_over_time" => at0(InnerFunc::Sum), + "stddev_over_time" => at0(InnerFunc::StdDev), + "stdvar_over_time" => at0(InnerFunc::Variance), + "count_over_time" => at0(InnerFunc::Count), + // `changes` (value changes) and `resets` (counter resets) are NOT + // sample counts — aliasing them to `count_over_time` silently produced + // the wrong number. Reject until they have distinct intents. + other => Err(LoweringError::UnsupportedFunction(other.to_string())), + } +} + +/// Assemble the Layer-2 tree from a lowered inner vector, the resolved group +/// keys, and the enclosing aggregator shape. +fn build(inner: Inner, keys: Vec, outer: Outer) -> Result { + match outer { + Outer::None => match &inner.func { + None => Ok(filtered_source(inner.metric, inner.matchers)), + Some(f) => { + let func = inner_func(f); + Ok(windowed_aggregate(inner, keys, func)) + } + }, + // An OUTER aggregation operator (`sum`/`avg`/…/`count`) over an inner + // range-vector function (`rate`/`increase`/`*_over_time`) is a + // two-level reduction: the inner func runs per series, the outer op + // then aggregates across series. Collapsing them into one aggregate + // silently drops a level — e.g. `sum(rate(m[w]))` must keep the `sum`. + Outer::Plain(outer_intent) => Ok(match &inner.func { + None => windowed_aggregate(inner, keys, outer_func(&outer_intent)), + Some(f) => { + let inner_f = inner_func(f); + let inner_agg = windowed_aggregate(inner, vec![], inner_f); + outer_aggregate(keys, outer_func(&outer_intent), inner_agg) + } + }), + Outer::Count => Ok(match &inner.func { + None => windowed_aggregate(inner, keys, AggFunc::CountDistinct), + Some(f) => { + let inner_f = inner_func(f); + let inner_agg = windowed_aggregate(inner, vec![], inner_f); + outer_aggregate(keys, AggFunc::CountDistinct, inner_agg) + } + }), + Outer::TopK { k, descending } => { + // Heavy-hitter only when ranking by frequency (`count`): a dedicated + // sketch serves it in one pass → first-class `TopK`. Any other + // ranking (topk over avg/quantile, all bottomk) is generic + // order-by-value + limit. + let heavy_hitter = descending && matches!(inner.func, Some(InnerFunc::Count)); + if heavy_hitter { + let scan = window_scan(inner); + Ok(L2::TopK { + k, + by: keys, + input: Box::new(scan), + }) + } else { + let func = match &inner.func { + Some(f) => inner_func(f), + None => AggFunc::Sum, + }; + let base = windowed_aggregate(inner, keys, func); + let sorted = L2::Sort { + keys: vec![L2SortKey { + expr: L2Expr::Column(ColumnRef::SampleValue), + ascending: !descending, + nulls_first: false, + }], + input: Box::new(base), + }; + Ok(L2::Limit { + n: k, + offset: 0, + input: Box::new(sorted), + }) + } + } + } +} + +/// `Aggregate{keys, [func]}` over `[Window{w}] → Filter(Source)`. Rate/Increase +/// carry their own window in the func, so no `Window` node is emitted. +fn windowed_aggregate(inner: Inner, keys: Vec, func: AggFunc) -> L2 { + let skip_window = matches!(func, AggFunc::Rate { .. } | AggFunc::Increase { .. }); + let window = inner.window; + let base = filtered_source(inner.metric, inner.matchers); + let input = match window { + Some(w) if !skip_window => L2::Window { + duration: w, + slide: None, + input: Box::new(base), + }, + _ => base, + }; + L2::Aggregate { + keys, + aggs: vec![AggItem { + // None alias → the converter keeps PromQL's intent-keyed output + // names ("sum", "quantile_0_99", …) instead of overriding them. + alias: None, + func, + col: ColumnRef::SampleValue, + }], + having: None, + input: Box::new(input), + } +} + +/// `Aggregate{keys, [func]}` directly over an existing L2 subtree — the OUTER +/// level of a two-level aggregation such as `sum(rate(…))` or the +/// `Aggregate{[Quantile]}` that wraps a `histogram_quantile` argument. +fn outer_aggregate(keys: Vec, func: AggFunc, input: L2) -> L2 { + L2::Aggregate { + keys, + aggs: vec![AggItem { + // None alias → the converter keeps PromQL's intent-keyed output + // names ("sum", "quantile_0_99", …) instead of overriding them. + alias: None, + func, + col: ColumnRef::SampleValue, + }], + having: None, + input: Box::new(input), + } +} + +/// `[Window{w}] → Filter(Source)` with no aggregate (the heavy-hitter TopK +/// child — the sketch counts directly off the scan). +fn window_scan(inner: Inner) -> L2 { + let base = filtered_source(inner.metric, inner.matchers); + match inner.window { + Some(w) => L2::Window { + duration: w, + slide: None, + input: Box::new(base), + }, + None => base, + } +} + +fn filtered_source(metric: String, matchers: Vec) -> L2 { + let source = L2::Source(SourceSpec::new(metric)); + if matchers.is_empty() { + source + } else { + let pred = if matchers.len() == 1 { + matchers.into_iter().next().unwrap() + } else { + L2Expr::BoolAnd(matchers) + }; + L2::Filter { + pred, + input: Box::new(source), + } + } +} + +fn inner_func(f: &InnerFunc) -> AggFunc { + match f { + InnerFunc::Quantile(q) => AggFunc::Quantile(*q), + InnerFunc::Avg => AggFunc::Avg, + InnerFunc::Min => AggFunc::Min, + InnerFunc::Max => AggFunc::Max, + InnerFunc::Sum => AggFunc::Sum, + InnerFunc::StdDev => AggFunc::StdDev { population: false }, + InnerFunc::Variance => AggFunc::Variance { population: false }, + InnerFunc::Count => AggFunc::Count, + InnerFunc::Rate(w) => AggFunc::Rate { window: *w }, + InnerFunc::Increase(w) => AggFunc::Increase { window: *w }, + } +} + +fn outer_func(o: &OuterIntent) -> AggFunc { + match o { + OuterIntent::Sum => AggFunc::Sum, + OuterIntent::Avg => AggFunc::Avg, + OuterIntent::Min => AggFunc::Min, + OuterIntent::Max => AggFunc::Max, + OuterIntent::StdDev => AggFunc::StdDev { population: false }, + OuterIntent::Variance => AggFunc::Variance { population: false }, + OuterIntent::Quantile(q) => AggFunc::Quantile(*q), + } +} + +/// Resolve `by(labels)` into a key list. `without(...)` needs the metric's +/// full label set, which the usage-derived schema model doesn't carry, so it +/// is rejected (a registry-backed `SchemaCatalog` would lift this). +fn resolve_group(agg: &AggregateExpr) -> Result> { + match &agg.modifier { + None => Ok(vec![]), + Some(LabelModifier::Include(ls)) => { + // Grouping labels are a set: `by (a, b)` ≡ `by (b, a)`. Canonicalise + // so equivalent groupings lower to identical keys. PromQL labels have + // no table qualifier → `ColumnRef::Named`. + let mut keys = ls.labels.clone(); + keys.sort(); + keys.dedup(); + Ok(keys.into_iter().map(ColumnRef::Named).collect()) + } + Some(LabelModifier::Exclude(_)) => Err(LoweringError::UnsupportedFeature( + "`without(...)` grouping requires a registry-backed catalog of the \ + metric's label set (the usage-derived schema can't enumerate the \ + complement)" + .into(), + )), + } +} + +// ── Free helpers ────────────────────────────────────────────────────────────── + +fn vs_parts(vs: &VectorSelector) -> Result<(String, Vec)> { + // `offset` / `@` shift the evaluation/lookback time. The intent algebra has + // no representation for either, so silently lowering them (as if absent) + // would change the query's meaning. Reject rather than mislower. + if vs.offset.is_some() || vs.at.is_some() { + return Err(LoweringError::UnsupportedFeature( + "`offset` / `@` time-shift modifiers have no intent-algebra representation".into(), + )); + } + let metric = vs.name.clone().unwrap_or_else(|| { + vs.matchers + .matchers + .iter() + .find(|m| m.name == "__name__") + .map(|m| m.value.clone()) + .unwrap_or_default() + }); + // Label matchers are an unordered set: `{a="1",b="2"}` and `{b="2",a="1"}` + // select the same series. Canonicalise by (name, value) so equivalent + // selectors lower to identical predicates. + let mut ms: Vec<&Matcher> = vs + .matchers + .matchers + .iter() + .filter(|m| m.name != "__name__") + .collect(); + ms.sort_by(|a, b| a.name.cmp(&b.name).then_with(|| a.value.cmp(&b.value))); + let matchers = ms.into_iter().map(matcher_to_l3expr).collect(); + Ok((metric, matchers)) +} + +fn matcher_to_l3expr(m: &Matcher) -> L2Expr { + let op = match &m.op { + MatchOp::Equal => CompareOp::Eq, + MatchOp::NotEqual => CompareOp::Ne, + MatchOp::Re(_) => CompareOp::Regex, + MatchOp::NotRe(_) => CompareOp::NotRegex, + }; + L2Expr::Compare { + left: Box::new(L2Expr::Column(ColumnRef::Named(m.name.clone()))), + op, + right: Box::new(L2Expr::Literal(L3Scalar::Utf8(m.value.clone()))), + } +} + +fn extract_matrix(expr: &Expr) -> Result<(String, Vec, Duration)> { + match expr { + Expr::MatrixSelector(ms) => { + let (metric, matchers) = vs_parts(&ms.vs)?; + Ok((metric, matchers, ms.range)) + } + Expr::Paren(p) => extract_matrix(&p.expr), + // A range-vector function argument must be a (parenthesised) matrix + // selector. Do NOT descend through an arbitrary `Call` — that would + // silently strip an unsupported wrapper (`rate(deriv(m[5m]))` lowering + // as `rate(m[5m])`). Reject instead. + other => Err(LoweringError::UnsupportedFeature(format!( + "expected a range-vector (matrix) argument, got {:?}", + std::mem::discriminant(other) + ))), + } +} + +fn arg(call: &Call, idx: usize) -> Result<&Expr> { + call.args + .args + .get(idx) + .map(|b| b.as_ref()) + .ok_or_else(|| LoweringError::MissingArgument(format!("{} arg #{idx}", call.func.name))) +} + +fn num_arg(call: &Call, idx: usize) -> Result { + num_expr(arg(call, idx)?) +} + +fn num_param(agg: &AggregateExpr) -> Result { + match &agg.param { + Some(e) => num_expr(e), + None => Err(LoweringError::MissingArgument( + "aggregate parameter (k / φ)".into(), + )), + } +} + +fn num_expr(expr: &Expr) -> Result { + match expr { + Expr::NumberLiteral(n) => Ok(n.val), + other => Err(LoweringError::InvalidParameter(format!( + "expected a numeric literal, got {:?}", + std::mem::discriminant(other) + ))), + } +} + +/// `topk`/`bottomk` count parameter — a non-negative integer. Rejects +/// fractional / negative / non-finite values rather than silently truncating +/// or saturating them via `as u64` (`topk(2.7, …)` ≠ `topk(2, …)`). +fn count_param(agg: &AggregateExpr) -> Result { + let v = num_param(agg)?; + if v.is_finite() && v >= 0.0 && v.fract() == 0.0 && v <= u64::MAX as f64 { + Ok(v as u64) + } else { + Err(LoweringError::InvalidParameter(format!( + "topk/bottomk k must be a non-negative integer, got {v}" + ))) + } +} + +/// Quantile φ — must be a finite value in `[0, 1]`. Rejects NaN/∞ and +/// out-of-range φ (which would otherwise propagate into a bogus intent and +/// output-column name like `quantile_NaN`). +fn quantile_param(q: f64) -> Result { + if q.is_finite() && (0.0..=1.0).contains(&q) { + Ok(q) + } else { + Err(LoweringError::InvalidParameter(format!( + "quantile φ must be in [0, 1], got {q}" + ))) + } +} + +fn binop(id: token::TokenId) -> Result { + Ok(if id == token::T_ADD { + BinaryOpKind::Arith(ArithOp::Add) + } else if id == token::T_SUB { + BinaryOpKind::Arith(ArithOp::Sub) + } else if id == token::T_MUL { + BinaryOpKind::Arith(ArithOp::Mul) + } else if id == token::T_DIV { + BinaryOpKind::Arith(ArithOp::Div) + } else if id == token::T_MOD { + BinaryOpKind::Arith(ArithOp::Mod) + } else if id == token::T_POW { + BinaryOpKind::Pow + } else if id == token::T_ATAN2 { + BinaryOpKind::Atan2 + } else if id == token::T_EQLC { + BinaryOpKind::Compare(CompareOp::Eq) + } else if id == token::T_NEQ { + BinaryOpKind::Compare(CompareOp::Ne) + } else if id == token::T_LSS { + BinaryOpKind::Compare(CompareOp::Lt) + } else if id == token::T_LTE { + BinaryOpKind::Compare(CompareOp::Le) + } else if id == token::T_GTR { + BinaryOpKind::Compare(CompareOp::Gt) + } else if id == token::T_GTE { + BinaryOpKind::Compare(CompareOp::Ge) + } else if id == token::T_LAND { + BinaryOpKind::And + } else if id == token::T_LOR { + BinaryOpKind::Or + } else if id == token::T_LUNLESS { + BinaryOpKind::Unless + } else { + return Err(LoweringError::UnsupportedFeature(format!( + "binary operator token {id}" + ))); + }) +} diff --git a/crates/lower/src/schema_pass.rs b/crates/lower/src/schema_pass.rs deleted file mode 100644 index 1ec5b0b1..00000000 --- a/crates/lower/src/schema_pass.rs +++ /dev/null @@ -1,166 +0,0 @@ -use std::sync::Arc; - -use asap_control_core::intent_algebra::expr::QueryExpr; -use asap_control_core::intent_algebra::schema::{HasSchema, L3Schema, SchemaCatalog}; -use asap_control_core::intent_algebra::L3Node; - -/// Recursively populate the `schema` field on every node in a `QueryExpr` tree. -/// -/// The lowerer creates every node with an empty schema (`make_node`). This -/// pass walks the tree bottom-up, computing each node's output schema from -/// its children's schemas and the catalog, and returns a fully typed -/// `Arc` tree. -pub fn populate_schemas(expr: QueryExpr, catalog: &SchemaCatalog) -> Arc { - let (rebuilt, child_schemas) = rebuild(expr, catalog); - let refs: Vec<&L3Schema> = child_schemas.iter().collect(); - let schema = rebuilt.output_schema(&refs, catalog); - Arc::new(L3Node { - expr: rebuilt, - schema, - }) -} - -/// Recursively rebuild the expression tree with populated child nodes. -/// Returns `(rebuilt_expr, child_output_schemas)` so the caller can pass -/// those schemas to `output_schema`. -fn rebuild(expr: QueryExpr, catalog: &SchemaCatalog) -> (QueryExpr, Vec) { - use QueryExpr::*; - - // Helper: process one child Arc → fresh Arc with schema set. - let proc = |node: Arc| populate_schemas(node.expr.clone(), catalog); - - match expr { - // Leaf: schema comes from the catalog, no child schemas needed. - Scan { .. } => (expr, vec![]), - - Filter { child, pred } => { - let c = proc(child); - let cs = c.schema.clone(); - (Filter { child: c, pred }, vec![cs]) - } - Project { child, cols } => { - let c = proc(child); - let cs = c.schema.clone(); - (Project { child: c, cols }, vec![cs]) - } - Aggregate { - child, - by, - aggs, - having, - output_names, - } => { - let c = proc(child); - let cs = c.schema.clone(); - ( - Aggregate { - child: c, - by, - aggs, - having, - output_names, - }, - vec![cs], - ) - } - Sort { child, keys } => { - let c = proc(child); - let cs = c.schema.clone(); - (Sort { child: c, keys }, vec![cs]) - } - Limit { child, n, offset } => { - let c = proc(child); - let cs = c.schema.clone(); - ( - Limit { - child: c, - n, - offset, - }, - vec![cs], - ) - } - Distinct { child, cols } => { - let c = proc(child); - let cs = c.schema.clone(); - (Distinct { child: c, cols }, vec![cs]) - } - Partition { child, keys } => { - let c = proc(child); - let cs = c.schema.clone(); - (Partition { child: c, keys }, vec![cs]) - } - TimeWindow { - child, - kind, - size, - slide, - } => { - let c = proc(child); - let cs = c.schema.clone(); - ( - TimeWindow { - child: c, - kind, - size, - slide, - }, - vec![cs], - ) - } - WindowFunc { - child, - func, - args, - partition_by, - order_by, - frame, - } => { - let c = proc(child); - let cs = c.schema.clone(); - ( - WindowFunc { - child: c, - func, - args, - partition_by, - order_by, - frame, - }, - vec![cs], - ) - } - SetOp { - kind, - all, - left, - right, - } => { - let l = proc(left); - let r = proc(right); - let ls = l.schema.clone(); - let rs = r.schema.clone(); - ( - SetOp { - kind, - all, - left: l, - right: r, - }, - vec![ls, rs], - ) - } - Merge { children } => { - let new_children: Vec> = children.into_iter().map(proc).collect(); - let schemas: Vec = new_children.iter().map(|c| c.schema.clone()).collect(); - ( - Merge { - children: new_children, - }, - schemas, - ) - } - // Unimplemented variants: return as-is; output_schema will todo!() if called. - other => (other, vec![]), - } -} diff --git a/crates/lower/src/sql/expr.rs b/crates/lower/src/sql/expr.rs index f7a8ddaa..d11d9a6a 100644 --- a/crates/lower/src/sql/expr.rs +++ b/crates/lower/src/sql/expr.rs @@ -1,7 +1,6 @@ use datafusion::logical_expr::{BinaryExpr, Expr, Operator}; -use asap_control_core::intent_algebra::expr::ColumnRef; -use asap_control_core::intent_algebra::{ArithOp, CompareOp, L3Expr, L3Scalar}; +use asap_control_core::intent_algebra::{ArithOp, ColumnRef, CompareOp, L2Expr, L3Scalar}; use crate::error::LoweringError; @@ -22,39 +21,34 @@ pub(super) fn split_conjuncts(expr: &Expr) -> Vec<&Expr> { } } -/// Translate a slice of DataFusion `Expr`s (non-time conjuncts) into a single -/// `L3Expr`. A single element is returned as-is; multiple elements are wrapped -/// in `L3Expr::BoolAnd`. -pub(super) fn conjuncts_to_l3expr(conjuncts: Vec<&Expr>) -> Result { - let parts: Result, _> = conjuncts.iter().map(|e| df_expr_to_l3(e)).collect(); - let mut parts = parts?; - if parts.len() == 1 { - Ok(parts.pop().unwrap()) - } else { - Ok(L3Expr::BoolAnd(parts)) - } -} - -/// Translate a DataFusion `Expr` to an `L3Expr`. +/// Translate a DataFusion `Expr` to an `L2Expr`. /// Returns `UnsupportedFeature` for anything not needed in v1. -pub(super) fn df_expr_to_l3(expr: &Expr) -> Result { +pub(super) fn df_expr_to_l2(expr: &Expr) -> Result { match expr { - Expr::Column(col) => Ok(L3Expr::Column(ColumnRef(col.name.clone()))), + // Preserve DataFusion's relation qualifier so a column name shared + // across a join (`a.k` vs `b.k`) resolves to the correct side. + Expr::Column(col) => Ok(L2Expr::Column(match &col.relation { + Some(rel) => ColumnRef::Qualified { + table: rel.to_string(), + name: col.name.clone(), + }, + None => ColumnRef::Named(col.name.clone()), + })), - Expr::Literal(sv) => scalar_value_to_l3(sv).map(L3Expr::Literal), + Expr::Literal(sv) => scalar_value_to_l3(sv).map(L2Expr::Literal), - Expr::Alias(a) => df_expr_to_l3(&a.expr), + Expr::Alias(a) => df_expr_to_l2(&a.expr), Expr::BinaryExpr(BinaryExpr { left, op, right }) => match op { Operator::And => { let parts = split_conjuncts(expr); - let l3_parts: Result, _> = parts.iter().map(|e| df_expr_to_l3(e)).collect(); - Ok(L3Expr::BoolAnd(l3_parts?)) + let l3_parts: Result, _> = parts.iter().map(|e| df_expr_to_l2(e)).collect(); + Ok(L2Expr::BoolAnd(l3_parts?)) } Operator::Or => { let parts = split_disjuncts(expr); - let l3_parts: Result, _> = parts.iter().map(|e| df_expr_to_l3(e)).collect(); - Ok(L3Expr::BoolOr(l3_parts?)) + let l3_parts: Result, _> = parts.iter().map(|e| df_expr_to_l2(e)).collect(); + Ok(L2Expr::BoolOr(l3_parts?)) } Operator::Eq => compare(left, CompareOp::Eq, right), Operator::NotEq => compare(left, CompareOp::Ne, right), @@ -91,13 +85,13 @@ pub(super) fn df_expr_to_l3(expr: &Expr) -> Result { // Unary minus: negate literals directly; wrap others in -1 * x. Expr::Negative(inner) => { - let inner_l3 = df_expr_to_l3(inner)?; + let inner_l3 = df_expr_to_l2(inner)?; match inner_l3 { - L3Expr::Literal(L3Scalar::Int64(v)) => Ok(L3Expr::Literal(L3Scalar::Int64(-v))), - L3Expr::Literal(L3Scalar::Float64(v)) => Ok(L3Expr::Literal(L3Scalar::Float64(-v))), - other => Ok(L3Expr::Arith { + L2Expr::Literal(L3Scalar::Int64(v)) => Ok(L2Expr::Literal(L3Scalar::Int64(-v))), + L2Expr::Literal(L3Scalar::Float64(v)) => Ok(L2Expr::Literal(L3Scalar::Float64(-v))), + other => Ok(L2Expr::Arith { op: ArithOp::Mul, - left: Box::new(L3Expr::Literal(L3Scalar::Int64(-1))), + left: Box::new(L2Expr::Literal(L3Scalar::Int64(-1))), right: Box::new(other), }), } @@ -108,35 +102,35 @@ pub(super) fn df_expr_to_l3(expr: &Expr) -> Result { let operand = c .expr .as_ref() - .map(|e| df_expr_to_l3(e).map(Box::new)) + .map(|e| df_expr_to_l2(e).map(Box::new)) .transpose()?; let branches = c .when_then_expr .iter() - .map(|(when, then)| Ok((df_expr_to_l3(when)?, df_expr_to_l3(then)?))) + .map(|(when, then)| Ok((df_expr_to_l2(when)?, df_expr_to_l2(then)?))) .collect::, LoweringError>>()?; let else_expr = c .else_expr .as_ref() - .map(|e| df_expr_to_l3(e).map(Box::new)) + .map(|e| df_expr_to_l2(e).map(Box::new)) .transpose()?; - Ok(L3Expr::Case { + Ok(L2Expr::Case { operand, branches, else_expr, }) } - Expr::Not(inner) => Ok(L3Expr::Not(Box::new(df_expr_to_l3(inner)?))), + Expr::Not(inner) => Ok(L2Expr::Not(Box::new(df_expr_to_l2(inner)?))), - Expr::IsNull(inner) => Ok(L3Expr::IsNull(Box::new(df_expr_to_l3(inner)?))), + Expr::IsNull(inner) => Ok(L2Expr::IsNull(Box::new(df_expr_to_l2(inner)?))), - Expr::IsNotNull(inner) => Ok(L3Expr::IsNotNull(Box::new(df_expr_to_l3(inner)?))), + Expr::IsNotNull(inner) => Ok(L2Expr::IsNotNull(Box::new(df_expr_to_l2(inner)?))), Expr::Cast(c) => { - let inner = df_expr_to_l3(&c.expr)?; + let inner = df_expr_to_l2(&c.expr)?; let to = arrow_to_l3(&c.data_type)?; - Ok(L3Expr::Cast { + Ok(L2Expr::Cast { expr: Box::new(inner), to, try_cast: false, @@ -145,9 +139,9 @@ pub(super) fn df_expr_to_l3(expr: &Expr) -> Result { // TRY_CAST returns NULL on conversion failure; preserve that semantic. Expr::TryCast(c) => { - let inner = df_expr_to_l3(&c.expr)?; + let inner = df_expr_to_l2(&c.expr)?; let to = arrow_to_l3(&c.data_type)?; - Ok(L3Expr::Cast { + Ok(L2Expr::Cast { expr: Box::new(inner), to, try_cast: true, @@ -155,9 +149,9 @@ pub(super) fn df_expr_to_l3(expr: &Expr) -> Result { } Expr::InList(il) => { - let expr = df_expr_to_l3(&il.expr)?; - let list: Result, _> = il.list.iter().map(df_expr_to_l3).collect(); - Ok(L3Expr::InList { + let expr = df_expr_to_l2(&il.expr)?; + let list: Result, _> = il.list.iter().map(df_expr_to_l2).collect(); + Ok(L2Expr::InList { expr: Box::new(expr), list: list?, negated: il.negated, @@ -173,15 +167,15 @@ pub(super) fn df_expr_to_l3(expr: &Expr) -> Result { // NOT BETWEEN: invert each side let lt = compare(&b.expr, CompareOp::Lt, &b.low)?; let gt = compare(&b.expr, CompareOp::Gt, &b.high)?; - Ok(L3Expr::BoolOr(vec![lt, gt])) + Ok(L2Expr::BoolOr(vec![lt, gt])) } else { - Ok(L3Expr::BoolAnd(vec![x_low, x_high])) + Ok(L2Expr::BoolAnd(vec![x_low, x_high])) } } Expr::ScalarFunction(sf) => { - let args: Result, _> = sf.args.iter().map(df_expr_to_l3).collect(); - Ok(L3Expr::FunctionCall { + let args: Result, _> = sf.args.iter().map(df_expr_to_l2).collect(); + Ok(L2Expr::FunctionCall { name: sf.func.name().to_string(), args: args?, }) @@ -194,19 +188,19 @@ pub(super) fn df_expr_to_l3(expr: &Expr) -> Result { } } -pub(super) fn compare(left: &Expr, op: CompareOp, right: &Expr) -> Result { - Ok(L3Expr::Compare { - left: Box::new(df_expr_to_l3(left)?), +pub(super) fn compare(left: &Expr, op: CompareOp, right: &Expr) -> Result { + Ok(L2Expr::Compare { + left: Box::new(df_expr_to_l2(left)?), op, - right: Box::new(df_expr_to_l3(right)?), + right: Box::new(df_expr_to_l2(right)?), }) } -pub(super) fn arith(left: &Expr, op: ArithOp, right: &Expr) -> Result { - Ok(L3Expr::Arith { +pub(super) fn arith(left: &Expr, op: ArithOp, right: &Expr) -> Result { + Ok(L2Expr::Arith { op, - left: Box::new(df_expr_to_l3(left)?), - right: Box::new(df_expr_to_l3(right)?), + left: Box::new(df_expr_to_l2(left)?), + right: Box::new(df_expr_to_l2(right)?), }) } diff --git a/crates/lower/src/sql/mod.rs b/crates/lower/src/sql/mod.rs index 4b85a129..a06353ec 100644 --- a/crates/lower/src/sql/mod.rs +++ b/crates/lower/src/sql/mod.rs @@ -1,73 +1,89 @@ +//! SQL → Layer-2 relational lowering. +//! +//! Parses SQL via DataFusion (over the catalog's registered tables), then walks +//! the unoptimized `LogicalPlan` and emits the language-independent +//! [`relational::QueryExpr`](asap_control_core::intent_algebra::relational) that +//! [`convert_root`](asap_control_core::intent_algebra::convert_root) lowers to +//! canonical L3. Positional column identity, accuracy threading, and the +//! window-over-aggregate fold all happen in that converter — this front end +//! only interprets SQL semantics into the shared L2 algebra. + use std::sync::Arc; use datafusion::common::ScalarValue; use datafusion::datasource::MemTable; -use datafusion::logical_expr::{self, Distinct, Expr, LogicalPlan, WindowFunctionDefinition}; +use datafusion::logical_expr::{ + self, Distinct, Expr, JoinType, LogicalPlan, WindowFunctionDefinition, +}; use datafusion::prelude::SessionContext; -use asap_control_core::intent_algebra::expr::{ - AggIntent, ColumnRef, GroupKey, L3Node, Predicate, ProjectItem, QueryExpr, SetOpKind, SortKey, - Source, TableRef, WindowFuncKind, +use asap_control_core::intent_algebra::relational::{ + AggFunc, AggItem, L2ProjectItem, L2SortKey, QueryExpr as L2, SourceSpec, +}; +use asap_control_core::intent_algebra::schema::Schema; +use asap_control_core::intent_algebra::{ + ColumnRef, CompareOp, JoinKind, L2Expr, L3Scalar, SetOpKind, WindowFuncKind, }; -use asap_control_core::intent_algebra::schema::{L3Schema, SchemaCatalog, TableSchema}; -use asap_control_core::intent_algebra::{L3Expr, L3Scalar}; -use asap_control_core::types::AccuracyTarget; use crate::error::LoweringError; mod expr; -mod time; mod types; -use self::expr::{conjuncts_to_l3expr, df_expr_to_l3, split_conjuncts}; -use self::time::extract_time_range_from_conjuncts; -use self::types::table_schema_to_arrow; +pub use types::SqlCatalog; +use self::expr::df_expr_to_l2; +use self::types::schema_to_arrow; + +/// Lowers SQL strings to the Layer-2 [`relational::QueryExpr`] over a table +/// [`SqlCatalog`]. Call [`convert_root`](asap_control_core::intent_algebra::convert_root) +/// on the result for canonical L3. pub struct SqlLowerer<'a> { - catalog: &'a SchemaCatalog, - accuracy: AccuracyTarget, + catalog: &'a SqlCatalog, } impl<'a> SqlLowerer<'a> { - pub fn new(catalog: &'a SchemaCatalog, accuracy: AccuracyTarget) -> Self { - Self { catalog, accuracy } + pub fn new(catalog: &'a SqlCatalog) -> Self { + Self { catalog } } - pub async fn lower(&self, sql: &str) -> Result { + /// Parse + lower a SQL query to Layer-2 relational form. + pub async fn lower(&self, sql: &str) -> Result { let ctx = self.build_context()?; let df = ctx.sql(sql).await?; let plan = df.into_unoptimized_plan(); self.lower_plan(&plan) } + /// Register the catalog tables (empty Arrow `MemTable`s) so DataFusion can + /// resolve table/column references during planning. fn build_context(&self) -> Result { let ctx = SessionContext::new(); - for (name, table_schema) in &self.catalog.tables { - let arrow_schema = Arc::new(table_schema_to_arrow(table_schema)); + for (name, schema) in &self.catalog.tables { + let arrow_schema = Arc::new(schema_to_arrow(schema)); let mem_table = MemTable::try_new(arrow_schema, vec![])?; ctx.register_table(name.as_str(), Arc::new(mem_table))?; } Ok(ctx) } - fn lower_plan(&self, plan: &LogicalPlan) -> Result { + fn lower_plan(&self, plan: &LogicalPlan) -> Result { match plan { LogicalPlan::TableScan(scan) => self.lower_table_scan(scan), - LogicalPlan::Filter(filter) => self.lower_filter(filter), + LogicalPlan::Filter(filter) => Ok(L2::Filter { + pred: df_expr_to_l2(&filter.predicate)?, + input: Box::new(self.lower_plan(&filter.input)?), + }), LogicalPlan::Projection(proj) => self.lower_projection(proj), LogicalPlan::Aggregate(agg) => self.lower_aggregate(agg), LogicalPlan::Sort(sort) => self.lower_sort(sort), LogicalPlan::Limit(limit) => self.lower_limit(limit), - LogicalPlan::Window(window) => self.lower_window(window), LogicalPlan::Distinct(d) => match d { Distinct::On(_) => Err(LoweringError::UnsupportedFeature("DISTINCT ON".into())), - Distinct::All(input) => { - let child = self.lower_plan(input)?; - Ok(QueryExpr::Distinct { - child: make_untyped_node(child), - cols: vec![], - }) - } + Distinct::All(input) => Ok(L2::Distinct { + cols: vec![], + input: Box::new(self.lower_plan(input)?), + }), }, LogicalPlan::Union(u) => { // Fold n inputs left-associatively into SetOp { Union, all: true }. @@ -77,25 +93,26 @@ impl<'a> SqlLowerer<'a> { .ok_or_else(|| LoweringError::InvalidExpression("empty union".into()))?; let first_expr = self.lower_plan(first)?; iter.try_fold(first_expr, |left, right_plan| { - let right = self.lower_plan(right_plan)?; - Ok(QueryExpr::SetOp { + Ok(L2::SetOp { kind: SetOpKind::Union, all: true, - left: make_untyped_node(left), - right: make_untyped_node(right), + left: Box::new(left), + right: Box::new(self.lower_plan(right_plan)?), }) }) } - LogicalPlan::Join(_) => Err(LoweringError::UnsupportedFeature("JOIN".into())), + LogicalPlan::Window(window) => self.lower_window(window), + LogicalPlan::Join(join) => self.lower_join(join), LogicalPlan::Subquery(_) => Err(LoweringError::UnsupportedFeature("subquery".into())), LogicalPlan::SubqueryAlias(alias) => { - // Simple table alias (wraps only a TableScan or another alias) is - // transparent. A derived table (wraps Projection, Aggregate, etc.) - // is an inline-view subquery — unsupported in v1. + // An alias over a table re-qualifies the scan's columns with the + // alias (so `a.col` / `b.col` in a self-join disambiguate). A + // derived table (inline view) is unsupported in v1. match alias.input.as_ref() { - LogicalPlan::TableScan(_) | LogicalPlan::SubqueryAlias(_) => { - self.lower_plan(&alias.input) + LogicalPlan::TableScan(scan) => { + self.scan_source(&scan.table_name.to_string(), &alias.alias.to_string()) } + LogicalPlan::SubqueryAlias(_) => self.lower_plan(&alias.input), _ => Err(LoweringError::UnsupportedFeature( "subquery (inline view / derived table)".into(), )), @@ -108,430 +125,460 @@ impl<'a> SqlLowerer<'a> { } } - fn lower_table_scan(&self, scan: &logical_expr::TableScan) -> Result { - let table_name = scan.table_name.to_string(); - let table_schema = self + /// Table leaf — carries the catalog's resolved schema so the L2→L3 Binder + /// has positional column identity. Projection pushdown is left to the + /// enclosing `Project` (DataFusion's unoptimized plan sets no projection). + fn lower_table_scan(&self, scan: &logical_expr::TableScan) -> Result { + let table = scan.table_name.to_string(); + self.scan_source(&table, &table) + } + + /// A `Source` over catalog table `table`, with its columns qualified by + /// `qualifier` (the table name, or an alias from a `SubqueryAlias`) so + /// `Qualified` column refs resolve to the right side across a join. + fn scan_source(&self, table: &str, qualifier: &str) -> Result { + let schema = self .catalog .tables - .get(&table_name) - .ok_or_else(|| LoweringError::TableNotFound(table_name.clone()))?; - let columns = projection_columns(scan, table_schema); - Ok(QueryExpr::Scan { - source: Source::Table { - table_ref: TableRef(table_name), - columns, - time_range: None, - }, - predicates: vec![], - }) + .get(table) + .ok_or_else(|| LoweringError::TableNotFound(table.to_string()))?; + let qualified = Schema { + columns: schema + .columns + .iter() + .cloned() + .map(|c| c.with_table(qualifier)) + .collect(), + time_index: schema.time_index, + unique_keys: schema.unique_keys.clone(), + }; + Ok(L2::Source(SourceSpec::with_schema( + table.to_string(), + qualified, + ))) } - fn lower_filter(&self, filter: &logical_expr::Filter) -> Result { - // Walk the full filter chain to find a TableScan at any depth, then - // collect predicates from all stacked filters (including the outermost). - let (inner_preds, maybe_scan) = collect_filter_chain(&filter.input); - - if let Some(scan) = maybe_scan { - let table_name = scan.table_name.to_string(); - if let Some(schema) = self.catalog.tables.get(&table_name) { - if let Some(time_col) = &schema.time_column { - if let Err(e) = schema.validate() { - return Err(LoweringError::InvalidExpression(format!( - "catalog table '{table_name}': {e}" - ))); - } - // Merge outermost predicate + all inner filter predicates into one - // flat conjunct list, then classify for time-range extraction. - let all_conjuncts: Vec<&Expr> = std::iter::once(&filter.predicate) - .chain(inner_preds) - .flat_map(|p| split_conjuncts(p)) - .collect(); - let (time_range, non_time) = - extract_time_range_from_conjuncts(all_conjuncts, time_col); - let columns = projection_columns(scan, schema); - let scan_expr = QueryExpr::Scan { - source: Source::Table { - table_ref: TableRef(table_name), - columns, - time_range, - }, - predicates: vec![], - }; - return if non_time.is_empty() { - Ok(scan_expr) - } else { - let pred_expr = conjuncts_to_l3expr(non_time)?; - Ok(QueryExpr::Filter { - child: make_untyped_node(scan_expr), - pred: Predicate(pred_expr), - }) - }; - } + /// ⋈ — equijoin. The `on` key pairs become `left = right` comparisons, + /// AND-ed with any non-equi `filter`, into the L2 join predicate. The L2→L3 + /// converter derives the concatenated output schema; the join predicate + /// stays name-based (like a `WHERE`). Semi/anti/mark joins have no L3 + /// counterpart yet and are rejected. + fn lower_join(&self, join: &logical_expr::Join) -> Result { + let kind = match join.join_type { + JoinType::Inner => JoinKind::Inner, + JoinType::Left => JoinKind::Left, + JoinType::Right => JoinKind::Right, + JoinType::Full => JoinKind::Full, + other => { + return Err(LoweringError::UnsupportedFeature(format!( + "join type: {other:?}" + ))) } + }; + let mut conjuncts = join + .on + .iter() + .map(|(l, r)| { + Ok(L2Expr::Compare { + left: Box::new(df_expr_to_l2(l)?), + op: CompareOp::Eq, + right: Box::new(df_expr_to_l2(r)?), + }) + }) + .collect::, LoweringError>>()?; + if let Some(filter) = &join.filter { + conjuncts.push(df_expr_to_l2(filter)?); } + let pred = match conjuncts.len() { + 0 => None, + 1 => Some(conjuncts.pop().unwrap()), + _ => Some(L2Expr::BoolAnd(conjuncts)), + }; + Ok(L2::Join { + kind, + pred, + left: Box::new(self.lower_plan(&join.left)?), + right: Box::new(self.lower_plan(&join.right)?), + }) + } - let pred_expr = df_expr_to_l3(&filter.predicate)?; - let child = self.lower_plan(&filter.input)?; - Ok(QueryExpr::Filter { - child: make_untyped_node(child), - pred: Predicate(pred_expr), + /// `func(args) OVER (PARTITION BY … ORDER BY …)`. One window function per + /// plan node; window frames are not modelled yet (default frame assumed). + fn lower_window(&self, window: &logical_expr::Window) -> Result { + if window.window_expr.len() > 1 { + return Err(LoweringError::UnsupportedFeature(format!( + "multiple window functions in one plan node (got {}); split them", + window.window_expr.len() + ))); + } + let input = Box::new(self.lower_plan(&window.input)?); + let first = window + .window_expr + .first() + .ok_or_else(|| LoweringError::InvalidExpression("empty window expression".into()))?; + let Expr::WindowFunction(wf) = first else { + return Err(LoweringError::InvalidExpression( + "expected a window function in Window plan node".into(), + )); + }; + let func = lower_window_func_kind(&wf.fun)?; + let mut args = wf + .args + .iter() + .map(df_expr_to_l2) + .collect::, _>>()?; + // Nth_value: lift N from the (literal) 2nd arg, keep only the column. + let func = if matches!(func, WindowFuncKind::NthValue(None)) { + let n = match args.get(1) { + Some(L2Expr::Literal(L3Scalar::Int64(n))) if *n > 0 => *n as u64, + other => { + return Err(LoweringError::InvalidExpression(format!( + "NTH_VALUE requires a positive integer literal 2nd arg, got {other:?}" + ))) + } + }; + args.truncate(1); + WindowFuncKind::NthValue(Some(n)) + } else { + func + }; + let partition_by = wf + .partition_by + .iter() + .map(expr_to_group_ref) + .collect::, _>>()?; + let order_by = wf + .order_by + .iter() + .map(|s| { + df_expr_to_l2(&s.expr).map(|expr| L2SortKey { + expr, + ascending: s.asc, + nulls_first: s.nulls_first, + }) + }) + .collect::, _>>()?; + // The window plan's schema is `[input fields …, window output]`; the last + // field is the window column's name (what an enclosing Project references). + let output_name = window + .schema + .fields() + .last() + .map(|f| f.name().clone()) + .unwrap_or_else(|| "window".into()); + Ok(L2::WindowFunc { + func, + args, + partition_by, + order_by, + output_name, + input, }) } - fn lower_projection( - &self, - proj: &logical_expr::Projection, - ) -> Result { - // SELECT * — all wildcards means "no column constraint". Pass through - // without a Project wrapper; an empty Scan.columns list means "all columns". + fn lower_projection(&self, proj: &logical_expr::Projection) -> Result { + // SELECT * — no column constraint; pass through without a Project. if proj.expr.iter().any(|e| matches!(e, Expr::Wildcard { .. })) { return self.lower_plan(&proj.input); } - - let child = self.lower_plan(&proj.input)?; + let input = Box::new(self.lower_plan(&proj.input)?); let cols = proj .expr .iter() .map(|e| match e { - Expr::Alias(a) => df_expr_to_l3(&a.expr).map(|expr| ProjectItem { + Expr::Alias(a) => df_expr_to_l2(&a.expr).map(|expr| L2ProjectItem { expr, alias: Some(a.name.clone()), }), - _ => df_expr_to_l3(e).map(|expr| ProjectItem { expr, alias: None }), + _ => df_expr_to_l2(e).map(|expr| L2ProjectItem { expr, alias: None }), }) .collect::, _>>()?; - - // DataFusion's unoptimized plan never sets TableScan.projection, so we - // derive the Scan's column list from the enclosing projection instead. - let child = push_columns_into_scan(child, &cols); - - Ok(QueryExpr::Project { - child: make_untyped_node(child), - cols, - }) + Ok(L2::Project { cols, input }) } - fn lower_aggregate(&self, agg: &logical_expr::Aggregate) -> Result { - let child = self.lower_plan(&agg.input)?; - let by = agg + fn lower_aggregate(&self, agg: &logical_expr::Aggregate) -> Result { + let input = Box::new(self.lower_plan(&agg.input)?); + let keys = agg .group_expr .iter() - .map(expr_to_group_key) + .map(expr_to_group_ref) .collect::, _>>()?; - let aggs = agg - .aggr_expr - .iter() - .map(|e| self.lower_agg_expr(e)) - .collect::, _>>()?; - // Use DataFusion's own aggregate output schema for column names — the same - // schema the enclosing Projection was built against when it wrote its column - // references (e.g. "MIN(metrics.ts)"). The first n_groups fields are the - // GROUP BY columns; the remaining fields are the aggregate outputs. - // TODO: output_names couples core's Aggregate IR to DataFusion's internal - // naming convention. Cleaner boundary: emit a Project on top of every - // Aggregate that renames DataFusion's names to user-visible aliases, so - // Aggregate.output_names can be removed and column resolution lives in Project. - let n_groups = agg.group_expr.len(); - let output_names: Vec = agg + // DataFusion names the aggregate outputs in its own schema (e.g. + // "sum(metrics.bytes)") — the same names the enclosing Projection + // references. The schema is [group fields …, aggregate fields …], so + // skip the group fields and thread the rest as L2 aliases → L3 + // `Aggregate.output_names`, letting that Projection resolve them. + let out_names: Vec = agg .schema .fields() .iter() - .skip(n_groups) - .take(agg.aggr_expr.len()) + .skip(agg.group_expr.len()) .map(|f| f.name().to_string()) .collect(); - Ok(QueryExpr::Aggregate { - child: make_untyped_node(child), - by, + let aggs = agg + .aggr_expr + .iter() + .enumerate() + .map(|(i, e)| { + let mut item = lower_agg_item(e)?; + if let Some(name) = out_names.get(i) { + item.alias = Some(name.clone()); + } + Ok(item) + }) + .collect::, LoweringError>>()?; + Ok(L2::Aggregate { + keys, aggs, having: None, - output_names, + input, }) } - fn lower_sort(&self, sort: &logical_expr::Sort) -> Result { - // TopK: Sort with a constant LIMIT folded in, all keys descending, on an Aggregate. - // Note: Sort.fetch is Option; Limit.fetch is Option>. + fn lower_sort(&self, sort: &logical_expr::Sort) -> Result { + // Heavy-hitter TopK only when ranking DESC by a single COUNT aggregate + // (the frequency sketch the `TopK` intent represents). Any other + // ranking — by a SUM/AVG/… output or a group column — keeps the real + // Aggregate under a generic Sort+Limit so its aggregate isn't discarded. if let Some(k) = sort.fetch { - if sort.expr.iter().all(|s| !s.asc) { - if let Some(agg) = find_aggregate(strip_projections_and_aliases(&sort.input)) { - return self.lower_as_topk(agg, k); + if let Some(agg) = find_aggregate(strip_projections_and_aliases(&sort.input)) { + if heavy_hitter_topk(sort, agg) { + return self.lower_as_topk(agg, k as u64); } } } - let child = self.lower_plan(&sort.input)?; let keys = sort .expr .iter() .map(|s| { - df_expr_to_l3(&s.expr).map(|expr| SortKey { + df_expr_to_l2(&s.expr).map(|expr| L2SortKey { expr, ascending: s.asc, nulls_first: s.nulls_first, }) }) .collect::, _>>()?; - Ok(QueryExpr::Sort { - child: make_untyped_node(child), + Ok(L2::Sort { keys, + input: Box::new(self.lower_plan(&sort.input)?), }) } - fn lower_limit(&self, limit: &logical_expr::Limit) -> Result { - // TopK: Limit on top of Sort on top of Aggregate, all sort keys DESC. + fn lower_limit(&self, limit: &logical_expr::Limit) -> Result { + // Heavy-hitter TopK only for a count-ranked Limit-over-Sort-over-Aggregate + // with no OFFSET (see `lower_sort`). Otherwise fall through to Limit+Sort. if let Some(k) = eval_fetch(&limit.fetch) { - if eval_fetch(&limit.skip).unwrap_or(0) > 0 { - return Err(LoweringError::UnsupportedFeature( - "LIMIT ... OFFSET is not supported with ORDER BY ... DESC aggregates (TopK)" - .into(), - )); - } - let inner = strip_aliases(&limit.input); - if let LogicalPlan::Sort(sort) = inner { - if sort.expr.iter().all(|s| !s.asc) { + if eval_fetch(&limit.skip).unwrap_or(0) == 0 { + if let LogicalPlan::Sort(sort) = strip_aliases(&limit.input) { if let Some(agg) = find_aggregate(strip_projections_and_aliases(&sort.input)) { - return self.lower_as_topk(agg, k); + if heavy_hitter_topk(sort, agg) { + return self.lower_as_topk(agg, k as u64); + } } } } } - let child = self.lower_plan(&limit.input)?; - Ok(QueryExpr::Limit { - child: make_untyped_node(child), - n: eval_fetch(&limit.fetch).map(|v| v as u64), + Ok(L2::Limit { + n: eval_fetch(&limit.fetch).unwrap_or(usize::MAX) as u64, offset: eval_fetch(&limit.skip).unwrap_or(0) as u64, + input: Box::new(self.lower_plan(&limit.input)?), }) } - fn lower_as_topk( - &self, - agg: &logical_expr::Aggregate, - k: usize, - ) -> Result { - let child = self.lower_plan(&agg.input)?; + fn lower_as_topk(&self, agg: &logical_expr::Aggregate, k: u64) -> Result { let by = agg .group_expr .iter() - .map(expr_to_col_ref) + .map(expr_to_group_ref) .collect::, _>>()?; - Ok(QueryExpr::Aggregate { - child: make_untyped_node(child), - by: vec![], - aggs: vec![AggIntent::TopK { - k, - by, - accuracy: self.accuracy.clone(), - }], - having: None, - output_names: vec![], + Ok(L2::TopK { + k, + by, + input: Box::new(self.lower_plan(&agg.input)?), }) } +} - fn lower_window(&self, window: &logical_expr::Window) -> Result { - if window.window_expr.len() > 1 { - return Err(LoweringError::UnsupportedFeature(format!( - "multiple window functions in one Window plan node (got {}); split into separate nodes", - window.window_expr.len() - ))); - } - let child = self.lower_plan(&window.input)?; - let first = window - .window_expr - .first() - .ok_or_else(|| LoweringError::InvalidExpression("empty window expressions".into()))?; - if let Expr::WindowFunction(wf) = first { - let func = lower_window_func_kind(&wf.fun)?; - let mut args = wf - .args - .iter() - .map(df_expr_to_l3) - .collect::, _>>()?; - - // For NthValue, extract N from args[1] and keep only the column (args[0]). - let func = if matches!(func, WindowFuncKind::NthValue(None)) { - let n = match args.get(1) { - Some(L3Expr::Literal(L3Scalar::Int64(n))) if *n > 0 => *n as u64, - other => { - return Err(LoweringError::InvalidExpression(format!( - "NthValue requires a positive integer literal as second arg, got: {other:?}" - ))) - } - }; - args.truncate(1); - WindowFuncKind::NthValue(Some(n)) - } else { - func - }; - debug_assert!( - !matches!(func, WindowFuncKind::NthValue(None)), - "NthValue sentinel not resolved; lower_window has a bug" - ); +// ── Aggregate / group-key helpers ─────────────────────────────────────────────── - let partition_by = wf - .partition_by - .iter() - .map(expr_to_group_key) - .collect::, _>>()?; - // In DataFusion 43, WindowFunction.order_by is Vec. - let order_by = wf - .order_by - .iter() - .map(|s| { - df_expr_to_l3(&s.expr).map(|expr| SortKey { - expr, - ascending: s.asc, - nulls_first: s.nulls_first, - }) - }) - .collect::, _>>()?; - return Ok(QueryExpr::WindowFunc { - child: make_untyped_node(child), +/// Map a DataFusion aggregate expression to a relational [`AggItem`]. The +/// L2→L3 converter resolves the input column to a positional id and applies the +/// workload accuracy target — so this only picks the `AggFunc` + input column. +fn lower_agg_item(expr: &Expr) -> Result { + match expr { + Expr::Alias(a) => lower_agg_item(&a.expr), + Expr::AggregateFunction(agg_fn) => { + let name = agg_fn.func.name().to_lowercase(); + // L3 has no DISTINCT modifier for the value reducers; only + // COUNT(DISTINCT) maps (to Cardinality). Reject DISTINCT elsewhere + // rather than silently lowering `SUM(DISTINCT x)` as `SUM(x)`. + if agg_fn.distinct && name != "count" { + return Err(LoweringError::UnsupportedAggregate(format!( + "DISTINCT {name}" + ))); + } + // Value reducers (`reducer_col`) require a real column — `SUM(a*b)` + // is rejected, not silently reduced over a probe column. + let (func, col) = match name.as_str() { + "count" if agg_fn.distinct => (AggFunc::CountDistinct, agg_col_ref(&agg_fn.args)), + "count" => (AggFunc::Count, ColumnRef::Wildcard), + "sum" => (AggFunc::Sum, reducer_col(&name, &agg_fn.args)?), + "min" => (AggFunc::Min, reducer_col(&name, &agg_fn.args)?), + "max" => (AggFunc::Max, reducer_col(&name, &agg_fn.args)?), + "avg" | "mean" => (AggFunc::Avg, reducer_col(&name, &agg_fn.args)?), + "stddev" | "stddev_samp" => ( + AggFunc::StdDev { population: false }, + reducer_col(&name, &agg_fn.args)?, + ), + "stddev_pop" => ( + AggFunc::StdDev { population: true }, + reducer_col(&name, &agg_fn.args)?, + ), + "var" | "variance" | "var_samp" => ( + AggFunc::Variance { population: false }, + reducer_col(&name, &agg_fn.args)?, + ), + "var_pop" => ( + AggFunc::Variance { population: true }, + reducer_col(&name, &agg_fn.args)?, + ), + "approx_percentile_cont" | "percentile_cont" => ( + AggFunc::Quantile(extract_percentile_q(&agg_fn.args)?), + agg_col_ref(&agg_fn.args), + ), + "approx_distinct" => (AggFunc::CountDistinct, agg_col_ref(&agg_fn.args)), + _ => return Err(LoweringError::UnsupportedAggregate(name)), + }; + Ok(AggItem { + alias: Some(name), func, - args, - partition_by, - order_by, - frame: None, - }); + col, + }) } - Err(LoweringError::UnsupportedFeature( - "unexpected non-WindowFunction expr in Window plan node".into(), - )) + _ => Err(LoweringError::UnsupportedAggregate(format!("{expr:?}"))), } +} - fn lower_agg_expr(&self, expr: &Expr) -> Result { - match expr { - Expr::AggregateFunction(agg_fn) => { - let name = agg_fn.func.name().to_lowercase(); - match name.as_str() { - "count" if agg_fn.distinct => Ok(AggIntent::Cardinality { - accuracy: self.accuracy.clone(), - }), - "count" => Ok(AggIntent::Count { - accuracy: self.accuracy.clone(), - }), - "sum" => Ok(AggIntent::Sum { - col: agg_col(&agg_fn.args), - }), - "min" => Ok(AggIntent::Min { - col: agg_col(&agg_fn.args), - }), - "max" => Ok(AggIntent::Max { - col: agg_col(&agg_fn.args), - }), - "avg" | "mean" => Ok(AggIntent::Avg { - col: agg_col(&agg_fn.args), - }), - "stddev" | "stddev_samp" => Ok(AggIntent::Stddev { - col: agg_col(&agg_fn.args), - population: false, - }), - "stddev_pop" => Ok(AggIntent::Stddev { - col: agg_col(&agg_fn.args), - population: true, - }), - "approx_percentile_cont" | "percentile_cont" => { - let q = extract_percentile_q(&agg_fn.args)?; - Ok(AggIntent::Quantile { - q, - accuracy: self.accuracy.clone(), - }) - } - "approx_distinct" => Ok(AggIntent::Cardinality { - accuracy: self.accuracy.clone(), - }), - _ => Err(LoweringError::UnsupportedAggregate(name)), - } - } - Expr::Alias(alias) => self.lower_agg_expr(&alias.expr), - _ => Err(LoweringError::UnsupportedAggregate(format!("{expr:?}"))), +/// The first aggregate argument's column name (bare / aliased / cast column), +/// or `None` for `*` / a non-column expression. +fn agg_col_name(args: &[Expr]) -> Option { + fn col_name(e: &Expr) -> Option { + match e { + Expr::Column(c) => Some(c.name.clone()), + Expr::Alias(a) => col_name(&a.expr), + Expr::Cast(c) => col_name(&c.expr), + _ => None, } } + args.first().and_then(col_name) } -// ── Free helpers ────────────────────────────────────────────────────────────── +/// The aggregated input column. `COUNT(*)` and non-column arguments yield +/// `Wildcard`; a bare/aliased/cast column yields its name. +fn agg_col_ref(args: &[Expr]) -> ColumnRef { + match agg_col_name(args) { + Some(name) => ColumnRef::Named(name), + None => ColumnRef::Wildcard, + } +} -/// Map a DataFusion `TableScan.projection` (column index list) back to -/// `ColumnRef` names from the catalog schema. -/// Returns an empty `Vec` when the projection is absent (full scan / `SELECT *`). -fn projection_columns(scan: &logical_expr::TableScan, schema: &TableSchema) -> Vec { - match &scan.projection { - Some(indices) => indices - .iter() - .filter_map(|&i| schema.columns.get(i)) - .map(|c| ColumnRef(c.name.clone())) - .collect(), - None => vec![], +/// The single input column of a value reducer (`SUM`/`MIN`/`MAX`/`AVG`/stddev/ +/// variance). Errors if the argument is not a column: L3 reduces a column, not +/// an arbitrary expression (`SUM(a*b)`), so silently picking a probe column +/// would compute the wrong result. +fn reducer_col(name: &str, args: &[Expr]) -> Result { + agg_col_name(args).map(ColumnRef::Named).ok_or_else(|| { + LoweringError::UnsupportedAggregate(format!("{name} over a non-column expression")) + }) +} + +fn expr_to_group_ref(expr: &Expr) -> Result { + match expr { + // Preserve the relation qualifier so a GROUP BY / PARTITION BY key over a + // join (`b.k` vs `a.k`) resolves to the correct side — the same rule the + // scalar predicate path uses (`df_expr_to_l2`). + Expr::Column(col) => Ok(match &col.relation { + Some(rel) => ColumnRef::Qualified { + table: rel.to_string(), + name: col.name.clone(), + }, + None => ColumnRef::Named(col.name.clone()), + }), + Expr::Alias(a) => expr_to_group_ref(&a.expr), + other => Err(LoweringError::UnsupportedFeature(format!( + "non-column GROUP BY expression: {other}" + ))), } } -/// If `child` (or a Filter wrapping it) contains a `Scan` with an empty -/// column list, populate it from the columns referenced in `cols`. -/// DataFusion's unoptimized plan never sets `TableScan.projection`, so this -/// compensates without requiring optimizer passes that could alter other -/// plan-node shapes our lowerer depends on. -/// -/// Handled topologies: `Project → Scan` and `Project → Filter → Scan`. -/// -/// **Known gap**: `Project → Aggregate → * → Scan` is NOT handled. Aggregate -/// lowering does not call this function, so `Scan.columns` stays empty in any -/// topology where an Aggregate sits between the Project and the Scan. Any -/// downstream stage that uses `Scan.columns` for pruning or cost estimation -/// will see an unconstrained (full) scan in those cases. TODO: propagate -/// column refs through the Aggregate child when implementing column-pruning. -fn push_columns_into_scan(child: QueryExpr, cols: &[ProjectItem]) -> QueryExpr { - match child { - // Recurse through Filter so that Project → Filter → Scan works. - QueryExpr::Filter { child: inner, pred } => { - let updated = push_columns_into_scan(inner.expr.clone(), cols); - QueryExpr::Filter { - child: Arc::new(L3Node { - expr: updated, - schema: inner.schema.clone(), - }), - pred, - } - } - QueryExpr::Scan { - source: - Source::Table { - table_ref, - columns, - time_range, - }, - predicates, - } if columns.is_empty() => { - let mut seen = std::collections::HashSet::::new(); - let col_refs: Vec = cols - .iter() - .flat_map(|item| item.expr.columns_referenced()) - .filter(|&c| seen.insert(c.0.clone())) - .cloned() - .collect(); - QueryExpr::Scan { - source: Source::Table { - table_ref, - columns: col_refs, - time_range, - }, - predicates, - } +fn extract_percentile_q(args: &[Expr]) -> Result { + let q = match args.get(1) { + Some(Expr::Literal(ScalarValue::Float64(Some(q)))) => *q, + Some(Expr::Literal(ScalarValue::Float32(Some(q)))) => *q as f64, + _ => { + return Err(LoweringError::InvalidExpression( + "percentile value must be a float literal (2nd arg)".into(), + )) } - other => other, + }; + if q.is_finite() && (0.0..=1.0).contains(&q) { + Ok(q) + } else { + Err(LoweringError::InvalidExpression(format!( + "percentile must be in [0, 1], got {q}" + ))) } } -fn make_untyped_node(expr: QueryExpr) -> Arc { - Arc::new(L3Node { - expr, - schema: L3Schema { - fields: vec![], - time_index: None, - }, - }) +/// True iff `sort` ranks **descending by a single `COUNT` aggregate** of `agg` +/// — the only shape the heavy-hitter (frequency) `TopK` sketch is correct for. +/// +/// Requires `agg` to have exactly one aggregate (a plain, non-`DISTINCT` +/// `COUNT`) and the sole sort key to reference *that* output column (not a +/// group key, a `SUM`/`AVG`/… output, or a multi-aggregate select). Anything +/// else stays a generic `Sort` + `Limit` over the real `Aggregate`, mirroring +/// the PromQL gate (`topk` is heavy-hitter only over `count_over_time`). +fn heavy_hitter_topk(sort: &logical_expr::Sort, agg: &logical_expr::Aggregate) -> bool { + let [key] = sort.expr.as_slice() else { + return false; + }; + if key.asc { + return false; + } + if agg.aggr_expr.len() != 1 || !is_count_aggregate(&agg.aggr_expr[0]) { + return false; + } + // The DESC key must rank by the count's output column, not a group key. The + // aggregate schema is `[group fields …, aggregate fields …]`, so the single + // count output sits at index `group_expr.len()`. + let count_name = agg + .schema + .fields() + .get(agg.group_expr.len()) + .map(|f| f.name().clone()); + column_name(&key.expr) == count_name } -/// Evaluate a constant fetch/skip expression to a `usize`. -/// Returns `None` for parametric (non-literal) fetch expressions. +/// The referenced column name of a bare/aliased column expression, else `None`. +fn column_name(expr: &Expr) -> Option { + match expr { + Expr::Column(c) => Some(c.name.clone()), + Expr::Alias(a) => column_name(&a.expr), + _ => None, + } +} + +/// Whether `expr` is a plain (non-`DISTINCT`) `COUNT` aggregate. +fn is_count_aggregate(expr: &Expr) -> bool { + match expr { + Expr::Alias(a) => is_count_aggregate(&a.expr), + Expr::AggregateFunction(f) => f.func.name().eq_ignore_ascii_case("count") && !f.distinct, + _ => false, + } +} + +// ── LogicalPlan navigation helpers ────────────────────────────────────────────── + fn eval_fetch(expr_opt: &Option>) -> Option { expr_opt.as_ref().and_then(|e| match e.as_ref() { Expr::Literal(ScalarValue::Int64(Some(v))) if *v >= 0 => Some(*v as usize), @@ -548,8 +595,7 @@ fn strip_aliases(plan: &LogicalPlan) -> &LogicalPlan { } } -/// Strip Projection and SubqueryAlias for TopK pattern-matching only. -/// Do NOT use when building the output tree. +/// Strip Projection + SubqueryAlias for TopK pattern-matching only. fn strip_projections_and_aliases(plan: &LogicalPlan) -> &LogicalPlan { match plan { LogicalPlan::SubqueryAlias(a) => strip_projections_and_aliases(&a.input), @@ -567,80 +613,13 @@ fn find_aggregate(plan: &LogicalPlan) -> Option<&logical_expr::Aggregate> { } } -fn expr_to_group_key(expr: &Expr) -> Result { - match expr { - Expr::Column(col) => Ok(GroupKey(col.name.clone())), - Expr::Alias(a) => expr_to_group_key(&a.expr), - other => Err(LoweringError::UnsupportedFeature(format!( - "non-column GROUP BY expression: {other}" - ))), - } -} - -fn expr_to_col_ref(expr: &Expr) -> Result { - match expr { - Expr::Column(col) => Ok(ColumnRef(col.name.clone())), - Expr::Alias(a) => expr_to_col_ref(&a.expr), - other => Err(LoweringError::UnsupportedFeature(format!( - "non-column reference in TopK by-list: {other}" - ))), - } -} - -/// Extract the aggregated column name from aggregate function args. -/// Returns `None` for wildcards (`COUNT(*)`) and non-column expressions. -fn agg_col(args: &[Expr]) -> Option { - match args.first() { - Some(Expr::Column(col)) => Some(ColumnRef(col.name.clone())), - Some(Expr::Alias(a)) => match a.expr.as_ref() { - Expr::Column(col) => Some(ColumnRef(col.name.clone())), - _ => None, - }, - Some(Expr::Cast(c)) => match c.expr.as_ref() { - Expr::Column(col) => Some(ColumnRef(col.name.clone())), - _ => None, - }, - Some(Expr::Wildcard { .. }) | None => None, - _ => None, - } -} - -fn extract_percentile_q(args: &[Expr]) -> Result { - let val = args.get(1).ok_or_else(|| { - LoweringError::InvalidExpression("percentile requires 2 arguments".into()) - })?; - match val { - Expr::Literal(ScalarValue::Float64(Some(q))) => Ok(*q), - Expr::Literal(ScalarValue::Float32(Some(q))) => Ok(*q as f64), - _ => Err(LoweringError::InvalidExpression( - "percentile value must be a float literal".into(), - )), - } -} - -/// Walk a `Filter(Filter(...(TableScan)))` chain. -/// Returns `(predicates_from_inner_filters, Some(scan))` when a TableScan is -/// found at any depth, or `(vec![], None)` if a non-Filter non-Scan node is -/// reached first. The outermost filter's predicate is NOT included — the -/// caller adds it. -fn collect_filter_chain(plan: &LogicalPlan) -> (Vec<&Expr>, Option<&logical_expr::TableScan>) { - let plan = strip_aliases(plan); - match plan { - LogicalPlan::TableScan(scan) => (vec![], Some(scan)), - LogicalPlan::Filter(f) => { - let (mut inner_preds, maybe_scan) = collect_filter_chain(&f.input); - if maybe_scan.is_some() { - inner_preds.push(&f.predicate); - } - (inner_preds, maybe_scan) - } - _ => (vec![], None), - } -} - +/// Map a DataFusion window-function definition to the L3 [`WindowFuncKind`]. +/// `NthValue` is returned with `None`; `lower_window` fills in `n` from args. fn lower_window_func_kind(fun: &WindowFunctionDefinition) -> Result { + let unsupported = |what: &str, name: &str| { + LoweringError::UnsupportedFeature(format!("window {what}: {name}")) + }; match fun { - // In DataFusion 43 most ranking/nav window functions are WindowUDF. WindowFunctionDefinition::WindowUDF(udf) => match udf.name().to_lowercase().as_str() { "row_number" => Ok(WindowFuncKind::RowNumber), "rank" => Ok(WindowFuncKind::Rank), @@ -649,11 +628,8 @@ fn lower_window_func_kind(fun: &WindowFunctionDefinition) -> Result Ok(WindowFuncKind::Lead), "first_value" => Ok(WindowFuncKind::FirstValue), "last_value" => Ok(WindowFuncKind::LastValue), - // NthValue(None) is a sentinel; lower_window extracts the real N from args. "nth_value" => Ok(WindowFuncKind::NthValue(None)), - other => Err(LoweringError::UnsupportedFeature(format!( - "window fn: {other}" - ))), + other => Err(unsupported("function", other)), }, WindowFunctionDefinition::AggregateUDF(udf) => match udf.name().to_lowercase().as_str() { "sum" => Ok(WindowFuncKind::Sum), @@ -661,12 +637,8 @@ fn lower_window_func_kind(fun: &WindowFunctionDefinition) -> Result Ok(WindowFuncKind::Count), "min" => Ok(WindowFuncKind::Min), "max" => Ok(WindowFuncKind::Max), - other => Err(LoweringError::UnsupportedFeature(format!( - "window agg: {other}" - ))), + other => Err(unsupported("aggregate", other)), }, - // In DataFusion 43, BuiltInWindowFunction covers FirstValue, LastValue, NthValue. - // NthValue(None) is a sentinel; the real N is extracted from args in lower_window. WindowFunctionDefinition::BuiltInWindowFunction(biwf) => { use datafusion::logical_expr::BuiltInWindowFunction; match biwf { @@ -677,45 +649,3 @@ fn lower_window_func_kind(fun: &WindowFunctionDefinition) -> Result LogicalPlan { - use datafusion::arrow::datatypes::{DataType, Field, Schema}; - use datafusion::logical_expr::builder::LogicalTableSource; - use datafusion::logical_expr::LogicalPlanBuilder; - - let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); - let source = Arc::new(LogicalTableSource::new(schema)); - LogicalPlanBuilder::scan("t", source, None) - .unwrap() - .build() - .unwrap() - } - - #[test] - fn collect_filter_chain_finds_scan_at_depth_zero() { - let scan = empty_scan(); - let (preds, maybe_scan) = collect_filter_chain(&scan); - assert!(maybe_scan.is_some(), "should find the TableScan"); - assert!(preds.is_empty(), "no inner predicates at depth zero"); - } - - #[test] - fn collect_filter_chain_returns_none_for_non_scan() { - use datafusion::common::DFSchema; - use datafusion::logical_expr::EmptyRelation; - - let empty = LogicalPlan::EmptyRelation(EmptyRelation { - produce_one_row: false, - schema: Arc::new(DFSchema::empty()), - }); - let (preds, maybe_scan) = collect_filter_chain(&empty); - assert!(maybe_scan.is_none()); - assert!(preds.is_empty()); - } -} diff --git a/crates/lower/src/sql/time.rs b/crates/lower/src/sql/time.rs deleted file mode 100644 index 58efaa12..00000000 --- a/crates/lower/src/sql/time.rs +++ /dev/null @@ -1,288 +0,0 @@ -use datafusion::common::ScalarValue; -use datafusion::logical_expr::{BinaryExpr, Expr, Operator}; - -use asap_control_core::intent_algebra::expr::TimeRange; - -/// Core: classify a pre-split list of conjuncts into time bounds + residual. -pub(super) fn extract_time_range_from_conjuncts<'a>( - conjuncts: Vec<&'a Expr>, - time_col: &str, -) -> (Option, Vec<&'a Expr>) { - let mut start_ms: Option = None; - let mut end_ms: Option = None; - let mut non_time: Vec<&'a Expr> = vec![]; - - for c in conjuncts { - match classify_time_pred(c, time_col) { - TimeClass::Start(ms) => { - start_ms = Some(start_ms.map_or(ms, |s: i64| s.max(ms))); - } - TimeClass::End(ms) => { - end_ms = Some(end_ms.map_or(ms, |e: i64| e.min(ms))); - } - TimeClass::Both(lo, hi) => { - start_ms = Some(start_ms.map_or(lo, |s: i64| s.max(lo))); - end_ms = Some(end_ms.map_or(hi, |e: i64| e.min(hi))); - } - TimeClass::NonTime => non_time.push(c), - } - } - - let range = if start_ms.is_some() || end_ms.is_some() { - Some(TimeRange { start_ms, end_ms }) - } else { - None - }; - (range, non_time) -} - -/// Convenience wrapper: split a single expression then classify conjuncts. -#[cfg(test)] -fn extract_time_range<'a>(expr: &'a Expr, time_col: &str) -> (Option, Vec<&'a Expr>) { - use super::expr::split_conjuncts; - extract_time_range_from_conjuncts(split_conjuncts(expr), time_col) -} - -enum TimeClass { - Start(i64), - End(i64), - /// BETWEEN low AND high on the time column — contributes both bounds at once. - Both(i64, i64), - NonTime, -} - -fn classify_time_pred(expr: &Expr, time_col: &str) -> TimeClass { - match expr { - // `ts BETWEEN low AND high` — contributes both a start and end bound. - // `ts NOT BETWEEN …` cannot be expressed as a contiguous TimeRange; treat as non-time. - Expr::Between(b) if !b.negated && is_time_col(&b.expr, time_col) => { - match (expr_to_ms(&b.low), expr_to_ms(&b.high)) { - (Some(lo), Some(hi)) => TimeClass::Both(lo, hi), - _ => TimeClass::NonTime, - } - } - - Expr::BinaryExpr(BinaryExpr { left, op, right }) => { - let (col_is_left, val_expr): (bool, &Expr) = if is_time_col(left, time_col) { - (true, right) - } else if is_time_col(right, time_col) { - (false, left) - } else { - return TimeClass::NonTime; - }; - let Some(ms) = expr_to_ms(val_expr) else { - return TimeClass::NonTime; - }; - match (op, col_is_left) { - (Operator::Gt | Operator::GtEq, true) | (Operator::Lt | Operator::LtEq, false) => { - TimeClass::Start(ms) - } - (Operator::Lt | Operator::LtEq, true) | (Operator::Gt | Operator::GtEq, false) => { - TimeClass::End(ms) - } - // Eq (exact timestamp equality) and all other operators cannot be - // expressed as a contiguous half-open range, so leave them as - // regular Filter predicates rather than time-range bounds. - _ => TimeClass::NonTime, - } - } - - _ => TimeClass::NonTime, - } -} - -fn is_time_col(expr: &Expr, time_col: &str) -> bool { - match expr { - Expr::Column(col) => col.name == time_col, - Expr::Cast(c) => is_time_col(&c.expr, time_col), - _ => false, - } -} - -fn expr_to_ms(expr: &Expr) -> Option { - match expr { - Expr::Literal(sv) => scalar_to_ms(sv), - Expr::Cast(c) => expr_to_ms(&c.expr), - Expr::TryCast(c) => expr_to_ms(&c.expr), - _ => None, - } -} - -/// Round `v` to the nearest millisecond and return it as `i64`. -/// Returns `None` if `v` is non-finite or outside the `i64` range. -fn float_to_ms(v: f64) -> Option { - let rounded = v.round(); - // i64::MAX as f64 rounds up to 2^63, which overflows i64 on cast. - // Use strict less-than for the upper bound. - if rounded.is_finite() && rounded >= i64::MIN as f64 && rounded < i64::MAX as f64 { - Some(rounded as i64) - } else { - None - } -} - -fn scalar_to_ms(sv: &ScalarValue) -> Option { - match sv { - ScalarValue::Int64(Some(v)) => Some(*v), - ScalarValue::Int32(Some(v)) => Some(*v as i64), - ScalarValue::Float64(Some(v)) => float_to_ms(*v), - ScalarValue::Float32(Some(v)) => float_to_ms(*v as f64), - ScalarValue::TimestampMillisecond(Some(ms), _) => Some(*ms), - ScalarValue::TimestampNanosecond(Some(ns), _) => Some(*ns / 1_000_000), - ScalarValue::TimestampMicrosecond(Some(us), _) => Some(*us / 1_000), - ScalarValue::TimestampSecond(Some(s), _) => Some(*s * 1_000), - _ => None, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use datafusion::common::ScalarValue; - use datafusion::logical_expr::{BinaryExpr, Expr, Operator}; - - fn col(name: &str) -> Expr { - Expr::Column(datafusion::common::Column::new_unqualified(name)) - } - fn int(v: i64) -> Expr { - Expr::Literal(ScalarValue::Int64(Some(v))) - } - fn float(v: f64) -> Expr { - Expr::Literal(ScalarValue::Float64(Some(v))) - } - fn bin(left: Expr, op: Operator, right: Expr) -> Expr { - Expr::BinaryExpr(BinaryExpr { - left: Box::new(left), - op, - right: Box::new(right), - }) - } - fn and(l: Expr, r: Expr) -> Expr { - bin(l, Operator::And, r) - } - - #[test] - fn col_left_gt_lower_bound() { - let expr = bin(col("ts"), Operator::Gt, int(1000)); - let (range, non_time) = extract_time_range(&expr, "ts"); - assert_eq!( - range, - Some(TimeRange { - start_ms: Some(1000), - end_ms: None - }) - ); - assert!(non_time.is_empty()); - } - - #[test] - fn col_right_lt_is_start_bound() { - // `1000 < ts` ≡ `ts > 1000` - let expr = bin(int(1000), Operator::Lt, col("ts")); - let (range, non_time) = extract_time_range(&expr, "ts"); - assert_eq!( - range, - Some(TimeRange { - start_ms: Some(1000), - end_ms: None - }) - ); - assert!(non_time.is_empty()); - } - - #[test] - fn col_right_gt_is_end_bound() { - // `2000 > ts` ≡ `ts < 2000` - let expr = bin(int(2000), Operator::Gt, col("ts")); - let (range, non_time) = extract_time_range(&expr, "ts"); - assert_eq!( - range, - Some(TimeRange { - start_ms: None, - end_ms: Some(2000) - }) - ); - assert!(non_time.is_empty()); - } - - #[test] - fn overlapping_repeated_bounds_tighten() { - // `ts > 500 AND ts > 1000` → start = 1000 (tighter) - let expr = and( - bin(col("ts"), Operator::Gt, int(500)), - bin(col("ts"), Operator::Gt, int(1000)), - ); - let (range, _) = extract_time_range(&expr, "ts"); - assert_eq!(range.unwrap().start_ms, Some(1000)); - } - - #[test] - fn overlapping_end_bounds_tighten() { - // `ts < 2000 AND ts < 1500` → end = 1500 (tighter) - let expr = and( - bin(col("ts"), Operator::Lt, int(2000)), - bin(col("ts"), Operator::Lt, int(1500)), - ); - let (range, _) = extract_time_range(&expr, "ts"); - assert_eq!(range.unwrap().end_ms, Some(1500)); - } - - #[test] - fn between_contributes_both_bounds() { - use datafusion::logical_expr::Between; - let expr = Expr::Between(Between { - expr: Box::new(col("ts")), - negated: false, - low: Box::new(int(1000)), - high: Box::new(int(2000)), - }); - let (range, non_time) = extract_time_range(&expr, "ts"); - assert_eq!( - range, - Some(TimeRange { - start_ms: Some(1000), - end_ms: Some(2000) - }) - ); - assert!(non_time.is_empty()); - } - - #[test] - fn not_between_is_non_time() { - use datafusion::logical_expr::Between; - let expr = Expr::Between(Between { - expr: Box::new(col("ts")), - negated: true, - low: Box::new(int(1000)), - high: Box::new(int(2000)), - }); - let (range, non_time) = extract_time_range(&expr, "ts"); - assert!(range.is_none()); - assert_eq!(non_time.len(), 1); - } - - #[test] - fn float_literal_extracted_as_ms() { - let expr = bin(col("ts"), Operator::Gt, float(1_000_000.0)); - let (range, non_time) = extract_time_range(&expr, "ts"); - assert_eq!( - range, - Some(TimeRange { - start_ms: Some(1_000_000), - end_ms: None - }) - ); - assert!(non_time.is_empty()); - } - - #[test] - fn non_time_conjunct_passes_through() { - let expr = and( - bin(col("ts"), Operator::Gt, int(1000)), - bin(col("value"), Operator::Gt, int(0)), - ); - let (range, non_time) = extract_time_range(&expr, "ts"); - assert!(range.is_some()); - assert_eq!(non_time.len(), 1); - } -} diff --git a/crates/lower/src/sql/types.rs b/crates/lower/src/sql/types.rs index 5f124891..21ba90d5 100644 --- a/crates/lower/src/sql/types.rs +++ b/crates/lower/src/sql/types.rs @@ -1,13 +1,41 @@ -use std::sync::Arc; +//! Type bridges between DataFusion's Arrow types and the L3 `DataType`, plus +//! the SQL table catalog used to register tables with DataFusion and to carry +//! resolved leaf schemas into the relational L2 tree. -use datafusion::arrow::datatypes::{DataType as ArrowDataType, Field, Fields, Schema, TimeUnit}; +use std::collections::HashMap; + +use datafusion::arrow::datatypes::{ + DataType as ArrowDataType, Field, Fields, Schema as ArrowSchema, +}; use datafusion::common::ScalarValue; -use asap_control_core::intent_algebra::schema::{L3DataType, TableSchema}; +use asap_control_core::intent_algebra::schema::{Column, DataType, Schema}; use asap_control_core::intent_algebra::L3Scalar; use crate::error::LoweringError; +/// Table catalog for SQL lowering: table name → resolved L3 [`Schema`]. +/// +/// Used twice: to register Arrow-backed `MemTable`s so DataFusion can resolve +/// `SELECT … FROM t`, and to attach each table's schema to the relational +/// `SourceSpec` so the L2→L3 Binder/converter has positional column identity. +#[derive(Debug, Clone, Default)] +pub struct SqlCatalog { + pub tables: HashMap, +} + +impl SqlCatalog { + pub fn new() -> Self { + Self::default() + } + + /// Builder: register `name` with its resolved L3 schema. + pub fn with_table(mut self, name: impl Into, schema: Schema) -> Self { + self.tables.insert(name.into(), schema); + self + } +} + pub(super) fn scalar_value_to_l3(sv: &ScalarValue) -> Result { match sv { ScalarValue::Int64(Some(v)) => Ok(L3Scalar::Int64(*v)), @@ -24,7 +52,6 @@ pub(super) fn scalar_value_to_l3(sv: &ScalarValue) -> Result Ok(L3Scalar::Boolean(*b)), - // Typed nulls and untyped null both become L3Scalar::Null _ if sv.is_null() => Ok(L3Scalar::Null), _ => Err(LoweringError::InvalidExpression(format!( "unsupported scalar: {sv:?}" @@ -32,53 +59,42 @@ pub(super) fn scalar_value_to_l3(sv: &ScalarValue) -> Result Result { +/// Arrow → L3 `DataType` (used for `CAST` targets). L3 is deliberately narrow. +pub(super) fn arrow_to_l3(dt: &ArrowDataType) -> Result { match dt { ArrowDataType::Int64 | ArrowDataType::Int32 | ArrowDataType::Int16 - | ArrowDataType::Int8 => Ok(L3DataType::Int64), - ArrowDataType::Float64 | ArrowDataType::Float32 => Ok(L3DataType::Float64), - ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 => Ok(L3DataType::Utf8), - ArrowDataType::Boolean => Ok(L3DataType::Boolean), - ArrowDataType::Timestamp(_, _) => Ok(L3DataType::Timestamp), - ArrowDataType::Duration(_) => Ok(L3DataType::Duration), + | ArrowDataType::Int8 => Ok(DataType::Int64), + ArrowDataType::Float64 | ArrowDataType::Float32 => Ok(DataType::Float64), + ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 => Ok(DataType::Utf8), + ArrowDataType::Boolean => Ok(DataType::Bool), + ArrowDataType::Timestamp(_, _) => Ok(DataType::Timestamp), other => Err(LoweringError::UnsupportedFeature(format!( - "Arrow type in cast: {other:?}" + "Arrow type: {other:?}" ))), } } -pub(super) fn table_schema_to_arrow(schema: &TableSchema) -> Schema { +/// L3 `DataType` → Arrow (for registering catalog tables with DataFusion). +pub(super) fn l3_to_arrow(dt: &DataType) -> ArrowDataType { + match dt { + DataType::Int64 => ArrowDataType::Int64, + DataType::Float64 => ArrowDataType::Float64, + DataType::Utf8 => ArrowDataType::Utf8, + DataType::Bool => ArrowDataType::Boolean, + DataType::Timestamp => { + ArrowDataType::Timestamp(datafusion::arrow::datatypes::TimeUnit::Millisecond, None) + } + } +} + +/// Build an Arrow schema from an L3 [`Schema`] (column name + type + nullability). +pub(super) fn schema_to_arrow(schema: &Schema) -> ArrowSchema { let fields: Fields = schema .columns .iter() - .map(|c| Field::new(&c.name, l3_to_arrow(&c.data_type), c.nullable)) + .map(|c: &Column| Field::new(&c.name, l3_to_arrow(&c.dtype), c.nullable)) .collect(); - Schema::new(fields) -} - -pub(super) fn l3_to_arrow(dt: &L3DataType) -> ArrowDataType { - match dt { - L3DataType::Int64 => ArrowDataType::Int64, - L3DataType::Float64 => ArrowDataType::Float64, - L3DataType::Utf8 => ArrowDataType::Utf8, - L3DataType::Boolean => ArrowDataType::Boolean, - L3DataType::Timestamp => ArrowDataType::Timestamp(TimeUnit::Millisecond, None), - L3DataType::Duration => ArrowDataType::Duration(TimeUnit::Millisecond), - L3DataType::Map(k, v) => ArrowDataType::Map( - Arc::new(Field::new( - "entries", - ArrowDataType::Struct(Fields::from(vec![ - Field::new("key", l3_to_arrow(k), false), - Field::new("value", l3_to_arrow(v), true), - ])), - false, - )), - false, - ), - L3DataType::List(item) => { - ArrowDataType::List(Arc::new(Field::new("item", l3_to_arrow(item), true))) - } - } + ArrowSchema::new(fields) } diff --git a/crates/lower/tests/data/promql_corpus_docs.txt b/crates/lower/tests/data/promql_corpus_docs.txt new file mode 100644 index 00000000..0f9daa7b --- /dev/null +++ b/crates/lower/tests/data/promql_corpus_docs.txt @@ -0,0 +1,86 @@ +# Verbatim example PromQL queries from the official docs and the PromLabs cheat sheet. +# Sources: +# https://prometheus.io/docs/prometheus/latest/querying/basics/ +# https://promlabs.com/promql-cheat-sheet/ +# Used only as lowering INPUT (we do not execute) — see promql_corpus.rs. + +# --- basics: selectors & matchers --- +http_requests_total +http_requests_total{job="prometheus",group="canary"} +http_requests_total{environment=~"staging|testing|development",method!="GET"} +http_requests_total{replica!="rep-a",replica=~"rep.*"} +{__name__=~"job:.*"} +http_requests_total{job="prometheus"}[5m] + +# --- basics: time-shift modifiers & subqueries --- +http_requests_total offset 5m +sum(http_requests_total{method="GET"} offset 5m) +http_requests_total @ 1609746000 +http_requests_total @ 1609746000 offset 5m +rate(http_requests_total[5m])[30m:1m] +rate(http_requests_total[5m])[30m:] + +# --- cheat sheet: selecting series --- +node_cpu_seconds_total +node_cpu_seconds_total[5m] +node_cpu_seconds_total{cpu="0",mode="idle"} + +# --- cheat sheet: rates of increase for counters --- +rate(demo_api_request_duration_seconds_count[5m]) +irate(demo_api_request_duration_seconds_count[1m]) +increase(demo_api_request_duration_seconds_count[1h]) + +# --- cheat sheet: aggregating over multiple series --- +sum(node_filesystem_size_bytes) +sum by(job, instance) (node_filesystem_size_bytes) +sum without(instance, job) (node_filesystem_size_bytes) + +# --- cheat sheet: math between series --- +node_memory_MemFree_bytes + node_memory_Cached_bytes +node_memory_MemFree_bytes + on(instance, job) node_memory_Cached_bytes +rate(demo_cpu_usage_seconds_total[1m]) / on(instance, job) group_left demo_num_cpus + +# --- cheat sheet: filtering series by value --- +node_filesystem_avail_bytes > 10*1024*1024 +go_goroutines > go_threads +go_goroutines > bool on(job, instance) go_threads + +# --- cheat sheet: set operations --- +up{job="prometheus"} or up{job="node"} +node_network_mtu_bytes and (node_network_address_assign_type == 0) +node_network_mtu_bytes unless (node_network_address_assign_type == 1) + +# --- cheat sheet: quantiles from histograms --- +histogram_quantile(0.9, rate(demo_api_request_duration_seconds_bucket[5m])) +histogram_quantile(0.9, sum by(le, path, method) (rate(demo_api_request_duration_seconds_bucket[5m]))) + +# --- cheat sheet: changes in gauges --- +deriv(demo_disk_usage_bytes[1h]) +delta(demo_disk_usage_bytes[1h]) +predict_linear(demo_disk_usage_bytes[4h], 3600) + +# --- cheat sheet: aggregating over time --- +avg_over_time(go_goroutines[5m]) +max_over_time(process_resident_memory_bytes[1d]) +count_over_time(process_resident_memory_bytes[5m]) + +# --- cheat sheet: time --- +time() +time() - demo_batch_last_success_timestamp_seconds +time() - demo_batch_last_success_timestamp_seconds > 3600 + +# --- cheat sheet: dealing with missing data --- +absent(up{job="some-job"}) +absent_over_time(up{job="some-job"}[5m]) + +# --- cheat sheet: manipulating labels --- +label_replace(up, "hostname", "$1", "instance", "(.+):(\\d+)") + +# --- cheat sheet: subqueries --- +rate(demo_api_request_duration_seconds_count[5m])[1h:] +rate(demo_api_request_duration_seconds_count[5m])[1h:15s] +max_over_time(rate(demo_api_request_duration_seconds_count[5m])[1h:]) + +# --- cheat sheet: sorting / top-k --- +topk(3, sum by(method, path) (rate(demo_api_request_duration_seconds_count[5m]))) +sort_desc(sum by(method, path) (rate(demo_api_request_duration_seconds_count[5m]))) diff --git a/crates/lower/tests/data/promql_corpus_testdata.txt b/crates/lower/tests/data/promql_corpus_testdata.txt new file mode 100644 index 00000000..b07c42d9 --- /dev/null +++ b/crates/lower/tests/data/promql_corpus_testdata.txt @@ -0,0 +1,1849 @@ +# PromQL expressions extracted from the Prometheus engine test corpus. +# Source: https://github.com/prometheus/prometheus/tree/main/promql/promqltest/testdata (Apache-2.0). +# Each line is the from an `eval ...` directive; deduped, grouped by source file. +# Used only as lowering INPUT (we do not execute) — see promql_corpus.rs. +# +# --- prometheus testdata: aggregators.test --- +SUM BY (group) (http_requests{job="api-server"}) +SUM BY (group) (((http_requests{job="api-server"}))) +sum by (group) (http_requests{job="api-server"}) +avg by (group) (http_requests{job="api-server"}) +count by (group) (http_requests{job="api-server"}) +sum without (instance) (http_requests{job="api-server"}) +sum by () (http_requests{job="api-server"}) +sum(http_requests{job="api-server"}) +sum without () (http_requests{job="api-server",group="production"}) +sum without (instance) (http_requests{job="api-server"} or foo) +sum(http_requests) by (job) + min(http_requests) by (job) + max(http_requests) by (job) + avg(http_requests) by (job) +sum(sum by (group) (http_requests{job="api-server"})) by (job) +SUM(http_requests) +SUM(http_requests{instance="0"}) BY(job) +SUM(http_requests) BY (job) +SUM(http_requests) BY (job, nonexistent) +COUNT(http_requests) BY (job) +SUM(http_requests) BY (job, group) +AVG(http_requests) BY (job) +MIN(http_requests) BY (job) +MAX(http_requests) BY (job) +abs(-1 * http_requests{group="production",job="api-server"}) +floor(0.004 * http_requests{group="production",job="api-server"}) +ceil(0.004 * http_requests{group="production",job="api-server"}) +round(0.004 * http_requests{group="production",job="api-server"}) +round(-1 * (0.004 * http_requests{group="production",job="api-server"})) +round(0.005 * http_requests{group="production",job="api-server"}) +round(-1 * (0.005 * http_requests{group="production",job="api-server"})) +round(1 + 0.005 * http_requests{group="production",job="api-server"}) +round(-1 * (1 + 0.005 * http_requests{group="production",job="api-server"})) +round(0.0005 * http_requests{group="production",job="api-server"}, 0.1) +round(2.1 + 0.0005 * http_requests{group="production",job="api-server"}, 0.1) +round(5.2 + 0.0005 * http_requests{group="production",job="api-server"}, 0.1) +round(-1 * (5.2 + 0.0005 * http_requests{group="production",job="api-server"}), 0.1) +round(0.025 * http_requests{group="production",job="api-server"}, 5) +round(0.045 * http_requests{group="production",job="api-server"}, 5) +stddev(http_requests) +stddev by (instance)(http_requests) +stdvar(http_requests) +stdvar by (instance)(http_requests) +sum(label_grouping_test) by (a, b) +max(http_requests) +max({job="api-server"}) +max(http_requests_histogram) +min(http_requests) +min({job="api-server"}) +min(http_requests_histogram) +max by (group) (http_requests) +min by (group) (http_requests) +topk(3, http_requests) +topk((3), (http_requests)) +topk(5, http_requests{group="canary",job="app-server"}) +bottomk(3, http_requests) +bottomk(5, http_requests{group="canary",job="app-server"}) +topk by (group) (1, http_requests) +bottomk by (group) (2, http_requests) +bottomk by (group) (2, http_requests{group="production"}) +topk(3, http_requests{job="api-server",group="production"}) +bottomk(3, http_requests{job="api-server",group="production"}) +bottomk(9999999999, http_requests{job="app-server",group="canary"}) +topk(9999999999, http_requests{job="api-server",group="production"}) +topk(scalar(foo), http_requests) +count(topk(scalar(foo), http_requests)) +count(bottomk(scalar(foo), http_requests)) +topk(100, http_requests_histogram) +topk(1, {__name__=~"http_requests(_histogram)?"}) +count(topk(1000, {__name__=~"http_requests(_histogram)?"})) +topk by (instance) (1, {__name__=~"http_requests(_histogram)?"}) +bottomk(100, http_requests_histogram) +bottomk(1, {__name__=~"http_requests(_histogram)?"}) +count(bottomk(1000, {__name__=~"http_requests(_histogram)?"})) +bottomk by (instance) (1, {__name__=~"http_requests(_histogram)?"}) +topk(NaN, non_existent) +limitk(NaN, non_existent) +limit_ratio(NaN, non_existent) +count_values("version", version) +count_values(((("version"))), version) +count_values without (instance)("version", version) +count_values without (instance)("job", version) +count_values by (job, group)("job", version) +count_values("a\xc5z", version) +quantile without(point)(0.8, data) +quantile without(point)(0.2, data) +quantile without(point)(0.8, {__name__=~"data(_histogram)?"}) +quantile(0.8, data_histogram) +quantile without(point)(scalar(foo), data) +quantile without(point)((scalar(foo)), data) +quantile without(point)(NaN, data) +quantile without(point) (scalar(foo), data) +group without(point)(data) +group(foo) +avg(data{test="ten"}) +avg(data{test="inf"}) +avg(data{test="inf2"}) +avg(data{test="inf3"}) +avg(data{test="-inf"}) +avg(data{test="-inf2"}) +avg(data{test="-inf3"}) +avg(data{test="nan"}) +avg(data{test="big"}) +avg(data{test="-big"}) +avg(data{test="bigzero"}) +avg(data) +sum(data{test="ten"}) +sum by (group) (data{test="pos_inf"}) +avg by (group) (data{test="pos_inf"}) +sum by (group) (data{test="neg_inf"}) +avg by (group) (data{test="neg_inf"}) +sum(data{test="inf_inf"}) +avg(data{test="inf_inf"}) +sum by (group) (data{test="nan"}) +avg by (group) (data{test="nan"}) +avg(foo) - 52 +avg(topk(11, foo)) - 52 +avg(topk(10, foo)) - 52 +avg(topk(9, foo)) - 52 +avg(topk(8, foo)) - 52 +avg(foo) == 52 +avg(topk(11, foo)) == 52 +avg(topk(10, foo)) == 52 +avg(topk(9, foo)) == 52 +avg(topk(8, foo)) == 52 +stddev(series) +stdvar(series) +stddev({label="c"}) +stdvar({label="c"}) +stddev by (label) (series) +stdvar by (label) (series) +stddev (series) +stdvar (series) +# --- prometheus testdata: at_modifier.test --- +metric @ 100 +metric @ 100s +metric @ 1m40s +metric @ 100 offset 50s +metric @ 100 offset 50 +metric offset 50s @ 100 +metric offset 50 @ 100 +metric @ 0 offset -50s +metric @ 0 offset -50 +metric offset -50s @ 0 +metric offset -50 @ 0 +-metric @ 100 +---metric @ 100 +metric_ms @ 1.234 +sum_over_time(metric{job="1"}[100s] @ 100) +sum_over_time(metric{job="1"}[100s] @ 100 offset 50s) +sum_over_time(metric{job="1"}[100s] offset 50s @ 100) +sum_over_time(metric{job="1"}[100] @ 100 offset 50) +sum_over_time(metric{job="1"}[100] offset 50s @ 100) +metric{job="1"} @ 50 + metric{job="1"} @ 100 +rate(metric{job="1"}[100s] @ 100) + label_replace(rate(metric{job="2"}[123s] @ 200), "job", "1", "", "") +sum_over_time(metric{job="1"}[100s] @ 100) + label_replace(sum_over_time(metric{job="2"}[100s] @ 100), "job", "1", "", "") +sum_over_time(metric{job="1"}[100] @ 100) + label_replace(sum_over_time(metric{job="2"}[100] @ 100), "job", "1", "", "") +sum_over_time(metric{job="1"}[100s:1s] @ 100) +sum_over_time(metric{job="1"}[100s:1s] @ 100 offset 20s) +sum_over_time(metric{job="1"}[100s:1s] offset 20s @ 100) +sum_over_time(metric{job="1"}[100:1] offset 20 @ 100) +sum_over_time(sum_over_time(metric{job="1"}[100s] @ 100)[100s:25s] @ 50) +sum_over_time(sum_over_time(sum_over_time(metric{job="1"}[100s] @ 100)[100s:25s] @ 50)[3s:1s] @ 3000) +sum_over_time(sum_over_time(sum_over_time(metric{job="1"}[10s])[100s:25s] @ 50)[3s:1s] @ 200) +sum_over_time(sum_over_time(sum_over_time(metric{job="1"}[10s])[100s:25s] @ 200)[3s:1s] @ 50) +sum_over_time(sum_over_time(sum_over_time(metric{job="1"}[20s])[20s:10s] offset 10s)[100s:25s] @ 1000) +minute(metric @ 1500) +timestamp(metric{job="1"} @ 10) +timestamp(timestamp(metric{job="1"} @ 10)) +sum_over_time(minute(metric @ 1500)[100s:10s]) +sum_over_time(minute()[50m:1m] @ 6000) +sum_over_time(minute()[50m:1m] @ 6000 offset 5m) +sum_over_time(vector(time())[100s:1s] @ 3000) +sum_over_time(vector(time())[100s:1s] @ 3000 offset 600s) +sum_over_time(timestamp(metric{job="1"} @ 10)[100s:10s] @ 3000) +sum_over_time(timestamp(timestamp(metric{job="1"} @ 999))[10s:1s] @ 10) +quantile_over_time(scalar(up) + 1, {__name__="up"}[1h:1m] @ 1111111) +predict_linear({__name__="up"}[1h:1m] @ 1111111, 0.1) +deriv({__name__="up"}[1h:1m] @ 1111111) +changes({__name__="up"}[1h:1m] @ 1111111) +resets({__name__="up"}[1h:1m] @ 1111111) +first_over_time({__name__="up"}[1h:1m] @ 1111111) +last_over_time({__name__="up"}[1h:1m] @ 1111111) +sum_over_time({__name__="up"}[1h:1m] @ 1111111) +avg_over_time({__name__="up"}[1h:1m] @ 1111111) +min_over_time({__name__="up"}[1h:1m] @ 1111111) +max_over_time({__name__="up"}[1h:1m] @ 1111111) +count_over_time({__name__="up"}[1h:1m] @ 1111111) +stddev_over_time({__name__="up"}[1h:1m] @ 1111111) +stdvar_over_time({__name__="up"}[1h:1m] @ 1111111) +mad_over_time({__name__="up"}[1h:1m] @ 1111111) +metric @ 11 +abs(metric @ 11) +timestamp(metric) +timestamp(metric @ 11) +timestamp(metric @ 19) +timestamp(metric @ 20) +timestamp(metric_missing @ 0) +timestamp(metric_missing @ 10) +timestamp(metric_missing @ 20) +timestamp(abs(metric @ 11)) +timestamp(abs(metric_missing @ 11)) +# --- prometheus testdata: collision.test --- +count by(namespace, pod, cpu) (node_cpu_seconds_total{cpu=~".*",job="node-exporter",mode="idle",namespace="observability",pod="node-exporter-l454v"}) * on(namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{namespace="observability",pod="node-exporter-l454v"} +ceil({__name__=~'testmetric1|testmetric2'}) +# --- prometheus testdata: duration_expression.test --- +changes(http_requests[30m]) +changes(http_requests[26m+4m]) +changes(http_requests[30m+0s]) +changes(http_requests[1800]) +changes(http_requests[60*30]) +changes(http_requests[2m*15]) +changes(http_requests[2m*(10+5)]) +changes(http_requests[29m+60s]) +changes(http_requests[24m+((1.5*2m)+2m)]) +changes(http_requests[-5m+35m]) +changes(http_requests[1h/2]) +changes(http_requests[1h30m % 1h]) +changes(http_requests[30m1s-30m1s % 1m]) +changes(http_requests[(9m30s+30s)*3]) +sum_over_time(metric1_total[29s+1s:5s+5s]) +sum_over_time(metric1_total[29s+1s:((((8 - 2) / 3) * 7s) % 4) + 8000ms]) +sum_over_time(metric1_total[29s+1s:20*500ms] offset (20*(((((8 - 2) / 3) * 7s) % 4) + 8000ms))) +sum_over_time(metric1_total[29s+1s:20*500ms] offset -(20*(((((8 - 2) / 3) * 7s) % 4) + 8000ms))) +metric1_total offset (100 + 2) +metric1_total offset 100 + 2 +(metric1_total offset 2) ^ 2 +metric1_total offset 2 ^ 2 +metric1_total offset -2 ^ 2 +metric1_total offset (2 ^ 2) +metric1_total offset (2 * 2) +metric1_total offset -2 * 2 +metric1_total offset (-2 * 2) +metric1_total offset -4 +metric1_total offset (-2 ^ 2) +count_over_time(metric1_total[step()]) +count_over_time(metric1_total[step()+1ms]) +count_over_time(metric1_total[(step())+1]) +count_over_time(metric1_total[1+(STep()-5)*2]) +count_over_time(metric1_total[step()+1]) +count_over_time(metric1_total[min_of(step()+1,1h)]) +count_over_time(metric1_total[max_of(min_of(step()+1,1h),1ms)]) +count_over_time(metric1_total[((max_of(min_of((step()+1),((1h))),1ms)))]) +metric1_total offset STEP() +metric1_total offset step() +metric1_total offset step()*0 +metric1_total offset (-step()*2) +metric1_total offset -step()*2 +metric1_total offset step()^0 +metric1_total offset (STEP()/10) +metric1_total offset (step()) +metric1_total offset min_of(step(), 1s) +metric1_total offset min_of(step(), 1s)+8000 +metric1_total offset -min_of(step(), 1s)+8000 +metric1_total offset -(min_of(step(), 1s))+8000 +metric1_total offset -min_of(step(), 1s)^0 +metric1_total offset +min_of(step(), 1s)^0 +metric1_total offset min_of(step(), 1s)^0 +metric1_total offset max_of(3s,min_of(step(), 1s))+8000 +metric1_total offset -(min_of(step(), 2s)-5)+8000 +count_over_time(metric1_total[range()]) +metric1_total offset range() +metric1_total offset min_of(range(), 8s) +# --- prometheus testdata: extended_vectors.test --- +increase(metric[1m]) +increase(metric[1m] anchored) +increase(metric[1m] smoothed) +delta(metric[1m]) +delta(metric[1m] anchored) +increase(metric[5m]) +increase(metric[5m] smoothed) +increase(metric[5m] anchored) +delta(metric[5m] smoothed) +changes(metric[5m]) +changes(metric[5m] anchored) +resets(metric[5m]) +resets(metric[5m] anchored) +changes(metric[1m]) +changes(metric[1m] anchored) +changes(metric[1m1ms] anchored) +resets(metric[1m]) +resets(metric[1m] anchored) +resets(metric[1m1ms] anchored) +increase(metric[2m] smoothed) +rate(metric[10s] smoothed) +deriv(foo[3m] smoothed) +resets(foo[3m] smoothed) +changes(foo[3m] smoothed) +max_over_time(foo[3m] smoothed) +predict_linear(foo[3m] smoothed, 4) +deriv(foo[3m] anchored) +resets(foo[3m] anchored) +changes(foo[3m] anchored) +max_over_time(foo[3m] anchored) +predict_linear(foo[3m] anchored, 4) +metric smoothed +withreset smoothed +notregular smoothed +rate(metric[5s] smoothed) +increase(metric[5s] smoothed) +increase(metric[10s] smoothed) +metric @ 100 smoothed +metric @ 100 smoothed + 0 +metric offset -100 +metric offset -100 smoothed +metric offset -100 smoothed + 0 +histogram_count(rate(hist_counter[1m])) +histogram_count(increase(hist_counter[1m])) +histogram_count(increase(hist_counter[1m] anchored)) +histogram_sum(increase(hist_counter[1m] anchored)) +histogram_count(increase(hist_counter[1m] smoothed)) +histogram_sum(increase(hist_counter[1m] smoothed)) +histogram_count(rate(hist_counter[1m] smoothed)) +histogram_count(delta(hist_counter[1m] smoothed)) +histogram_count(rate(hist_counter[1m] anchored)) +histogram_count(delta(hist_counter[1m] anchored)) +histogram_count(hist_counter smoothed) +rate(mixed_hist[1m] anchored) +rate(mixed_hist[1m] smoothed) +histogram_count(increase(reset_custom_hist[90s])) +histogram_sum(increase(reset_custom_hist[90s])) +histogram_count(increase(reset_custom_hist[90s] anchored)) +histogram_sum(increase(reset_custom_hist[90s] anchored)) +histogram_count(rate(mid_gauge_hist[90s] anchored)) +histogram_count(increase(mid_gauge_hist[90s] smoothed)) +histogram_count(delta(mid_gauge_hist[90s] anchored)) +histogram_count(increase(custom_only[1m] smoothed)) +histogram_sum(increase(custom_only[1m] smoothed)) +histogram_count(reset_middle smoothed) +histogram_count(rate(reset_boundary[1s] smoothed)) +histogram_count(increase(anchored_reset_at_end[70s] anchored)) +histogram_count(increase(anchored_two_sample_reset[30s] anchored)) +histogram_count(increase(smoothed_double_reset[14s] smoothed)) +histogram_count(increase(smoothed_two_sample_both_interp[10s] smoothed)) +histogram_count(smoothed_mix smoothed) +sort(mixed_types smoothed) +histogram_count(rate(right_boundary_reset[10s] smoothed)) +# --- prometheus testdata: fill-modifier.test --- +left_vector + fill(0) right_vector +left_vector + fill_left(0) right_vector +left_vector + fill_right(0) right_vector +left_vector + fill_left(5) fill_right(7) right_vector +left_vector + fill(NaN) right_vector +left_vector + fill(Inf) right_vector +left_vector + fill(-Inf) right_vector +left_vector == fill(30) right_vector +left_vector != fill(30) right_vector +left_vector > fill(25) right_vector +left_vector == bool fill(30) right_vector +left_vector != bool fill(30) right_vector +left_vector > bool fill(25) right_vector +left_vector + on(job, instance) fill(0) right_vector +left_vector + on(job, instance) fill_right(0) right_vector +left_vector + on(job, instance) fill_left(0) right_vector +left_vector + ignoring(job) group_left fill(0) right_vector +requests / on(status) group_left fill_right(1) limits +requests + on(status) group_left fill_left(0) limits +requests + on(status) group_left fill(0) limits +node_meta * on(instance) group_right fill_left(1) cpu_info +node_meta * on(instance) group_right fill_right(0) cpu_info +node_meta * on(instance) group_right fill(1) cpu_info +requests + on(status) group_left(owner) fill_right(0) limits +only_left + fill(0) only_right +only_left + fill_left(0) only_right +only_left + fill_right(0) only_right +complete_left + fill(99) complete_right +range_left + fill(0) range_right +range_left + fill_right(0) range_right +range_left + fill_left(0) range_right +intermittent_left + fill(0) intermittent_right +intermittent_left + fill_right(0) intermittent_right +intermittent_left + fill_left(0) intermittent_right +non_empty + fill_right(0) nonexistent +non_empty + fill_left(0) nonexistent +nonexistent + fill_left(0) non_empty +nonexistent + fill_right(0) non_empty +non_empty + fill(0) nonexistent +nonexistent + fill(0) non_empty +fill + fill(0) other +other + fill +other + fill(0) fill +other + fill_left(0) fill_left +other + fill_right(0) fill_right +# --- prometheus testdata: functions.test --- +resets(http_requests[5m]) +resets(http_requests[10m]) +resets(http_requests[600]) +resets(http_requests[20m]) +resets(http_requests[30m]) +resets(http_requests[32m]) +resets(http_requests[50m]) +resets(nonexistent_metric[50m]) +resets(http_requests_histogram[6m]) +resets(http_requests_histogram[60m]) +changes(http_requests[5m]) +changes(http_requests[6m]) +changes(http_requests[20m]) +changes(http_requests[50m]) +changes((http_requests[50m])) +changes(nonexistent_metric[50m]) +changes(http_requests_histogram[5m]) +changes(http_requests_histogram[6m]) +changes(http_requests_histogram[60m]) +changes(x[20m]) +increase(http_requests_total[50m]) +increase(http_requests_total[100m]) +increase(http_requests_total[30m]) +rate(testcounter_reset_middle_total[50m]) +rate(testcounter_reset_end_total[5m]) +rate(testcounter_reset_end_total[6m]) +rate(calculate_rate_window_total[50m]) +rate(calculate_rate_offset_total[10m] offset 5m) +rate(testcounter_zero_cutoff_total[20m]) +irate(http_requests_total[50m]) +irate(http_requests_nan[15m1s]) +irate(http_requests_histogram{path="/a"}[20m]) +irate(http_requests_histogram{path="/b"}[20m]) +irate(http_requests_histogram{path="/b"}[6m]) +irate(http_requests_histogram{path="/c"}[20m]) +irate(http_requests_histogram{path="/d"}[20m]) +irate(http_requests_histogram{path="/e"}[20m]) +irate(http_requests_histogram{path="/f"}[20m]) +irate(http_requests_histogram{path="/g"}[20m]) +delta(http_requests[20m]) +delta(http_requests_gauge[20m]) +delta(http_requests_counter[20m]) +delta(http_requests_mix[20m]) +idelta(http_requests[20m]) +idelta(http_requests_nan[15m1s]) +idelta(http_requests_histogram{path="/a"}[20m]) +idelta(http_requests_histogram{path="/b"}[20m]) +idelta(http_requests_histogram{path="/b"}[6m]) +idelta(http_requests_histogram{path="/c"}[20m]) +idelta(http_requests_histogram{path="/d"}[20m]) +idelta(http_requests_histogram{path="/e"}[20m]) +idelta(http_requests_histogram{path="/f"}[20m]) +idelta(http_requests_histogram{path="/g"}[20m]) +rate(http_requests_total{group="canary", instance="1", job="app-server"}[50m]) +deriv(http_requests_total{group="canary", instance="1", job="app-server"}[50m]) +deriv(testcounter_reset_middle_total[100m]) +deriv(http_requests_mix{group="canary", instance="1", job="app-server"}[110m]) +deriv(testcounter_reset_middle_mix[110m]) +deriv(http_requests_histogram[60m]) +deriv(http_requests_inf[100m]) +predict_linear(testcounter_reset_middle_total[50m], 3600) +predict_linear(testcounter_reset_middle_total[50m], 1h) +predict_linear(testcounter_reset_middle_total[55m] @ 3000, 3600) +predict_linear(testcounter_reset_middle_total[55m] @ 3000, 1h) +predict_linear(testcounter_reset_middle_mix[60m], 3000) +predict_linear(testcounter_reset_middle_mix[60m], 50m) +predict_linear(http_requests_histogram[60m], 50m) +predict_linear(http_requests_inf[100m], 6000) +predict_linear(http_requests_total[50m], 3600) - (http_requests_total + deriv(http_requests_total[50m]) * 3600) +label_replace(testmetric, "dst", "destination-value-$1", "src", "source-value-(.*)") +label_replace(testmetric, "dst", "destination-value-$1", "src", "value-(.*)") +label_replace(testmetric, "dst", "$1-value-$2", "src", "(.*)-value-(.*)") +label_replace(testmetric, "dst", "value-$1", "nonexistent-src", "source-value-(.*)") +label_replace(testmetric, "dst", "value-$1", "nonexistent-src", "(.*)") +label_replace(testmetric, "dst", "value-$1", "src", "non-matching-regex") +label_replace((((testmetric))), (("dst")), (("value-$1")), (("src")), (("non-matching-regex"))) +label_replace(testmetric, "dst", "", "dst", ".*") +label_replace(testmetric, "dst", "value-$1", "src", "(.*") +label_replace(testmetric, "\xff", "", "src", "(.*)") +label_replace(testmetric, "src", "", "", "") +timestamp(((metric))) +label_join(testmetric, "dst", "-", "src", "src1", "src2") +label_join(testmetric, "dst", "-", "src", "src3", "src1") +label_join(testmetric, "dst", "", "emptysrc", "emptysrc1", "emptysrc2") +label_join(testmetric, "dst", ", ") +label_join(testmetric1, "dst", ", ", "src", "src1", "src2") +label_join(dup, "label", "", "this") +vector(1) +vector(time()) +clamp_max(test_clamp, 75) +clamp_min(test_clamp, -25) +clamp(test_clamp, -25, 75) +clamp_max(clamp_min(test_clamp, -20), 70) +clamp_max((clamp_min(test_clamp, (-20))), (70)) +clamp(test_clamp, 0, NaN) +clamp(test_clamp, NaN, 0) +clamp(test_clamp, 5, -5) +clamp(mixed_metric, 2, 5) +clamp_min(mixed_metric, 2) +clamp_max(mixed_metric, 2) +sgn(test_sgn) +sort(http_requests) +sort_desc(http_requests) +sort_by_label(http_requests, "instance") +sort_by_label(http_requests, "instance", "group") +sort_by_label(http_requests, "group", "instance", "job") +sort_by_label(http_requests, "job", "instance", "group") +sort_by_label_desc(http_requests, "instance") +sort_by_label_desc(http_requests, "instance", "group") +sort_by_label_desc(http_requests, "instance", "group", "job") +sort_by_label(cpu_time_total, "cpu") +sort_by_label(node_uname_info, "instance") +sort_by_label(node_uname_info, "release") +double_exponential_smoothing(http_requests[1m], 0.01, 0.1) +double_exponential_smoothing(http_requests_mix[1m], 0.01, 0.1) +double_exponential_smoothing(http_requests_histogram[1m], 0.01, 0.1) +avg_over_time(metric[10s]) +avg_over_time(metric[20s]) +avg_over_time(metric[1m]) +sum_over_time(metric[1m])/count_over_time(metric[1m]) +avg_over_time(metric2[1m]) +sum_over_time(metric2[1m])/count_over_time(metric2[1m]) +avg_over_time(metric3[1m]) +sum_over_time(metric3[1m])/count_over_time(metric3[1m]) +avg_over_time(metric4[1m]) +sum_over_time(metric4[1m])/count_over_time(metric4[1m]) +avg_over_time(metric5[1m]) +sum_over_time(metric5[1m])/count_over_time(metric5[1m]) +avg_over_time(metric5b[1m]) +sum_over_time(metric5b[1m])/count_over_time(metric5b[1m]) +avg_over_time(metric5c[1m]) +sum_over_time(metric5c[1m])/count_over_time(metric5c[1m]) +avg_over_time(metric6[1m]) +sum_over_time(metric6[1m])/count_over_time(metric6[1m]) +avg_over_time(metric6b[1m]) +sum_over_time(metric6b[1m])/count_over_time(metric6b[1m]) +avg_over_time(metric6c[1m]) +sum_over_time(metric6c[1m])/count_over_time(metric6c[1m]) +avg_over_time(metric7[1m]) +sum_over_time(metric7[1m])/count_over_time(metric7[1m]) +avg_over_time(metric8[1m]) +sum_over_time(metric8[1m])/count_over_time(metric8[1m]) +avg_over_time(metric9[1m]) +sum_over_time(metric9[1m])/count_over_time(metric9[1m]) +avg_over_time(metric10[1m]) +sum_over_time(metric10[1m])/count_over_time(metric10[1m]) +avg_over_time(metric11[1m]) +sum_over_time(metric11[1m])/count_over_time(metric11[1m]) +sum_over_time(metric12[1m]) +avg_over_time(metric12[1m]) +sum_over_time(metric13[1m]) +avg_over_time(metric13[1m]) +sum_over_time(metric13[1m])/count_over_time(metric13[1m]) +sum_over_time(metric[2m]) +avg_over_time(metric[2m]) +avg_over_time(metric1[1m]) +avg_over_time(foo[100s]) - 52 +avg_over_time(foo[110s]) - 52 +avg_over_time(foo[120s]) - 52 +avg_over_time(foo[130s]) - 52 +avg_over_time(foo[100s]) == 52 +avg_over_time(foo[110s]) == 52 +avg_over_time(foo[120s]) == 52 +avg_over_time(foo[130s]) == 52 +sum_over_time(metric[1000ms]) +sum_over_time(metric[1001ms]) +sum_over_time(metric[1002ms]) +sum_over_time(metric[1003ms]) +sum_over_time(metric[2000ms]) +sum_over_time(metric[2001ms]) +sum_over_time(metric[2002ms]) +sum_over_time(metric[2003ms]) +sum_over_time(metric[3000ms]) +sum_over_time(metric[3001ms]) +sum_over_time(metric[3002ms]) +sum_over_time(metric[3003ms]) +stdvar_over_time(metric[2m]) +stddev_over_time(metric[2m]) +stddev_over_time((metric[2m])) +stddev_over_time(metric_histogram{type="only_histogram"}[2m]) +stddev_over_time(metric_histogram{type="mix"}[2m]) +stdvar_over_time(metric_histogram{type="only_histogram"}[2m]) +stdvar_over_time(metric_histogram{type="mix"}[2m]) +stdvar_over_time(metric[1m]) +stddev_over_time(metric[1m]) +mad_over_time(metric[70s]) +mad_over_time(metric_histogram{type="only_histogram"}[70s]) +mad_over_time(metric_histogram{type="mix"}[70s]) +ts_of_min_over_time(metric[90s]) +ts_of_max_over_time(metric[90s]) +ts_of_last_over_time(metric[90s]) +ts_of_last_over_time(metric_histogram{type="only_histogram"}[90s]) +ts_of_last_over_time(metric_histogram{type="mix"}[90s]) +ts_of_first_over_time(metric[90s]) +ts_of_first_over_time(metric_histogram{type="only_histogram"}[90s]) +ts_of_first_over_time(metric_histogram{type="mix"}[90s]) +quantile_over_time(0, data[2m]) +quantile_over_time(0.5, data[2m]) +quantile_over_time(0.75, data[2m]) +quantile_over_time(0.8, data[2m]) +quantile_over_time(1, data[2m]) +quantile_over_time(-1, data[2m]) +quantile_over_time(2, data[2m]) +(quantile_over_time(2, (data[2m]))) +quantile_over_time(0.5, data_histogram{test="only histogram samples"}[2m]) +quantile_over_time(0.5, data_histogram{test="mix samples"}[2m]) +year() +time() +year(vector(1136239445)) +month() +month(vector(1136239445)) +day_of_month() +day_of_month(vector(1136239445)) +day_of_year() +day_of_year(vector(1136239445)) +day_of_week() +day_of_week(vector(1136239445)) +hour() +hour(vector(1136239445)) +minute() +minute(vector(1136239445)) +year(vector(1230767999)) +year(vector(1230768000)) +month(vector(1456790399)) + day_of_month(vector(1456790399)) / 100 +month(vector(1456790400)) + day_of_month(vector(1456790400)) / 100 +day_of_year(vector(1483191420)) +day_of_year(vector(1672493820)) +days_in_month(vector(1454284800)) +days_in_month(vector(1485907200)) +day_of_month(histogram_sample) +day_of_week(histogram_sample) +day_of_year(histogram_sample) +days_in_month(histogram_sample) +hour(histogram_sample) +minute(histogram_sample) +month(histogram_sample) +year(histogram_sample) +changes({__name__=~'testmetric1|testmetric2'}[5m]) +min_over_time(data[2m]) +min_over_time(data_histogram{type="only_histogram"}[2m]) +min_over_time(data_histogram{type=~"mix_samples.*"}[2m]) +min_over_time(data_sparse[2m]) +max_over_time(data[2m]) +max_over_time(data_histogram{type="only_histogram"}[2m]) +max_over_time(data_histogram{type=~"mix_samples.*"}[2m]) +max_over_time(data_sparse[2m]) +last_over_time({__name__=~"data(_histogram|_sparse|_empty)?"}[2m]) +first_over_time({__name__=~"data(_histogram|_sparse|_empty)?"}[2m]) +count_over_time({__name__=~"data(_histogram|_sparse|_empty)?"}[2m]) +abs(data) +ceil(data) +floor(data) +round(data) +absent(nonexistent) +absent(nonexistent{job="testjob", instance="testinstance", method=~".x"}) +absent(nonexistent{job="testjob",job="testjob2",foo="bar"}) +absent(nonexistent{job="testjob",job="testjob2",job="three",foo="bar"}) +absent(nonexistent{job="testjob",job=~"testjob2",foo="bar"}) +absent(http_requests) +absent(sum(http_requests)) +absent(http_requests_histogram) +absent(sum(http_requests_histogram)) +absent(sum(nonexistent{job="testjob", instance="testinstance"})) +absent(max(nonexistent)) +absent(nonexistent > 1) +absent(a + b) +absent(a and b) +absent(rate(nonexistent[5m])) +absent_over_time(http_requests_total[5m]) +absent_over_time(http_requests_total{handler="/foo"}[5m]) +absent_over_time(http_requests_total{handler!="/foo"}[5m]) +absent_over_time(http_requests_total{handler="/foo", handler="/bar", handler="/foobar"}[5m]) +absent_over_time(rate(nonexistent[5m])[5m:]) +absent_over_time(http_requests_total{handler="/foo", handler="/bar", instance="127.0.0.1"}[5m]) +absent_over_time(rate(http_requests_total[5m])[5m:1m]) +absent_over_time(httpd_log_lines_total[30s]) +absent_over_time(http_requests_total[10m]) +absent_over_time(http_requests_total[6m]) +absent_over_time(http_requests_total[16m]) +absent_over_time(httpd_handshake_failures_total[1m]) +absent_over_time(httpd_handshake_failures_total[2m]) +absent_over_time({instance="127.0.0.1"}[5m]) +absent_over_time({instance="127.0.0.1"}[20m]) +absent_over_time({job="grok"}[20m]) +absent_over_time({instance="127.0.0.1"}[5m:5s]) +absent_over_time({job="ingress"}[4m]) +absent_over_time(http_requests_histogram[5m]) +absent_over_time(rate(http_requests_histogram[5m])[5m:1m]) +present_over_time(http_requests_total[5m]) +present_over_time(http_requests_total{handler="/foo"}[5m]) +present_over_time(http_requests_total{handler!="/foo"}[5m]) +present_over_time(http_requests_total{handler="/foo", handler="/bar", handler="/foobar"}[5m]) +present_over_time(rate(nonexistent[5m])[5m:]) +present_over_time(http_requests_total{handler="/foo", handler="/bar", instance="127.0.0.1"}[5m]) +present_over_time(rate(http_requests_total[5m])[5m:1m]) +present_over_time(httpd_log_lines_total[30s]) +present_over_time(http_requests_total[10m]) +present_over_time(http_requests_total[6m]) +present_over_time(http_requests_total[16m]) +present_over_time(httpd_handshake_failures_total[1m]) +present_over_time({instance="127.0.0.1"}[5m]) +present_over_time({job="grok"}[20m]) +present_over_time({instance="127.0.0.1"}[5m:5s]) +present_over_time({job="ingress"}[4m]) +exp(exp_root_log) +exp({__name__=~"exp_root_log(_h)?"}) +exp(exp_root_log - 10) +exp(exp_root_log - 20) +ln(exp_root_log) +ln({__name__=~"exp_root_log(_h)?"}) +ln(exp_root_log - 10) +ln(exp_root_log - 20) +exp(ln(exp_root_log)) +exp(ln({__name__=~"exp_root_log(_h)?"})) +sqrt(exp_root_log) +sqrt({__name__=~"exp_root_log(_h)?"}) +log2(exp_root_log) +log2({__name__=~"exp_root_log(_h)?"}) +log2(exp_root_log - 10) +log2(exp_root_log - 20) +log10(exp_root_log) +log10({__name__=~"exp_root_log(_h)?"}) +log10(exp_root_log - 10) +log10(exp_root_log - 20) +round(mixed_metric) +scalar(metric) +scalar({type="histogram"}) +scalar({l="x"}) +label_replace(series, "idx", "replaced", "idx", ".*") +label_join(series, "idx", ",", "label", "label") +label_replace(overlap, "idx", "same", "idx", ".*") +label_join(overlap, "idx", ",", "label", "label") +step() +range() +vector(step()) +vector(range()) +metric * step() +metric + range() +start() +end() +vector(start()) +vector(end()) +end() - start() +(end() + start()) / 2 +start() + range() +end() - range() +metric_for_at @ start() +metric_for_at @ end() +min_of(3, 5) +min_of(5, 3) +max_of(3, 5) +max_of(5, 3) +min_of(4, 4) +max_of(4, 4) +min_of(-2, -5) +max_of(-2, -5) +min_of(0, 1) +max_of(0, 1) +min_of(NaN, 3) +min_of(3, NaN) +max_of(NaN, 3) +max_of(3, NaN) +# --- prometheus testdata: histograms.test --- +histogram_count(testhistogram3) +testhistogram3_count +histogram_sum(testhistogram3) +testhistogram3_sum +histogram_avg(testhistogram3) +histogram_stddev(testhistogram3) +histogram_stdvar(testhistogram3) +histogram_fraction(0, 4, testhistogram2) +histogram_fraction(0, 4, testhistogram2_bucket) +histogram_fraction(0, 6, testhistogram2) +histogram_fraction(0, 6, testhistogram2_bucket) +histogram_fraction(0, 3.5, testhistogram2) +histogram_fraction(0, 3.5, testhistogram2_bucket) +histogram_fraction(0, 0.2, testhistogram3) +histogram_fraction(0, 0.2, testhistogram3_bucket) +histogram_fraction(0, 0.2, rate(testhistogram3[10m])) +histogram_fraction(0, 0.2, rate(testhistogram3_bucket[10m])) +histogram_fraction(0, 1.5, positive_buckets_lower_falls_in_the_first_bucket_bucket) +histogram_fraction(0, 1.5, positive_buckets_lower_falls_in_the_first_bucket) +histogram_fraction(-4, -2, negative_buckets_lower_falls_in_the_first_bucket_bucket) +histogram_fraction(-4, -2, negative_buckets_lower_falls_in_the_first_bucket) +histogram_fraction(-Inf, -1.5, lower_is_negative_Inf_bucket) +histogram_fraction(-Inf, -1.5, lower_is_negative_Inf) +histogram_fraction(-Inf, +Inf, lower_is_negative_Inf_and_upper_is_positive_Inf__positive_buckets__bucket) +histogram_fraction(-Inf, +Inf, lower_is_negative_Inf_and_upper_is_positive_Inf__positive_buckets_) +histogram_fraction(-Inf, +Inf, lower_is_negative_Inf_and_upper_is_positive_Inf__negative_buckets__bucket) +histogram_fraction(-Inf, +Inf, lower_is_negative_Inf_and_upper_is_positive_Inf__negative_buckets_) +histogram_fraction(4, 5, lower_and_upper_fall_in_last_bucket__positive_buckets__bucket) +histogram_fraction(4, 5, lower_and_upper_fall_in_last_bucket__positive_buckets_) +histogram_fraction(0, 1, lower_and_upper_fall_in_last_bucket__negative_buckets__bucket) +histogram_fraction(0, 1, lower_and_upper_fall_in_last_bucket__negative_buckets_) +histogram_fraction(2, 5, upper_falls_in_last_bucket_bucket) +histogram_fraction(2, 5, upper_falls_in_last_bucket) +histogram_fraction(400, +Inf, upper_is_positive_Inf_bucket) +histogram_fraction(400, +Inf, upper_is_positive_Inf) +histogram_fraction(2, 2, lower_equals_upper_bucket) +histogram_fraction(2, 2, lower_equals_upper) +histogram_fraction(3, 2, lower_greater_than_upper_bucket) +histogram_fraction(3, 2, lower_greater_than_upper) +histogram_fraction(0, 1, single_bucket_bucket) +histogram_fraction(0, 1, single_bucket) +histogram_fraction(0, 5, all_zero_counts_bucket) +histogram_fraction(0, 5, all_zero_counts) +histogram_fraction(2, 3.5, lower_exactly_on_bucket_boundary_bucket) +histogram_fraction(2, 3.5, lower_exactly_on_bucket_boundary) +histogram_fraction(0.5, 2, upper_exactly_on_bucket_boundary_bucket) +histogram_fraction(0.5, 2, upper_exactly_on_bucket_boundary) +histogram_fraction(1, 3, both_bounds_exactly_on_bucket_boundaries_bucket) +histogram_fraction(1, 3, both_bounds_exactly_on_bucket_boundaries) +histogram_fraction(0.1, 0.75, fractional_bucket_bounds_bucket) +histogram_fraction(0.1, 0.75, fractional_bucket_bounds) +histogram_fraction(-1, 1, range_crosses_zero_bucket) +histogram_fraction(-1, 1, range_crosses_zero) +histogram_fraction(NaN, 1, lower_is_NaN_bucket) +histogram_fraction(NaN, 1, lower_is_NaN) +histogram_fraction(0, NaN, upper_is_NaN_bucket) +histogram_fraction(0, NaN, upper_is_NaN) +histogram_fraction(-10, -5, range_entirely_below_all_buckets_bucket) +histogram_fraction(-10, -5, range_entirely_below_all_buckets) +histogram_fraction(5, 10, range_entirely_above_all_buckets_bucket) +histogram_fraction(5, 10, range_entirely_above_all_buckets) +testhistogram3_bucket{le=".2"} / ignoring(le) testhistogram3_count +rate(testhistogram3_bucket{le=".2"}[10m]) / ignoring(le) rate(testhistogram3_count[10m]) +histogram_quantile(0, testhistogram3) +histogram_quantile(0, testhistogram3_bucket) +histogram_quantile(0.25, testhistogram3) +histogram_quantile(0.25, testhistogram3_bucket) +histogram_quantile(0.5, testhistogram3) +histogram_quantile(0.5, testhistogram3_bucket) +histogram_quantile(0.75, testhistogram3) +histogram_quantile(0.75, testhistogram3_bucket) +histogram_quantile(1, testhistogram3) +histogram_quantile(1, testhistogram3_bucket) +histogram_quantiles(testhistogram3, "q", 0, 0.25, 0.5, 0.75, 1) +histogram_quantiles(testhistogram3_bucket, "q", 0, 0.25, 0.5, 0.75, 1) +histogram_quantiles(testhistogram3, "start", 0, 0.25, 0.5, 0.75, 1) +histogram_quantiles(testhistogram3_bucket, "start", 0, 0.25, 0.5, 0.75, 1) +histogram_quantile(-0.1, testhistogram) +histogram_quantile(-0.1, testhistogram_bucket) +histogram_quantiles(testhistogram, "q", -0.1) +histogram_quantiles(testhistogram_bucket, "q", -0.1) +histogram_quantile(1.01, testhistogram) +histogram_quantile(1.01, testhistogram_bucket) +histogram_quantiles(testhistogram, "q", 1.01) +histogram_quantiles(testhistogram_bucket, "q", 1.01) +histogram_quantile(NaN, testhistogram) +histogram_quantile(NaN, testhistogram_bucket) +histogram_quantiles(testhistogram, "q", NaN) +histogram_quantiles(testhistogram_bucket, "q", NaN) +histogram_quantile(NaN, non_existent) +histogram_quantiles(non_existent, "q", NaN) +histogram_quantile(0, testhistogram) +histogram_quantile(0, testhistogram_bucket) +histogram_quantile(1, testhistogram) +histogram_quantile(1, testhistogram_bucket) +histogram_quantile(0.2, testhistogram) +histogram_quantile(0.2, testhistogram_bucket) +histogram_quantile(0.5, testhistogram) +histogram_quantile(0.5, testhistogram_bucket) +histogram_quantile(0.8, testhistogram) +histogram_quantile(0.8, testhistogram_bucket) +histogram_quantile(0.2, rate(testhistogram[10m])) +histogram_quantile(0.2, rate(testhistogram_bucket[10m])) +histogram_quantile(0.5, rate(testhistogram[10m])) +histogram_quantile(0.5, rate(testhistogram_bucket[10m])) +histogram_quantile(0.8, rate(testhistogram[10m])) +histogram_quantile(0.8, rate(testhistogram_bucket[10m])) +histogram_quantile(1./6., testhistogram2) +histogram_quantile(1./6., testhistogram2_bucket) +histogram_quantile(0.5, testhistogram2) +histogram_quantile(0.5, testhistogram2_bucket) +histogram_quantile(5./6., testhistogram2) +histogram_quantile(5./6., testhistogram2_bucket) +histogram_quantile(1./6., rate(testhistogram2[15m])) +histogram_quantile(1./6., rate(testhistogram2_bucket[15m])) +histogram_quantile(0.5, rate(testhistogram2[15m])) +histogram_quantile(0.5, rate(testhistogram2_bucket[15m])) +histogram_quantile(5./6., rate(testhistogram2[15m])) +histogram_quantile(5./6., rate(testhistogram2_bucket[15m])) +histogram_quantile(0.3, sum(rate(request_duration_seconds[10m]))) +histogram_quantile(0.3, sum(rate(request_duration_seconds_bucket[10m])) by (le)) +histogram_quantile(0.5, sum(rate(request_duration_seconds[10m]))) +histogram_quantile(0.5, sum(rate(request_duration_seconds_bucket[10m])) by (le)) +histogram_quantile(0.3, avg(rate(request_duration_seconds[10m]))) +histogram_quantile(0.3, avg(rate(request_duration_seconds_bucket[10m])) by (le)) +histogram_quantile(0.5, avg(rate(request_duration_seconds[10m]))) +histogram_quantile(0.5, avg(rate(request_duration_seconds_bucket[10m])) by (le)) +histogram_quantile(0.3, sum(rate(request_duration_seconds[10m])) by (instance)) +histogram_quantile(0.3, sum(rate(request_duration_seconds_bucket[10m])) by (le, instance)) +histogram_quantile(0.5, sum(rate(request_duration_seconds[10m])) by (instance)) +histogram_quantile(0.5, sum(rate(request_duration_seconds_bucket[10m])) by (le, instance)) +histogram_quantile(0.3, sum(rate(request_duration_seconds[10m])) by (job)) +histogram_quantile(0.3, sum(rate(request_duration_seconds_bucket[10m])) by (le, job)) +histogram_quantile(0.5, sum(rate(request_duration_seconds[10m])) by (job)) +histogram_quantile(0.5, sum(rate(request_duration_seconds_bucket[10m])) by (le, job)) +histogram_quantile(0.3, sum(rate(request_duration_seconds[10m])) by (job, instance)) +histogram_quantile(0.3, sum(rate(request_duration_seconds_bucket[10m])) by (le, job, instance)) +histogram_quantile(0.5, sum(rate(request_duration_seconds[10m])) by (job, instance)) +histogram_quantile(0.5, sum(rate(request_duration_seconds_bucket[10m])) by (le, job, instance)) +histogram_quantile(0.3, rate(request_duration_seconds[10m])) +histogram_quantile(0.3, rate(request_duration_seconds_bucket[10m])) +histogram_quantile(0.5, rate(request_duration_seconds[10m])) +histogram_quantile(0.5, rate(request_duration_seconds_bucket[10m])) +sum(request_duration_seconds) +sum(request_duration_seconds{job="job1",instance="ins1"} + ignoring(job,instance) request_duration_seconds{job="job1",instance="ins2"} + ignoring(job,instance) request_duration_seconds{job="job2",instance="ins1"} + ignoring(job,instance) request_duration_seconds{job="job2",instance="ins2"}) +avg(request_duration_seconds) +avg (request_duration_seconds_bucket{le="0.1"}) +avg (request_duration_seconds_bucket{le="0.2"}) - avg (request_duration_seconds_bucket{le="0.1"}) +avg (request_duration_seconds_bucket{le="+Inf"}) - avg (request_duration_seconds_bucket{le="0.2"}) +count(request_duration_seconds) +histogram_quantile(0.01, nonmonotonic_bucket) +histogram_quantile(0.5, nonmonotonic_bucket) +histogram_quantile(0.99, nonmonotonic_bucket) +histogram_quantiles(nonmonotonic_bucket, "q", 0.01, 0.5, 0.99) +histogram_quantile(0.5, rate(mixed_bucket[10m])) +histogram_quantile(0.5, rate(mixed[10m])) +histogram_quantile(0.75, rate(mixed_bucket[10m])) +histogram_quantile(1, rate(mixed_bucket[10m])) +histogram_quantile(0.2, rate(empty_bucket[10m])) +histogram_quantile(0.99, {__name__=~"request_duration_seconds\\d*_bucket"}) +histogram_quantiles({__name__=~"request_duration_seconds\\d*_bucket"}, "q", 0.99) +histogram_quantile(0.99, {__name__=~"request_duration_seconds\\d*"}) +histogram_quantiles({__name__=~"request_duration_seconds\\d*"}, "q", 0.99) +rate(const_histogram_bucket[5m]) +rate(const_histogram[5m]) +histogram_quantile(1.0, sum by (le) (rate(const_histogram_bucket[5m]))) +histogram_quantile(1.0, sum(rate(const_histogram[5m]))) +sum_over_time(histogram_over_time[4m:1m]) +avg_over_time(histogram_over_time[4m:1m]) +increase(histogram_with_reset[15m]) +resets(histogram_with_reset[15m]) +histogram_count(increase(histogram_with_reset[15m])) +histogram_sum(increase(histogram_with_reset[15m])) +histogram_quantile(0.8, series) +histogram_quantiles(series, "q", 0.1, 0.2) +histogram_fraction(-Inf, 1, series) +# --- prometheus testdata: info.test --- +info(metric, {data=~".+"}) +info(metric) +info(metric_not_matching_target_info) +info(metric_not_matching_target_info, {data=~".*"}) +info(metric_not_matching_target_info, {data=~".+"}) +info(metric, {non_existent=~".+"}) +info(metric, {non_existent=~".*"}) +info(metric_with_overlapping_label) +info(metric_with_overlapping_label, {data="info"}) +info(metric_with_overlapping_label, {data=~".+"}) +info(metric_with_overlapping_label, {instance="a"}) +info(metric, {__name__="target_info"}) +info(metric, {__name__="non_existent"}) +info(metric, {__name__="non_existent", data=~".+"}) +info(metric, {__name__="build_info"}) +info(metric, {__name__=~".+_info"}) +info(build_info, {__name__=~".+_info", another_data=~".+"}) +info(build_info, {__name__=~".+_info"}) +info(metric, {__name__!~".+_info", data=~".+"}) +info(metric, {__name__!~".+_info", data=~".*"}) +info(metric, {__name__!="target_info"}) +info(build_info, {__name__=~"target_.+", __name__=~".+_info"}) +info({__name__=~"websvc_.+"}, {__name__=~".+_info", __name__!~"websvc_.+"}) +info(metric, {__name__=~"target_.+", __name__=~".+_info"}) +info(metric, {__name__=~".+_info", __name__!~".*build.*"}) +info(metric, {data=~".*"}) +info(metric, {__name__="histogram"}) +info(metric @ 60) +info(metric offset 1m) +info(data_metric, {__name__="info_metric"}) +info({job="work"}, {__name__="info_metric"}) +info(metric, {__name__="custom_info"}) +# --- prometheus testdata: limit.test --- +count(limitk by (group) (0, http_requests)) +count(limitk by (group) (-1, http_requests)) +count(limitk by (group) (1, http_requests) and http_requests) +count(limitk by (group) (2, http_requests) and http_requests) +count(limitk(100, http_requests) and http_requests) +limitk(1, http_requests{instance="histogram_1"}) +limitk(8, http_requests{instance=~"(histogram_2|0)"}) +count(limitk(2, http_requests{instance=~"histogram_[0-9]"})) +count(limitk(1000, http_requests{instance=~"histogram_[0-9]"})) +count(limitk(scalar(foo), http_requests)) +count(limit_ratio(0.0, http_requests)) +count(limitk(2, http_requests) and http_requests) +count(limit_ratio(0.5, http_requests) and http_requests) <= bool (4+1) +count(limit_ratio(0.5, http_requests) and http_requests) >= bool (4-1) +count(limit_ratio(1.0, http_requests) and http_requests) +count(limit_ratio(-1.0, http_requests) and http_requests) +count(limit_ratio(1.1, http_requests) and http_requests) +count(limit_ratio(-1.1, http_requests) and http_requests) +count(limit_ratio(0.2, http_requests) or limit_ratio(-0.8, http_requests)) +count(limit_ratio(0.2, http_requests) and limit_ratio(-0.8, http_requests)) +count(limit_ratio(0.5, http_requests) or limit_ratio(-0.5, http_requests)) +count(limit_ratio(0.5, http_requests) and limit_ratio(-0.5, http_requests)) +count(limit_ratio(0.8, http_requests) or limit_ratio(-0.2, http_requests)) +count(limit_ratio(0.8, http_requests) and limit_ratio(-0.2, http_requests)) +count(limit_ratio(time() % 17/17, http_requests) or limit_ratio( - (1.0 - (time() % 17/17)), http_requests)) +count(limit_ratio(time() % 17/17, http_requests) and limit_ratio( - (1.0 - (time() % 17/17)), http_requests)) +abs(avg(limit_ratio(0.5, http_requests{instance!~"histogram_[0-9]"})) - avg(limit_ratio(-0.5, http_requests{instance!~"histogram_[0-9]"}))) <= bool stddev(http_requests{instance!~"histogram_[0-9]"}) +limit_ratio(1, http_requests{instance="histogram_1"}) +count(limit_ratio(scalar(bar), http_requests)) +# --- prometheus testdata: literals.test --- +12.34e6 +12.34e+6 +12.34e-6 +1+1 +1-1 +1 - -1 +.2 ++0.2 +-0.2e-6 ++Inf +inF +-inf +NaN +nan +2. +1 / 0 +((1) / (0)) +-1 / 0 +0 / 0 +1 % 0 +("Foo") +"Foo" +" Foo " +("") +"" +# --- prometheus testdata: name_label_dropping.test --- +metric_total{env="1"} +-metric_total +metric_total + another_metric_total +metric_total <= another_metric_total +metric_total <= bool another_metric_total +metric_total * 2 +clamp(metric_total, 0, 100) +round(metric_total) +rate(metric_total{env="1"}[10m]) +last_over_time(metric_total{env="1"}[10m]) +first_over_time(metric_total{env="1"}[10m]) +last_over_time(abs(metric_total{env="1"})[10m:]) +max_over_time(metric_total{env="1"}[10m]) +label_replace(rate({env="1"}[10m]), "my_name", "rate_$1", "__name__", "(.+)") +label_replace(rate({env="1"}[10m]), "__name__", "rate_$1", "__name__", "(.+)") +label_join(rate({env="1"}[10m]), "my_name", "_", "__name__") +label_join(rate({env="1"}[10m]), "__name__", "_", "__name__", "env") +sum by (__name__, env) (metric_total{env="1"}) +sum by (__name__) (rate({env="1"}[10m])) +sum(rate({env="1"}[10m])) by (env) +topk(10, sum by (__name__, env) (metric_total{env="1"})) +topk(10, sum by (__name__, env) (rate(metric_total{env="1"}[10m]))) +sum by (__name__) (metric_total{env="1"}) +sum by (__name__) (rate(metric_total{env="2"}[5m])) +label_replace(sum by (__name__) (rate(metric_total{env="2"}[5m])), "__name__", "$1", "__name__", "(.+)") +sum by (__name__) (metric_total{env="1"} or rate(metric_total{env="2"}[5m])) +sum by (__name__) (rate(metric_total{env="2"}[5m]) or metric_total{env="1"}) +sum by (__name__) (metric_total{env="3"} or rate(metric_total{env="2"}[5m])) +sum by (__name__) (rate(metric_total{env="3"}[5m]) or metric_total{env="1"}) +-metric_a or -metric_b +# --- prometheus testdata: native_histograms.test --- +empty_histogram +histogram_count(empty_histogram) +histogram_sum(empty_histogram) +histogram_avg(empty_histogram) +histogram_fraction(-Inf, +Inf, empty_histogram) +histogram_fraction(0, 8, empty_histogram) +histogram_count(single_histogram) +histogram_sum(single_histogram) +histogram_avg(single_histogram) +histogram_fraction(1, 2, single_histogram) +histogram_fraction(0, 8, single_histogram) +histogram_quantile(0.5, single_histogram) +histogram_quantiles(single_histogram, "q", 0.5) +histogram_count(multi_histogram) +histogram_sum(multi_histogram) +histogram_avg(multi_histogram) +histogram_fraction(1, 2, multi_histogram) +histogram_quantile(0.5, multi_histogram) +histogram_count(incr_histogram) +histogram_sum(incr_histogram) +histogram_avg(incr_histogram) +histogram_fraction(1, 2, incr_histogram) +histogram_quantile(0.5, incr_histogram) +incr_histogram +rate(incr_histogram[10m]) +histogram_quantile(0.5, rate(incr_histogram[10m])) +low_res_histogram +histogram_count(low_res_histogram) +histogram_sum(low_res_histogram) +histogram_avg(low_res_histogram) +histogram_fraction(1, 4, low_res_histogram) +histogram_count(single_zero_histogram) +histogram_sum(single_zero_histogram) +histogram_avg(single_zero_histogram) +histogram_fraction(-0.5, 0.5, single_zero_histogram) +histogram_quantile(0.5, single_zero_histogram) +histogram_count(negative_histogram) +histogram_sum(negative_histogram) +histogram_avg(negative_histogram) +histogram_fraction(-2, -1, negative_histogram) +histogram_quantile(0.5, negative_histogram) +histogram_count(two_samples_histogram) +histogram_sum(two_samples_histogram) +histogram_avg(two_samples_histogram) +histogram_fraction(-2, -1, two_samples_histogram) +histogram_quantile(0.5, two_samples_histogram) +histogram_count(balanced_histogram) +histogram_sum(balanced_histogram) +histogram_avg(balanced_histogram) +histogram_fraction(0, 4, balanced_histogram) +histogram_quantile(0.5, balanced_histogram) +histogram_sum(sum(incr_sum_histogram)) +histogram_sum(sum(last_over_time(incr_sum_histogram[5m]))) +rate(histogram_rate[45s]) +histogram_count(histogram_count_sum_2) +histogram_sum(histogram_count_sum_2) +histogram_stddev(histogram_stddev_stdvar_1) +histogram_stdvar(histogram_stddev_stdvar_1) +histogram_stddev(histogram_stddev_stdvar_2) +histogram_stdvar(histogram_stddev_stdvar_2) +histogram_stddev(histogram_stddev_stdvar_3) +histogram_stdvar(histogram_stddev_stdvar_3) +histogram_stddev(histogram_stddev_stdvar_4) +histogram_stdvar(histogram_stddev_stdvar_4) +histogram_stddev(histogram_stddev_stdvar_5) +histogram_stdvar(histogram_stddev_stdvar_5) +histogram_stddev(histogram_stddev_stdvar_6) +histogram_stdvar(histogram_stddev_stdvar_6) +histogram_stddev(histogram_stddev_stdvar_7) +histogram_stdvar(histogram_stddev_stdvar_7) +histogram_quantile(1.001, histogram_quantile_1) +histogram_quantile(1, histogram_quantile_1) +histogram_quantile(0.99, histogram_quantile_1) +histogram_quantile(0.9, histogram_quantile_1) +histogram_quantile(0.6, histogram_quantile_1) +histogram_quantile(0.5, histogram_quantile_1) +histogram_quantile(0.1, histogram_quantile_1) +histogram_quantile(0, histogram_quantile_1) +histogram_quantile(-1, histogram_quantile_1) +histogram_quantile(1.001, histogram_quantile_2) +histogram_quantile(1, histogram_quantile_2) +histogram_quantile(0.99, histogram_quantile_2) +histogram_quantile(0.9, histogram_quantile_2) +histogram_quantile(0.5, histogram_quantile_2) +histogram_quantile(0.1, histogram_quantile_2) +histogram_quantile(0, histogram_quantile_2) +histogram_quantile(-1, histogram_quantile_2) +histogram_quantile(1.001, histogram_quantile_3) +histogram_quantile(1, histogram_quantile_3) +histogram_quantile(0.99, histogram_quantile_3) +histogram_quantile(0.9, histogram_quantile_3) +histogram_quantile(0.7, histogram_quantile_3) +histogram_quantile(0.55, histogram_quantile_3) +histogram_quantile(0.5, histogram_quantile_3) +histogram_quantile(0.45, histogram_quantile_3) +histogram_quantile(0.3, histogram_quantile_3) +histogram_quantile(0.1, histogram_quantile_3) +histogram_quantile(0.01, histogram_quantile_3) +histogram_quantile(0, histogram_quantile_3) +histogram_quantile(-1, histogram_quantile_3) +histogram_quantile(0.5, var_res_histogram) +histogram_fraction(0, 2, var_res_histogram{schema="-1"}) +histogram_fraction(0, 1.4142135623730951, var_res_histogram{schema="0"}) +histogram_fraction(0, 1.189207, var_res_histogram{schema="+1"}) +histogram_fraction(0, 8, var_res_histogram{schema="-1"}) +histogram_fraction(0, 2.82842712474619, var_res_histogram{schema="0"}) +histogram_fraction(0, 1.6817928305074292, var_res_histogram{schema="+1"}) +histogram_fraction(-2, 0, var_res_histogram{schema="-1"}) +histogram_fraction(-1.4142135623730951, 0, var_res_histogram{schema="0"}) +histogram_fraction(-1.189207, 0, var_res_histogram{schema="+1"}) +histogram_fraction(-8, 0, var_res_histogram{schema="-1"}) +histogram_fraction(-2.82842712474619, 0, var_res_histogram{schema="0"}) +histogram_fraction(-1.6817928305074292, 0, var_res_histogram{schema="+1"}) +histogram_fraction(3.1415, 42, histogram_fraction_1) +histogram_fraction(0, +Inf, histogram_fraction_2) +histogram_fraction(-Inf, 0, histogram_fraction_2) +histogram_fraction(-0.001, 0, histogram_fraction_2) +histogram_fraction(0, 0.001, histogram_fraction_2) +histogram_fraction(0.001, inf, histogram_fraction_2) +histogram_fraction(0, 0.0005, histogram_fraction_2) +histogram_quantile(0.08333333333333333, histogram_fraction_2) +histogram_fraction(-inf, -0.001, histogram_fraction_2) +histogram_fraction(1, 2, histogram_fraction_2) +histogram_fraction(0, 1.5, histogram_fraction_2) +histogram_fraction(1.5, 2, histogram_fraction_2) +histogram_fraction(1, 8, histogram_fraction_2) +histogram_fraction(0, 6, histogram_fraction_2) +histogram_quantile(0.6320802083934297, histogram_fraction_2) +histogram_fraction(1, 6, histogram_fraction_2) +histogram_fraction(1.5, 6, histogram_fraction_2) +histogram_fraction(-2, -1, histogram_fraction_2) +histogram_fraction(-2, -1.5, histogram_fraction_2) +histogram_fraction(-8, -1, histogram_fraction_2) +histogram_fraction(-6, -1, histogram_fraction_2) +histogram_fraction(-6, -1.5, histogram_fraction_2) +histogram_fraction(42, 3.1415, histogram_fraction_2) +histogram_fraction(0, 0, histogram_fraction_2) +histogram_fraction(0.000001, 0.000001, histogram_fraction_2) +histogram_fraction(42, 42, histogram_fraction_2) +histogram_fraction(-3.1, -3.1, histogram_fraction_2) +histogram_fraction(3.1415, NaN, histogram_fraction_2) +histogram_fraction(NaN, 42, histogram_fraction_2) +histogram_fraction(NaN, NaN, histogram_fraction_2) +histogram_fraction(-Inf, +Inf, histogram_fraction_2) +histogram_fraction(0, +Inf, histogram_fraction_3) +histogram_fraction(-Inf, 0, histogram_fraction_3) +histogram_fraction(-0.001, 0, histogram_fraction_3) +histogram_fraction(0, 0.001, histogram_fraction_3) +histogram_fraction(-0.0005, 0, histogram_fraction_3) +histogram_fraction(-inf, -0.0005, histogram_fraction_3) +histogram_quantile(0.9166666666666666, histogram_fraction_3) +histogram_fraction(0.001, inf, histogram_fraction_3) +histogram_fraction(-inf, -0.001, histogram_fraction_3) +histogram_fraction(1, 2, histogram_fraction_3) +histogram_fraction(1.5, 2, histogram_fraction_3) +histogram_fraction(1, 8, histogram_fraction_3) +histogram_fraction(1, 6, histogram_fraction_3) +histogram_fraction(1.5, 6, histogram_fraction_3) +histogram_fraction(-2, -1, histogram_fraction_3) +histogram_fraction(-2, -1.5, histogram_fraction_3) +histogram_fraction(-8, -1, histogram_fraction_3) +histogram_fraction(-inf, -6, histogram_fraction_3) +histogram_quantile(0.36791979160657035, histogram_fraction_3) +histogram_fraction(-6, -1, histogram_fraction_3) +histogram_fraction(-6, -1.5, histogram_fraction_3) +histogram_fraction(42, 3.1415, histogram_fraction_3) +histogram_fraction(0, 0, histogram_fraction_3) +histogram_fraction(0.000001, 0.000001, histogram_fraction_3) +histogram_fraction(42, 42, histogram_fraction_3) +histogram_fraction(-3.1, -3.1, histogram_fraction_3) +histogram_fraction(3.1415, NaN, histogram_fraction_3) +histogram_fraction(NaN, 42, histogram_fraction_3) +histogram_fraction(NaN, NaN, histogram_fraction_3) +histogram_fraction(-Inf, +Inf, histogram_fraction_3) +histogram_fraction(0, +Inf, histogram_fraction_4) +histogram_fraction(-Inf, 0, histogram_fraction_4) +histogram_fraction(-0.001, 0, histogram_fraction_4) +histogram_fraction(0, 0.001, histogram_fraction_4) +histogram_fraction(-0.0005, 0.0005, histogram_fraction_4) +histogram_fraction(-inf, 0.0005, histogram_fraction_4) +histogram_quantile(0.5416666666666666, histogram_fraction_4) +histogram_fraction(-inf, -0.0005, histogram_fraction_4) +histogram_quantile(0.4583333333333333, histogram_fraction_4) +histogram_fraction(0.001, inf, histogram_fraction_4) +histogram_fraction(-inf, -0.001, histogram_fraction_4) +histogram_fraction(1, 2, histogram_fraction_4) +histogram_fraction(1.5, 2, histogram_fraction_4) +histogram_fraction(1, 8, histogram_fraction_4) +histogram_fraction(1, 6, histogram_fraction_4) +histogram_fraction(1.5, 6, histogram_fraction_4) +histogram_fraction(-2, -1, histogram_fraction_4) +histogram_fraction(-2, -1.5, histogram_fraction_4) +histogram_fraction(-8, -1, histogram_fraction_4) +histogram_fraction(-6, -1, histogram_fraction_4) +histogram_fraction(-6, -1.5, histogram_fraction_4) +histogram_fraction(42, 3.1415, histogram_fraction_4) +histogram_fraction(0, 0, histogram_fraction_4) +histogram_fraction(0.000001, 0.000001, histogram_fraction_4) +histogram_fraction(42, 42, histogram_fraction_4) +histogram_fraction(-3.1, -3.1, histogram_fraction_4) +histogram_fraction(3.1415, NaN, histogram_fraction_4) +histogram_fraction(NaN, 42, histogram_fraction_4) +histogram_fraction(NaN, NaN, histogram_fraction_4) +histogram_fraction(-Inf, +Inf, histogram_fraction_4) +histogram_sum(scalar(histogram_fraction(-Inf, +Inf, sum(histogram_fraction_4))) * histogram_fraction_4) +histogram_mul_div*3 +histogram_mul_div*-1 +-histogram_mul_div +histogram_mul_div*-3 +3*histogram_mul_div +histogram_mul_div*float_series_3 +float_series_3*histogram_mul_div +histogram_mul_div/3 +histogram_mul_div/-3 +histogram_mul_div/float_series_3 +histogram_mul_div*0 +0*histogram_mul_div +histogram_mul_div*float_series_0 +float_series_0*histogram_mul_div +histogram_mul_div/0 +histogram_mul_div/float_series_0 +histogram_mul_div*0/0 +histogram_mul_div*histogram_mul_div +histogram_mul_div/histogram_mul_div +float_series_3/histogram_mul_div +0/histogram_mul_div +float_sample+histogram_sample +histogram_sample+float_sample +float_sample-histogram_sample +histogram_sample-float_sample +increase(reset_in_bucket[15m]) +histogram_count(increase(reset_in_bucket[15m])) +histogram_sum(increase(reset_in_bucket[15m])) +histogram_fraction(5, 10, custom_buckets_histogram) +histogram_quantile(0.5, custom_buckets_histogram) +sum(custom_buckets_histogram) +rate(some_metric[1m]) +rate(some_metric[1m30s]) +histogram_count(rate(some_metric[1m30s])) +histogram_avg(rate(const_histogram[5m])) +histogram_count(rate(const_histogram[5m])) +histogram_sum(rate(const_histogram[5m])) +histogram_fraction(0.0, 1.0, rate(const_histogram[5m])) +histogram_count(rate(const_histogram[5m])) == 0.0 or histogram_fraction(0.0, 1.0, rate(const_histogram[5m])) * histogram_count(rate(const_histogram[5m])) +histogram_quantile(1.0, rate(const_histogram[5m])) +histogram_stddev(rate(const_histogram[5m])) +histogram_stdvar(rate(const_histogram[5m])) +sum(metric) +avg(metric) +count(metric) +group(metric) +count(limitk(1, metric)) +limitk(3, metric) +limit_ratio(1, metric) +metric{series="1"} and ignoring(series) metric{series="2"} +metric{series="1"} or ignoring(series) metric{series="2"} +metric{series="2"} + ignoring (series) metric{series="3"} +metric{series="2"} - ignoring (series) metric{series="3"} +metric1 == metric2 +metric1 != metric2 +metric2 > metric2 +sum_over_time(nhcb_metric[13m]) +avg_over_time(nhcb_metric[13m]) +last_over_time(nhcb_metric[13m]) +count_over_time(nhcb_metric[13m]) +present_over_time(nhcb_metric[13m]) +changes(nhcb_metric[13m]) +delta(nhcb_metric[13m]) +increase(nhcb_metric[13m]) +rate(nhcb_metric[13m]) +resets(nhcb_metric[13m]) +sum by (group) (metric) +sum(histogram_sum) +sum({idx="0"}) +sum(histogram_sum{idx="0"} + ignoring(idx) histogram_sum{idx="3"}) +count(histogram_sum) +avg(histogram_sum) +avg(histogram_avg_incremental) +sum_over_time(histogram_sum_over_time[4m:1m]) +avg_over_time(histogram_sum_over_time[4m:1m]) +sum_over_time(histogram_sum_over_time_2[8m:1m]) +avg_over_time(histogram_sum_over_time_2[8m:1m]) +sum_over_time(histogram_sum_over_time_3[4m:1m]) +avg_over_time(histogram_sum_over_time_3[4m:1m]) +sum_over_time(histogram_sum_over_time_4[7m:1m]) +avg_over_time(histogram_sum_over_time_4[7m:1m]) +sum_over_time(histogram_sum_over_time_incremental[8m:1m]) +avg_over_time(histogram_sum_over_time_incremental[8m:1m]) +sum_over_time(histogram_sum_over_time_incremental_2[7m:1m]) +avg_over_time(histogram_sum_over_time_incremental_2[7m:1m]) +sum_over_time(histogram_sum_over_time_incremental_3[7m:1m]) +avg_over_time(histogram_sum_over_time_incremental_3[7m:1m]) +sum_over_time(histogram_sum_over_time_incremental_4[7m:1m]) +avg_over_time(histogram_sum_over_time_incremental_4[7m:1m]) +sum_over_time(histogram_sum_over_time_incremental_6[4m:1m]) +avg_over_time(histogram_sum_over_time_incremental_6[4m:1m]) +avg_over_time(single_histogram_sample[1m]) +avg_over_time(single_nhcb_sample[1m]) +histogram_sub_1{idx="0"} - ignoring(idx) histogram_sub_1{idx="1"} +histogram_sub_2{idx="0"} - ignoring(idx) histogram_sub_2{idx="1"} +histogram_sub_3{idx="0"} - ignoring(idx) histogram_sub_3{idx="1"} +last_over_time({__name__="http_request_duration_seconds"} @ start()[1h:1m] offset 1m16s) +histogram_quantile(1, histogram_nan) +histogram_quantile(0.81, histogram_nan) +histogram_quantiles(histogram_nan, "q", 0.81) +histogram_quantile(0.8, histogram_nan{case="100% NaNs"}) +histogram_quantile(0.8, histogram_nan{case="20% NaNs"}) +histogram_quantile(0.4, histogram_nan{case="100% NaNs"}) +histogram_quantile(0.4, histogram_nan{case="20% NaNs"}) +histogram_fraction(-Inf, 0.7071067811865475, histogram_nan) +histogram_fraction(-Inf, +Inf, histogram_nan) +increase(metric[90m]) +increase(metric[55m15s]) +increase(metric[54m45s]) +histogram_count(increase(metric[90m])) +histogram_count(increase(metric[55m15s])) +histogram_count(increase(metric[54m45s])) +-metric +metric - 0.5 * metric +metric - 2 * metric +sum_over_time(mixed[3m]) +avg_over_time(mixed[3m]) +sum_over_time(mixed[10m]) +avg_over_time(mixed[10m]) +sum(metric{type=~"counter.*"}) +avg(metric{type=~"counter.*"}) +histogram_count(sum(metric)) +histogram_count(avg(metric)) +histogram_count(sum_over_time(mixed[10m])) +histogram_count(avg_over_time(mixed[10m])) +histogram_count(sum_over_time(mixed[2m])) +histogram_count(avg_over_time(mixed[2m])) +histogram_quantile(0.5, myHistogram1) +histogram_quantile(0.5, myHistogram2) +histogram_quantile(0.5, mixedHistogram) +histogram_quantiles(mixedHistogram, "q", 0.5) +histogram_count(increase(h[40m:9m])) +increase(h[40m:9m]) +histogram_count(sum_over_time(reset{timing="late"}[5m])) +histogram_count(sum(reset)) +histogram_count(avg(reset)) +histogram_count(rate(reset{timing="late"}[5m])) +histogram_count(histogram unless histogram_quantile(0.5, histogram) < 3) +histogram_quantile(0.5, histogram unless histogram_count(histogram) == 0) +histogram_quantiles(histogram unless histogram_count(histogram) == 0, "q", 0.5) +mixed_metric1 +mixed_metric2 +irate(nhcb_add_buckets[2m]) * 60 +irate(nhcb_remove_buckets[2m]) * 60 +irate(nhcb_add_bucket[2m]) * 60 +h_test >/ -Inf +h_test / +Inf +h_test / 0 +h_test / 1.4142135624 +h_test_2 / 1.13 +h_test_2 >/ -1.3 +h_test_2 / 2 +h_test >/ -1 +h_test / 0.5 +h_positive_buckets >/ 0.1 +h_positive_buckets >/ 0 +h_positive_buckets / -0.5 +h_negative_buckets >/ -0.1 +h_negative_buckets >/ 0 +zero_bucket_only >/ 0.1 +zero_bucket_only / 0.05 +zero_bucket_only / 0 +zero_bucket_only / -0.05 +zero_bucket_only / -0.1 +cbh / 15 +cbh / 13 +cbh / +Inf +cbh / -Inf +cbh >/ 0 +cbh / 0 +cbh_one_bucket / 10.0 +cbh_one_bucket / +Inf +cbh_one_bucket >/ -Inf +cbh_one_bucket / -10.0 +cbh_two_buckets_split_at_zero >/ 0.0 +cbh_two_buckets_split_at_zero >/ 10.0 +cbh_two_buckets_split_at_positive / -10.0 +cbh_two_buckets_split_at_positive >/ 0.0 +cbh_two_buckets_split_at_positive >/ 2.0 +cbh_two_buckets_split_at_positive >/ 10.0 +cbh_two_buckets_split_at_negative / -10.0 +cbh_two_buckets_split_at_negative >/ -2.0 +cbh_two_buckets_split_at_negative >/ 0.0 +cbh_two_buckets_split_at_negative >/ 10.0 +histogram_sum(cbh_two_buckets_split_at_negative >/ 10.0) +histogram_count(cbh_two_buckets_split_at_negative >/ 10.0) +cbh_for_join >/ on (label) float_for_join +empty / -Inf +empty >/ -5 +empty >/ 0 +empty >/ 5 +empty >/ +Inf +histogram_count(h_test / 2) +histogram_fraction(2, +Inf, h_test) * histogram_count(h_test) +histogram_count(h_test / -1) +histogram_fraction(-1, +Inf, h_test) * histogram_count(h_test) +histogram_count(h_test / 0) +histogram_fraction(0, +Inf, h_test) * histogram_count(h_test) +histogram_count(h_test / 1.4142135624) +histogram_fraction(1.4142135624, +Inf, h_test) * histogram_count(h_test) +histogram_count(h_test_2 / 1.13) +histogram_fraction(1.13, +Inf, h_test_2) * histogram_count(h_test_2) +histogram_count(cbh / 15) +histogram_fraction(15, +Inf, cbh) * histogram_count(cbh) +histogram_count(cbh / 13) +histogram_fraction(13, +Inf, cbh) * histogram_count(cbh) +# --- prometheus testdata: operators.test --- +SUM(http_requests_total) BY (job) - COUNT(http_requests_total) BY (job) +2 - SUM(http_requests_total) BY (job) +-http_requests_total{job="api-server",instance="0",group="production"} ++http_requests_total{job="api-server",instance="0",group="production"} +- - - SUM(http_requests_total) BY (job) +- - - 1 +-2^---1*3 +2/-2^---1*3+2 +-10^3 * - SUM(http_requests_total) BY (job) ^ -1 +1000 / SUM(http_requests_total) BY (job) +SUM(http_requests_total) BY (job) - 2 +SUM(http_requests_total) BY (job) % 3 +SUM(http_requests_total) BY (job) % 0.3 +SUM(http_requests_total) BY (job) ^ 2 +SUM(http_requests_total) BY (job) % 3 ^ 2 +SUM(http_requests_total) BY (job) % 2 ^ (3 ^ 2) +SUM(http_requests_total) BY (job) % 2 ^ 3 ^ 2 +SUM(http_requests_total) BY (job) % 2 ^ 3 ^ 2 ^ 2 +COUNT(http_requests_total) BY (job) ^ COUNT(http_requests_total) BY (job) +SUM(http_requests_total) BY (job) / 0 +http_requests_total{group="canary", instance="0", job="api-server"} / 0 +-1 * http_requests_total{group="canary", instance="0", job="api-server"} / 0 +0 * http_requests_total{group="canary", instance="0", job="api-server"} / 0 +0 * http_requests_total{group="canary", instance="0", job="api-server"} % 0 +SUM(http_requests_total) BY (job) + SUM(http_requests_total) BY (job) +(SUM((http_requests_total)) BY (job)) + SUM(http_requests_total) BY (job) +http_requests_total{job="api-server", group="canary"} +http_requests_total{job="api-server", group="canary"} + rate(http_requests_total{job="api-server"}[10m]) * 5 * 60 +rate(http_requests_total[25m]) * 25 * 60 +(rate((http_requests_total[25m])) * 25) * 60 +http_requests_total{group="canary"} and http_requests_total{instance="0"} +(http_requests_total{group="canary"} + 1) and http_requests_total{instance="0"} +(http_requests_total{group="canary"} + 1) and on(instance, job) http_requests_total{instance="0", group="production"} +(http_requests_total{group="canary"} + 1) and on(instance) http_requests_total{instance="0", group="production"} +(http_requests_total{group="canary"} + 1) and ignoring(group) http_requests_total{instance="0", group="production"} +(http_requests_total{group="canary"} + 1) and ignoring(group, job) http_requests_total{instance="0", group="production"} +http_requests_total{group="canary"} or http_requests_total{group="production"} +(http_requests_total{group="canary"} + 1) or http_requests_total{instance="1"} +(http_requests_total{group="canary"} + 1) or on(instance) (http_requests_total or cpu_count or vector_matching_a) +(http_requests_total{group="canary"} + 1) or ignoring(l, group, job) (http_requests_total or cpu_count or vector_matching_a) +http_requests_total{group="canary"} unless http_requests_total{instance="0"} +http_requests_total{group="canary"} unless on(job) http_requests_total{instance="0"} +http_requests_total{group="canary"} unless on(job, instance) http_requests_total{instance="0"} +http_requests_total{group="canary"} / on(instance,job) http_requests_total{group="production"} +http_requests_total{group="canary"} unless ignoring(group, instance) http_requests_total{instance="0"} +http_requests_total{group="canary"} unless ignoring(group) http_requests_total{instance="0"} +http_requests_total{group="canary"} / ignoring(group) http_requests_total{group="production"} +http_requests_total AND ON (dummy) vector(1) +http_requests_total AND IGNORING (group, instance, job) vector(1) +SUM(http_requests_total) BY (job) > 1000 +1000 < SUM(http_requests_total) BY (job) +SUM(http_requests_total) BY (job) <= 1000 +SUM(http_requests_total) BY (job) != 1000 +SUM(http_requests_total) BY (job) == 1000 +SUM(http_requests_total) BY (job) == bool 1000 +SUM(http_requests_total) BY (job) == bool SUM(http_requests_total) BY (job) +SUM(http_requests_total) BY (job) != bool SUM(http_requests_total) BY (job) +0 == bool 1 +1 == bool 1 +http_requests_total{job="api-server", instance="0", group="production"} == bool 100 +{job="app-server"} == 80 +http_requests_histogram != 80 +http_requests_histogram > 80 +http_requests_histogram < 80 +http_requests_histogram >= 80 +http_requests_histogram <= 80 +http_requests_histogram == http_requests_histogram +http_requests_histogram != http_requests_histogram +many_side > on(job) one_side +many_side >= on(job) one_side +many_side <= on(job) one_side +node_role * on (instance) group_right (role) node_var +node_var * on (instance) group_left (role) node_role +node_var * ignoring (role) group_left (role) node_role +node_role * ignoring (role) group_right (role) node_var +node_cpu * ignoring (role, mode) group_left (role) node_role +node_cpu * on (instance) group_left (role) node_role +node_cpu / on (instance) group_left sum by (instance,job)(node_cpu) +sum by (mode, job)(node_cpu) / on (job) group_left sum by (job)(node_cpu) +sum(sum by (mode, job)(node_cpu) / on (job) group_left sum by (job)(node_cpu)) +node_cpu / ignoring (mode) group_left sum without (mode)(node_cpu) +node_cpu / ignoring (mode) group_left(dummy) sum without (mode)(node_cpu) +sum without (instance)(node_cpu) / ignoring (mode) group_left sum without (instance, mode)(node_cpu) +sum(sum without (instance)(node_cpu) / ignoring (mode) group_left sum without (instance, mode)(node_cpu)) +node_cpu + on(dummy) group_left(foo) random*0 +node_cpu > on(job, instance) group_left(target) threshold +node_cpu > on(job, instance) group_left(target) (threshold or on (job, instance) (sum by (job, instance)(node_cpu) * 0 + 1)) +node_cpu + 2 +node_cpu - 2 +node_cpu / 2 +node_cpu * 2 +node_cpu ^ 2 +node_cpu % 2 +random + on() metricA +metricA + ignoring() metricB +metricA + metricB +-{__name__=~'testmetric1|testmetric2'} +test_total > bool test_smaller +test_total > test_smaller +test_total < bool test_smaller +test_total < test_smaller +trigy atan2 trigx +trigy atan2 trigNaN +10 atan2 20 +10 atan2 NaN +left_floats == right_floats +left_floats == bool right_floats +left_floats == does_not_match +left_histograms == right_histograms +left_histograms == bool right_histograms +left_histograms == right_floats_for_histograms +left_histograms == bool right_floats_for_histograms +left_floats != right_floats +left_floats != bool right_floats +left_histograms != right_histograms +left_histograms != bool right_histograms +left_histograms != right_floats_for_histograms +left_histograms != bool right_floats_for_histograms +left_floats > right_floats +left_floats > bool right_floats +left_histograms > right_histograms +left_histograms > bool right_histograms +left_histograms > right_floats_for_histograms +left_histograms > bool right_floats_for_histograms +left_floats >= right_floats +left_floats >= bool right_floats +left_histograms >= right_histograms +left_histograms >= bool right_histograms +left_histograms >= right_floats_for_histograms +left_histograms >= bool right_floats_for_histograms +left_floats < right_floats +left_floats < bool right_floats +left_histograms < right_histograms +left_histograms < bool right_histograms +left_histograms < right_floats_for_histograms +left_histograms < bool right_floats_for_histograms +left_floats <= right_floats +left_floats <= bool right_floats +left_histograms <= right_histograms +left_histograms <= bool right_histograms +left_histograms <= right_floats_for_histograms +left_histograms <= bool right_floats_for_histograms +left_floats == 3 +left_floats != 3 +left_floats > 3 +left_floats >= 3 +left_floats < 3 +left_floats <= 3 +left_floats == bool 3 +left_floats == Inf +left_floats == bool Inf +left_floats == NaN +left_floats == bool NaN +left_histograms == 3 +left_histograms == 0 +left_histograms != 3 +left_histograms != 0 +left_histograms > 3 +left_histograms > 0 +left_histograms >= 3 +left_histograms >= 0 +left_histograms < 3 +left_histograms < 0 +left_histograms <= 3 +left_histograms <= 0 +left_histograms == bool 3 +left_histograms == bool 0 +left_histograms != bool 3 +left_histograms != bool 0 +left_histograms > bool 3 +left_histograms > bool 0 +left_histograms >= bool 3 +left_histograms >= bool 0 +left_histograms < bool 3 +left_histograms < bool 0 +left_histograms <= bool 3 +left_histograms <= bool 0 +3 == left_floats +3 != left_floats +3 < left_floats +3 <= left_floats +3 > left_floats +3 >= left_floats +3 == bool left_floats +Inf == left_floats +Inf == bool left_floats +NaN == left_floats +NaN == bool left_floats +3 == left_histograms +0 == left_histograms +3 != left_histograms +0 != left_histograms +3 < left_histograms +0 < left_histograms +3 > left_histograms +0 > left_histograms +3 >= left_histograms +0 >= left_histograms +(testhistogram_bucket) and on() (vector(1) == 1) +(testhistogram_bucket) and on() (vector(-1) == 1) +(testhistogram) and on() (vector(1) == 1) +(testhistogram) and on() (vector(-1) == 1) +-{job="api"} +# --- prometheus testdata: range_queries.test --- +sum_over_time(bar[30s]) +metric +foo > 2 or bar +requests * 2 +some_metric[1m] +some_metric[2m] +some_metric_with_stale_marker[3m] +some_nonexistent_metric[1m] +sort(series) +sort_desc(series) +sort_by_label(series) +sort_by_label_desc(series) +sum(sort(series)) +# --- prometheus testdata: selectors.test --- +rate(http_requests_total[1m]) +rate(http_requests_total{group=~"pro.*"}[1m]) +rate(http_requests_total{group=~".*ry", instance="1"}[1m]) +rate(http_requests_total{instance!="3"}[1m] offset 10000s) +rate(http_requests_total{instance!="3"}[1m] offset -4000s) +rate(http_requests_total[40s]) - rate(http_requests_total[1m] offset 10000s) +http_requests_total{foo!="bar"} +http_requests_total{foo!="bar", job="api-server"} +http_requests_total{foo!~"bar", job="api-server"} +http_requests_total{foo!~"bar", job="api-server", instance="1", x!="y", z="", group!=""} +rate(http_requests_total{group=~"(?i:PRO).*"}[1m]) +rate(http_requests_total{group=~".*?(?i:PRO).*"}[1m]) +rate(http_requests_total{group=~".*(?i:DUC).*"}[1m]) +rate(http_requests_total{group=~".*(?i:TION)"}[1m]) +rate(http_requests_total{group=~".*(?i:TION).*?"}[1m]) +rate(http_requests_total{group=~"((?i)PRO).*"}[1m]) +rate(http_requests_total{group=~".*((?i)DUC).*"}[1m]) +rate(http_requests_total{group=~".*((?i)TION)"}[1m]) +rate(http_requests_total{group=~"(?i:PRODUCTION)"}[1m]) +rate(http_requests_total{group=~".*(?i:C).*"}[1m]) +metric1 offset 15m or metric2 offset 45m +x{y="testvalue"} +{__name__=~".+"} +{job=~".+-server", job!~"api-.+"} +http_requests_total{group!="canary"} +http_requests_total{job=~".+-server",group!="canary"} +http_requests_total{job!~"api-.+",group!="canary"} +http_requests_total{group="production",job=~"api-.+"} +http_requests_total{group="production",job="api-server"} offset 5m +testmetric +# --- prometheus testdata: staleness.test --- +count_over_time(metric[1m]) +count_over_time(metric[1s]) +count_over_time(metric[10s]) +count_over_time(metric[20s]) +count_over_time(metric[10]) +count_over_time(metric[20]) +# --- prometheus testdata: start_timestamps.test --- +increase(cumulative[5m]) +rate(cumulative[5m]) +irate(cumulative[5m]) +round(increase(cumulative[5m1ms])) +increase(cumulative[5m:1m]) +resets(cumulative[5m]) +increase(delta[5m]) +rate(delta[5m]) +irate(delta[5m]) +increase(delta[5m:1m]) +round(increase(series[1m1ms])) +# --- prometheus testdata: subquery.test --- +sum_over_time(metric_total[50s:10s]) +sum_over_time(metric_total[50s:5s]) +sum_over_time(metric_total[60s:10s]) +rate(metric_total[20s:10s]) +rate(metric_total[20s:5s]) +rate(http_requests_total{group=~"pro.*"}[1m:10s]) +avg_over_time(rate(http_requests_total[1m])[1m:1s]) +sum_over_time(metric1_total[30s:10s]) +sum_over_time(metric1_total[30s:5s]) +sum_over_time(metric1_total[30s:10s] offset 10s) +sum_over_time(metric1_total[30s:10s] offset 9s) +sum_over_time(metric1_total[30s:10s] offset 7s) +sum_over_time(metric1_total[30s:10s] offset 5s) +sum_over_time(metric1_total[30s:10s] offset 3s) +sum_over_time((metric1_total)[30s:10s] offset 3s) +sum_over_time(metric1_total[30:10] offset 3) +sum_over_time((metric1_total)[30:10s] offset 3s) +sum_over_time((metric1_total)[30:10] offset 3s) +sum_over_time((metric1_total)[30:10] offset 3) +rate(sum_over_time(metric1_total[30s:10s])[50s:10s]) +rate(sum_over_time(metric2_total[30s:10s])[50s:10s]) +rate(sum_over_time(metric3_total[30s:10s])[50s:10s]) +rate(sum_over_time((metric1_total+metric2_total+metric3_total)[30s:10s])[30s:10s]) +rate(metric_total[1m]) +rate(metric_total[1m500ms:10s]) +rate(metric_total[1m1s:10s]) +min_over_time(metric_total[10s]) +min_over_time(metric_total[15s:10s]) +min_over_time(rate(metric_total[5m])[20m:1m]) +increase(native_histogram[10m:3m]) +increase(native_histogram[10m:15s]) +min_over_time((topk(1, foo))[1m:5m]) +# --- prometheus testdata: trig_functions.test --- +sin(trig) +cos(trig) +tan(trig) +asin(trig - 10.1) +acos(trig - 10.1) +atan(trig) +sinh(trig) +cosh(trig) +tanh(trig) +asinh(trig) +acosh(trig) +atanh(trig - 10.1) +rad(trig) +rad(trig - 10) +rad(trig - 20) +deg(trig) +deg(trig - 10) +deg(trig - 20) +pi() +# --- prometheus testdata: type_and_unit.test --- +SUM(http_requests_total) BY (job) +SUM(http_requests_total{__type__="counter", __unit__="request"}) BY (job) +SUM({__type__="counter"}) BY (job) +SUM({__unit__="request"}) BY (job) +SUM({__type__="counter", __unit__="request"}) BY (job) +http_requests_total{__type__="counter", job="api-server", group="canary"} diff --git a/crates/lower/tests/promql_conformance.rs b/crates/lower/tests/promql_conformance.rs new file mode 100644 index 00000000..32a29b33 --- /dev/null +++ b/crates/lower/tests/promql_conformance.rs @@ -0,0 +1,603 @@ +//! PromQL **semantic conformance** for the L1→L3 lowering. +//! +//! We *lower* PromQL to the intent algebra; we do not *execute* it. So "same +//! semantic job as Prometheus" here means: for each canonical query, does the +//! L3 tree encode the **documented PromQL meaning** — and where we knowingly +//! diverge (reject, approximate, or drop a modifier), is that pinned by a test +//! so it stays visible? +//! +//! Sources for the queries + their semantics: +//! - PromQL basics (data types, selectors, offset/@/subquery): +//! +//! - PromLabs PromQL cheat sheet (common real-world queries by category): +//! +//! - Prometheus' own engine test corpus (these are *execution* tests — +//! load → eval → expect values — so they define semantics we mirror as +//! *structure*): +//! Relevant files, mapped to the sections below: selectors.test, +//! aggregators.test, functions.test, histograms.test, operators.test, +//! subquery.test, at_modifier.test, literals.test, limit.test +//! +//! Legend used in test names: +//! - (no suffix) — we lower it and the L3 intent matches PromQL. +//! - `__GAP` — a PromQL capability we don't *yet* support. It is **cleanly +//! rejected** (never silently mislowered), and pinned here so adding support +//! later flips the assertion deliberately. +//! +//! NOTE: the formerly-silent divergences (`group`→sum, dropped `offset`/`@`, +//! `changes`/`resets`→count) are now rejected rather than mislowered — see the +//! equivalence suite (`promql_equivalence.rs`) and section L below. + +// `__GAP`-suffixed test names intentionally SHOUT the documented divergences. +#![allow(non_snake_case)] + +use std::time::Duration; + +use asap_control_core::intent_algebra::{ + AggIntent, ArithOp, BinaryOpKind, CompareOp, QueryExpr, Source, +}; +use asap_control_core::types::AccuracyTarget; +use asap_control_lower::{lower_promql, LoweringError}; + +// ── harness helpers ───────────────────────────────────────────────────────────── + +/// Lower, expecting success. +fn ok(q: &str) -> QueryExpr { + lower_promql(q, AccuracyTarget::Exact) + .unwrap_or_else(|e| panic!("expected {q:?} to lower, got error: {e}")) +} + +/// Lower, expecting a clean `LoweringError` (an unsupported capability). +fn rejected(q: &str) -> LoweringError { + match lower_promql(q, AccuracyTarget::Exact) { + Err(e) => e, + Ok(tree) => panic!("expected {q:?} to be rejected, but it lowered to: {tree:?}"), + } +} + +/// Every `AggIntent` anywhere in the tree, root-to-leaf. +fn intents(e: &QueryExpr) -> Vec { + let mut out = Vec::new(); + collect(e, &mut out); + out +} + +fn collect(e: &QueryExpr, out: &mut Vec) { + match e { + QueryExpr::Aggregate { aggs, child, .. } => { + out.extend(aggs.iter().cloned()); + collect(child, out); + } + QueryExpr::Window { child, .. } + | QueryExpr::Partition { child, .. } + | QueryExpr::Filter { child, .. } + | QueryExpr::Sort { child, .. } + | QueryExpr::Limit { child, .. } + | QueryExpr::Subquery { child, .. } + | QueryExpr::Distinct { child, .. } + | QueryExpr::WindowFunc { child, .. } + | QueryExpr::Project { child, .. } => collect(child, out), + QueryExpr::BinaryOp { lhs, rhs, .. } => { + collect(lhs, out); + collect(rhs, out); + } + QueryExpr::Join { left, right, .. } | QueryExpr::SetOp { left, right, .. } => { + collect(left, out); + collect(right, out); + } + QueryExpr::Merge { children } => children.iter().for_each(|c| collect(c, out)), + QueryExpr::LetBinding { expr, child, .. } => { + collect(expr, out); + collect(child, out); + } + QueryExpr::Scan { .. } | QueryExpr::Ref { .. } => {} + } +} + +/// The first `Scan` reached by descending single-child nodes, with its metric +/// name and predicate count. +fn first_scan(e: &QueryExpr) -> (String, usize) { + match e { + QueryExpr::Scan { + source, predicates, .. + } => { + let name = match source { + Source::TimeSeries { metric } => metric.clone(), + Source::Table { table_ref } => table_ref.clone(), + }; + (name, predicates.len()) + } + QueryExpr::Window { child, .. } + | QueryExpr::Aggregate { child, .. } + | QueryExpr::Partition { child, .. } + | QueryExpr::Filter { child, .. } + | QueryExpr::Sort { child, .. } + | QueryExpr::Limit { child, .. } + | QueryExpr::Subquery { child, .. } => first_scan(child), + other => panic!("no Scan reachable from {other:?}"), + } +} + +fn has bool>(e: &QueryExpr, pred: F) -> bool { + intents(e).iter().any(pred) +} + +// ───────────────────────────────────────────────────────────────────────────── +// A. Selectors & label matchers (basics §"Instant/Range Vector +// Selectors"; selectors.test) +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn instant_vector_selector() { + // SEMANTICS: bare metric → instant vector (latest sample per series). + let (metric, preds) = first_scan(&ok("node_cpu_seconds_total")); + assert_eq!(metric, "node_cpu_seconds_total"); + assert_eq!(preds, 0, "no label matchers → no predicates"); +} + +#[test] +fn label_matchers_become_scan_predicates() { + // SEMANTICS: `=`, `!=`, `=~`, `!~` filter series; one conjunct per matcher. + let (_, preds) = first_scan(&ok( + r#"http_requests_total{job!="x",path=~"/api/.*",env!~"dev"}"#, + )); + assert_eq!(preds, 3, "three matchers → three Scan predicates"); +} + +#[test] +fn name_label_selects_the_metric() { + // SEMANTICS: the metric name is the internal `__name__` label. + let (metric, preds) = first_scan(&ok(r#"{__name__="up"}"#)); + assert_eq!(metric, "up"); + assert_eq!(preds, 0, "__name__ is the metric, not a residual predicate"); +} + +#[test] +fn range_vector_selector_is_a_window() { + // SEMANTICS: `[5m]` turns an instant vector into a range vector. + let qe = ok("node_cpu_seconds_total[5m]"); + let QueryExpr::Window { size, .. } = &qe else { + panic!("expected Window for a range-vector selector, got {qe:?}"); + }; + assert_eq!(*size, Duration::from_secs(300)); +} + +// ───────────────────────────────────────────────────────────────────────────── +// B. Counters: rate / irate / increase (cheat sheet "Rates of Increase"; +// functions.test) +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn rate_carries_its_window_in_the_intent() { + // SEMANTICS: per-second average rate over the range; the window IS the rate + // parameter, so no separate Window node. + let qe = ok("rate(http_requests_total[5m])"); + assert!(matches!(&qe, QueryExpr::Aggregate { .. })); + assert!(has( + &qe, + |i| matches!(i, AggIntent::Rate { window } if *window == Duration::from_secs(300)) + )); +} + +#[test] +fn irate_maps_to_rate_intent() { + // SEMANTICS: instant rate from the last two samples; same intent vocabulary. + assert!(has(&ok("irate(http_requests_total[1m])"), |i| matches!( + i, + AggIntent::Rate { .. } + ))); +} + +#[test] +fn increase_maps_to_increase_intent() { + assert!(has(&ok("increase(http_requests_total[1h])"), |i| matches!( + i, + AggIntent::Increase { window } if *window == Duration::from_secs(3600) + ))); +} + +// ───────────────────────────────────────────────────────────────────────────── +// C. Aggregation across series (cheat sheet "Aggregating Over +// Multiple Series"; aggregators.test) +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn sum_collapses_all_series() { + // SEMANTICS: `sum(v)` → one output series. No grouping → no Partition. + let qe = ok("sum(node_filesystem_size_bytes)"); + assert!(matches!(&qe, QueryExpr::Aggregate { .. })); + assert!(has(&qe, |i| matches!(i, AggIntent::Sum { .. }))); +} + +#[test] +fn sum_by_groups_via_positional_aggregate() { + // SEMANTICS: `by(job,instance)` keeps those labels; the grouping lives on a + // positional `Aggregate.by` — the same shape SQL `GROUP BY` produces (not a + // name-based Partition). Binder leaf = [ts, value, instance, job] (referenced + // keys appended sorted), so the keys resolve to columns [2, 3]. + let qe = ok("sum by(job, instance) (node_filesystem_size_bytes)"); + let QueryExpr::Aggregate { + by, aggs, child, .. + } = &qe + else { + panic!("expected positional Aggregate for `by(...)`, got {qe:?}"); + }; + assert_eq!( + by, + &vec![2, 3], + "group keys resolve to positional ColumnIds" + ); + assert!(matches!(aggs.as_slice(), [AggIntent::Sum { .. }])); + assert!(matches!(child.as_ref(), QueryExpr::Scan { .. })); +} + +#[test] +fn count_is_cardinality() { + assert!(has(&ok("count(up)"), |i| matches!( + i, + AggIntent::Cardinality { .. } + ))); +} + +#[test] +fn avg_min_max_stddev_stdvar_quantile_aggregators() { + assert!(has(&ok("avg(up)"), |i| matches!(i, AggIntent::Avg { .. }))); + assert!(has(&ok("min(up)"), |i| matches!(i, AggIntent::Min { .. }))); + assert!(has(&ok("max(up)"), |i| matches!(i, AggIntent::Max { .. }))); + assert!(has(&ok("stddev(up)"), |i| matches!( + i, + AggIntent::StdDev { .. } + ))); + assert!(has(&ok("stdvar(up)"), |i| matches!( + i, + AggIntent::Variance { .. } + ))); + assert!(has(&ok("quantile(0.5, up)"), |i| matches!( + i, + AggIntent::Quantile { .. } + ))); +} + +#[test] +fn sum_without_is_rejected() { + // SEMANTICS: `without(instance)` = group by all labels EXCEPT instance. + // We can't enumerate a metric's full label set (usage-derived schema), so + // the complement is rejected rather than silently mis-grouped. + let e = rejected("sum without(instance) (node_filesystem_size_bytes)"); + assert!(format!("{e}").contains("without"), "got {e}"); +} + +#[test] +fn group_aggregator_is_rejected() { + // SEMANTICS (PromQL): `group(v)` returns a constant 1 per group (presence), + // NOT a sum. Rather than fold it onto `Sum` (wrong value) we reject it. + let _ = rejected("group by (job) (up)"); +} + +// ───────────────────────────────────────────────────────────────────────────── +// D. Two-level: outer aggregation OVER an inner counter (the canonical +// `sum(rate(...))` shape; aggregators.test + functions.test) +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn sum_of_rate_is_two_levels() { + // SEMANTICS: per-series rate, THEN cross-series sum. Both must survive. + let qe = ok("sum(rate(http_requests_total[5m]))"); + let QueryExpr::Aggregate { aggs, child, .. } = &qe else { + panic!("expected outer Aggregate{{Sum}}, got {qe:?}"); + }; + assert!(matches!(aggs.as_slice(), [AggIntent::Sum { .. }])); + assert!(matches!( + child.as_ref(), + QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Rate { .. }]) + )); +} + +#[test] +fn sum_by_of_rate_groups_outer_level() { + // Outer cross-series Sum grouped on positional `Aggregate.by` over the + // label-preserving inner Rate. Leaf = [ts, value, instance] → by = [2]. + let qe = ok("sum by(instance) (rate(node_network_receive_bytes_total[5m]))"); + let QueryExpr::Aggregate { + by, aggs, child, .. + } = &qe + else { + panic!("expected outer Aggregate grouped by instance, got {qe:?}"); + }; + assert_eq!(by, &vec![2]); + assert!(matches!(aggs.as_slice(), [AggIntent::Sum { .. }])); + // child is the inner per-series Rate aggregate. + assert!(matches!( + child.as_ref(), + QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Rate { .. }]) + )); +} + +// ───────────────────────────────────────────────────────────────────────────── +// E. Aggregation over time (per-series) (cheat sheet "Aggregating Over +// Time"; functions.test) +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn over_time_functions_window_then_reduce() { + // SEMANTICS: reduce the samples WITHIN each series over the range → Window + // over the matching reduce intent. + for (q, want) in [ + ("avg_over_time(go_goroutines[5m])", "avg"), + ("max_over_time(process_resident_memory_bytes[1d])", "max"), + ("min_over_time(go_goroutines[5m])", "min"), + ("sum_over_time(go_goroutines[5m])", "sum"), + ("count_over_time(go_goroutines[5m])", "count"), + ] { + let qe = ok(q); + assert!( + matches!(&qe, QueryExpr::Window { .. }), + "{q}: expected Window" + ); + let matched = intents(&qe).iter().any(|i| match want { + "avg" => matches!(i, AggIntent::Avg { .. }), + "max" => matches!(i, AggIntent::Max { .. }), + "min" => matches!(i, AggIntent::Min { .. }), + "sum" => matches!(i, AggIntent::Sum { .. }), + "count" => matches!(i, AggIntent::Count { .. }), + _ => unreachable!(), + }); + assert!(matched, "{q}: missing {want} intent"); + } +} + +#[test] +fn quantile_over_time_is_window_over_quantile() { + let qe = ok("quantile_over_time(0.9, request_latency_seconds[5m])"); + assert!(matches!(&qe, QueryExpr::Window { .. })); + assert!(has( + &qe, + |i| matches!(i, AggIntent::Quantile { q, .. } if (*q - 0.9).abs() < 1e-9) + )); +} + +// ───────────────────────────────────────────────────────────────────────────── +// F. Histograms (cheat sheet "Quantiles from +// Histograms"; histograms.test) +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn histogram_quantile_over_rate() { + // SEMANTICS: φ-quantile estimated from bucket rates. + let qe = ok("histogram_quantile(0.9, rate(demo_api_request_duration_seconds_bucket[5m]))"); + let QueryExpr::Aggregate { aggs, .. } = &qe else { + panic!("expected Aggregate{{Quantile}}, got {qe:?}"); + }; + assert!(matches!(aggs.as_slice(), [AggIntent::Quantile { q, .. }] if (*q - 0.9).abs() < 1e-9)); + assert!(has(&qe, |i| matches!(i, AggIntent::Rate { .. }))); +} + +#[test] +fn histogram_quantile_over_sum_by_le_preserves_le_grouping() { + // SEMANTICS: the standard pattern — bucket rates summed by `le`, then the + // quantile. The `sum by (le)` aggregation must survive into L3. + let qe = ok( + "histogram_quantile(0.99, sum by(le) (rate(demo_api_request_duration_seconds_bucket[5m])))", + ); + let QueryExpr::Aggregate { aggs, child, .. } = &qe else { + panic!("expected outer Aggregate{{Quantile}}, got {qe:?}"); + }; + assert!(matches!(aggs.as_slice(), [AggIntent::Quantile { .. }])); + // `sum by(le)` now survives as a positional Aggregate (by = [2], `le`), over + // the inner Rate — no name-based Partition. + let QueryExpr::Aggregate { by, aggs, .. } = child.as_ref() else { + panic!("expected `sum by(le)` as a positional Aggregate, got {child:?}"); + }; + assert_eq!(by, &vec![2]); + assert!(matches!(aggs.as_slice(), [AggIntent::Sum { .. }])); +} + +// ───────────────────────────────────────────────────────────────────────────── +// G. Binary ops: math, matching, comparison (cheat sheet "Math Between +// Series" / "Filtering Series by Value"; operators.test) +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn vector_arithmetic() { + let qe = ok("node_memory_MemFree_bytes + node_memory_Cached_bytes"); + let QueryExpr::BinaryOp { op, .. } = &qe else { + panic!("expected BinaryOp, got {qe:?}"); + }; + assert_eq!(*op, BinaryOpKind::Arith(ArithOp::Add)); +} + +#[test] +fn on_matching_with_group_left() { + // SEMANTICS: many-to-one matching on a label subset. + let qe = + ok("rate(demo_cpu_usage_seconds_total[1m]) / on(instance, job) group_left demo_num_cpus"); + let QueryExpr::BinaryOp { + op, vector_match, .. + } = &qe + else { + panic!("expected BinaryOp, got {qe:?}"); + }; + assert_eq!(*op, BinaryOpKind::Arith(ArithOp::Div)); + let vm = vector_match.as_ref().expect("on(...) group_left present"); + assert_eq!(vm.labels, vec!["instance".to_string(), "job".to_string()]); + assert!( + vm.grouping.is_some(), + "group_left should set the grouping side" + ); +} + +#[test] +fn vector_comparison_filters() { + // SEMANTICS: `>` between two vectors keeps the LHS series where it holds. + let qe = ok("go_goroutines > go_threads"); + assert!( + matches!(&qe, QueryExpr::BinaryOp { op, .. } if *op == BinaryOpKind::Compare(CompareOp::Gt)) + ); +} + +#[test] +fn unary_negation_is_rejected__GAP() { + // SEMANTICS (PromQL): `-expr` flips the sign of every sample (and `-rate(…)` + // negates the rate). With no negate/scalar node in the L2 PromQL path we + // can't model that, so it's rejected rather than silently lowered as `+expr` + // (which would compute the wrong result). `-` folds into the + // literal at parse time and is caught by the bare-scalar rejection instead. + let _ = rejected("-rate(http_errors_total[5m])"); + let _ = rejected("-some_metric"); + let _ = rejected("-metric_a or -metric_b"); + // Negation nested inside a larger expression propagates the rejection, + // rather than lowering the rest with the inner sign silently dropped. + let _ = rejected("http_requests_total - -http_errors_total"); + let _ = rejected("sum(-node_cpu_seconds_total)"); +} + +#[test] +fn count_maps_to_cardinality_and_inherits_accuracy() { + // SEMANTICS (review #2): PromQL `count by (...)` counts distinct series → the + // `Cardinality` intent. The workload's AccuracyTarget threads onto it: + // `Exact` stays exact (no silent HLL substitution); an approximate target is + // carried through for L4 to honor. This pins the intentional count→Cardinality + // mapping and its accuracy gating. + let exact = lower_promql("count by (job) (up)", AccuracyTarget::Exact).unwrap(); + assert!( + has(&exact, |i| matches!( + i, + AggIntent::Cardinality { + accuracy: AccuracyTarget::Exact + } + )), + "count→Cardinality must stay Exact under AccuracyTarget::Exact, got {:?}", + intents(&exact) + ); + + let approx = lower_promql("count by (job) (up)", AccuracyTarget::Epsilon(0.01)).unwrap(); + assert!( + has(&approx, |i| matches!( + i, + AggIntent::Cardinality { + accuracy: AccuracyTarget::Epsilon(e) + } if (*e - 0.01).abs() < 1e-9 + )), + "count→Cardinality must carry the approximate target, got {:?}", + intents(&approx) + ); +} + +#[test] +fn scalar_literal_operand_is_rejected__GAP() { + // SEMANTICS (PromQL): `v > 10*1024*1024` filters by a scalar threshold. + // We have no scalar/number-literal expression in L2, so a literal operand + // is rejected. Common real-world thresholds therefore don't lower yet. + let _ = rejected("node_filesystem_avail_bytes > 10*1024*1024"); +} + +// ───────────────────────────────────────────────────────────────────────────── +// H. Set operations (cheat sheet "Set Operations"; +// operators.test) +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn set_ops_lower_to_binaryop() { + // SEMANTICS: or = union of label sets; and = intersection; unless = difference. + assert!(matches!(&ok("up{job=\"a\"} or up{job=\"b\"}"), + QueryExpr::BinaryOp { op, .. } if *op == BinaryOpKind::Or)); + assert!(matches!(&ok("node_network_mtu_bytes and node_up"), + QueryExpr::BinaryOp { op, .. } if *op == BinaryOpKind::And)); + assert!(matches!(&ok("node_network_mtu_bytes unless node_down"), + QueryExpr::BinaryOp { op, .. } if *op == BinaryOpKind::Unless)); +} + +// ───────────────────────────────────────────────────────────────────────────── +// I. Sorting / top-k (cheat sheet "Sorting"/topk; +// functions.test, limit.test) +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn topk_over_count_is_heavy_hitter() { + // SEMANTICS: top-k by frequency → single-pass heavy-hitter sketch. + let qe = ok("topk(10, count_over_time(http_requests_total[1m]))"); + assert!(has( + &qe, + |i| matches!(i, AggIntent::TopK { k, .. } if *k == 10) + )); +} + +#[test] +fn bottomk_is_generic_sort_limit() { + // SEMANTICS: bottom-k → generic ascending order + limit (no sketch). + let qe = ok("bottomk(3, count_over_time(http_requests_total[5m]))"); + assert!(matches!(&qe, QueryExpr::Limit { .. })); +} + +#[test] +fn topk_over_aggregate_arg_is_rejected__GAP() { + // SEMANTICS (PromQL): `topk(3, sum by(x)(rate(...)))` is extremely common. + // Our aggregate-argument lowering only accepts selectors/calls, not a + // nested aggregate, so this is rejected today. + let _ = rejected("topk(3, sum by(instance) (rate(node_cpu_seconds_total[5m])))"); +} + +// ───────────────────────────────────────────────────────────────────────────── +// J. Subqueries (basics §Subqueries; subquery.test) +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn subquery_wraps_inner_query() { + // SEMANTICS: `[range:res]` evaluates the inner query across a range. + let qe = ok("rate(demo_api_request_duration_seconds_count[5m])[1h:]"); + assert!(matches!(&qe, QueryExpr::Subquery { .. })); + assert!(has(&qe, |i| matches!(i, AggIntent::Rate { .. }))); +} + +#[test] +fn over_time_of_subquery_is_rejected__GAP() { + // SEMANTICS (PromQL): `max_over_time(rate(...)[1h:])` chains a subquery into + // a range-vector function. `extract_matrix` doesn't accept a subquery arg, + // so this canonical pattern is rejected today. + let _ = rejected("max_over_time(rate(demo_api_request_duration_seconds_count[5m])[1h:])"); +} + +// ───────────────────────────────────────────────────────────────────────────── +// K. Time-shift modifiers (basics §Offset/@; at_modifier.test) +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn offset_modifier_is_rejected() { + // SEMANTICS (PromQL): `offset 5m` shifts the lookback 5m into the past. The + // intent algebra can't represent it, so we reject rather than silently drop + // it (which would change the query's meaning). + let _ = rejected("http_requests_total offset 5m"); +} + +#[test] +fn at_modifier_is_rejected() { + // SEMANTICS (PromQL): `@ ` pins the evaluation time. Rejected for the + // same reason as `offset`. + let _ = rejected("http_requests_total @ 1609746000"); +} + +// ───────────────────────────────────────────────────────────────────────────── +// L. Unsupported functions (functions.test) — clean rejection +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn unsupported_functions_are_rejected() { + // These parse fine but have no intent-algebra lowering yet. Each must return + // a clean LoweringError rather than mislower. + for q in [ + "time()", + "timestamp(up)", + "absent(up)", + "absent_over_time(up[5m])", + "deriv(demo_disk_usage_bytes[1h])", + "delta(demo_disk_usage_bytes[1h])", + "predict_linear(demo_disk_usage_bytes[4h], 3600)", + r#"label_replace(up, "host", "$1", "instance", "(.+):.*")"#, + "clamp_max(go_goroutines, 5)", + // changes / resets are NOT sample counts (formerly aliased to Count). + "changes(demo_disk_usage_bytes[1h])", + "resets(http_requests_total[1h])", + ] { + let _ = rejected(q); + } +} diff --git a/crates/lower/tests/promql_corpus.rs b/crates/lower/tests/promql_corpus.rs new file mode 100644 index 00000000..458aa84d --- /dev/null +++ b/crates/lower/tests/promql_corpus.rs @@ -0,0 +1,86 @@ +//! Exhaustive PromQL **corpus** — every query string from the three sources, +//! run through the lowerer. +//! +//! Two corpora (in `tests/data/`): +//! - `promql_corpus_docs.txt` — verbatim examples from the PromQL basics +//! docs and the PromLabs cheat sheet. +//! - `promql_corpus_testdata.txt` — every `eval` expression (deduped) from the +//! Prometheus engine test suite. +//! +//! We *lower* (not execute), so the property under test is **totality**: for +//! every real-world PromQL string, `lower_promql` returns `Ok` or a clean +//! `Err` and **never panics**. A panic anywhere in the loop fails the test — +//! that is the guarantee. A coverage floor guards against a change silently +//! tanking how much of the corpus we can lower. + +use asap_control_core::types::AccuracyTarget; +use asap_control_lower::{lower_promql, LoweringError}; + +const DOCS: &str = include_str!("data/promql_corpus_docs.txt"); +const TESTDATA: &str = include_str!("data/promql_corpus_testdata.txt"); + +/// Non-comment, non-blank query lines. +fn queries(corpus: &str) -> impl Iterator { + corpus + .lines() + .map(str::trim) + .filter(|l| !l.is_empty() && !l.starts_with('#')) +} + +#[derive(Default, Debug)] +struct Tally { + lowered: usize, + rejected: usize, + unparseable: usize, +} + +impl Tally { + fn total(&self) -> usize { + self.lowered + self.rejected + self.unparseable + } +} + +/// Lower every query; a panic here fails the test (the totality guarantee). +fn tally(corpus: &str) -> Tally { + let mut t = Tally::default(); + for q in queries(corpus) { + match lower_promql(q, AccuracyTarget::Exact) { + Ok(_) => t.lowered += 1, + Err(LoweringError::Parse(_)) => t.unparseable += 1, + Err(_) => t.rejected += 1, + } + } + t +} + +#[test] +fn lowering_is_total_over_the_entire_corpus() { + let docs = tally(DOCS); + let td = tally(TESTDATA); + eprintln!("docs corpus: {docs:?}"); + eprintln!("testdata corpus: {td:?}"); + + // Totality: reaching here means no query panicked. Sanity-check that every + // query was classified into exactly one bucket. + assert!( + docs.total() >= 45, + "docs corpus unexpectedly small: {docs:?}" + ); + assert!( + td.total() > 1500, + "testdata corpus unexpectedly small: {td:?}" + ); + + // Coverage tripwire: a code change that breaks lowering for a large slice of + // real PromQL trips this. Set well below the current numbers (docs≈29, + // testdata≈574 lowered / 1014 rejected / 235 unparseable on the private + // promql-parser `asap` branch); it guards regressions, not an exact count. + assert!( + docs.lowered >= 20, + "docs lowering coverage regressed: {docs:?}" + ); + assert!( + td.lowered >= 520, + "testdata lowering coverage regressed: {td:?}" + ); +} diff --git a/crates/lower/tests/promql_equivalence.rs b/crates/lower/tests/promql_equivalence.rs new file mode 100644 index 00000000..b8423a9e --- /dev/null +++ b/crates/lower/tests/promql_equivalence.rs @@ -0,0 +1,172 @@ +//! PromQL **semantic-equivalence proving** for the L1→L3 lowering. +//! +//! The lowering is a *normalizer*: it should map a whole class of +//! semantically-equivalent PromQL strings to **one** canonical L3 tree, and +//! must keep semantically-*distinct* queries distinct. This suite proves: +//! +//! 1. Equivalence classes collapse to identical L3 (`assert_equiv`). +//! 2. Distinct meanings stay distinct (`assert_distinct`). +//! 3. The lowering never *wrongly* equates distinct semantics — the cases it +//! cannot faithfully distinguish are **rejected**, not silently merged. +//! +//! Equivalences are grammar/spec-level facts, drawn from: +//! - PromQL basics: +//! - PromLabs cheat sheet: +//! - Prometheus engine tests (operators.test, aggregators.test, selectors.test): +//! + +#![allow(non_snake_case)] + +use asap_control_core::intent_algebra::QueryExpr; +use asap_control_core::types::AccuracyTarget; +use asap_control_lower::lower_promql; + +fn lo(q: &str) -> QueryExpr { + lower_promql(q, AccuracyTarget::Exact).unwrap_or_else(|e| panic!("{q:?} should lower: {e}")) +} + +/// Every member of an equivalence class must lower to the *same* L3 tree. +fn assert_equiv(class: &[&str]) { + let first = lo(class[0]); + for q in &class[1..] { + assert_eq!( + lo(q), + first, + "expected {q:?} ≡ {:?}, but they lowered to different L3", + class[0] + ); + } +} + +/// Two semantically-distinct queries must lower to *different* L3 trees. +fn assert_distinct(a: &str, b: &str) { + assert_ne!( + lo(a), + lo(b), + "{a:?} and {b:?} must not collapse to the same L3" + ); +} + +/// A query whose semantics we can't faithfully represent must be rejected +/// (never silently mislowered into a different meaning). +fn assert_rejected(q: &str) { + assert!( + lower_promql(q, AccuracyTarget::Exact).is_err(), + "{q:?} should be rejected, not silently lowered" + ); +} + +// ───────────────────────────────────────────────────────────────────────────── +// 1. Equivalence classes the lowering canonicalises to one L3. +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn aggregation_modifier_placement_is_equivalent() { + // ` by (..) (expr)` and `(expr) by (..)` are the same query. + assert_equiv(&["sum by (job) (up)", "sum(up) by (job)"]); + assert_equiv(&["count by (instance) (up)", "count(up) by (instance)"]); +} + +#[test] +fn parentheses_are_transparent() { + assert_equiv(&[ + "sum(rate(http_requests_total[5m]))", + "(sum(rate(http_requests_total[5m])))", + "sum((rate(http_requests_total[5m])))", + ]); +} + +#[test] +fn whitespace_is_irrelevant() { + assert_equiv(&[ + "rate(http_requests_total[5m])", + "rate( http_requests_total [5m] )", + "rate(http_requests_total[5m] )", + ]); +} + +#[test] +fn label_matcher_order_is_equivalent() { + // A matcher set is unordered: same series, so same L3 (FIX: predicates are + // now canonicalised by (name, value) at lowering time). + assert_equiv(&[r#"up{job="a",env="prod"}"#, r#"up{env="prod",job="a"}"#]); +} + +#[test] +fn group_key_order_is_equivalent() { + // Grouping labels are a set: `by (a, b)` ≡ `by (b, a)` (FIX: keys sorted). + assert_equiv(&["sum by (instance, job) (up)", "sum by (job, instance) (up)"]); +} + +// ───────────────────────────────────────────────────────────────────────────── +// 2. Distinct semantics must NOT collapse. +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn distinct_semantics_stay_distinct() { + // Outer aggregation matters (the sum(rate) two-level fix). + assert_distinct("sum(rate(m[5m]))", "rate(m[5m])"); + // Window size matters. + assert_distinct("rate(m[5m])", "rate(m[10m])"); + // Operand order matters for non-commutative binary ops. + assert_distinct("a / b", "b / a"); + // Operator identity matters. + assert_distinct("a and b", "a or b"); + // Quantile parameter matters. + assert_distinct( + "quantile_over_time(0.9, m[5m])", + "quantile_over_time(0.5, m[5m])", + ); + // Grouping dimension matters. + assert_distinct("sum by (job) (up)", "sum by (instance) (up)"); + // Aggregator identity matters. + assert_distinct("sum(up)", "avg(up)"); + // Heavy-hitter topk vs generic bottomk are different plans. + assert_distinct( + "topk(5, count_over_time(m[5m]))", + "bottomk(5, count_over_time(m[5m]))", + ); +} + +// ───────────────────────────────────────────────────────────────────────────── +// 3. Intentional intent-level equivalence (documented, not a bug). +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn rate_and_irate_share_the_same_intent() { + // L3 captures *intent* ("per-second rate of a counter"), not the estimation + // method. `rate` (windowed average) and `irate` (last two samples) differ + // only in HOW the rate is estimated — an L4/execution concern — so they + // share one L3 intent by design. + assert_equiv(&["rate(m[5m])", "irate(m[5m])"]); +} + +// ───────────────────────────────────────────────────────────────────────────── +// 4. Distinct semantics we cannot faithfully represent are REJECTED, not +// silently merged into a wrong intent. (Each previously mislowered.) +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn changes_and_resets_are_not_count_over_time() { + // PromQL: count_over_time = #samples, changes = #value-changes, + // resets = #counter-resets. They previously all collapsed to `Count`. + assert_rejected("changes(m[5m])"); + assert_rejected("resets(m[5m])"); +} + +#[test] +fn group_is_not_sum() { + // PromQL `group` returns a constant 1 per group; it previously collapsed + // onto `sum` (sum of values). + assert_rejected("group(up)"); + assert_rejected("group by (job) (up)"); +} + +#[test] +fn offset_and_at_are_not_dropped() { + // Time-shift modifiers change the query's meaning; they previously lowered + // identically to the un-shifted query (silent loss). + assert_rejected("http_requests_total offset 5m"); + assert_rejected("http_requests_total @ 1609746000"); + assert_rejected("rate(http_requests_total[5m] offset 1h)"); +} diff --git a/crates/lower/tests/promql_lowering.rs b/crates/lower/tests/promql_lowering.rs new file mode 100644 index 00000000..d99fbd4e --- /dev/null +++ b/crates/lower/tests/promql_lowering.rs @@ -0,0 +1,577 @@ +//! End-to-end tests for PromQL → L2 → canonical L3 lowering. + +use std::time::Duration; + +use asap_control_core::intent_algebra::{ + AggIntent, ArithOp, BinaryOpKind, CompareOp, L3Expr, L3Scalar, PartitionKeys, QueryExpr, + Source, WindowKind, +}; +use asap_control_core::types::AccuracyTarget; +use asap_control_core::workload::{ + BatchEntry, Query, QueryLanguage, QueryRequirements, QueryWorkload, +}; + +use asap_control_lower::{lower_promql, lower_promql_batch, LoweringError}; + +fn lower(q: &str) -> QueryExpr { + lower_promql(q, AccuracyTarget::Exact).unwrap_or_else(|e| panic!("lower failed for {q:?}: {e}")) +} + +// ── Bare selectors & label matchers (folded onto Scan.predicates) ─────────────── + +#[test] +fn bare_selector_is_scan_with_predicates() { + let qe = lower(r#"http_requests_total{env="prod",status!="500"}"#); + let QueryExpr::Scan { + source, predicates, .. + } = &qe + else { + panic!("expected Scan, got {qe:?}"); + }; + assert!(matches!(source, Source::TimeSeries { metric } if metric == "http_requests_total")); + // The converter splits the matcher conjunction into one predicate per + // conjunct on the Scan. + assert_eq!(predicates.len(), 2); + assert!(predicates + .iter() + .all(|p| matches!(&p.0, L3Expr::Compare { .. }))); +} + +#[test] +fn regex_matcher_lowers_to_regex_compareop() { + let qe = lower(r#"http_requests_total{path=~"/api/.*"}"#); + let QueryExpr::Scan { + predicates, schema, .. + } = &qe + else { + panic!("expected Scan, got {qe:?}"); + }; + let L3Expr::Compare { left, op, right } = &predicates[0].0 else { + panic!("expected Compare, got {:?}", predicates[0].0); + }; + assert_eq!(*op, CompareOp::Regex); + // The label matcher's column is resolved positionally against the scan schema. + let path_id = schema.column_id("path").expect("path in scan schema"); + assert!(matches!(left.as_ref(), L3Expr::Column(id) if *id == path_id)); + assert!(matches!(right.as_ref(), L3Expr::Literal(L3Scalar::Utf8(v)) if v == "/api/.*")); +} + +// ── *_over_time → Window over Aggregate ───────────────────────────────────────── + +#[test] +fn quantile_over_time_is_window_over_aggregate() { + let qe = lower(r#"quantile_over_time(0.99, http_request_duration{env="prod"}[5m])"#); + let QueryExpr::Window { + kind, size, child, .. + } = &qe + else { + panic!("expected Window, got {qe:?}"); + }; + assert_eq!(*kind, WindowKind::Tumbling); + assert_eq!(*size, Duration::from_secs(300)); + let QueryExpr::Aggregate { + by, aggs, child, .. + } = child.as_ref() + else { + panic!("expected Aggregate under Window, got {child:?}"); + }; + assert!(by.is_empty()); + assert!(matches!(aggs.as_slice(), [AggIntent::Quantile { q, .. }] if (*q - 0.99).abs() < 1e-9)); + // The label matcher folded onto the Scan. + assert!(matches!(child.as_ref(), QueryExpr::Scan { predicates, .. } if predicates.len() == 1)); +} + +#[test] +fn outer_sum_by_wraps_in_partition() { + // `sum by (host) (quantile_over_time(...))` is a two-level reduction: an + // inner per-series quantile-over-time, then an outer cross-series sum. + // Grouping rides on a `Partition` wrapping the outer Sum (backend model). + let qe = lower(r#"sum by (host) (quantile_over_time(0.99, latency{service="web"}[5m]))"#); + let QueryExpr::Partition { keys, child } = &qe else { + panic!("expected Partition, got {qe:?}"); + }; + assert_eq!(keys, &PartitionKeys::By(vec!["host".into()])); + // Outer cross-series Sum. + let QueryExpr::Aggregate { aggs, child, .. } = child.as_ref() else { + panic!("expected outer Aggregate{{Sum}} under Partition, got {child:?}"); + }; + assert!(matches!(aggs.as_slice(), [AggIntent::Sum { .. }])); + // Inner: Window over Aggregate{Quantile}. + let QueryExpr::Window { child, .. } = child.as_ref() else { + panic!("expected Window under the outer Sum, got {child:?}"); + }; + assert!(matches!( + child.as_ref(), + QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Quantile { .. }]) + )); +} + +#[test] +fn avg_over_time_maps_to_avg_intent() { + let qe = lower("avg_over_time(cpu_seconds_total[10m])"); + let QueryExpr::Window { size, child, .. } = &qe else { + panic!("expected Window, got {qe:?}"); + }; + assert_eq!(*size, Duration::from_secs(600)); + assert!(matches!( + child.as_ref(), + QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Avg { .. }]) + )); +} + +#[test] +fn stddev_and_stdvar_over_time() { + let qe = lower("stddev_over_time(m[5m])"); + let QueryExpr::Window { child, .. } = &qe else { + panic!("expected Window"); + }; + assert!(matches!( + child.as_ref(), + QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::StdDev { population: false, .. }]) + )); + + let qe = lower("stdvar_over_time(m[5m])"); + let QueryExpr::Window { child, .. } = &qe else { + panic!("expected Window"); + }; + assert!(matches!( + child.as_ref(), + QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Variance { population: false, .. }]) + )); +} + +#[test] +fn histogram_quantile_wraps_inner_in_quantile() { + // The argument's structure (here `rate`) is preserved *under* the Quantile, + // not squashed away — `Aggregate{Quantile}` over `Aggregate{Rate}` over Scan. + let qe = lower(r#"histogram_quantile(0.95, rate(http_duration_seconds_bucket{le="0.5"}[5m]))"#); + let QueryExpr::Aggregate { aggs, child, .. } = &qe else { + panic!("expected outer Aggregate{{Quantile}}, got {qe:?}"); + }; + assert!(matches!(aggs.as_slice(), [AggIntent::Quantile { q, .. }] if (*q - 0.95).abs() < 1e-9)); + let QueryExpr::Aggregate { aggs, child, .. } = child.as_ref() else { + panic!("expected inner Aggregate{{Rate}}, got {child:?}"); + }; + assert!( + matches!(aggs.as_slice(), [AggIntent::Rate { window }] if *window == Duration::from_secs(300)) + ); + assert!(matches!(child.as_ref(), QueryExpr::Scan { predicates, .. } if predicates.len() == 1)); +} + +#[test] +fn histogram_quantile_over_sum_by_le_preserves_grouping() { + // The canonical Prometheus histogram pattern. Previously returned + // UnsupportedFeature because `extract_matrix` couldn't see through the + // `sum by (le)` aggregate; now the `le` grouping survives into L3. + let qe = lower(r#"histogram_quantile(0.99, sum by (le) (rate(http_requests_bucket[5m])))"#); + let QueryExpr::Aggregate { aggs, child, .. } = &qe else { + panic!("expected outer Aggregate{{Quantile}}, got {qe:?}"); + }; + assert!(matches!(aggs.as_slice(), [AggIntent::Quantile { q, .. }] if (*q - 0.99).abs() < 1e-9)); + // `sum by (le)` survives as a positional Aggregate (by = [2], `le`) over the + // inner Rate — no name-based Partition. + let QueryExpr::Aggregate { by, aggs, .. } = child.as_ref() else { + panic!("expected `sum by (le)` as a positional Aggregate, got {child:?}"); + }; + assert_eq!(by, &vec![2]); + assert!(matches!(aggs.as_slice(), [AggIntent::Sum { .. }])); +} + +// ── rate / increase carry their own window (no Window node) ───────────────────── + +#[test] +fn rate_has_no_window_node() { + let qe = lower("rate(http_requests_total[5m])"); + let QueryExpr::Aggregate { aggs, child, .. } = &qe else { + panic!("expected Aggregate (no Window) for rate, got {qe:?}"); + }; + assert!(matches!( + aggs.as_slice(), + [AggIntent::Rate { window }] if *window == Duration::from_secs(300) + )); + assert!(matches!(child.as_ref(), QueryExpr::Scan { .. })); +} + +#[test] +fn increase_maps_to_increase_intent() { + let qe = lower("increase(errors_total[1h])"); + let QueryExpr::Aggregate { aggs, .. } = &qe else { + panic!("expected Aggregate for increase, got {qe:?}"); + }; + assert!(matches!( + aggs.as_slice(), + [AggIntent::Increase { window }] if *window == Duration::from_secs(3600) + )); +} + +// ── outer aggregation over an inner range-vector func is two levels ───────────── + +#[test] +fn sum_over_rate_keeps_both_levels() { + // Regression: `sum(rate(m[w]))` — the most common PromQL shape — must keep + // the cross-series Sum, not collapse to a bare per-series Rate. + let qe = lower("sum(rate(http_requests_total[5m]))"); + let QueryExpr::Aggregate { aggs, child, .. } = &qe else { + panic!("expected outer Aggregate{{Sum}}, got {qe:?}"); + }; + assert!(matches!(aggs.as_slice(), [AggIntent::Sum { .. }])); + let QueryExpr::Aggregate { aggs, child, .. } = child.as_ref() else { + panic!("expected inner Aggregate{{Rate}}, got {child:?}"); + }; + assert!( + matches!(aggs.as_slice(), [AggIntent::Rate { window }] if *window == Duration::from_secs(300)) + ); + assert!(matches!(child.as_ref(), QueryExpr::Scan { .. })); +} + +#[test] +fn sum_by_over_rate_groups_the_outer_sum() { + // `sum by (job) (rate(...))`: the grouping belongs to the OUTER sum and lands + // on a positional `Aggregate.by` (the same shape SQL produces) over the + // label-preserving inner Rate. Leaf = [ts, value, job] → by = [2]. + let qe = lower("sum by (job) (rate(http_requests_total[5m]))"); + let QueryExpr::Aggregate { + by, aggs, child, .. + } = &qe + else { + panic!("expected outer Aggregate grouped by job, got {qe:?}"); + }; + assert_eq!(by, &vec![2]); + assert!(matches!(aggs.as_slice(), [AggIntent::Sum { .. }])); + assert!(matches!( + child.as_ref(), + QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Rate { .. }]) + )); +} + +#[test] +fn count_over_rate_keeps_both_levels() { + // The `Outer::Count` sibling of the `sum(rate(...))` bug. + let qe = lower("count(rate(http_requests_total[5m]))"); + let QueryExpr::Aggregate { aggs, child, .. } = &qe else { + panic!("expected outer Aggregate{{Cardinality}}, got {qe:?}"); + }; + assert!(matches!(aggs.as_slice(), [AggIntent::Cardinality { .. }])); + assert!(matches!( + child.as_ref(), + QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Rate { .. }]) + )); +} + +// ── count / cardinality ─────────────────────────────────────────────────────── + +#[test] +fn count_over_time_is_count_intent() { + let qe = lower("count_over_time(m[5m])"); + let QueryExpr::Window { child, .. } = &qe else { + panic!("expected Window"); + }; + assert!(matches!( + child.as_ref(), + QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Count { .. }]) + )); +} + +#[test] +fn outer_count_is_cardinality() { + // `count by (symbol) (count_over_time(...))`: inner per-series sample count + // over the window, outer cross-series cardinality grouped by symbol. + let qe = lower("count by (symbol) (count_over_time(financial_last_trade_price[5m]))"); + let QueryExpr::Partition { keys, child } = &qe else { + panic!("expected Partition, got {qe:?}"); + }; + assert_eq!(keys, &PartitionKeys::By(vec!["symbol".into()])); + // Outer cardinality (count of series). + let QueryExpr::Aggregate { aggs, child, .. } = child.as_ref() else { + panic!("expected outer Aggregate{{Cardinality}} under Partition, got {child:?}"); + }; + assert!(matches!(aggs.as_slice(), [AggIntent::Cardinality { .. }])); + // Inner: Window over Aggregate{Count} (count_over_time). + let QueryExpr::Window { child, .. } = child.as_ref() else { + panic!("expected Window under the outer cardinality, got {child:?}"); + }; + assert!(matches!( + child.as_ref(), + QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Count { .. }]) + )); +} + +// ── topk / bottomk ──────────────────────────────────────────────────────────── + +#[test] +fn topk_over_count_is_heavy_hitter_topk() { + let qe = lower(r#"topk by (service) (10, count_over_time(requests{env="prod"}[1m]))"#); + // Heavy-hitter: Aggregate{TopK} with grouping resolved to positional ids. + let QueryExpr::Aggregate { + by, aggs, child, .. + } = &qe + else { + panic!("expected Aggregate with TopK, got {qe:?}"); + }; + // `service` is the only group key → resolved to a positional ColumnId. + assert_eq!(by.len(), 1); + assert!(matches!(aggs.as_slice(), [AggIntent::TopK { k: 10, .. }])); + // The heavy-hitter sketch counts directly off the windowed scan. + let QueryExpr::Window { size, child, .. } = child.as_ref() else { + panic!("expected Window under TopK Aggregate, got {child:?}"); + }; + assert_eq!(*size, Duration::from_secs(60)); + assert!(matches!(child.as_ref(), QueryExpr::Scan { .. })); +} + +#[test] +fn topk_over_avg_is_generic_sort_limit() { + let qe = lower("topk by (host) (5, avg_over_time(cpu[5m]))"); + let QueryExpr::Limit { n, offset, child } = &qe else { + panic!("expected Limit, got {qe:?}"); + }; + assert_eq!(*n, 5); + assert_eq!(*offset, 0); + let QueryExpr::Sort { keys, child } = child.as_ref() else { + panic!("expected Sort under Limit, got {child:?}"); + }; + assert_eq!(keys.len(), 1); + assert!(!keys[0].ascending, "topk ranks descending"); + // Underneath: the windowed avg aggregate, grouped via Partition. + assert!( + matches!(child.as_ref(), QueryExpr::Partition { keys, .. } if *keys == PartitionKeys::By(vec!["host".into()])) + ); +} + +#[test] +fn bottomk_is_always_generic_sort_ascending() { + let qe = lower("bottomk(3, count_over_time(m[5m]))"); + let QueryExpr::Limit { n, child, .. } = &qe else { + panic!("expected Limit, got {qe:?}"); + }; + assert_eq!(*n, 3); + let QueryExpr::Sort { keys, .. } = child.as_ref() else { + panic!("expected Sort"); + }; + assert!(keys[0].ascending, "bottomk ranks ascending"); +} + +// ── binary ops ──────────────────────────────────────────────────────────────── + +#[test] +fn binary_op_division() { + let qe = lower("rate(a[5m]) / rate(b[5m])"); + let QueryExpr::BinaryOp { op, lhs, rhs, .. } = &qe else { + panic!("expected BinaryOp, got {qe:?}"); + }; + assert_eq!(*op, BinaryOpKind::Arith(ArithOp::Div)); + assert!( + matches!(lhs.as_ref(), QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Rate { .. }])) + ); + assert!( + matches!(rhs.as_ref(), QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Rate { .. }])) + ); +} + +#[test] +fn binary_op_with_on_grouping() { + let qe = lower("a / on(host) b"); + let QueryExpr::BinaryOp { vector_match, .. } = &qe else { + panic!("expected BinaryOp, got {qe:?}"); + }; + let vm = vector_match.as_ref().expect("vector_match present"); + use asap_control_core::intent_algebra::VectorMatchKind; + assert_eq!(vm.kind, VectorMatchKind::On); + assert_eq!(vm.labels, vec!["host".to_string()]); +} + +#[test] +fn binary_op_binds_each_branch_against_its_own_schema() { + // Each side scans a different metric and groups by a different label. With a + // single root schema threaded to both branches, the left scan would leak the + // right's group key (and vice-versa). Per-branch binding keeps them separate. + let qe = lower("count by (job) (a) / count by (region) (b)"); + let QueryExpr::BinaryOp { lhs, rhs, .. } = &qe else { + panic!("expected BinaryOp, got {qe:?}"); + }; + let lcols = scan_columns(lhs); + let rcols = scan_columns(rhs); + assert!( + lcols.iter().any(|c| c == "job") && !lcols.iter().any(|c| c == "region"), + "lhs scan schema leaked the rhs key: {lcols:?}" + ); + assert!( + rcols.iter().any(|c| c == "region") && !rcols.iter().any(|c| c == "job"), + "rhs scan schema leaked the lhs key: {rcols:?}" + ); +} + +/// Column names on the first `Scan` reachable by descending single-child nodes. +fn scan_columns(e: &QueryExpr) -> Vec { + match e { + QueryExpr::Scan { schema, .. } => schema.columns.iter().map(|c| c.name.clone()).collect(), + QueryExpr::Partition { child, .. } + | QueryExpr::Aggregate { child, .. } + | QueryExpr::Window { child, .. } + | QueryExpr::Filter { child, .. } + | QueryExpr::Sort { child, .. } + | QueryExpr::Limit { child, .. } => scan_columns(child), + _ => vec![], + } +} + +// ── without is unsupported (no label registry) ────────────────────────────────── + +#[test] +fn without_grouping_is_unsupported() { + let err = lower_promql( + "sum without (instance) (rate(m[5m]))", + AccuracyTarget::Exact, + ) + .unwrap_err(); + assert!(format!("{err}").contains("without"), "got {err}"); +} + +// ── parameter validation (reject rather than silently truncate/garble) ────────── + +#[test] +fn fractional_or_negative_topk_k_is_rejected() { + // `as u64` would silently truncate 2.7→2 / saturate -1→0. + assert!(lower_promql("topk(2.7, count_over_time(m[1m]))", AccuracyTarget::Exact).is_err()); + assert!(lower_promql("bottomk(2.5, sum_over_time(m[1m]))", AccuracyTarget::Exact).is_err()); +} + +#[test] +fn out_of_range_quantile_phi_is_rejected() { + // φ outside [0,1] would otherwise yield a bogus `quantile_1_5` column. + assert!(lower_promql("quantile(1.5, up)", AccuracyTarget::Exact).is_err()); + assert!(lower_promql("quantile_over_time(1.5, m[5m])", AccuracyTarget::Exact).is_err()); + assert!(lower_promql( + "histogram_quantile(2.0, rate(b[5m]))", + AccuracyTarget::Exact + ) + .is_err()); +} + +#[test] +fn function_wrapped_range_vector_is_rejected_not_stripped() { + // `rate(abs(m[5m]))` must NOT silently lower as `rate(m[5m])` — the wrapper + // is rejected (here, at parse or in extract_matrix), never stripped. + assert!( + lower_promql("rate(abs(http_requests_total[5m]))", AccuracyTarget::Exact).is_err(), + "function-wrapped range vector should be rejected" + ); +} + +#[test] +fn pathologically_nested_query_is_rejected_not_stack_overflow() { + // 300 nested parens parse fine but exceed the walker's depth limit (256); + // it must return an error, not overflow the stack. + let q = format!("{}m{}", "(".repeat(300), ")".repeat(300)); + let err = lower_promql(&q, AccuracyTarget::Exact).unwrap_err(); + assert!(format!("{err}").contains("nesting"), "got {err}"); +} + +// ── accuracy propagation ────────────────────────────────────────────────────── + +#[test] +fn accuracy_target_flows_into_quantile_intent() { + let qe = lower_promql( + "quantile_over_time(0.9, m[5m])", + AccuracyTarget::Epsilon(0.01), + ) + .unwrap(); + let QueryExpr::Window { child, .. } = &qe else { + panic!("expected Window"); + }; + let QueryExpr::Aggregate { aggs, .. } = child.as_ref() else { + panic!("expected Aggregate"); + }; + assert!(matches!( + &aggs[0], + AggIntent::Quantile { accuracy: AccuracyTarget::Epsilon(e), .. } if (*e - 0.01).abs() < 1e-12 + )); +} + +// ── schema flow (positional, carried on Scan; derived on demand) ───────────────── + +#[test] +fn aggregate_output_schema_is_single_quantile_column() { + let qe = lower(r#"quantile_over_time(0.99, http_request_duration{env="prod"}[5m])"#); + // Window requires its child to carry a time axis; the Aggregate beneath it + // strips it, so derive the schema at the Aggregate node. + let QueryExpr::Window { child, .. } = &qe else { + panic!("expected Window"); + }; + let schema = child.output_schema().expect("aggregate schema"); + let names: Vec<&str> = schema.columns.iter().map(|c| c.name.as_str()).collect(); + assert_eq!(names, vec!["quantile_0_99"]); + assert!( + schema.time_index.is_none(), + "aggregate strips the time axis" + ); +} + +#[test] +fn scan_schema_carries_ts_value_and_group_keys() { + // `service` is a group key → the Binder lands it in the self-contained + // Scan schema (positional). `env` is only a filter, so it is not a column. + let qe = lower("count by (service) (count_over_time(requests[1m]))"); + fn find_scan(n: &QueryExpr) -> &QueryExpr { + match n { + QueryExpr::Scan { .. } => n, + QueryExpr::Partition { child, .. } + | QueryExpr::Window { child, .. } + | QueryExpr::Aggregate { child, .. } + | QueryExpr::Filter { child, .. } => find_scan(child), + other => panic!("unexpected node {other:?}"), + } + } + let QueryExpr::Scan { schema, .. } = find_scan(&qe) else { + unreachable!() + }; + let mut names: Vec<&str> = schema.columns.iter().map(|c| c.name.as_str()).collect(); + names.sort(); + assert_eq!(names, vec!["service", "ts", "value"]); + assert_eq!(schema.time_index, Some(0)); // ts +} + +// ── batch entry point ───────────────────────────────────────────────────────── + +#[test] +fn batch_lowers_each_entry_and_reads_per_query_accuracy() { + let workload = QueryWorkload { + language: QueryLanguage::PromQL, + query_batch: Some(vec![ + BatchEntry { + query: Query("rate(a[5m])".into()), + requirements: None, + }, + BatchEntry { + query: Query("quantile_over_time(0.9, b[5m])".into()), + requirements: Some(QueryRequirements { + accuracy: Some(AccuracyTarget::Epsilon(0.02)), + latency_ms: None, + }), + }, + ]), + repeating_queries: None, + data_characteristics: None, + }; + let results = lower_promql_batch(&workload); + assert_eq!(results.len(), 2); + assert!(results[0].is_ok()); + assert!(results[1].is_ok()); +} + +#[test] +fn batch_rejects_non_promql_language() { + use asap_control_core::workload::SqlDialect; + let workload = QueryWorkload { + language: QueryLanguage::SQL(SqlDialect::DataFusionSQL), + query_batch: Some(vec![BatchEntry { + query: Query("SELECT 1".into()), + requirements: None, + }]), + repeating_queries: None, + data_characteristics: None, + }; + let results = lower_promql_batch(&workload); + assert_eq!(results.len(), 1); + assert!(matches!(results[0], Err(LoweringError::WrongLanguage(_)))); +} diff --git a/crates/lower/tests/sql_lowering.rs b/crates/lower/tests/sql_lowering.rs index b8f3d888..e37e81c4 100644 --- a/crates/lower/tests/sql_lowering.rs +++ b/crates/lower/tests/sql_lowering.rs @@ -1,2027 +1,505 @@ -use std::collections::HashMap; - -use asap_control_core::intent_algebra::expr::{ - AggIntent, Predicate, ProjectItem, QueryExpr, SortKey, Source, -}; -use asap_control_core::intent_algebra::schema::{ - ColumnDef, L3DataType, SchemaCatalog, TableSchema, +//! End-to-end SQL → L2 → canonical L3 lowering tests (positional IR). +//! +//! Validates the re-targeted DataFusion front end: SQL parses + plans, lowers to +//! the relational L2 algebra, and the shared `convert_root` produces positional +//! canonical L3 (the same converter the PromQL path uses). + +use asap_control_core::intent_algebra::schema::{Column, DataType, Schema}; +use asap_control_core::intent_algebra::{ + AggIntent, CompareOp, JoinKind, L3Expr, QueryExpr, Source, WindowFuncKind, }; -use asap_control_core::intent_algebra::{CompareOp, L3Expr, SetOpKind}; use asap_control_core::types::AccuracyTarget; -use asap_control_core::workload::{BatchEntry, Query, QueryLanguage, QueryWorkload, SqlDialect}; -use asap_control_lower::{lower_batch, populate_schemas, LoweringError, SqlLowerer}; - -// ── Catalog helpers ─────────────────────────────────────────────────────────── +use asap_control_lower::{lower_sql, SqlCatalog}; -fn metrics_catalog() -> SchemaCatalog { - let mut tables = HashMap::new(); - tables.insert( - "metrics".to_string(), - TableSchema { - columns: vec![ - ColumnDef { - name: "ts".to_string(), - data_type: L3DataType::Int64, - nullable: false, - }, - ColumnDef { - name: "value".to_string(), - data_type: L3DataType::Float64, - nullable: true, - }, - ColumnDef { - name: "region".to_string(), - data_type: L3DataType::Utf8, - nullable: true, - }, - ColumnDef { - name: "host".to_string(), - data_type: L3DataType::Utf8, - nullable: true, - }, - ], - time_column: Some("ts".to_string()), - }, - ); - SchemaCatalog { tables } +fn col(name: &str, dtype: DataType) -> Column { + Column::new(name, dtype, false) } -fn no_time_catalog() -> SchemaCatalog { - let mut tables = HashMap::new(); - tables.insert( - "events".to_string(), - TableSchema { - columns: vec![ - ColumnDef { - name: "id".to_string(), - data_type: L3DataType::Int64, - nullable: false, - }, - ColumnDef { - name: "value".to_string(), - data_type: L3DataType::Float64, - nullable: true, - }, - ColumnDef { - name: "name".to_string(), - data_type: L3DataType::Utf8, - nullable: true, - }, - ], - time_column: None, - }, - ); - SchemaCatalog { tables } +/// `metrics(ts, service, latency, bytes)` + `hosts(service, region)`. +fn catalog() -> SqlCatalog { + SqlCatalog::new() + .with_table( + "metrics", + Schema::with_time_index( + vec![ + col("ts", DataType::Timestamp), + col("service", DataType::Utf8), + col("latency", DataType::Float64), + col("bytes", DataType::Int64), + ], + 0, + vec![], + ), + ) + .with_table( + "hosts", + Schema::new(vec![ + col("service", DataType::Utf8), + col("region", DataType::Utf8), + ]), + ) } -// ── Tree-walking helpers ────────────────────────────────────────────────────── +async fn lower(sql: &str) -> QueryExpr { + lower_sql(sql, &catalog(), AccuracyTarget::Exact) + .await + .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")) +} -/// Walk through Project/Filter/Sort/Limit wrappers to find the first Aggregate. -fn find_aggregate( - expr: &QueryExpr, -) -> Option<( - &Vec, - &Vec, -)> { - match expr { +/// Find the first `Aggregate` node along the single-child spine. +fn find_aggregate(qe: &QueryExpr) -> Option<(&Vec, &Vec)> { + match qe { QueryExpr::Aggregate { by, aggs, .. } => Some((by, aggs)), QueryExpr::Project { child, .. } | QueryExpr::Filter { child, .. } + | QueryExpr::Window { child, .. } + | QueryExpr::Partition { child, .. } + | QueryExpr::Distinct { child, .. } | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } => find_aggregate(&child.expr), - _ => None, - } -} - -/// Walk through wrappers to find the predicate of the first Filter node. -fn find_predicate(expr: &QueryExpr) -> Option<&L3Expr> { - match expr { - QueryExpr::Filter { - pred: Predicate(e), .. - } => Some(e), - QueryExpr::Project { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } => find_predicate(&child.expr), - _ => None, - } -} - -/// Walk through wrappers to find the cols of the first Project node. -fn find_project_items(expr: &QueryExpr) -> Option<&[ProjectItem]> { - match expr { - QueryExpr::Project { cols, .. } => Some(cols), - QueryExpr::Sort { child, .. } - | QueryExpr::Filter { child, .. } - | QueryExpr::Limit { child, .. } => find_project_items(&child.expr), + | QueryExpr::Limit { child, .. } + | QueryExpr::Subquery { child, .. } => find_aggregate(child), _ => None, } } -/// Walk through wrappers to find the keys of the first Sort node. -fn find_sort_keys(expr: &QueryExpr) -> Option<&[SortKey]> { - match expr { - QueryExpr::Sort { keys, .. } => Some(keys), +/// Find the first `Join` node along the single-child spine. +fn find_join(qe: &QueryExpr) -> Option<&QueryExpr> { + match qe { + QueryExpr::Join { .. } => Some(qe), QueryExpr::Project { child, .. } | QueryExpr::Filter { child, .. } - | QueryExpr::Limit { child, .. } => find_sort_keys(&child.expr), + | QueryExpr::Aggregate { child, .. } + | QueryExpr::Window { child, .. } + | QueryExpr::Partition { child, .. } + | QueryExpr::Distinct { child, .. } + | QueryExpr::Sort { child, .. } + | QueryExpr::Limit { child, .. } + | QueryExpr::Subquery { child, .. } => find_join(child), _ => None, } } -/// Walk through wrappers to find the first Scan source. -fn find_source(expr: &QueryExpr) -> Option<&Source> { - match expr { - QueryExpr::Scan { source, .. } => Some(source), +/// The first `Filter` node along the single-child spine. +fn find_filter(qe: &QueryExpr) -> Option<&QueryExpr> { + match qe { + QueryExpr::Filter { .. } => Some(qe), QueryExpr::Project { child, .. } - | QueryExpr::Filter { child, .. } | QueryExpr::Aggregate { child, .. } + | QueryExpr::Window { child, .. } + | QueryExpr::Partition { child, .. } + | QueryExpr::Distinct { child, .. } | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } => find_source(&child.expr), + | QueryExpr::Limit { child, .. } + | QueryExpr::Subquery { child, .. } => find_filter(child), _ => None, } } -fn make_workload(queries: Vec<&str>) -> QueryWorkload { - QueryWorkload { - language: QueryLanguage::SQL(SqlDialect::DataFusionSQL), - query_batch: Some( - queries - .into_iter() - .map(|q| BatchEntry { - query: Query(q.to_string()), - requirements: None, - }) - .collect(), - ), - repeating_queries: None, - data_characteristics: None, - } +#[tokio::test] +async fn select_star_with_where_folds_predicate_onto_scan() { + // SELECT * elides the projection; WHERE folds onto the Scan predicates. + let qe = lower("SELECT * FROM metrics WHERE service = 'api'").await; + let QueryExpr::Scan { + source, predicates, .. + } = &qe + else { + panic!("expected Scan at root, got {qe:?}"); + }; + assert!(matches!(source, Source::Table { table_ref } if table_ref == "metrics")); + assert_eq!(predicates.len(), 1, "WHERE clause folded onto the scan"); } -mod scan_filter { - use super::*; - - // ── Tests: Scan / Projection ────────────────────────────────────────────────── - - #[tokio::test] - async fn test_scan_table_name() { - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT ts, value FROM metrics") - .await - .unwrap(); - - let source = find_source(&result).expect("expected a Scan node"); - let Source::Table { table_ref, .. } = source else { - panic!("expected Source::Table, got {source:?}"); - }; - assert_eq!(table_ref.0, "metrics"); - } - - #[tokio::test] - async fn test_projection_wraps_scan() { - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT ts, value FROM metrics") - .await - .unwrap(); - - // DataFusion always emits a Projection for explicit SELECT lists. - let QueryExpr::Project { cols, child } = result else { - panic!("expected Project at root, got {:?}", result); - }; - assert_eq!(cols.len(), 2); - assert!(matches!(child.expr, QueryExpr::Scan { .. })); - } - - // ── Tests: Filter / time extraction ────────────────────────────────────────── - - #[tokio::test] - async fn test_non_time_filter_stays_as_filter() { - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - // region is not the time column → stays as Filter - let result = lowerer - .lower("SELECT ts FROM metrics WHERE region = 'us-east'") - .await - .unwrap(); - - fn has_filter(e: &QueryExpr) -> bool { - match e { - QueryExpr::Filter { .. } => true, - QueryExpr::Project { child, .. } => has_filter(&child.expr), - _ => false, - } - } - assert!(has_filter(&result), "expected a Filter node"); - } - - #[tokio::test] - async fn test_time_predicate_extracted_to_source() { - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - // ts is the time column → extracted into Source::Table.time_range - let result = lowerer - .lower("SELECT value FROM metrics WHERE ts > 1000 AND ts < 2000") - .await - .unwrap(); - - let source = find_source(&result).unwrap(); - let Source::Table { time_range, .. } = source else { - panic!("expected Source::Table"); - }; - let tr = time_range.as_ref().expect("expected time_range to be Some"); - assert_eq!(tr.start_ms, Some(1000)); - assert_eq!(tr.end_ms, Some(2000)); - } - - #[tokio::test] - async fn test_time_predicate_no_filter_wrapper() { - // When ALL predicates are time bounds, no Filter node should appear. - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT value FROM metrics WHERE ts > 1000 AND ts < 2000") - .await - .unwrap(); - - fn has_filter(e: &QueryExpr) -> bool { - match e { - QueryExpr::Filter { .. } => true, - QueryExpr::Project { child, .. } => has_filter(&child.expr), - _ => false, - } - } - assert!( - !has_filter(&result), - "expected no Filter node when only time predicates" - ); - } - - #[tokio::test] - async fn test_mixed_filter_keeps_filter_and_time_range() { - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT value FROM metrics WHERE ts > 1000 AND region = 'eu'") - .await - .unwrap(); - - // The Source should have a time_range... - let source = find_source(&result).unwrap(); - let Source::Table { time_range, .. } = source else { - panic!("expected Source::Table"); - }; - assert!(time_range.is_some(), "expected time_range extracted"); - - // ...and there should also be a Filter node for the non-time predicate. - fn has_filter(e: &QueryExpr) -> bool { - match e { - QueryExpr::Filter { .. } => true, - QueryExpr::Project { child, .. } => has_filter(&child.expr), - _ => false, - } - } - assert!( - has_filter(&result), - "expected Filter node for non-time predicate" - ); - } +#[tokio::test] +async fn multi_aggregate_group_by_binds_columns_positionally() { + // SUM(bytes)=col 3, AVG(latency)=col 2, GROUP BY service=col 1. + let qe = lower("SELECT service, SUM(bytes), AVG(latency) FROM metrics GROUP BY service").await; + let (by, aggs) = find_aggregate(&qe).expect("expected an Aggregate in the tree"); + assert_eq!(by, &vec![1], "GROUP BY service → column 1"); + assert!( + aggs.contains(&AggIntent::Sum { col: Some(3) }), + "SUM(bytes) → Sum{{col:3}}, got {aggs:?}" + ); + assert!( + aggs.contains(&AggIntent::Avg { col: Some(2) }), + "AVG(latency) → Avg{{col:2}}, got {aggs:?}" + ); } -mod aggregates_and_topk { - use super::*; - - // ── Tests: Aggregates ───────────────────────────────────────────────────────── - - #[tokio::test] - async fn test_count_star_exact() { - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer.lower("SELECT COUNT(*) FROM metrics").await.unwrap(); - - let (by, aggs) = find_aggregate(&result).expect("expected Aggregate"); - assert!(by.is_empty(), "no GROUP BY expected"); - assert_eq!(aggs.len(), 1); - assert!(matches!( - aggs[0], - AggIntent::Count { - accuracy: AccuracyTarget::Exact - } - )); - } - - #[tokio::test] - async fn test_count_inherits_accuracy() { - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Epsilon(0.01)); - let result = lowerer.lower("SELECT COUNT(*) FROM metrics").await.unwrap(); - - let (_, aggs) = find_aggregate(&result).unwrap(); - assert!( - matches!(aggs[0], AggIntent::Count { accuracy: AccuracyTarget::Epsilon(e) } if (e - 0.01).abs() < 1e-12) - ); - } - - #[tokio::test] - async fn test_sum_aggregate() { - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT SUM(value) FROM metrics") - .await - .unwrap(); - - let (_, aggs) = find_aggregate(&result).unwrap(); - assert_eq!(aggs.len(), 1); - assert!(matches!(aggs[0], AggIntent::Sum { .. })); - } - - #[tokio::test] - async fn test_min_max_aggregates() { - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT MIN(value), MAX(value) FROM metrics") - .await - .unwrap(); - - let (_, aggs) = find_aggregate(&result).unwrap(); - assert_eq!(aggs.len(), 2); - assert!(aggs.iter().any(|a| matches!(a, AggIntent::Min { .. }))); - assert!(aggs.iter().any(|a| matches!(a, AggIntent::Max { .. }))); - } - - #[tokio::test] - async fn test_avg_aggregate() { - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT AVG(value) FROM metrics") - .await - .unwrap(); - - let (_, aggs) = find_aggregate(&result).unwrap(); - assert!(aggs.iter().any(|a| matches!(a, AggIntent::Avg { .. }))); - } - - #[tokio::test] - async fn test_stddev_sample() { - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT STDDEV(value) FROM metrics") - .await - .unwrap(); - - let (_, aggs) = find_aggregate(&result).unwrap(); - assert!(aggs.iter().any(|a| matches!( - a, - AggIntent::Stddev { - population: false, - .. - } - ))); - } - - #[tokio::test] - async fn test_group_by_extracted() { - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT region, COUNT(*) FROM metrics GROUP BY region") - .await - .unwrap(); - - let (by, aggs) = find_aggregate(&result).unwrap(); - assert_eq!(by.len(), 1); - assert_eq!(by[0].0, "region"); - assert!(matches!(aggs[0], AggIntent::Count { .. })); - } - - #[tokio::test] - async fn test_multiple_aggregates_in_one_node() { - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT COUNT(*), SUM(value), MIN(value), MAX(value) FROM metrics") - .await - .unwrap(); - - let (_, aggs) = find_aggregate(&result).unwrap(); - assert_eq!(aggs.len(), 4); - } - - // ── Tests: Approximate intents ──────────────────────────────────────────────── - - #[tokio::test] - async fn test_count_distinct_becomes_cardinality() { - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT COUNT(DISTINCT host) FROM metrics") - .await - .unwrap(); - - let (_, aggs) = find_aggregate(&result).unwrap(); - assert!(aggs - .iter() - .any(|a| matches!(a, AggIntent::Cardinality { .. }))); - } - - #[tokio::test] - async fn test_approx_percentile_becomes_quantile() { - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new( - &catalog, - AccuracyTarget::EpsilonDelta { - epsilon: 0.01, - delta: 0.001, - }, - ); - let result = lowerer - .lower("SELECT approx_percentile_cont(value, 0.99) FROM metrics") - .await - .unwrap(); - - let (_, aggs) = find_aggregate(&result).unwrap(); - assert!(aggs - .iter() - .any(|a| matches!(a, AggIntent::Quantile { q, .. } if (*q - 0.99).abs() < 1e-12))); - } - - // ── Test: TopK heavy-hitter pattern ────────────────────────────────────────── - - #[tokio::test] - async fn test_order_by_desc_limit_becomes_topk() { - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower( - "SELECT host, COUNT(*) AS cnt FROM metrics \ - GROUP BY host ORDER BY cnt DESC LIMIT 10", - ) - .await - .unwrap(); - - let (_, aggs) = find_aggregate(&result).expect("expected Aggregate"); - assert_eq!(aggs.len(), 1, "TopK should produce exactly one AggIntent"); - let AggIntent::TopK { k, by, .. } = &aggs[0] else { - panic!("expected TopK, got {:?}", aggs[0]); - }; - assert_eq!(*k, 10); - assert_eq!(by.len(), 1, "TopK should have exactly 1 by-column"); - assert!(by.iter().any(|c| c.0 == "host")); - } - - // ── Test: TopK + OFFSET returns error (#1) ─────────────────────────────────── - - #[tokio::test] - async fn test_order_by_desc_limit_with_offset_returns_error() { - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let err = lowerer - .lower( - "SELECT host, COUNT(*) FROM metrics \ - GROUP BY host ORDER BY 2 DESC LIMIT 10 OFFSET 5", - ) - .await - .unwrap_err(); - assert!( - matches!(err, LoweringError::UnsupportedFeature(ref msg) if msg.contains("OFFSET")), - "expected UnsupportedFeature(OFFSET), got: {err}" - ); - } +#[tokio::test] +async fn projection_over_aggregate_resolves_output_types_via_output_names() { + // The enclosing Projection references the aggregates by DataFusion's + // generated names (e.g. "sum(metrics.bytes)"); output_names threads those + // onto the L3 Aggregate so the Project resolves real types — not the Utf8 + // fallback that an unresolved column would get. + let qe = lower("SELECT SUM(bytes), AVG(latency) FROM metrics").await; + let schema = qe + .output_schema() + .expect("root projection schema derivation"); + assert_eq!(schema.columns.len(), 2); + assert_eq!( + schema.columns[0].dtype, + DataType::Int64, + "SUM(bytes:Int64) resolves to Int64, not the Utf8 fallback" + ); + assert_eq!( + schema.columns[1].dtype, + DataType::Float64, + "AVG(latency) resolves to Float64" + ); } -mod window_and_errors { - use super::*; - - // ── Test: Window functions ──────────────────────────────────────────────────── - - #[tokio::test] - async fn test_window_function_produces_window_func_node() { - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower( - "SELECT region, AVG(value) OVER (PARTITION BY region ORDER BY ts) \ - FROM metrics", - ) - .await - .unwrap(); - - fn has_window_func(e: &QueryExpr) -> bool { - match e { - QueryExpr::WindowFunc { .. } => true, - QueryExpr::Project { child, .. } => has_window_func(&child.expr), - _ => false, - } - } - assert!(has_window_func(&result), "expected a WindowFunc node"); - } - - // ── Tests: Error cases ──────────────────────────────────────────────────────── - - #[tokio::test] - async fn test_join_returns_error() { - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let err = lowerer - .lower("SELECT a.ts FROM metrics a JOIN metrics b ON a.ts = b.ts") - .await - .unwrap_err(); - assert!( - matches!(err, LoweringError::UnsupportedFeature(ref msg) if msg.contains("JOIN")), - "unexpected error: {err}" - ); - } - - #[tokio::test] - async fn test_subquery_returns_error() { - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let err = lowerer - .lower("SELECT * FROM (SELECT value FROM metrics) sub") - .await - .unwrap_err(); - assert!( - matches!(err, LoweringError::UnsupportedFeature(ref msg) if msg.to_lowercase().contains("subquery")), - "unexpected error: {err}" - ); - } +#[tokio::test] +async fn single_agg_group_by_keeps_key_in_output_schema() { + // A tabular single-aggregate GROUP BY routes through the positional + // Aggregate.by path (not the PromQL fused-Partition shape), so the group + // key is a real output column the enclosing SELECT projection resolves. + let qe = lower("SELECT service, SUM(bytes) FROM metrics GROUP BY service").await; + let (by, aggs) = find_aggregate(&qe).expect("expected an Aggregate (not a Partition)"); + assert_eq!(by, &vec![1], "GROUP BY service → Aggregate.by column 1"); + assert!(matches!(aggs.as_slice(), [AggIntent::Sum { col: Some(3) }])); + + // Both the group key and the aggregate resolve in the root projection schema. + let schema = qe.output_schema().expect("root projection schema"); + assert_eq!(schema.columns.len(), 2); + assert_eq!( + schema.columns[0].dtype, + DataType::Utf8, + "service is in the output" + ); + assert_eq!(schema.columns[1].dtype, DataType::Int64, "SUM(bytes)"); } -mod predicates_and_exprs { - use super::*; - - // ── Tests: Predicate / ProjectItem / SortKey content ───────────────────────── - - #[tokio::test] - async fn test_filter_predicate_references_filtered_column() { - // WHERE value > 5.0 should produce a Predicate that references "value". - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT ts FROM metrics WHERE value > 5.0") - .await - .unwrap(); - - let pred = find_predicate(&result).expect("expected Filter predicate"); - let refs = pred.columns_referenced(); - assert!( - refs.iter().any(|r| r.0 == "value"), - "expected 'value' column ref in predicate" - ); - } - - #[tokio::test] - async fn test_filter_two_non_time_predicates_is_bool_and() { - // WHERE value > 0 AND region = 'us' — both non-time → BoolAnd with 2 conjuncts. - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT ts FROM metrics WHERE value > 0 AND region = 'us'") - .await - .unwrap(); - - let pred = find_predicate(&result).expect("expected Filter predicate"); - assert_eq!( - pred.conjuncts().len(), - 2, - "expected BoolAnd with 2 conjuncts, got: {pred:?}" - ); - } - - #[tokio::test] - async fn test_project_items_carry_column_refs() { - // SELECT id, value FROM events → two ProjectItems, each with a Column expr. - let catalog = no_time_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer.lower("SELECT id, value FROM events").await.unwrap(); - - let items = find_project_items(&result).expect("expected Project node"); - assert_eq!(items.len(), 2, "expected 2 ProjectItems"); - - let col_names: Vec<&str> = items - .iter() - .filter_map(|pi| { - if let L3Expr::Column(c) = &pi.expr { - Some(c.0.as_str()) - } else { - None - } - }) - .collect(); - assert!(col_names.contains(&"id"), "expected 'id' in project items"); - assert!( - col_names.contains(&"value"), - "expected 'value' in project items" - ); - } - - #[tokio::test] - async fn test_sort_key_ascending_flag() { - // ORDER BY id ASC → SortKey with ascending = true. - let catalog = no_time_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT id FROM events ORDER BY id ASC") - .await - .unwrap(); - - let keys = find_sort_keys(&result).expect("expected Sort node"); - assert_eq!(keys.len(), 1); - assert!(keys[0].ascending, "expected ascending sort key"); - } - - #[tokio::test] - async fn test_sort_key_descending_flag() { - // ORDER BY value DESC → SortKey with ascending = false. - let catalog = no_time_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT id FROM events ORDER BY value DESC") - .await - .unwrap(); - - let keys = find_sort_keys(&result).expect("expected Sort node"); - assert_eq!(keys.len(), 1); - assert!(!keys[0].ascending, "expected descending sort key"); - } - - // ── Tests: df_expr_to_l3 edge cases ────────────────────────────────────────── - - #[tokio::test] - async fn test_filter_is_null_predicate() { - // WHERE value IS NULL → IsNull(Column("value")) - let catalog = no_time_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT id FROM events WHERE value IS NULL") - .await - .unwrap(); - - let pred = find_predicate(&result).expect("expected Filter predicate"); - assert!( - matches!(pred, L3Expr::IsNull(inner) if matches!(inner.as_ref(), L3Expr::Column(c) if c.0 == "value")), - "expected IsNull(Column(\"value\")), got: {pred:?}" - ); - } - - #[tokio::test] - async fn test_filter_in_list_predicate() { - // WHERE id IN (1, 2, 3) → InList { expr: Column("id"), list: [..], negated: false } - let catalog = no_time_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT id FROM events WHERE id IN (1, 2, 3)") - .await - .unwrap(); - - let pred = find_predicate(&result).expect("expected Filter predicate"); - let L3Expr::InList { - expr, - list, - negated, - } = pred - else { - panic!("expected InList, got: {pred:?}"); - }; - assert!(matches!(expr.as_ref(), L3Expr::Column(c) if c.0 == "id")); - assert_eq!(list.len(), 3); - assert!(!negated); - } - - #[tokio::test] - async fn test_filter_between_normalizes_to_bool_and() { - // WHERE value BETWEEN 0 AND 100 → BoolAnd([Compare(Ge, 0), Compare(Le, 100)]) - let catalog = no_time_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT id FROM events WHERE value BETWEEN 0.0 AND 100.0") - .await - .unwrap(); - - let pred = find_predicate(&result).expect("expected Filter predicate"); - let conjuncts = pred.conjuncts(); - assert_eq!( - conjuncts.len(), - 2, - "BETWEEN should produce 2 conjuncts, got: {pred:?}" - ); - // First conjunct: value >= 0, second: value <= 100 - assert!(matches!( - &conjuncts[0], - L3Expr::Compare { - op: CompareOp::Ge, - .. - } - )); - assert!(matches!( - &conjuncts[1], - L3Expr::Compare { - op: CompareOp::Le, - .. - } - )); - } +#[tokio::test] +async fn count_ranked_topk_is_heavy_hitter() { + // `ORDER BY COUNT(*) DESC LIMIT k` over a single COUNT aggregate is the one + // case the heavy-hitter (frequency) sketch is correct for. (The key must + // reference the count output directly; an alias would safely fall back to a + // generic Sort+Limit.) + let qe = lower( + "SELECT service, COUNT(*) FROM metrics GROUP BY service ORDER BY COUNT(*) DESC LIMIT 10", + ) + .await; + let (by, aggs) = find_aggregate(&qe).expect("expected an Aggregate"); + assert_eq!(by, &vec![1], "GROUP BY service → col 1"); + assert!( + matches!(aggs.as_slice(), [AggIntent::TopK { k: 10, .. }]), + "count-ranked topk → heavy-hitter TopK, got {aggs:?}" + ); } -mod batch_and_catalog { - use super::*; - - // ── Tests: lower_batch ──────────────────────────────────────────────────────── - - #[tokio::test] - async fn test_lower_batch_empty_returns_empty_vec() { - let catalog = metrics_catalog(); - let workload = QueryWorkload { - language: QueryLanguage::SQL(SqlDialect::DataFusionSQL), - query_batch: None, - repeating_queries: None, - data_characteristics: None, - }; - let results = lower_batch(&workload, &catalog).await; - assert!(results.is_empty()); - } - - #[tokio::test] - async fn test_lower_batch_two_valid_queries() { - let catalog = metrics_catalog(); - let workload = make_workload(vec![ - "SELECT COUNT(*) FROM metrics", - "SELECT SUM(value) FROM metrics", - ]); - let results = lower_batch(&workload, &catalog).await; - assert_eq!(results.len(), 2); - assert!(results[0].is_ok(), "first query should succeed"); - assert!(results[1].is_ok(), "second query should succeed"); - } - - #[tokio::test] - async fn test_lower_batch_error_is_per_query() { - // A bad query in the batch should not prevent the good ones from being lowered. - let catalog = metrics_catalog(); - let workload = make_workload(vec!["SELECT COUNT(*) FROM metrics", "NOT VALID SQL !!!"]); - let results = lower_batch(&workload, &catalog).await; - assert_eq!(results.len(), 2); - assert!(results[0].is_ok(), "first (valid) query should succeed"); - assert!(results[1].is_err(), "second (invalid) query should fail"); - } - - #[tokio::test] - async fn test_unknown_table_returns_error() { - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let err = lowerer - .lower("SELECT x FROM ghost_table") - .await - .unwrap_err(); - // DataFusion rejects the unknown table at plan time before our lowerer runs. - assert!( - matches!(err, LoweringError::DataFusion(_)), - "unexpected error variant: {err}" - ); - } - - // ── Tests: per-table catalog validation (#9) ────────────────────────────────── - - #[tokio::test] - async fn test_bad_time_column_fails_query_touching_that_table() { - let mut tables = HashMap::new(); - // Valid table. - tables.insert( - "metrics".to_string(), - TableSchema { - columns: vec![ColumnDef { - name: "value".to_string(), - data_type: L3DataType::Float64, - nullable: true, - }], - time_column: None, - }, - ); - // Table whose time_column doesn't exist in its columns list. - tables.insert( - "broken".to_string(), - TableSchema { - columns: vec![ColumnDef { - name: "id".to_string(), - data_type: L3DataType::Int64, - nullable: false, - }], - time_column: Some("nonexistent".to_string()), - }, - ); - let catalog = SchemaCatalog { tables }; - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - - // Query on the valid table must succeed. - let result = lowerer.lower("SELECT value FROM metrics").await; - assert!( - result.is_ok(), - "query on valid table should succeed: {result:?}" - ); - - // Query on the broken table must fail with InvalidExpression. - let err = lowerer - .lower("SELECT id FROM broken WHERE id > 0") - .await - .unwrap_err(); - assert!( - matches!(err, LoweringError::InvalidExpression(_)), - "expected InvalidExpression for bad time_column, got: {err}" - ); - } - - // ── Tests: language guard ───────────────────────────────────────────────────── - - #[tokio::test] - async fn test_promql_workload_returns_wrong_language() { - let catalog = metrics_catalog(); - let workload = QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: Some(vec![BatchEntry { - query: Query("some_metric".into()), - requirements: None, - }]), - repeating_queries: None, - data_characteristics: None, - }; - let results = lower_batch(&workload, &catalog).await; - assert_eq!(results.len(), 1); - assert!( - matches!(results[0], Err(LoweringError::WrongLanguage(_))), - "expected WrongLanguage, got: {:?}", - results[0] - ); - } - - #[tokio::test] - async fn test_elastic_dsl_workload_returns_wrong_language() { - let catalog = metrics_catalog(); - let workload = QueryWorkload { - language: QueryLanguage::ElasticDSL, - query_batch: Some(vec![BatchEntry { - query: Query("{\"query\":{}}".into()), - requirements: None, - }]), - repeating_queries: None, - data_characteristics: None, - }; - let results = lower_batch(&workload, &catalog).await; - assert_eq!(results.len(), 1); - assert!(matches!(results[0], Err(LoweringError::WrongLanguage(_)))); - } - - #[tokio::test] - async fn test_datafusion_language_accepted() { - let catalog = metrics_catalog(); - let workload = QueryWorkload { - language: QueryLanguage::DataFusion, - query_batch: Some(vec![BatchEntry { - query: Query("SELECT COUNT(*) FROM metrics".into()), - requirements: None, - }]), - repeating_queries: None, - data_characteristics: None, - }; - let results = lower_batch(&workload, &catalog).await; - assert_eq!(results.len(), 1); - assert!(results[0].is_ok(), "DataFusion language should be accepted"); - } +#[tokio::test] +async fn non_count_ranked_limit_keeps_the_aggregate() { + // Ranking by AVG (not a count) must NOT become a frequency heavy-hitter — + // the AVG aggregate has to survive as a generic Sort+Limit. + let qe = lower( + "SELECT service, AVG(latency) AS a FROM metrics GROUP BY service ORDER BY a DESC LIMIT 10", + ) + .await; + let (_, aggs) = find_aggregate(&qe).expect("expected an Aggregate"); + assert!( + aggs.iter().any(|a| matches!(a, AggIntent::Avg { .. })), + "AVG must be preserved, got {aggs:?}" + ); + assert!( + !aggs.iter().any(|a| matches!(a, AggIntent::TopK { .. })), + "AVG ranking must not become a frequency heavy-hitter, got {aggs:?}" + ); } -mod time_and_dialects { - use super::*; - - // ── Tests: BETWEEN time extraction ─────────────────────────────────────────── - - #[tokio::test] - async fn test_time_between_extracted_to_source() { - // WHERE ts BETWEEN 1000 AND 2000 should set start_ms=1000, end_ms=2000. - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT value FROM metrics WHERE ts BETWEEN 1000 AND 2000") - .await - .unwrap(); - - let source = find_source(&result).unwrap(); - let Source::Table { time_range, .. } = source else { - panic!("expected Source::Table"); - }; - let tr = time_range - .as_ref() - .expect("expected time_range from BETWEEN"); - assert_eq!(tr.start_ms, Some(1000)); - assert_eq!(tr.end_ms, Some(2000)); - } - - #[tokio::test] - async fn test_time_between_leaves_no_filter_wrapper() { - // A BETWEEN-only predicate on the time column has no non-time residual, - // so no Filter node should appear. - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT value FROM metrics WHERE ts BETWEEN 1000 AND 2000") - .await - .unwrap(); - - fn has_filter(e: &QueryExpr) -> bool { - match e { - QueryExpr::Filter { .. } => true, - QueryExpr::Project { child, .. } => has_filter(&child.expr), - _ => false, - } - } - assert!( - !has_filter(&result), - "BETWEEN on time col should leave no Filter wrapper" - ); - } - - #[tokio::test] - async fn test_time_between_mixed_with_non_time_predicate() { - // WHERE ts BETWEEN 1000 AND 2000 AND region = 'us': - // time range should be extracted; Filter stays for the non-time conjunct. - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT value FROM metrics WHERE ts BETWEEN 1000 AND 2000 AND region = 'us'") - .await - .unwrap(); - - let source = find_source(&result).unwrap(); - let Source::Table { time_range, .. } = source else { - panic!("expected Source::Table"); - }; - let tr = time_range.as_ref().expect("expected time_range"); - assert_eq!(tr.start_ms, Some(1000)); - assert_eq!(tr.end_ms, Some(2000)); - - fn has_filter(e: &QueryExpr) -> bool { - match e { - QueryExpr::Filter { .. } => true, - QueryExpr::Project { child, .. } => has_filter(&child.expr), - _ => false, - } - } - assert!( - has_filter(&result), - "expected Filter for non-time predicate" - ); - } - - // ── Tests: multi-dialect SQL ────────────────────────────────────────────────── - - #[tokio::test] - async fn test_clickhouse_dialect_returns_unsupported_dialect() { - let catalog = metrics_catalog(); - let workload = QueryWorkload { - language: QueryLanguage::SQL(SqlDialect::ClickhouseSQL), - query_batch: Some(vec![BatchEntry { - query: Query("SELECT COUNT(*) FROM metrics".into()), - requirements: None, - }]), - repeating_queries: None, - data_characteristics: None, - }; - let results = lower_batch(&workload, &catalog).await; - assert_eq!(results.len(), 1); - assert!( - matches!(results[0], Err(LoweringError::UnsupportedDialect(_))), - "expected UnsupportedDialect, got: {:?}", - results[0] - ); - } - - #[tokio::test] - async fn test_elastic_sql_dialect_returns_unsupported_dialect() { - let catalog = metrics_catalog(); - let workload = QueryWorkload { - language: QueryLanguage::SQL(SqlDialect::ElasticSQL), - query_batch: Some(vec![BatchEntry { - query: Query("{\"query\":{}}".into()), - requirements: None, - }]), - repeating_queries: None, - data_characteristics: None, - }; - let results = lower_batch(&workload, &catalog).await; - assert_eq!(results.len(), 1); - assert!( - matches!(results[0], Err(LoweringError::UnsupportedDialect(_))), - "expected UnsupportedDialect, got: {:?}", - results[0] - ); - } - - #[tokio::test] - async fn test_datafusion_sql_dialect_accepted() { - let catalog = metrics_catalog(); - let workload = QueryWorkload { - language: QueryLanguage::SQL(SqlDialect::DataFusionSQL), - query_batch: Some(vec![BatchEntry { - query: Query("SELECT COUNT(*) FROM metrics".into()), - requirements: None, - }]), - repeating_queries: None, - data_characteristics: None, - }; - let results = lower_batch(&workload, &catalog).await; - assert_eq!(results.len(), 1); - assert!( - results[0].is_ok(), - "SQL(DataFusionSQL) should be accepted, got: {:?}", - results[0] - ); - } +#[tokio::test] +async fn distinct_value_reducer_is_rejected_not_dropped() { + // L3 has no distinct-Sum; SUM(DISTINCT x) must be rejected, not silently + // lowered as SUM(x). + let res = lower_sql( + "SELECT SUM(DISTINCT bytes) FROM metrics", + &catalog(), + AccuracyTarget::Exact, + ) + .await; + assert!(res.is_err(), "SUM(DISTINCT ...) should be rejected"); } -mod column_and_set_ops { - use super::*; - - // ── Tests: Source::Table.columns ───────────────────────────────────────────── - - #[tokio::test] - async fn test_scan_columns_populated_from_projection() { - // SELECT ts, value FROM metrics — DataFusion should push the projection - // into the TableScan; Source::Table.columns should name the two columns. - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT ts, value FROM metrics") - .await - .unwrap(); - - let source = find_source(&result).expect("expected a Scan"); - let Source::Table { columns, .. } = source else { - panic!("expected Source::Table"); - }; - // At minimum, each named column should appear in the list. - let names: Vec<&str> = columns.iter().map(|c| c.0.as_str()).collect(); - assert!( - names.contains(&"ts"), - "expected 'ts' in columns, got: {names:?}" - ); - assert!( - names.contains(&"value"), - "expected 'value' in columns, got: {names:?}" - ); - } - - #[tokio::test] - async fn test_scan_columns_empty_for_select_star() { - // SELECT * — no projection pushdown; Source::Table.columns stays empty - // (meaning "all columns"; cost estimator treats empty as unconstrained). - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer.lower("SELECT * FROM metrics").await.unwrap(); - - let source = find_source(&result).expect("expected a Scan"); - let Source::Table { columns, .. } = source else { - panic!("expected Source::Table"); - }; - assert!( - columns.is_empty(), - "SELECT * should produce no column constraints, got: {columns:?}" - ); - } - - // ── Tests: push_columns_into_scan gap for aggregate topology (#4) ──────────── - - #[tokio::test] - async fn test_scan_columns_empty_for_aggregate_topology() { - // push_columns_into_scan only rewrites a Scan that is the direct child of - // a Project. When there is an Aggregate between Project and Scan (the normal - // GROUP BY shape), column pushdown is skipped and Scan.columns stays empty - // ("all columns" semantics). This test documents the known gap so that any - // future fix must also update this assertion. - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT region, COUNT(*) FROM metrics GROUP BY region") - .await - .unwrap(); - - let source = find_source(&result).expect("expected a Scan node"); - let Source::Table { columns, .. } = source else { - panic!("expected Source::Table"); - }; - assert!( - columns.is_empty(), - "Project→Aggregate→Scan: Scan.columns should be empty (all-columns semantics), got: {columns:?}" - ); - } - - // ── Tests: UNION / UNION ALL ────────────────────────────────────────────────── - - #[tokio::test] - async fn test_union_all_produces_set_op_union_node() { - // UNION ALL — no dedup — maps to SetOp { kind: Union, all: true }. - let catalog = no_time_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT id FROM events UNION ALL SELECT id FROM events") - .await - .unwrap(); - - fn find_set_op(e: &QueryExpr) -> Option<(&SetOpKind, bool)> { - match e { - QueryExpr::SetOp { kind, all, .. } => Some((kind, *all)), - QueryExpr::Project { child, .. } => find_set_op(&child.expr), - _ => None, - } - } - let (kind, all) = find_set_op(&result).expect("expected SetOp node"); - assert!(matches!(kind, SetOpKind::Union)); - assert!(all, "UNION ALL should set all=true"); - } - - #[tokio::test] - async fn test_union_distinct_produces_distinct_over_set_op() { - // UNION (without ALL) = UNION DISTINCT = Distinct wrapping SetOp. - let catalog = no_time_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT id FROM events UNION SELECT id FROM events") - .await - .unwrap(); - - fn has_distinct(e: &QueryExpr) -> bool { - match e { - QueryExpr::Distinct { .. } => true, - QueryExpr::Project { child, .. } => has_distinct(&child.expr), - _ => false, - } - } - assert!( - has_distinct(&result), - "UNION DISTINCT should produce a Distinct node" - ); - } +#[tokio::test] +async fn aggregate_over_non_column_expression_is_rejected() { + // L3 reduces a column, not an arbitrary expression — SUM(bytes + 1) must be + // rejected rather than silently reducing a probe column. + let res = lower_sql( + "SELECT SUM(bytes + 1) FROM metrics", + &catalog(), + AccuracyTarget::Exact, + ) + .await; + assert!(res.is_err(), "SUM() should be rejected"); } -mod expressions { - use super::*; - - // ── Tests: arithmetic / negative / LIKE / CASE in expressions ──────────────── - - #[tokio::test] - async fn test_arithmetic_in_projection_succeeds() { - // SELECT value * 2 FROM events — should not return UnsupportedFeature. - let catalog = no_time_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer.lower("SELECT value * 2.0 FROM events").await; - assert!( - result.is_ok(), - "arithmetic in projection should succeed, got: {result:?}" - ); - } - - #[tokio::test] - async fn test_arithmetic_in_predicate_succeeds() { - // WHERE value * 0.9 > 5.0 — should not return UnsupportedFeature. - let catalog = no_time_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT id FROM events WHERE value * 0.9 > 5.0") - .await; - assert!( - result.is_ok(), - "arithmetic in predicate should succeed, got: {result:?}" - ); - } - - #[tokio::test] - async fn test_arithmetic_predicate_references_column() { - // Predicate from `value * 0.9 > 5` should still reference "value". - let catalog = no_time_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT id FROM events WHERE value * 0.9 > 5.0") - .await - .unwrap(); - let pred = find_predicate(&result).expect("expected Filter predicate"); - let refs = pred.columns_referenced(); - assert!( - refs.iter().any(|r| r.0 == "value"), - "expected 'value' in predicate refs" - ); - } - - #[tokio::test] - async fn test_negative_literal_in_predicate_succeeds() { - // WHERE value > -1 — unary minus should not return UnsupportedFeature. - let catalog = no_time_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT id FROM events WHERE value > -1.0") - .await; - assert!( - result.is_ok(), - "unary minus in predicate should succeed, got: {result:?}" - ); - } - - #[tokio::test] - async fn test_like_predicate_produces_compare_like() { - // WHERE name LIKE 'a%' → Compare { op: Like, .. } - let catalog = no_time_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT id FROM events WHERE name LIKE 'a%'") - .await - .unwrap(); - let pred = find_predicate(&result).expect("expected Filter predicate"); - assert!( - matches!( - pred, - L3Expr::Compare { - op: CompareOp::Like, - .. - } - ), - "expected Compare(Like), got: {pred:?}" - ); - } - - #[tokio::test] - async fn test_ilike_predicate_produces_compare_ilike() { - // WHERE name ILIKE 'A%' → Compare { op: ILike, .. } - let catalog = no_time_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT id FROM events WHERE name ILIKE 'A%'") - .await - .unwrap(); - let pred = find_predicate(&result).expect("expected Filter predicate"); - assert!( - matches!( - pred, - L3Expr::Compare { - op: CompareOp::ILike, - .. - } - ), - "expected Compare(ILike), got: {pred:?}" - ); - } - - #[tokio::test] - async fn test_case_in_projection_succeeds() { - // CASE WHEN value > 5 THEN 1 ELSE 0 END — should not return UnsupportedFeature. - let catalog = no_time_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT CASE WHEN value > 5.0 THEN 1 ELSE 0 END AS tier FROM events") - .await; - assert!( - result.is_ok(), - "CASE WHEN in projection should succeed, got: {result:?}" - ); - } - - #[tokio::test] - async fn test_case_projection_item_is_case_expr() { - let catalog = no_time_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT CASE WHEN value > 5.0 THEN 1 ELSE 0 END AS tier FROM events") - .await - .unwrap(); - let items = find_project_items(&result).expect("expected Project node"); - let has_case = items - .iter() - .any(|pi| matches!(pi.expr, L3Expr::Case { .. })); - assert!( - has_case, - "expected a Case expr in project items, got: {items:?}" - ); - } +#[tokio::test] +async fn count_star_is_count_intent() { + let qe = lower("SELECT COUNT(*) FROM metrics").await; + let (by, aggs) = find_aggregate(&qe).expect("expected an Aggregate"); + assert!(by.is_empty()); + assert!(matches!(aggs.as_slice(), [AggIntent::Count { .. }])); } -mod schemas { - use super::*; - - // ── Tests: populate_schemas ─────────────────────────────────────────────────── - - #[tokio::test] - async fn test_populate_schemas_scan_gets_catalog_schema() { - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let expr = lowerer.lower("SELECT * FROM metrics").await.unwrap(); - - let typed = populate_schemas(expr, &catalog); - assert_eq!( - typed.schema.fields.len(), - 4, - "scan schema should have 4 fields from catalog" - ); - assert_eq!(typed.schema.fields[0].name, "ts"); - assert_eq!(typed.schema.time_index, Some(0)); - } - - #[tokio::test] - async fn test_populate_schemas_project_gives_subset_schema() { - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let expr = lowerer - .lower("SELECT ts, value FROM metrics") - .await - .unwrap(); - - let typed = populate_schemas(expr, &catalog); - assert_eq!(typed.schema.fields.len(), 2); - assert_eq!(typed.schema.fields[0].name, "ts"); - assert_eq!(typed.schema.fields[1].name, "value"); - assert_eq!(typed.schema.time_index, Some(0)); - } - - #[tokio::test] - async fn test_populate_schemas_inner_nodes_are_typed() { - // The child of the root Project (a Scan) should also have a non-empty schema. - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let expr = lowerer - .lower("SELECT ts, value FROM metrics") - .await - .unwrap(); - - let typed = populate_schemas(expr, &catalog); - let QueryExpr::Project { child, .. } = &typed.expr else { - panic!("expected Project at root"); - }; - assert!( - !child.schema.fields.is_empty(), - "child Scan should have a populated schema after populate_schemas" - ); - } - - #[tokio::test] - async fn test_populate_schemas_aggregate() { - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let expr = lowerer - .lower("SELECT region, COUNT(*) FROM metrics GROUP BY region") - .await - .unwrap(); - - let typed = populate_schemas(expr, &catalog); - // Root is Project wrapping Aggregate; two fields: region + count col. - assert_eq!(typed.schema.fields.len(), 2); - assert_eq!(typed.schema.fields[0].name, "region"); - } - - // ── Tests: AggIntent column-type propagation ────────────────────────────────── - - #[tokio::test] - async fn test_min_int_col_gives_int64_in_aggregate_schema() { - // MIN(ts) where ts: Int64 → the root schema (through the outer Projection) - // should resolve the aggregate output type as Int64, not Float64. - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let expr = lowerer.lower("SELECT MIN(ts) FROM metrics").await.unwrap(); - let typed = populate_schemas(expr, &catalog); - - let dtype = typed - .schema - .fields - .last() - .map(|f| f.dtype.clone()) - .expect("root schema should have at least one field"); - assert_eq!( - dtype, - L3DataType::Int64, - "MIN(ts: Int64) should propagate Int64 through to root schema, got {:?}", - dtype - ); - } - - #[tokio::test] - async fn test_max_utf8_col_gives_utf8_in_aggregate_schema() { - // MAX(region) where region: Utf8 → root schema should carry Utf8. - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let expr = lowerer - .lower("SELECT MAX(region) FROM metrics") - .await - .unwrap(); - let typed = populate_schemas(expr, &catalog); - - let dtype = typed - .schema - .fields - .last() - .map(|f| f.dtype.clone()) - .expect("root schema should have at least one field"); - assert_eq!( - dtype, - L3DataType::Utf8, - "MAX(region: Utf8) should propagate Utf8 through to root schema, got {:?}", - dtype - ); - } - - #[tokio::test] - async fn test_sum_float_col_gives_float64_in_aggregate_schema() { - // SUM(value) where value: Float64 → Float64. - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let expr = lowerer - .lower("SELECT SUM(value) FROM metrics") - .await - .unwrap(); - let typed = populate_schemas(expr, &catalog); - - let dtype = typed - .schema - .fields - .last() - .map(|f| f.dtype.clone()) - .expect("root schema should have at least one field"); - assert_eq!(dtype, L3DataType::Float64); - } - - #[tokio::test] - async fn test_agg_col_name_tracked_via_col_field() { - // Verify that AggIntent.col() carries the aggregated column name so - // schema derivation can resolve types without re-parsing SQL. - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer.lower("SELECT MIN(ts) FROM metrics").await.unwrap(); - - let (_, aggs) = find_aggregate(&result).expect("expected Aggregate"); - let AggIntent::Min { col } = &aggs[0] else { - panic!("expected Min, got {:?}", aggs[0]); - }; - assert_eq!( - col.as_ref().map(|c| c.0.as_str()), - Some("ts"), - "Min should track the aggregated column name" - ); - } - - // ── Tests: NthValue N extraction (#10) ─────────────────────────────────────── - - #[tokio::test] - async fn test_nth_value_extracts_n_from_args() { - // NTH_VALUE(value, 2) OVER (ORDER BY ts) → WindowFuncKind::NthValue(2), not NthValue(0). - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT NTH_VALUE(value, 2) OVER (ORDER BY ts) FROM metrics") - .await - .unwrap(); - - fn find_window_kind( - e: &QueryExpr, - ) -> Option<&asap_control_core::intent_algebra::expr::WindowFuncKind> { - match e { - QueryExpr::WindowFunc { func, .. } => Some(func), - QueryExpr::Project { child, .. } => find_window_kind(&child.expr), - _ => None, - } - } - let kind = find_window_kind(&result).expect("expected WindowFunc node"); - assert!( - matches!( - kind, - asap_control_core::intent_algebra::expr::WindowFuncKind::NthValue(Some(2)) - ), - "expected NthValue(Some(2)), got: {kind:?}" - ); - } - - // ── Tests: lowering-time failure (#11) ─────────────────────────────────────── - - #[tokio::test] - async fn test_unsupported_aggregate_returns_error_at_lowering_time() { - // array_agg parses fine but is not in our AggIntent mapping → UnsupportedAggregate. - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let err = lowerer - .lower("SELECT array_agg(value) FROM metrics") - .await - .unwrap_err(); - assert!( - matches!(err, LoweringError::UnsupportedAggregate(_)), - "expected UnsupportedAggregate, got: {err}" - ); - } - - #[tokio::test] - async fn test_unsupported_agg_isolation_in_batch() { - // The bad query (unsupported agg) should not prevent the good one from succeeding. - let catalog = metrics_catalog(); - let workload = make_workload(vec![ - "SELECT COUNT(*) FROM metrics", - "SELECT array_agg(value) FROM metrics", - ]); - let results = lower_batch(&workload, &catalog).await; - assert_eq!(results.len(), 2); - assert!(results[0].is_ok(), "valid query should succeed"); - assert!(results[1].is_err(), "unsupported agg should fail"); - assert!( - matches!(results[1], Err(LoweringError::UnsupportedAggregate(_))), - "expected UnsupportedAggregate, got: {:?}", - results[1] - ); - } - - // ── Tests: 4-predicate WHERE with time extraction (#13) ────────────────────── - - #[tokio::test] - async fn test_four_predicate_where_extracts_time_and_keeps_filter() { - // WHERE ts > 1000 AND ts < 2000 AND region = 'us' AND value > 0.0 - // All four predicates in a single Filter node (DataFusion keeps them - // as one conjunctive Filter on the unoptimized plan). Time bounds should - // be extracted; the remaining two non-time predicates should stay as Filter. - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower( - "SELECT value FROM metrics \ - WHERE ts > 1000 AND ts < 2000 AND region = 'us' AND value > 0.0", - ) - .await - .unwrap(); - - let source = find_source(&result).expect("expected a Scan node"); - let Source::Table { time_range, .. } = source else { - panic!("expected Source::Table"); - }; - let tr = time_range.as_ref().expect("expected time_range extracted"); - assert_eq!(tr.start_ms, Some(1000)); - assert_eq!(tr.end_ms, Some(2000)); - - fn has_filter(e: &QueryExpr) -> bool { - match e { - QueryExpr::Filter { .. } => true, - QueryExpr::Project { child, .. } => has_filter(&child.expr), - _ => false, - } - } - assert!( - has_filter(&result), - "expected Filter node for the two non-time predicates" - ); - } - - #[tokio::test] - async fn test_filter_predicate_conjuncts_count_for_two_non_time_preds() { - // Same query: the remaining Filter predicate should have 2 conjuncts. - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower( - "SELECT value FROM metrics \ - WHERE ts > 1000 AND ts < 2000 AND region = 'us' AND value > 0.0", - ) - .await - .unwrap(); - - let pred = find_predicate(&result).expect("expected Filter predicate"); - assert_eq!( - pred.conjuncts().len(), - 2, - "expected 2 non-time conjuncts, got: {pred:?}" - ); - } +#[tokio::test] +async fn count_distinct_is_cardinality() { + let qe = lower("SELECT COUNT(DISTINCT service) FROM metrics").await; + let (_, aggs) = find_aggregate(&qe).expect("expected an Aggregate"); + assert!(matches!(aggs.as_slice(), [AggIntent::Cardinality { .. }])); +} - // ── Tests: multi-window error (#14) ────────────────────────────────────────── +#[tokio::test] +async fn select_distinct_lowers_to_distinct_with_positional_cols() { + // SELECT DISTINCT → a `Distinct` node whose `cols` are positional ColumnIds + // (not name-based ColumnRefs). DataFusion's `Distinct::All` dedups on every + // column, so `cols` is empty here — but the field type is now `Vec`. + let qe = lower("SELECT DISTINCT service FROM metrics").await; + let QueryExpr::Distinct { cols, .. } = &qe else { + panic!("expected a Distinct at the root, got {qe:?}"); + }; + let _: &Vec = cols; // compile-time: positional ids, not ColumnRefs + assert!(cols.is_empty(), "DISTINCT * dedups on all columns"); +} - #[tokio::test] - async fn test_multiple_window_funcs_same_over_returns_error() { - // DataFusion may fold two window functions with identical OVER clauses - // into one Window plan node. Our lowerer should surface an explicit error - // rather than silently dropping the second function. - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower( - "SELECT \ - SUM(value) OVER (PARTITION BY region ORDER BY ts), \ - AVG(value) OVER (PARTITION BY region ORDER BY ts) \ - FROM metrics", - ) - .await; +#[tokio::test] +async fn inner_join_lowers_to_join_over_two_scans() { + // INNER JOIN over two distinct tables → L3 Join with both leaves as Scans. + let qe = lower( + "SELECT metrics.bytes, hosts.region \ + FROM metrics JOIN hosts ON metrics.service = hosts.service", + ) + .await; + let join = find_join(&qe).expect("expected a Join in the tree"); + let QueryExpr::Join { + kind, left, right, .. + } = join + else { + unreachable!("find_join only returns Join"); + }; + assert_eq!(*kind, JoinKind::Inner); + assert!(matches!(left.as_ref(), QueryExpr::Scan { .. })); + assert!(matches!(right.as_ref(), QueryExpr::Scan { .. })); +} - match &result { - Err(e) => assert!( - matches!(e, LoweringError::UnsupportedFeature(msg) if msg.contains("multiple window")), - "unexpected error: {e}" - ), - Ok(expr) => { - // DataFusion split them into separate Window nodes — both must be present. - fn count_window_funcs(e: &QueryExpr) -> usize { - match e { - QueryExpr::WindowFunc { child, .. } => 1 + count_window_funcs(&child.expr), - QueryExpr::Project { child, .. } => count_window_funcs(&child.expr), - _ => 0, - } - } - assert_eq!( - count_window_funcs(expr), - 2, - "when DataFusion splits window funcs both must appear in the lowered tree" - ); - } +/// The two `ColumnId`s an equijoin predicate `Column(l) = Column(r)` binds to, +/// returned sorted so the assertion is independent of left/right ordering. +fn join_eq_columns(join: &QueryExpr) -> [usize; 2] { + let QueryExpr::Join { pred, .. } = join else { + unreachable!("expected a Join"); + }; + let L3Expr::Compare { + left, + op: CompareOp::Eq, + right, + } = &pred.0 + else { + panic!("expected an equijoin Compare, got {:?}", pred.0); + }; + match (left.as_ref(), right.as_ref()) { + (L3Expr::Column(l), L3Expr::Column(r)) => { + let mut cols = [*l, *r]; + cols.sort_unstable(); + cols } + other => panic!("expected Column = Column, got {other:?}"), } +} - // ── Tests: missing predicate coverage ──────────────────────────────────────── - - #[tokio::test] - async fn test_filter_is_not_null_predicate() { - // WHERE value IS NOT NULL → IsNotNull(Column("value")) - let catalog = no_time_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT id FROM events WHERE value IS NOT NULL") - .await - .unwrap(); - let pred = find_predicate(&result).expect("expected Filter predicate"); - assert!( - matches!(pred, L3Expr::IsNotNull(inner) if matches!(inner.as_ref(), L3Expr::Column(c) if c.0 == "value")), - "expected IsNotNull(Column(\"value\")), got: {pred:?}" - ); - } - - #[tokio::test] - async fn test_filter_not_in_list() { - // WHERE id NOT IN (1, 2, 3) → InList { negated: true } - let catalog = no_time_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT id FROM events WHERE id NOT IN (1, 2, 3)") - .await - .unwrap(); - let pred = find_predicate(&result).expect("expected Filter predicate"); - let L3Expr::InList { - expr, - list, - negated, - } = pred - else { - panic!("expected InList, got: {pred:?}"); - }; - assert!(matches!(expr.as_ref(), L3Expr::Column(c) if c.0 == "id")); - assert_eq!(list.len(), 3); - assert!(negated, "NOT IN should set negated=true"); - } - - #[tokio::test] - async fn test_not_between_normalizes_to_bool_or() { - // WHERE value NOT BETWEEN 0 AND 100 → BoolOr([Compare(Lt), Compare(Gt)]) - let catalog = no_time_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT id FROM events WHERE value NOT BETWEEN 0.0 AND 100.0") - .await - .unwrap(); - let pred = find_predicate(&result).expect("expected Filter predicate"); - let disjuncts = pred.disjuncts(); - assert_eq!( - disjuncts.len(), - 2, - "NOT BETWEEN should produce 2 disjuncts, got: {pred:?}" - ); - assert!(matches!( - &disjuncts[0], - L3Expr::Compare { - op: CompareOp::Lt, - .. - } - )); - assert!(matches!( - &disjuncts[1], - L3Expr::Compare { - op: CompareOp::Gt, - .. - } - )); - } - - #[tokio::test] - async fn test_try_cast_sets_try_cast_flag() { - // TRY_CAST(value AS BIGINT) → Cast { try_cast: true } - let catalog = no_time_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT TRY_CAST(value AS BIGINT) FROM events") - .await - .unwrap(); - let items = find_project_items(&result).expect("expected Project node"); - let has_try_cast = items - .iter() - .any(|pi| matches!(pi.expr, L3Expr::Cast { try_cast: true, .. })); - assert!( - has_try_cast, - "TRY_CAST should produce Cast {{ try_cast: true }}, got: {items:?}" - ); - } - - #[tokio::test] - async fn test_regular_cast_sets_try_cast_false() { - // CAST(value AS BIGINT) → Cast { try_cast: false } - let catalog = no_time_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let result = lowerer - .lower("SELECT CAST(value AS BIGINT) FROM events") - .await - .unwrap(); - let items = find_project_items(&result).expect("expected Project node"); - let has_cast = items.iter().any(|pi| { - matches!( - pi.expr, - L3Expr::Cast { - try_cast: false, - .. - } - ) - }); - assert!( - has_cast, - "CAST should produce Cast {{ try_cast: false }}, got: {items:?}" - ); - } - - // ── Tests: populate_schemas for additional node types ───────────────────────── - - #[tokio::test] - async fn test_populate_schemas_filter_node_typed() { - // Project → Filter → Scan: both the root Project and the inner Filter - // should have non-empty schemas after populate_schemas. - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let expr = lowerer - .lower("SELECT ts FROM metrics WHERE value > 0.0") - .await - .unwrap(); - let typed = populate_schemas(expr, &catalog); - // Root Project selects 1 column. - assert_eq!(typed.schema.fields.len(), 1); - assert_eq!(typed.schema.fields[0].name, "ts"); - // The inner Filter child should also carry the full scan schema. - let QueryExpr::Project { child, .. } = &typed.expr else { - panic!("expected Project at root"); - }; - assert!( - !child.schema.fields.is_empty(), - "Filter child schema should be populated" - ); - } - - #[tokio::test] - async fn test_populate_schemas_sort_node_typed() { - // Sort passes through its child schema. - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let expr = lowerer - .lower("SELECT ts, value FROM metrics ORDER BY ts") - .await - .unwrap(); - let typed = populate_schemas(expr, &catalog); - // Root should have 2 fields regardless of whether Sort or Project is on top. - assert_eq!(typed.schema.fields.len(), 2); - } - - #[tokio::test] - async fn test_populate_schemas_limit_node_typed() { - // Limit passes through its child schema. - let catalog = no_time_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let expr = lowerer - .lower("SELECT id FROM events LIMIT 5") - .await - .unwrap(); - let typed = populate_schemas(expr, &catalog); - assert_eq!(typed.schema.fields.len(), 1); - assert_eq!(typed.schema.fields[0].name, "id"); - } - - #[tokio::test] - async fn test_populate_schemas_set_op_uses_left_schema() { - // SetOp output schema matches the left child. - let catalog = no_time_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let expr = lowerer - .lower("SELECT id FROM events UNION ALL SELECT id FROM events") - .await - .unwrap(); - let typed = populate_schemas(expr, &catalog); - assert!( - !typed.schema.fields.is_empty(), - "SetOp schema should be populated after populate_schemas" - ); - assert!( - typed.schema.fields.iter().any(|f| f.name == "id"), - "expected 'id' field in SetOp output schema, got: {:?}", - typed.schema.fields - ); - } - - #[tokio::test] - async fn test_populate_schemas_window_func_appends_column() { - // WindowFunc appends one column to the child schema. - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let expr = lowerer - .lower( - "SELECT region, ROW_NUMBER() OVER (PARTITION BY region ORDER BY ts) \ - FROM metrics", - ) - .await - .unwrap(); - let typed = populate_schemas(expr, &catalog); - // Root may be a Project; all nodes in the tree should have non-empty schemas. - assert!( - !typed.schema.fields.is_empty(), - "root schema should be populated after populate_schemas" - ); - } - - // ── Tests: CR findings #1, #2, #5 ──────────────────────────────────────────── - - #[tokio::test] - async fn test_having_filter_fallback_preserves_inner_time_range() { - // Finding #1: lower_filter's collect_filter_chain stops at Aggregate - // (not a scan-chain node), so the HAVING Filter takes the fallback path. - // The WHERE Filter sitting below the Aggregate must still fold ts predicates - // into Source::Table.time_range via the normal scan-chain path. - // - // This test would catch a regression where the HAVING fallback accidentally - // short-circuits the inner time extraction. - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let expr = lowerer - .lower( - "SELECT region, COUNT(*) FROM metrics \ - WHERE ts >= 1000 AND ts < 2000 \ - GROUP BY region \ - HAVING region = 'us'", - ) - .await - .unwrap(); - - // Inner WHERE predicates must still be folded into time_range. - let source = find_source(&expr).expect("expected Scan in tree"); - let Source::Table { time_range, .. } = source else { - panic!("expected Source::Table, got {source:?}"); - }; - assert!( - time_range.is_some(), - "time_range should be extracted from WHERE ts >= 1000 AND ts < 2000; \ - the HAVING-path fallback must not prevent inner time extraction" - ); - let range = time_range.as_ref().unwrap(); - assert_eq!(range.start_ms, Some(1000), "start_ms from ts >= 1000"); - assert_eq!(range.end_ms, Some(2000), "end_ms from ts < 2000"); +#[tokio::test] +async fn join_predicate_disambiguates_shared_column_name() { + // Issue #7: `metrics.service = hosts.service` shares a column name across the + // join. The qualified refs must bind to two *distinct* positions in the + // concatenated schema, not collapse onto the first `service`. + // metrics(ts,service,latency,bytes) ++ hosts(service,region) + // → metrics.service = col 1, hosts.service = col 4. + let qe = lower( + "SELECT metrics.bytes, hosts.region \ + FROM metrics JOIN hosts ON metrics.service = hosts.service", + ) + .await; + let join = find_join(&qe).expect("expected a Join in the tree"); + assert_eq!( + join_eq_columns(join), + [1, 4], + "join key must bind to distinct positions, not the same `service`" + ); +} - // The HAVING predicate must appear as a Filter node directly above an Aggregate. - fn has_filter_over_aggregate(expr: &QueryExpr) -> bool { - match expr { - QueryExpr::Filter { child, .. } => { - matches!(child.expr, QueryExpr::Aggregate { .. }) - } - QueryExpr::Project { child, .. } | QueryExpr::Sort { child, .. } => { - has_filter_over_aggregate(&child.expr) - } - _ => false, - } - } - assert!( - has_filter_over_aggregate(&expr), - "HAVING should produce a Filter node directly wrapping an Aggregate; got: {expr:?}" - ); - } +#[tokio::test] +async fn self_join_disambiguates_via_aliases() { + // A self-join shares *every* column name; the alias qualifiers (`a`/`b`) are + // the only way to tell the two `service` columns apart. + // metrics ++ metrics → a.service = col 1, b.service = col 5 (4 cols/side). + let qe = lower( + "SELECT a.bytes, b.latency \ + FROM metrics a JOIN metrics b ON a.service = b.service", + ) + .await; + let join = find_join(&qe).expect("expected a self-Join in the tree"); + assert_eq!( + join_eq_columns(join), + [1, 5], + "self-join keys must bind to distinct sides" + ); +} - #[tokio::test] - async fn test_scan_columns_empty_when_aggregate_between_project_and_scan() { - // Finding #2: push_columns_into_scan recurses only through Filter nodes. - // For Project → Aggregate → Scan, push_columns_into_scan encounters the - // Aggregate and falls through to `other => other`, leaving Scan.columns empty. - // - // An empty Scan.columns means "all columns" (semantically safe), but prevents - // downstream stages from pruning unneeded columns. - // - // TODO: when this gap is fixed, update the assertion to check that columns - // contains ["value", "region"] (the columns actually referenced). - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let expr = lowerer - .lower("SELECT MAX(value) FROM metrics GROUP BY region") - .await - .unwrap(); +#[tokio::test] +async fn qualified_where_over_join_resolves_to_right_side() { + // Issue #7 beyond the join key: a WHERE on the *duplicated* column name + // (`service` exists on both sides) must bind to the qualified side, not the + // first match. metrics.service = col 1, hosts.service = col 4 → `hosts.service` + // must resolve to 4. (Unoptimized plan keeps the Filter above the Join — no + // predicate pushdown — so it binds against the concatenated schema.) + let qe = lower( + "SELECT metrics.bytes FROM metrics JOIN hosts ON metrics.service = hosts.service \ + WHERE hosts.service = 'api'", + ) + .await; + let filter = find_filter(&qe).expect("expected a Filter over the join"); + let QueryExpr::Filter { pred, .. } = filter else { + unreachable!("find_filter only returns Filter"); + }; + assert!( + matches!(&pred.0, L3Expr::Compare { left, op: CompareOp::Eq, .. } + if matches!(left.as_ref(), L3Expr::Column(4))), + "hosts.service must bind to concatenated position 4 (not the first `service`), got {:?}", + pred.0 + ); +} - let source = find_source(&expr).expect("expected Scan in tree"); - let Source::Table { columns, .. } = source else { - panic!("expected Source::Table, got {source:?}"); - }; - assert!( - columns.is_empty(), - "known gap: Scan.columns should be empty (full scan) when Aggregate sits \ - between Project and Scan — if this fails the gap has been fixed, \ - update to assert columns == [value, region]; got {columns:?}" - ); - } +#[tokio::test] +async fn self_join_group_by_disambiguates_via_qualifier() { + // L2 group-key qualifier fix: GROUP BY on the *duplicated* column over a + // self-join must bind to the qualified side, not first-match. metrics ⋈ + // metrics → a.service = col 1, b.service = col 5. (Without qualified keys, + // both `GROUP BY a.service` and `GROUP BY b.service` collapsed to col 1.) + let qe_b = lower( + "SELECT b.service, COUNT(*) FROM metrics a JOIN metrics b \ + ON a.service = b.service GROUP BY b.service", + ) + .await; + let (by, _) = find_aggregate(&qe_b).expect("expected an Aggregate over the self-join"); + assert_eq!( + by, + &vec![5], + "GROUP BY b.service binds to the b side (col 5)" + ); - #[tokio::test] - async fn test_max_of_expression_arg_schema_reports_float64_known_bug() { - // Finding #5: agg_col() returns None for non-bare-column aggregate args - // (e.g. `ts + 1`). Schema derivation falls back to a Float64 dummy field, - // so MAX(ts + 1) reports Float64 even though ts: Int64 → correct type is Int64. - // - // This test pins the current (incorrect) Float64 output as a known bug. - // TODO: fix agg_col / output_type so MAX of an Int64 expression → Int64. - // When fixed, change the assertion below to L3DataType::Int64. - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let expr = lowerer - .lower("SELECT MAX(ts + 1) FROM metrics") - .await - .unwrap(); - let typed = populate_schemas(expr, &catalog); + let qe_a = lower( + "SELECT a.service, COUNT(*) FROM metrics a JOIN metrics b \ + ON a.service = b.service GROUP BY a.service", + ) + .await; + let (by, _) = find_aggregate(&qe_a).expect("expected an Aggregate over the self-join"); + assert_eq!( + by, + &vec![1], + "GROUP BY a.service binds to the a side (col 1)" + ); +} - // Root is Project → Aggregate → Scan; first schema field is the MAX output. - assert_eq!( - typed.schema.fields.len(), - 1, - "expected one output field for SELECT MAX(...)" - ); - let dtype = &typed.schema.fields[0].dtype; - // BUG: currently Float64 because agg_col returns None → float64_dummy fallback. - // Should be Int64 (MAX of Int64 expression preserves the type). - assert_eq!( - *dtype, - L3DataType::Float64, - "known bug: MAX(ts + 1) schema reports {dtype:?} — expected Float64 \ - (the current wrong value); update to Int64 when the agg_col bug is fixed" - ); - } +#[tokio::test] +async fn aggregate_over_join_binds_against_concatenated_schema() { + // GROUP BY a right-table column over a join: the key must resolve against + // the concatenated schema, exercising the bottom-up converter end to end. + // Two aggregates → the multi-agg path, which carries GROUP BY keys as + // positional `Aggregate.by` (the single-agg path folds them into Partition). + let qe = lower( + "SELECT hosts.region, SUM(metrics.bytes), COUNT(*) \ + FROM metrics JOIN hosts ON metrics.service = hosts.service \ + GROUP BY hosts.region", + ) + .await; + let (by, aggs) = find_aggregate(&qe).expect("expected an Aggregate over the join"); + // metrics(ts,service,latency,bytes) ++ hosts(service,region) → + // region is column 5, bytes is column 3 of the concatenated schema. + assert_eq!( + by, + &vec![5], + "GROUP BY hosts.region → concatenated column 5" + ); + assert!( + aggs.contains(&AggIntent::Sum { col: Some(3) }), + "SUM(metrics.bytes) → Sum{{col:3}}, got {aggs:?}" + ); +} - // ── Tests: CR findings #11 and #12 ─────────────────────────────────────────── +#[tokio::test] +async fn semi_join_is_rejected_not_mislowered() { + // No L3 counterpart for semi/anti joins yet → reject rather than mislower. + let res = lower_sql( + "SELECT service FROM metrics WHERE service IN (SELECT service FROM hosts)", + &catalog(), + AccuracyTarget::Exact, + ) + .await; + assert!( + res.is_err(), + "semi-join / IN-subquery should be rejected in v1" + ); +} - #[tokio::test] - async fn test_invalid_time_column_in_catalog_returns_error() { - // Finding #11: lower_filter calls schema.validate() when a table has a - // time_column set. If time_column names a column that doesn't exist in - // TableSchema.columns, validate() returns Err, which lower_filter propagates - // as LoweringError::InvalidExpression. - let mut tables = HashMap::new(); - tables.insert( - "metrics".to_string(), - TableSchema { - columns: vec![ColumnDef { - name: "value".to_string(), - data_type: L3DataType::Float64, - nullable: true, - }], - // "ghost_ts" doesn't exist in columns — validate() will reject this. - time_column: Some("ghost_ts".to_string()), - }, - ); - let catalog = SchemaCatalog { tables }; - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - // Any WHERE clause triggers lower_filter → collect_filter_chain finds the - // TableScan → catalog lookup → time_column is Some → validate() fires. - let result = lowerer - .lower("SELECT value FROM metrics WHERE value > 0.0") - .await; - assert!( - matches!(result, Err(LoweringError::InvalidExpression(_))), - "expected InvalidExpression for catalog with invalid time_column, got: {result:?}" - ); +/// Find the first `WindowFunc` node along the single-child spine. +fn find_windowfunc(qe: &QueryExpr) -> Option<&QueryExpr> { + match qe { + QueryExpr::WindowFunc { .. } => Some(qe), + QueryExpr::Project { child, .. } + | QueryExpr::Filter { child, .. } + | QueryExpr::Aggregate { child, .. } + | QueryExpr::Window { child, .. } + | QueryExpr::Partition { child, .. } + | QueryExpr::Distinct { child, .. } + | QueryExpr::Sort { child, .. } + | QueryExpr::Limit { child, .. } + | QueryExpr::Subquery { child, .. } => find_windowfunc(child), + _ => None, } +} - #[tokio::test] - async fn test_agg_col_is_none_for_expression_arg() { - // Finding #12: agg_col() returns None when the aggregate argument is not - // a bare column reference (e.g. SUM(value + 1) uses a BinaryExpr, not Column). - // This documents the boundary: only Column, Alias(Column), and Cast(Column) - // produce a Some(ColumnRef); any other expression shape produces None. - let catalog = metrics_catalog(); - let lowerer = SqlLowerer::new(&catalog, AccuracyTarget::Exact); - let expr = lowerer - .lower("SELECT SUM(value + 1) FROM metrics GROUP BY region") - .await - .unwrap(); +#[tokio::test] +async fn window_function_lowers_to_positional_windowfunc() { + // ROW_NUMBER() OVER (PARTITION BY service ORDER BY bytes DESC). + let qe = lower( + "SELECT service, ROW_NUMBER() OVER (PARTITION BY service ORDER BY bytes DESC) \ + FROM metrics", + ) + .await; + let win = find_windowfunc(&qe).expect("expected a WindowFunc node"); + let QueryExpr::WindowFunc { + func, + partition_by, + order_by, + .. + } = win + else { + unreachable!("find_windowfunc only returns WindowFunc"); + }; + assert_eq!(*func, WindowFuncKind::RowNumber); + assert_eq!(partition_by, &vec![1], "PARTITION BY service → col 1"); + assert_eq!(order_by.len(), 1); + assert_eq!( + order_by[0].expr, + L3Expr::Column(3), + "ORDER BY bytes → col 3" + ); + assert!(!order_by[0].ascending, "DESC"); + + // The window output column is appended to the schema (Int64 for ROW_NUMBER), + // and the enclosing projection resolves it (output_name threading). + let schema = qe.output_schema().expect("root schema"); + assert!( + schema.columns.iter().any(|c| c.dtype == DataType::Int64), + "row_number output column present, got {:?}", + schema.columns + ); +} - let (_, aggs) = find_aggregate(&expr).expect("expected Aggregate in tree"); - assert_eq!(aggs.len(), 1); - let AggIntent::Sum { col } = &aggs[0] else { - panic!("expected AggIntent::Sum, got {:?}", aggs[0]); - }; - assert!( - col.is_none(), - "agg_col should return None for SUM(value + 1) — the arg is a BinaryExpr, \ - not a bare ColumnRef; got {col:?}" - ); - } +#[tokio::test] +async fn window_aggregate_lowers_to_windowfunc() { + let qe = lower("SELECT service, SUM(bytes) OVER (PARTITION BY service) FROM metrics").await; + let win = find_windowfunc(&qe).expect("expected a WindowFunc node"); + let QueryExpr::WindowFunc { func, args, .. } = win else { + unreachable!(); + }; + assert_eq!(*func, WindowFuncKind::Sum); + assert_eq!(args, &vec![L3Expr::Column(3)], "SUM(bytes) → arg col 3"); } diff --git a/docs/intent-algebra-reconciliation.md b/docs/intent-algebra-reconciliation.md new file mode 100644 index 00000000..de35153e --- /dev/null +++ b/docs/intent-algebra-reconciliation.md @@ -0,0 +1,108 @@ +# `intent_algebra` reconciliation plan (ASAPController ⇄ ASAPQuery-backend) + +> **Status (prerequisite done):** the *intra-repo* reconciliation — PR #4's +> name-based SQL L3 ⇄ PR #5's positional L3 — is complete on `feat/promql-l1-l3`. +> Both front ends (PromQL + SQL) now lower onto **one positional IR** via the +> shared `convert_root`: `expr_ir` is the SQL∪PromQL scalar superset, `AggIntent` +> carries `col: Option`, the converter is bottom-up schema-aware, and +> the DataFusion front end emits relational L2. The plan below (ASAP ⇄ +> control_plane) is the *next* step, now unblocked. + +ASAPController's `crates/core/src/intent_algebra` is a slimmed, refactored fork +of the canonical L3 IR in `ASAPQuery-backend/control_plane/src/intent_algebra`. +This is the first concrete step of the L4/L5 consolidation: produce **one** +canonical `intent_algebra` so that (a) both repos stop drifting, and (b) +control_plane's L4 (optimizer + sketch_algebra) and L5 (physical + emit) can be +ported onto the shared IR the PromQL/SQL front-ends already lower into. + +**Base = control_plane's L3** (the richer original); ASAPController's +correctness fixes and multi-language front-end are layered on top. + +## Drift summary (evidence) + +Line counts, `intent_algebra/*.rs`, control_plane (CP) vs ASAPController (ASAP): + +| file | CP | ASAP | finding | +|---|---:|---:|---| +| `schema.rs` | 332 | 332 | **byte-identical** (already shared) | +| `query_expr.rs` | 1193 | 445 | ASAP relational nodes ⊆ CP; CP's "extra" variants are the **inlined scalar IR** ASAP split into `expr_ir.rs` | +| `agg_intent.rs` | 673 | 245 | **additive both ways**: CP has 11 ASAP lacks; ASAP has `StdDev`/`Variance` CP lacks | +| `relational.rs` | 920 | 212 | CP superset, **incl. `Project`** (ASAP lacks; SQL needs it) | +| `lower.rs` | 845 | 294 | CP richer; ASAP carries the per-branch-binding fix + accuracy threading | +| `binder.rs` | 300 | 189 | CP richer; same `SchemaCatalog`/`UsageDerivedCatalog` API | +| `cse.rs` | 324 | 228 | ASAP carries the structural-`PartialEq` key fix | +| `column_resolution.rs` | 465 | 177 | CP richer | +| `mod.rs` | 125 | 41 | **same export shape**, CP exports more | +| `expr_ir.rs`, `names.rs` | inlined | separate | file-organization difference | + +Verdict: a **contained merge**, not a rewrite. `schema.rs` is already shared, the +node set is a clean superset, `AggIntent` is a union, and the public module API +matches. Cost concentrates in `lower.rs` and `expr_ir.rs`. + +Behavioral specifics found in CP: +- `convert_root(legacy)` takes **no accuracy** — it hardcodes `Exact` / + `Epsilon(0.05)` in the converter. ASAP threads a per-query `AccuracyTarget`. +- CP's converter threads **one root schema to all branches** — i.e. it has the + same per-branch-binding bug ASAP already fixed. +- CP `AggIntent` carries `accuracy` and the 11 extra intents + (`Changes/Resets/Deriv/Delta/PredictLinear/Absent/Present/Idelta/Irate/HoltWinters/Frequency`), + but **lacks `StdDev`/`Variance`** — it fans those out into a `Merge` of + quantile aggregates in `lower.rs`. +- CP sources `AccuracyTarget` / `BindingName` from a `types_v2` module. + +Two payoffs from basing on CP's L3: +- Several functions ASAPController currently **rejects** (`changes`, `resets`, + `deriv`, `delta`, `predict_linear`, `absent`, `present`) gain intents → become + lowerable. +- `irate` becomes distinguishable from `rate` (CP has a separate `Irate` intent); + our `rate≡irate` equivalence was a consequence of the slimmer vocabulary. + +## Decisions to settle (sign-off needed) + +| # | Fork | Options | Recommendation | +|---|---|---|---| +| **D1** | `StdDev` / `Variance` | CP's `Merge`-of-quantiles fan-out **vs** ASAP's first-class `AggIntent` | **CP fan-out** (less L4 work); add first-class intents only if L4 can bind them | +| **D2** | scalar IR location | CP inlines in `query_expr`/`relational` **vs** ASAP's separate `expr_ir.rs` (`L3Expr`) | **ASAP's `expr_ir.rs`**, extended to CP's scalar superset | +| **D3** | shared scalar types home | CP `types_v2` **vs** ASAP `names.rs` + `types.rs` (`AccuracyTarget`, `BindingName`, `QueryId`) | **one `core::types` module**; both already expose the same names | + +## File-by-file tasks + +Base = control_plane's file unless noted; "port" = bring ASAP's delta onto the CP base. + +| file | base | tasks | effort | risk | +|---|---|---|---|---| +| `schema.rs` | identical | **no-op** — adopt as-is | none | none | +| `query_expr.rs` | CP | adopt CP node set (ASAP ⊆ CP); per **D2** reference `expr_ir::L3Expr` instead of inline scalars; confirm CP covers ASAP methods (`output_schema_in`, …) | low | low | +| `agg_intent.rs` | CP | adopt CP's full vocabulary; resolve **D1**; reconcile `output_column` / `agg_accuracy` | low–med | low | +| `relational.rs` | CP | adopt CP (incl. `Project`); per **D2** point scalar refs at `expr_ir` | low | low | +| `lower.rs` | CP | **main work**: (1) port per-branch binding to `BinaryOp`/`Join`/`SetOp` arms; (2) add `acc: &AccuracyTarget` to `convert`/`convert_root`, replace hardcoded defaults; (3) verify nested-aggregate convert supports the two-level `sum(rate)` shape; (4) reflect **D1** | **med** | med | +| `binder.rs` | CP | adopt CP; verify `SchemaCatalog`/`UsageDerivedCatalog` parity | low | low | +| `cse.rs` | CP | **port** the structural-`PartialEq` key fix if CP keys by `Debug` | low | low | +| `column_resolution.rs` | CP | adopt CP | low | low | +| `expr_ir.rs` | ASAP (keep, **D2**) | **med work**: extend `L3Expr` to CP's scalar superset (`FunctionCall`, `InList`, `Between`, `IsNull`, `ScalarSubquery`, `Cast`) + our `Regex`/`NotRegex` | med | low | +| `names.rs` | merge → `types` | fold `BindingName`/`QueryId` into the shared types module (**D3**); update CP's `types_v2` imports | low | low | +| `mod.rs` | CP + ASAP | union the exports; keep `expr_ir` + shared `types` | low | low | + +## Cross-cutting (part of the merge, outside `intent_algebra`) + +- **`types_v2` → `core::types`** (D3): one home for `AccuracyTarget`, `BindingName`, `QueryId`. +- **Front-ends**: keep ASAPController's **PromQL + SQL** lowering (CP has PromQL only) and the correctness fixes (`sum(rate)`, `histogram_quantile`, matcher canonicalization) — retarget onto the unified `QueryExpr`. Lives in `crates/lower`. +- **Tests**: bring the `conformance` / `equivalence` / `corpus` suites onto the unified crate and merge with CP's `lower.rs` / `cse.rs` unit tests. Use the full suite as the acceptance gate. + +## Recommended order + +1. `schema.rs` (free) + `types`/`names.rs` (D3) — shared primitives. +2. `expr_ir.rs` to CP's scalar superset (D2) — unblocks the type layer. +3. `query_expr.rs` + `relational.rs` + `agg_intent.rs` (D1) — the type layer. +4. `binder.rs` + `column_resolution.rs` + `cse.rs` (+ port the CSE key fix). +5. `lower.rs` — per-branch binding + accuracy threading (the one med-risk file). +6. Retarget front-ends + bring tests; run the full ASAPController suite against the unified crate. + +## Effort summary + +- **adopt-CP (low):** schema, query_expr, relational, binder, column_resolution, mod. +- **port-our-fix (low):** cse, names/types. +- **real work (med):** `lower.rs` (per-branch binding + accuracy), `expr_ir.rs` (scalar superset). +- **decision with semantic weight:** D1 only. + +Open prerequisite (tracked separately): host the unified crate as **(A)** a cross-repo shared crate or **(B)** absorb control_plane into the ASAPController monorepo (design.md §5/§8). Recommendation: **B** for the core IR — a cross-repo git dependency on the central IR forces lock-step two-repo changes (cf. the `promql-parser` private-mirror friction). diff --git a/docs/promql-lowering.md b/docs/promql-lowering.md new file mode 100644 index 00000000..7d5536b8 --- /dev/null +++ b/docs/promql-lowering.md @@ -0,0 +1,379 @@ +# PromQL lowering — positional `ColumnId`, the Binder, and CSE + +How `asap-control-lower` turns a PromQL string into the canonical Layer-3 +intent algebra, and *why* the IR uses positional column identity, an explicit +name-resolution pass, and unique-key metadata. + +This is the PromQL companion to [`design.md` §6](design.md) ("Core crate +details — Layer 3"). It mirrors the `asapquery-backend` control-plane IR: +**two IRs joined by a Binder.** + +## The pipeline at a glance + +``` +PromQL string + │ parse promql-parser 0.8 + ▼ +Expr AST ← L1 + │ front-end lowering crates/lower/src/promql.rs + ▼ +relational::QueryExpr ← L2 — columns are NAMES (ColumnRef::Named, Aggregate.keys: Vec) + │ Binder pass: build Schema + crates/core/src/intent_algebra/binder.rs + │ resolve names → ColumnId + column_resolution.rs + ▼ +query_expr::QueryExpr ← L3 — columns are POSITIONS (Aggregate.by: Vec) + + a self-contained Schema rides on each Scan +``` + +The three layers are **L1 (`Expr AST`) → L2 (`relational::QueryExpr`, names) → +L3 (`query_expr::QueryExpr`, positions)**. The **Binder is not a layer** — it is +the pass that sits on the L2→L3 edge, turning names into positional `ColumnId`s. +`convert_root` (`intent_algebra/lower.rs`) runs it first, then converts the L2 +tree structurally: + +```rust +pub fn convert_root(legacy: &LQueryExpr, accuracy: &AccuracyTarget) + -> Result +{ + let schema = Binder::new().bind(legacy); // ← L2→L3 name resolution, once + convert(legacy, &schema, accuracy) // ← purely structural after this +} +``` + +The worked example at the end traces a query through all three layers (with the +Binder pass shown explicitly between L2 and L3). + +--- + +## Why positional `ColumnId` + +`ColumnId = usize` (`schema.rs`), an index into `Schema::columns`. Everywhere +the *canonical* IR names a column — `Aggregate.by`, `Schema::unique_keys`, +`Schema::time_index` — it is a position, not a string. + +The IR is deliberately split in two: + +| | Layer-2 `relational::QueryExpr` | Canonical `query_expr::QueryExpr` | +|---|---|---| +| Column identity | `ColumnRef::Named(String)`, `Aggregate.keys: Vec` | `ColumnId = usize`, `Aggregate.by: Vec` | +| Source of names | whatever the PromQL parser emits | resolved against a `Schema` | + +Why convert names → positions at all: + +1. **Identity is settled once.** A string `"service"` means nothing until you + know which schema it lives in and at what offset. If every downstream pass + (schema flow, push-down, CSE, cost model, L5 emitters) carried names, each + would re-resolve and each would own a "column not found" failure path. A + `ColumnId` is an array index that **cannot dangle** — resolution already + happened. +2. **The canonical tree is self-describing.** The `Scan` node carries the + `Schema`, so any sub-tree's output schema is computable without surrounding + context (`QueryExpr::output_schema_in`). Positions index straight into it. +3. **It matches the backend wire format.** `ColumnId` is aliased to `usize` + specifically to line up with `design.md`'s `unique_keys: Vec>`. + The point of the restructure was convergence with the backend IR, not a + parallel L3. + +The named alias is kept (rather than a bare `usize`) so code can still pattern +on intent — *"this is a column position, not just any number."* + +--- + +## Why the Binder is its own pass + +`Binder::bind` (`binder.rs`) walks the L2 tree and returns **one** +self-contained `Schema { columns, time_index, unique_keys }` that every +`ColumnId` in the converted tree indexes into. It: + +1. Seeds columns from the catalog (`SchemaCatalog::columns_for`), or the + `(ts, value)` floor if the catalog knows nothing. +2. Guarantees that `(ts, value)` floor is present. +3. Appends one `Utf8` column per referenced-but-unknown name — collected from + `Aggregate.keys`, `TopK.by`, `Partition.keys` (`collect_referenced_columns`). + +Why isolate this instead of resolving inline during lowering: + +- **The converter becomes purely structural and total.** Once the schema + exists, positional resolution downstream can't fail to *find* a column. Every + failure mode (`ResolveError::NotFound`, `NoSampleValue`, `WildcardNotPositional`) + is concentrated in this one pass. +- **Schema/catalog policy is swappable without touching lowering.** The default + `UsageDerivedCatalog` knows nothing — honest for observability, where metric + label sets are open-ended. A registry-backed `SchemaCatalog` is future work, + and the Binder pass does not change when it lands — only the catalog impl + swaps. +- **It is the natural home for resolution-policy errors.** `without(...)` is + rejected here: a usage-derived schema can't enumerate "all labels *except* + these," so the error belongs in binding, not smeared across lowering. + +--- + +## Why `unique_keys` / CSE + +`unique_keys: Vec>` (`schema.rs`). Each inner vec is a set of +column positions that *together* uniquely identify a row; the outer vec allows +several such sets. It is populated by the per-node output-schema rule — +`Aggregate { by, .. }` emits `unique_keys = [by-positions]` when `by` is +non-empty (`query_expr.rs`), most other nodes pass through. + +**What it is for: workload-level CSE.** When several queries are planned +together, `cse::dedupe_subtrees` hoists a shared sub-DAG into a `LetBinding` so +the cost model credits the producer once, with each root referencing it via +`Ref`. But sharing is only sound if the producer emits *the same rows* for every +consumer — and a unique key is exactly what proves that. + +`cse_reuse_is_legal` is the gatekeeper (`schema.rs`): + +```rust +pub fn cse_reuse_is_legal(producer_schema: &Schema, consumer_count: usize) + -> Result<(), CseError> +{ + if consumer_count < 2 { return Err(CseError::InsufficientConsumers(consumer_count)); } + if !producer_schema.has_unique_key() { return Err(CseError::NoUniqueKeys); } + Ok(()) +} +``` + +Why `unique_keys` rather than just deduping structurally-identical subtrees: +structural identity (`format!("{child:?}")`) tells you two consumers *want* the +same producer — it does **not** tell you the producer's output is *stable across +reads*. Without a provable unique key, two `Ref`s could observe different row +sets, and crediting the sharing would be unsound. Structural identity is the +candidate-finder; `unique_keys` is the correctness predicate. Expressing keys as +`ColumnId` sets is what lets the deduper assert this cheaply — another reason +positions exist. + +**Status:** this PR lands the scaffolding (`dedupe_subtrees`, +`cse_reuse_is_legal`, `Schema::unique_keys`, `LetBinding`/`Ref`). The cost-model +integration that makes it influence planning is tracked in **#6**. Single-query +plans never read `unique_keys`. + +--- + +## Worked example — one query through L1 → L2 → L3 + +Four steps: the three layers, plus the Binder pass shown explicitly on the +L2→L3 edge. + +```promql +topk by (service) (10, count_over_time(requests{env="prod"}[1m])) +``` + +This is the heavy-hitter case (`topk` over `count` → one-pass sketch), exercised +by `topk_over_count_is_heavy_hitter_topk` in `crates/lower/tests/promql_lowering.rs`. + +### Stage 1 — L1 parse (`promql-parser`) + +``` +Expr::Aggregate { + op: topk, + param: NumberLiteral(10), + modifier: by (service), + expr: Expr::Call { + func: count_over_time, + args: [ Expr::MatrixSelector { vs: requests{env="prod"}, range: 1m } ], + }, +} +``` + +### Stage 2 — L2 relational IR (names) · `promql.rs` + +`walk_aggregate` resolves the group modifier to `keys = ["service"]`, lowers the +inner `count_over_time(...[1m])` to `Inner { metric: "requests", +matchers: [env=="prod"], window: 1m, func: Count }`, and — because the op is +`topk` *and* the inner func is `Count` — picks the heavy-hitter branch +(`Outer::TopK { k: 10, descending: true }` → `heavy_hitter == true`): + +``` +TopK { k: 10, by: ["service"], ← columns are still NAMES + input: Window { duration: 1m, slide: None, + input: Filter { pred: Compare { left: Column("env"), op: Eq, right: "prod" }, + input: Source(SourceSpec { name: "requests" }) } } } +``` + +No `Aggregate` wraps the scan — the heavy-hitter sketch counts directly off the +windowed scan (`window_scan`). Grouping rides as a *name list* on `TopK.by`, +awaiting resolution. + +### Stage 3 — Binder pass (L2→L3 edge): build the Schema, resolve names → `ColumnId` + +`Binder::bind` walks the L2 tree: + +- `source_name() == "requests"`; `UsageDerivedCatalog` returns `None` → start + from the `(ts, value)` floor. +- `collect_referenced_columns` finds `TopK.by = ["service"]` → append `service` + as a `Utf8` column. + +Result — the single self-contained schema: + +``` +Schema { + columns: [ ts:Timestamp(0), value:Float64(1), service:Utf8(2) ], + time_index: Some(0), + unique_keys: [], ← UsageDerivedCatalog proves no unique key +} +``` + +`resolve_named_keys(["service"], schema)` → `"service"` is at position 2 → +`by = [2]`. + +### Stage 4 — L3 canonical IR (positions) · `lower.rs convert` + +The `TopK` arm rewrites to the canonical `Aggregate{TopK}`, threading the +accuracy target and the resolved positional `by`; the `Filter`-over-`Source` +folds into `Scan.predicates`; the bound schema rides on the `Scan`: + +``` +Aggregate { + by: [2], ← service, POSITIONAL now + aggs: [ TopK { k: 10, accuracy: } ], + having: None, + child: Window { kind: Tumbling, size: 1m, slide: None, + child: Scan { + source: TimeSeries { metric: "requests" }, + predicates: [ Compare { left: Column("env"), op: Eq, right: "prod" } ], + schema: Schema { [ts, value, service], time_index: Some(0), unique_keys: [] }, + } } } +``` + +### Schema flow & the CSE gate on this tree + +`output_schema_in` for the top `Aggregate`: `by = [2]` is non-empty, so its +output schema is + +``` +Schema { + columns: [ service:Utf8, topk_10:Utf8 ], ← group key + TopK output + time_index: None, + unique_keys: [[0]], ← the group key is now a unique key +} +``` + +Now suppose a second query shared the same `Window → Scan` producer. The deduper +would propose hoisting it and call +`cse_reuse_is_legal(window.output_schema(), 2)`. The window passes the Scan's +schema through unchanged — and that schema's `unique_keys` is **empty** (the +`UsageDerivedCatalog` couldn't prove one). So the gate returns +`Err(NoUniqueKeys)` and the producer is **not** shared — each consumer +recomputes it. + +That refusal is the design working as intended: under the default catalog we +cannot assert that a raw windowed scan yields identical rows across reads, so we +decline to share rather than risk an unsound plan. A registry-backed +`SchemaCatalog` that declared, say, `(ts, service)` unique on `requests` would +populate `Scan.schema.unique_keys`, flip the gate green, and let the windowed +scan be hoisted into a `LetBinding` — without any change to the Binder or +converter (cf. the `dedupe_subtrees_basic` test, which constructs exactly such a +Scan). The next section traces exactly that. + +--- + +## `unique_keys` propagation + +`unique_keys` is **not** something the Binder computes — `Binder::bind` always +emits `unique_keys: Vec::new()`. It enters at the leaf (from the catalog) and is +then derived edge-by-edge by each operator's output-schema rule +(`QueryExpr::output_schema_in`). The rules: + +| Operator | `unique_keys` of its output | +|---|---| +| `Scan` | **verbatim** from the schema the Binder/catalog built | +| `Window`, `Filter`, `Partition`, `Sort`, `Limit`, `Subquery`, `Project` | **pass through** the child's unchanged | +| `Aggregate { by }` | **replaced** with `[[0..by.len()]]` — the group keys, *re-based to output positions*; empty when `by` is empty | +| `Distinct { cols }` | child's keys **plus** `cols` added as a new key (`add_unique_key`) | +| `Merge` | first child's | +| `SetOp`, `Join`, `BinaryOp` | left / `lhs` child's | + +Two rules carry the weight: leaf-bearing operators **pass keys through** untouched, +while `Aggregate` **manufactures** a key — grouping by a column makes that column +unique in the result, so it becomes the new key (and the input's keys are +dropped, because the grouped output no longer has those rows). + +### Same tree, under a registry catalog + +Take the worked example's canonical tree, but bind it with a `SchemaCatalog` that +declares `(ts, service)` unique on `requests`. Now the leaf schema arrives with a +key, and we can watch it flow up (bottom → top): + +``` + ── unique_keys on this edge ── +Scan { requests, [[0, 2]] ← from catalog + schema: [ts(0), value(1), service(2)], (ts, service) + unique_keys = [[0, 2]] } + ▲ +Window { 1m } [[0, 2]] ← pass-through + ▲ (time_index present) +Aggregate { by:[2]=service, aggs:[TopK{10}] } [[0]] ← REPLACED + output cols: [service(0), topk_10(1)] by re-based to + output position 0 +``` + +Three things to read off this: + +1. **Scan** hands up the catalog's `[[0, 2]]` verbatim. +2. **Window** (and any `Filter`/`Sort`/`Limit` between) passes `[[0, 2]]` straight + through — these operators don't change which rows are distinct. +3. **Aggregate** does *not* forward `[[0, 2]]`. After `GROUP BY service`, the old + per-sample identity is gone; what's unique now is `service` itself — and in the + output schema `service` sits at **position 0**, so the derived key is `[[0]]`, + not `[[2]]`. This re-basing is why keys are positional `ColumnId`s, not names: + the same column is id `2` below the aggregate and id `0` above it. + +### `ColumnId` is relative to a schema — the `2` vs `0` + +A `ColumnId` is a position *within one schema*. The input and output edges of +`Aggregate` are **different schemas**, so the *same* logical `service` column +gets a different id on each. Watch it with sample rows. + +**Input edge** (below the aggregate) — `service` is column **2**: + +| `ts` · id 0 | `value` · id 1 | `service` · id 2 | +|---|---|---| +| 100 | 0.5 | api | +| 100 | 0.3 | web | +| 200 | 0.7 | api | +| 200 | 0.4 | web | + +→ "group by `service`" is written `by = [2]`. And every `(ts, service)` combo +occurs once, so the edge carries `unique_keys = [[0, 2]]`. + +**Output edge** (above the aggregate) — a *brand-new* table; `service` is now +column **0**: + +| `service` · id 0 | `topk_10` · id 1 | +|---|---| +| api | … | +| web | … | + +→ after grouping, each `service` appears exactly once, so `service` *alone* is +the key — at its **new** position: `unique_keys = [[0]]`. + +So `2` and `0` both name `service`; they differ only because input and output are +different schemas. This is also the cleanest way to see how the two concepts +divide up: + +- **`ColumnId`** answers *"which column"* — a pointer (`by = [2]`: group by the + column at position 2). Used everywhere a column must be named. +- **`unique_keys`** answers *"which set(s) of columns are jointly non-duplicating"* + — a fact about the data, *written using* `ColumnId`s (`[[0, 2]]`: columns 0 and + 2 together identify a row). Read only by the CSE gate. + +The outer `Vec` allows several such sets: `[[0, 2]]` = one key (the pair); +`[[0], [1, 2]]` = two independent keys. + +### Why this makes CSE legal here + +The shared producer the deduper would hoist is the `Window → Scan` sub-tree. Its +output edge now carries `unique_keys = [[0, 2]]`, so: + +``` +cse_reuse_is_legal( window.output_schema() // unique_keys = [[0, 2]] + , 2 /* consumers */ ) ==> Ok(()) +``` + +The gate fires green, the `Window → Scan` is hoisted into a `LetBinding`, and both +queries `Ref` it — scan + window computed once. Under the default +`UsageDerivedCatalog` the very same tree carries `unique_keys = []` at every edge +(nothing manufactures a key below the top `Aggregate`), so the gate returns +`Err(NoUniqueKeys)` and each consumer recomputes. **The only thing that changed +was the leaf key the catalog supplied; propagation and the gate did the rest.**