From 36854f3dbb5afc0b2226cc07a7c2d3ec2588f8a8 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 20 May 2026 06:39:37 -0600 Subject: [PATCH 01/40] feat(promql): L1-L3 PromQL lowering to intent algebra Add an `asap-control-lower` crate that lowers PromQL through all three language-independent layers, ending at the shared intent-algebra IR: - L1 parse is delegated to `promql-parser` 0.8. - L2 is that parser's own AST (no separate tree), mirroring how the SQL path (PR #4) reuses DataFusion's `LogicalPlan` as a free L2. - L3 lowers to `intent_algebra::QueryExpr` with intent-only `AggIntent`s over a `Source::TimeSeries` leaf, in the canonical Window-over-Aggregate shape. Heavy-hitter `topk(k, count_over_time(..))` becomes the first-class `AggIntent::TopK`; ranking by any other value (and all `bottomk`) lowers to generic `Sort + Limit`, per the L3 design rule. Fills in the core types the layer needs (previously stubs): concrete `MetricRef`/`ColumnRef`/`GroupKey`/`Predicate`, a language-independent `L3Expr` scalar IR with PromQL regex compare ops, a metric-aware `SchemaCatalog`, and `Source::TimeSeries` schema derivation. Label matchers ride on `Scan.predicates` rather than a PromQL-flavored `LabelFilter`, keeping every node above the leaf data-model-agnostic. Co-Authored-By: Claude Opus 4.7 (1M context) --- Cargo.lock | 674 ++++++++++++++++++++++ Cargo.toml | 1 + crates/core/src/intent_algebra/expr.rs | 374 ++++++++++-- crates/core/src/intent_algebra/expr_ir.rs | 97 ++++ crates/core/src/intent_algebra/mod.rs | 12 +- crates/core/src/intent_algebra/schema.rs | 68 ++- crates/lower/Cargo.toml | 8 + crates/lower/src/error.rs | 35 ++ crates/lower/src/lib.rs | 67 +++ crates/lower/src/promql.rs | 668 +++++++++++++++++++++ crates/lower/src/schema_pass.rs | 159 +++++ crates/lower/tests/promql_lowering.rs | 496 ++++++++++++++++ 12 files changed, 2591 insertions(+), 68 deletions(-) create mode 100644 crates/core/src/intent_algebra/expr_ir.rs create mode 100644 crates/lower/Cargo.toml create mode 100644 crates/lower/src/error.rs create mode 100644 crates/lower/src/lib.rs create mode 100644 crates/lower/src/promql.rs create mode 100644 crates/lower/src/schema_pass.rs create mode 100644 crates/lower/tests/promql_lowering.rs diff --git a/Cargo.lock b/Cargo.lock index 8b1e8423..d2c6cc16 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,680 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + [[package]] name = "asap-control-core" version = "0.1.0" + +[[package]] +name = "asap-control-lower" +version = "0.1.0" +dependencies = [ + "asap-control-core", + "promql-parser", +] + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "cactus" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbc26382d871df4b7442e3df10a9402bf3cf5e55cbd66f12be38861425f0564" + +[[package]] +name = "cc" +version = "1.2.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfgrammar" +version = "0.13.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe45e18904af7af10e4312df7c97251e98af98c70f42f1f2587aecfcbee56bf" +dependencies = [ + "indexmap", + "lazy_static", + "num-traits", + "regex", + "serde", + "vob", +] + +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getopts" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" +dependencies = [ + "cfg-if", + "futures-util", + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "lrlex" +version = "0.13.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c71364e868116ee891b0f93559eb9eca5675bec28b22d33c58481e66c3951d7e" +dependencies = [ + "cfgrammar", + "getopts", + "lazy_static", + "lrpar", + "num-traits", + "quote", + "regex", + "regex-syntax", + "serde", + "vergen", +] + +[[package]] +name = "lrpar" +version = "0.13.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b265a81193d94c92d1c9c715498d6fa505bce3f789ceecb24ab5d6fa2dbc71" +dependencies = [ + "bincode", + "cactus", + "cfgrammar", + "filetime", + "indexmap", + "lazy_static", + "lrtable", + "num-traits", + "packedvec", + "regex", + "serde", + "static_assertions", + "vergen", + "vob", +] + +[[package]] +name = "lrtable" +version = "0.13.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc36d15214ca997a5097845be1f932b7ee6125c36f5c5e55f6c49e027ddeb6de" +dependencies = [ + "cfgrammar", + "fnv", + "num-traits", + "serde", + "sparsevec", + "vob", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "packedvec" +version = "1.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69e0a534dd2e6aefce319af62a0aa0066a76bdfcec0201dfe02df226bc9ec70" +dependencies = [ + "num-traits", + "serde", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "promql-parser" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df2791a28f8ea7e48f2838999c06d089184d44adb860feab682d45dd190ef718" +dependencies = [ + "cfgrammar", + "chrono", + "lazy_static", + "lrlex", + "lrpar", + "regex", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "sparsevec" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68b4a8ce3045f0fe173fb5ae3c6b7dcfbec02bfa650bb8618b2301f52af0134d" +dependencies = [ + "num-traits", + "packedvec", + "serde", + "vob", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "libc", + "num-conv", + "num_threads", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "vergen" +version = "8.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2990d9ea5967266ea0ccf413a4aa5c42a93dbcfda9cb49a97de6931726b12566" +dependencies = [ + "anyhow", + "rustversion", + "time", +] + +[[package]] +name = "vob" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc936b5a7202a703aeaf7ce05e7931db2e0c8126813f97db3e9e06d867b0bb38" +dependencies = [ + "num-traits", + "serde", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] diff --git a/Cargo.toml b/Cargo.toml index c3365c5b..7feaef0f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [workspace] members = [ "crates/core", + "crates/lower", ] resolver = "2" diff --git a/crates/core/src/intent_algebra/expr.rs b/crates/core/src/intent_algebra/expr.rs index cadcdf43..a39248c0 100644 --- a/crates/core/src/intent_algebra/expr.rs +++ b/crates/core/src/intent_algebra/expr.rs @@ -1,48 +1,96 @@ -use std::rc::Rc; +use std::sync::Arc; use std::time::Duration; +use super::expr_ir::L3Expr; use super::schema::{HasSchema, L3Schema, SchemaCatalog}; use crate::types::AccuracyTarget; -// ── Stub leaf / supporting types ────────────────────────────────────────────── -// Full definitions will be added as the respective layers are implemented. +// ── Leaf / supporting types ─────────────────────────────────────────────────── /// A row-level filter predicate (WHERE clause / PromQL label matcher). #[derive(Debug, Clone)] -pub struct Predicate; +pub struct Predicate(pub L3Expr); + /// One item in a SELECT projection list. #[derive(Debug, Clone)] -pub struct ProjectItem; -/// A GROUP BY key reference. -#[derive(Debug, Clone)] -pub struct GroupKey; -/// A reference to a column by name. -#[derive(Debug, Clone)] -pub struct ColumnRef; +pub struct ProjectItem { + pub expr: L3Expr, + pub alias: Option, +} + +/// A GROUP BY key reference (column / label name). +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct GroupKey(pub String); + +/// A reference to a column / label by name. For time-series sources this is +/// a label name or the synthetic sample-value / timestamp column. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct ColumnRef(pub String); + /// A set of partitioning keys (sharding hint for L5 stage allocator). #[derive(Debug, Clone)] pub struct PartitionKeys; + /// One key in an ORDER BY clause. #[derive(Debug, Clone)] -pub struct SortKey; +pub struct SortKey { + pub expr: L3Expr, + pub ascending: bool, + pub nulls_first: bool, +} + /// An analytic window frame (ROWS / RANGE BETWEEN …). #[derive(Debug, Clone)] pub struct WindowFrame; -/// PromQL vector-match modifiers (`on`/`ignoring` + `group_left`/`group_right`). -#[derive(Debug, Clone)] -pub struct VectorMatch; + +/// PromQL vector-match modifiers: `on(...)` / `ignoring(...)` for label-set +/// matching, plus `group_left(...)` / `group_right(...)` for many-to-one and +/// one-to-many cardinality. Carried by `QueryExpr::BinaryOp`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VectorMatch { + /// `true` for `on(labels)`, `false` for `ignoring(labels)`. + pub on: bool, + /// The labels named in the `on` / `ignoring` clause. + pub labels: Vec, + /// Grouping side for many-to-one / one-to-many matches, if any. + pub grouping: Option, +} + +/// `group_left(labels)` / `group_right(labels)` modifier on a vector match. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VectorGrouping { + pub side: GroupSide, + pub labels: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum GroupSide { + Left, + Right, +} + /// Reference to a metric by name (PromQL / OTLP). -#[derive(Debug, Clone)] -pub struct MetricRef; -/// Closed time interval for a time-series scan. -#[derive(Debug, Clone)] -pub struct TimeRange; -/// Label matchers applied to a time-series scan. -#[derive(Debug, Clone)] -pub struct LabelFilter; +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct MetricRef(pub String); + +/// Closed time interval `[start_ms, end_ms]` in milliseconds since the Unix +/// epoch. Either bound may be `None`, meaning unbounded on that side. +/// +/// For PromQL this is the query's absolute evaluation range, which is supplied +/// by the query API (`/query_range` `start`/`end`), **not** by the query +/// string — so a PromQL `Scan` carries `time = None` until that context is +/// threaded in. The range-vector duration (`m[5m]`) is a *window* and lives on +/// `QueryExpr::TimeWindow`, not here. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TimeRange { + pub start_ms: Option, + pub end_ms: Option, +} + /// Reference to a relational table by name. -#[derive(Debug, Clone)] -pub struct TableRef; +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct TableRef(pub String); + /// Join key specification (USING / ON column reference). #[derive(Debug, Clone)] pub struct JoinKey; @@ -73,6 +121,8 @@ pub enum BinaryOpKind { Mul, Div, Mod, + Pow, + Atan2, // Comparison Eq, NotEq, @@ -135,10 +185,15 @@ pub enum TimeWindowKind { #[derive(Debug, Clone)] pub enum Source { /// Time-series input — deployment-model-asapquery / asaplifecycle shape. + /// + /// Label matchers are **not** carried here: they are language-independent + /// row filters and live in `QueryExpr::Scan.predicates` (the same place SQL + /// `WHERE` conjuncts on a table scan would go), so every node above the leaf + /// stays data-model-agnostic. TimeSeries { metric: MetricRef, - time: TimeRange, - labels: LabelFilter, + /// Absolute evaluation range; `None` when not yet bound (see `TimeRange`). + time: Option, }, /// Tabular input — deployment-model-asapfusion / future-OLAP shape. Table { @@ -179,6 +234,16 @@ pub enum AggIntent { Sum, Min, Max, + Avg, + /// Sample standard deviation when `population == false`; population stddev + /// otherwise. PromQL `stddev` / `stddev_over_time`. + StdDev { + population: bool, + }, + /// Variance — PromQL `stdvar` / `stdvar_over_time`. + Variance { + population: bool, + }, Quantile { q: f64, accuracy: AccuracyTarget, @@ -211,12 +276,31 @@ impl AggIntent { /// Which data model this intent semantically requires. L4 rules consult /// this to skip non-applicable intents (e.g. `Rate` over a `Source::Table`). pub fn requires(&self) -> DataModel { - todo!() + match self { + Self::Rate { .. } | Self::Increase { .. } => DataModel::TimeSeries, + _ => DataModel::Any, + } } - /// Output column type — used by L3 schema derivation for `Aggregate`. - pub fn output_type(&self, _input: &super::schema::L3Field) -> super::schema::L3DataType { - todo!() + /// Output column type for a single-column aggregate result. + /// + /// `input` is the field this intent reduces (the sample-value field for a + /// time-series window). Returns `None` for `TopK`, which produces multiple + /// output columns — use `QueryExpr::output_schema` for its schema. + pub fn output_type(&self, input: &super::schema::L3Field) -> Option { + use super::schema::L3DataType; + match self { + Self::Count { .. } | Self::Cardinality { .. } => Some(L3DataType::Int64), + Self::Min | Self::Max => Some(input.dtype.clone()), + Self::Sum + | Self::Avg + | Self::StdDev { .. } + | Self::Variance { .. } + | Self::Quantile { .. } + | Self::Rate { .. } + | Self::Increase { .. } => Some(L3DataType::Float64), + Self::TopK { .. } => None, + } } } @@ -224,7 +308,7 @@ impl AggIntent { /// A node in the L3 DAG. Wraps the expression and its derived output schema /// so that every edge implicitly carries a typed schema: holding an -/// `Rc` gives you both the child expression and the schema of the +/// `Arc` gives you both the child expression and the schema of the /// data flowing on that edge. #[derive(Debug, Clone)] pub struct L3Node { @@ -239,11 +323,13 @@ pub struct L3Node { /// Language- and deployment-independent intent-only IR. No sketch types, /// no sketch parameters, no language-specific operators. Traversing from /// the root node yields a DAG; shared sub-expressions appear as multiple -/// `Rc` references to the same `L3Node`. +/// `Arc` references to the same `L3Node`. #[derive(Debug, Clone)] pub enum QueryExpr { // ── Base relations ──────────────────────────────────────────────────────── - /// Outermost leaf. `source` carries the data-model-specific leaf shape. + /// Outermost leaf. `source` carries the data-model-specific leaf shape; + /// `predicates` are leaf-level row filters (PromQL label matchers, SQL + /// pushed-down `WHERE` conjuncts). Scan { source: Source, predicates: Vec, @@ -254,12 +340,12 @@ pub enum QueryExpr { // ── Filtering & projection ──────────────────────────────────────────────── /// σ — row-level filter. Output schema = child schema (unchanged). Filter { - child: Rc, + child: Arc, pred: Predicate, }, /// π — column projection. Output schema = child schema projected to `cols`. Project { - child: Rc, + child: Arc, cols: Vec, }, @@ -267,7 +353,7 @@ pub enum QueryExpr { /// γ + α — GROUP BY + aggregate intents. Concrete operator (HashAgg / /// SortAgg / SketchAgg) chosen by L4; `aggs` carry intent only. Aggregate { - child: Rc, + child: Arc, by: Vec, aggs: Vec, having: Option, @@ -275,10 +361,11 @@ pub enum QueryExpr { // ── Time / streaming windows ────────────────────────────────────────────── /// ψ — tumbling / sliding / session window over the time axis. Defines - /// the flush / reset lifecycle for aggregates in its sub-DAG. SQL analytic + /// the flush / reset lifecycle for aggregates in its sub-DAG. PromQL range + /// vectors (`m[5m]`) and subqueries (`m[5m:1m]`) lower here. SQL analytic /// frames are a different node (`WindowFunc`). TimeWindow { - child: Rc, + child: Arc, kind: TimeWindowKind, size: Duration, slide: Option, @@ -288,19 +375,19 @@ pub enum QueryExpr { /// Logical-only partitioning marker. Output schema = child schema. /// Carries a sharding hint for the L5 stage allocator. Partition { - child: Rc, + child: Arc, keys: PartitionKeys, }, /// δ — SQL `DISTINCT` / row deduplication. Distinct { - child: Rc, + child: Arc, cols: Vec, }, /// ⊕ — exact union of sub-results from independent stages or shards. /// Sketch unions are a separate node in `SummaryExpr` because they carry /// sketch-family / params type constraints. Merge { - children: Vec>, + children: Vec>, }, // ── Joins ───────────────────────────────────────────────────────────────── @@ -309,8 +396,8 @@ pub enum QueryExpr { /// accuracy target. Join { kind: JoinKind, - left: Rc, - right: Rc, + left: Arc, + right: Arc, pred: Option, }, @@ -318,26 +405,27 @@ pub enum QueryExpr { SetOp { kind: SetOpKind, all: bool, - left: Rc, - right: Rc, + left: Arc, + right: Arc, }, // ── Ordering & limiting ─────────────────────────────────────────────────── /// Generic order-by for non-heavy-hitter cases (`ORDER BY name LIMIT 10`). /// Heavy-hitter shapes lower to `AggIntent::TopK` instead. Sort { - child: Rc, + child: Arc, keys: Vec, }, Limit { - child: Rc, - n: u64, + child: Arc, + /// `None` means no upper bound (only an offset applies). + n: Option, offset: u64, }, // ── Subquery / CTE ──────────────────────────────────────────────────────── Subquery { - child: Rc, + child: Arc, alias: String, }, /// SQL `WITH name AS (expr) … body`; lowering target for PromQL @@ -345,8 +433,8 @@ pub enum QueryExpr { /// via `Ref(name)` in `body`, giving the DAG its fan-in. LetBinding { name: String, - expr: Rc, - body: Rc, + expr: Arc, + body: Arc, }, // ── Analytic window functions ───────────────────────────────────────────── @@ -354,7 +442,7 @@ pub enum QueryExpr { /// Distinct from `TimeWindow` — that is a streaming window over the time /// axis; this is an analytic frame over already-grouped rows. WindowFunc { - child: Rc, + child: Arc, func: WindowFuncKind, partition_by: Vec, order_by: Vec, @@ -366,14 +454,186 @@ pub enum QueryExpr { /// including `and` / `or` / `unless`, SQL boolean composition). BinaryOp { op: BinaryOpKind, - lhs: Rc, - rhs: Rc, + lhs: Arc, + rhs: Arc, vector_match: Option, }, } impl HasSchema for QueryExpr { - fn output_schema(&self, _input_schemas: &[&L3Schema], _catalog: &SchemaCatalog) -> L3Schema { - todo!() + fn output_schema(&self, input_schemas: &[&L3Schema], catalog: &SchemaCatalog) -> L3Schema { + use super::schema::{L3DataType, L3Field}; + + // Shorthand: first child's schema (most nodes have exactly one child). + let child = || input_schemas[0]; + + match self { + // ── Leaf: schema comes from the catalog ─────────────────────────── + QueryExpr::Scan { source, predicates } => match source { + // Time-series leaf: synthesize `timestamp` (the time axis) and + // the sample-`value` column, then one `Utf8` label column per + // label known from the catalog, unioned with any label + // referenced by the scan's predicates (so a filter on an + // unregistered label still type-checks). + Source::TimeSeries { metric, .. } => { + let meta = catalog.metrics.get(&metric.0); + let value_dtype = meta + .map(|m| m.value_type.clone()) + .unwrap_or(L3DataType::Float64); + + let mut fields = vec![ + L3Field { + name: "timestamp".to_string(), + dtype: L3DataType::Timestamp, + nullable: false, + }, + L3Field { + name: "value".to_string(), + dtype: value_dtype, + nullable: false, + }, + ]; + + let mut label_names: Vec = + meta.map(|m| m.labels.clone()).unwrap_or_default(); + for p in predicates { + for c in p.0.columns_referenced() { + if c.0 != "value" && c.0 != "timestamp" && !label_names.contains(&c.0) { + label_names.push(c.0.clone()); + } + } + } + for name in label_names { + fields.push(L3Field { + name, + dtype: L3DataType::Utf8, + nullable: true, + }); + } + L3Schema { + fields, + time_index: Some(0), + } + } + Source::Table { table_ref, .. } => { + let table = catalog + .tables + .get(&table_ref.0) + .unwrap_or_else(|| panic!("table '{}' not in catalog", table_ref.0)); + let fields: Vec = table + .columns + .iter() + .map(|c| L3Field { + name: c.name.clone(), + dtype: c.data_type.clone(), + nullable: c.nullable, + }) + .collect(); + let time_index = table + .time_column + .as_ref() + .and_then(|tc| fields.iter().position(|f| &f.name == tc)); + L3Schema { fields, time_index } + } + Source::Join { .. } => { + todo!("schema derivation for Join sources") + } + }, + + // ── Pass-through: output schema == child schema ─────────────────── + QueryExpr::Filter { .. } + | QueryExpr::Sort { .. } + | QueryExpr::Limit { .. } + | QueryExpr::Distinct { .. } + | QueryExpr::Partition { .. } + | QueryExpr::TimeWindow { .. } => child().clone(), + + // ── Aggregate: GROUP BY cols + one output col per AggIntent ─────── + QueryExpr::Aggregate { by, aggs, .. } => { + let cs = child(); + + // TopK is the only multi-column AggIntent: produces the TopK + // by-columns looked up from the child schema, followed by a + // synthetic "count" Int64 column. + if let [AggIntent::TopK { by: topk_by, .. }] = aggs.as_slice() { + let mut fields: Vec = topk_by + .iter() + .map(|col| { + cs.fields + .iter() + .find(|f| f.name == col.0) + .cloned() + .unwrap_or(L3Field { + name: col.0.clone(), + dtype: L3DataType::Utf8, + nullable: true, + }) + }) + .collect(); + fields.push(L3Field { + name: "count".to_string(), + dtype: L3DataType::Int64, + nullable: false, + }); + return L3Schema { + fields, + time_index: None, + }; + } + + // General case: GROUP BY fields (preserving child type) followed + // by one output field per AggIntent. The sample-`value` field is + // the canonical reduction input for a time-series window. + let value_field = cs + .fields + .iter() + .find(|f| f.name == "value") + .cloned() + .unwrap_or(L3Field { + name: "value".to_string(), + dtype: L3DataType::Float64, + nullable: true, + }); + let by_fields: Vec = by + .iter() + .filter_map(|key| cs.fields.iter().find(|f| f.name == key.0).cloned()) + .collect(); + let agg_fields: Vec = aggs + .iter() + .enumerate() + .map(|(i, agg)| { + let name = if aggs.len() == 1 { + "value".to_string() + } else { + format!("value_{i}") + }; + L3Field { + name, + dtype: agg.output_type(&value_field).unwrap_or(L3DataType::Float64), + nullable: true, + } + }) + .collect(); + let all_fields: Vec = by_fields.into_iter().chain(agg_fields).collect(); + L3Schema { + fields: all_fields, + time_index: None, + } + } + + // ── BinaryOp: left operand shape, value column re-typed ─────────── + // Arithmetic/comparison between two vectors yields the left vector's + // shape (label set + value); set ops (`and`/`or`/`unless`) likewise. + QueryExpr::BinaryOp { .. } => input_schemas[0].clone(), + + // ── Merge / SetOp: union-compatible; representative is the first ── + QueryExpr::Merge { .. } | QueryExpr::SetOp { .. } => input_schemas[0].clone(), + + // ── Everything else: not yet implemented ────────────────────────── + _ => todo!( + "output_schema not yet implemented for {:?}", + std::mem::discriminant(self) + ), + } } } diff --git a/crates/core/src/intent_algebra/expr_ir.rs b/crates/core/src/intent_algebra/expr_ir.rs new file mode 100644 index 00000000..03b01ba8 --- /dev/null +++ b/crates/core/src/intent_algebra/expr_ir.rs @@ -0,0 +1,97 @@ +use super::expr::ColumnRef; + +// ── Scalar literals ─────────────────────────────────────────────────────────── + +/// A typed scalar constant. Used in `L3Expr::Literal`. +#[derive(Debug, Clone, PartialEq)] +pub enum L3Scalar { + Int64(i64), + Float64(f64), + Utf8(String), + Boolean(bool), + Null, +} + +// ── Comparison operators ────────────────────────────────────────────────────── + +/// Binary comparison operators for `L3Expr::Compare`. +/// +/// `Regex` / `NotRegex` carry PromQL/RE2 regex-match semantics (`=~` / `!~`): +/// the right-hand side is a regular-expression pattern, not a literal value. +/// SQL `LIKE` / `ILIKE` are kept as separate operators because their +/// wildcard grammar (`%` / `_`) differs from regex. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CompareOp { + Eq, + Ne, + Lt, + Le, + Gt, + Ge, + /// RHS is a regular-expression pattern; matches PromQL `=~`. + Regex, + /// RHS is a regular-expression pattern; matches PromQL `!~`. + NotRegex, +} + +// ── Expression IR ───────────────────────────────────────────────────────────── + +/// A scalar expression used in filter predicates and (eventually) projection +/// lists and sort keys. Language-independent: PromQL label matchers, SQL +/// `WHERE` conjuncts, and Elastic term filters all lower to the same shape. +/// +/// Flat conjunctions (`BoolAnd`) / disjunctions (`BoolOr`) make per-conjunct +/// selectivity estimation and label-matcher lowering straightforward without +/// recursive descent. +#[derive(Debug, Clone, PartialEq)] +pub enum L3Expr { + /// Reference to a named column. For time-series sources a label name + /// (e.g. `Column("env")`) or the synthetic sample-value column. + Column(ColumnRef), + /// A constant literal value. + Literal(L3Scalar), + /// `left op right` — binary comparison. + Compare { + left: Box, + op: CompareOp, + right: Box, + }, + /// Flat conjunction (logical AND). An empty list is vacuously true. + BoolAnd(Vec), + /// Flat disjunction (logical OR). An empty list is vacuously false. + BoolOr(Vec), + /// Logical NOT. + Not(Box), +} + +impl L3Expr { + /// If this expression is a `BoolAnd`, return its elements; otherwise a + /// single-element slice containing `self`. Lets callers iterate all + /// top-level conjuncts without cloning. + pub fn conjuncts(&self) -> &[L3Expr] { + match self { + L3Expr::BoolAnd(v) => v.as_slice(), + _ => std::slice::from_ref(self), + } + } + + /// Recursively collect every `ColumnRef` referenced anywhere in this + /// expression. Used by L4 for column-lineage and selectivity estimation, + /// and by schema derivation to discover label columns referenced by a + /// time-series scan's predicates. + pub fn columns_referenced(&self) -> Vec<&ColumnRef> { + match self { + L3Expr::Column(c) => vec![c], + L3Expr::Literal(_) => vec![], + L3Expr::Compare { left, right, .. } => { + let mut v = left.columns_referenced(); + v.extend(right.columns_referenced()); + v + } + L3Expr::BoolAnd(parts) | L3Expr::BoolOr(parts) => { + parts.iter().flat_map(|e| e.columns_referenced()).collect() + } + L3Expr::Not(e) => e.columns_referenced(), + } + } +} diff --git a/crates/core/src/intent_algebra/mod.rs b/crates/core/src/intent_algebra/mod.rs index 5cb42f17..85521293 100644 --- a/crates/core/src/intent_algebra/mod.rs +++ b/crates/core/src/intent_algebra/mod.rs @@ -1,9 +1,13 @@ pub mod expr; +pub mod expr_ir; pub mod schema; pub use expr::{ - AggIntent, BinaryOpKind, ColumnRef, DataModel, GroupKey, JoinKey, JoinKind, L3Node, - LabelFilter, MetricRef, PartitionKeys, Predicate, ProjectItem, QueryExpr, SetOpKind, SortKey, - Source, TableRef, TimeRange, TimeWindowKind, VectorMatch, WindowFrame, WindowFuncKind, + AggIntent, BinaryOpKind, ColumnRef, DataModel, GroupKey, GroupSide, JoinKey, JoinKind, L3Node, + MetricRef, PartitionKeys, Predicate, ProjectItem, QueryExpr, SetOpKind, SortKey, Source, + TableRef, TimeRange, TimeWindowKind, VectorGrouping, VectorMatch, WindowFrame, WindowFuncKind, +}; +pub use expr_ir::{CompareOp, L3Expr, L3Scalar}; +pub use schema::{ + ColumnDef, HasSchema, L3DataType, L3Field, L3Schema, MetricSchema, SchemaCatalog, TableSchema, }; -pub use schema::{HasSchema, L3DataType, L3Field, L3Schema, SchemaCatalog}; diff --git a/crates/core/src/intent_algebra/schema.rs b/crates/core/src/intent_algebra/schema.rs index 4f6d1fc0..0220a45a 100644 --- a/crates/core/src/intent_algebra/schema.rs +++ b/crates/core/src/intent_algebra/schema.rs @@ -1,8 +1,58 @@ -/// Opaque handle to the external data-source catalog (Prometheus metric -/// metadata, SQL `information_schema`, DataFusion catalog). Used only by -/// `Scan` schema derivation to resolve leaf column types; all other nodes -/// derive their output schemas purely from their input schemas. -pub struct SchemaCatalog; +use std::collections::HashMap; + +/// Catalog of external data-source metadata. Used by L1→L3 lowering and by +/// `Scan` schema derivation to resolve leaf column / label types; all other +/// nodes derive their output schemas purely from their input schemas. +/// +/// Holds both relational tables (SQL / DataFusion sources) and time-series +/// metrics (PromQL / OTLP sources). A deployment model populates only the +/// half it needs. +#[derive(Debug, Clone, Default)] +pub struct SchemaCatalog { + /// Relational tables, keyed by table name. + pub tables: HashMap, + /// Time-series metrics, keyed by metric name. + pub metrics: HashMap, +} + +/// Schema for a single relational table. +#[derive(Debug, Clone)] +pub struct TableSchema { + pub columns: Vec, + /// Name of the column that holds the row timestamp, if any. + pub time_column: Option, +} + +/// One column in a `TableSchema`. +#[derive(Debug, Clone)] +pub struct ColumnDef { + pub name: String, + pub data_type: L3DataType, + pub nullable: bool, +} + +/// Schema for a single time-series metric. +/// +/// A metric scan always produces a `timestamp` (the time axis) and a sample +/// `value` column; this metadata names the label set carried alongside and the +/// value's type. When a metric is absent from the catalog the lowerer falls +/// back to `value: Float64` with labels discovered from the query's matchers. +#[derive(Debug, Clone)] +pub struct MetricSchema { + /// Label names exposed by this metric (e.g. `["service", "host", "env"]`). + pub labels: Vec, + /// Type of the sample value column. Usually `Float64`. + pub value_type: L3DataType, +} + +impl Default for MetricSchema { + fn default() -> Self { + Self { + labels: Vec::new(), + value_type: L3DataType::Float64, + } + } +} // ── Data types ──────────────────────────────────────────────────────────────── @@ -24,7 +74,7 @@ pub enum L3DataType { // ── Schema ──────────────────────────────────────────────────────────────────── -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub struct L3Field { pub name: String, pub dtype: L3DataType, @@ -35,7 +85,7 @@ pub struct L3Field { /// flowing between two operators. Type-checked at plan construction time: /// a node whose predicate references a column absent from its child's /// `L3Schema` is a plan-time error. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub struct L3Schema { pub fields: Vec, /// Index into `fields` for the time axis, if any. @@ -49,5 +99,9 @@ pub struct L3Schema { /// its children's output schemas. The `L3Node` wrapper stores the derived /// schema so derivation runs once at construction, not on every traversal. pub trait HasSchema { + /// # Panics + /// Panics if a `Scan` over a `Source::Table` references a table absent from + /// `catalog`. Time-series scans never panic — an unregistered metric falls + /// back to a `value: Float64` default schema. fn output_schema(&self, input_schemas: &[&L3Schema], catalog: &SchemaCatalog) -> L3Schema; } diff --git a/crates/lower/Cargo.toml b/crates/lower/Cargo.toml new file mode 100644 index 00000000..9e266588 --- /dev/null +++ b/crates/lower/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "asap-control-lower" +version = "0.1.0" +edition = "2021" + +[dependencies] +asap-control-core = { path = "../core" } +promql-parser = "0.8" diff --git a/crates/lower/src/error.rs b/crates/lower/src/error.rs new file mode 100644 index 00000000..8ce9857a --- /dev/null +++ b/crates/lower/src/error.rs @@ -0,0 +1,35 @@ +use std::fmt; + +#[derive(Debug)] +pub enum LoweringError { + /// The `promql-parser` crate rejected the query string (L1 parse failure). + Parse(String), + /// A PromQL function (`rate`, `*_over_time`, …) not supported in this version. + UnsupportedFunction(String), + /// A PromQL aggregation operator (`sum`, `topk`, …) not supported. + UnsupportedAggregateOp(String), + /// A structural PromQL feature (offset, `@`, `without` w/o catalog, …) not supported. + UnsupportedFeature(String), + /// A required function / aggregator argument was missing. + MissingArgument(String), + /// An argument had the wrong shape (e.g. a non-numeric `topk` parameter). + InvalidParameter(String), + /// The workload's query language is not handled by this lowerer. + WrongLanguage(String), +} + +impl fmt::Display for LoweringError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Parse(e) => write!(f, "PromQL parse error: {e}"), + Self::UnsupportedFunction(n) => write!(f, "unsupported PromQL function: {n}"), + Self::UnsupportedAggregateOp(n) => write!(f, "unsupported PromQL aggregate op: {n}"), + Self::UnsupportedFeature(m) => write!(f, "unsupported PromQL feature: {m}"), + Self::MissingArgument(m) => write!(f, "missing argument: {m}"), + Self::InvalidParameter(m) => write!(f, "invalid parameter: {m}"), + Self::WrongLanguage(l) => write!(f, "unsupported query language: {l}"), + } + } +} + +impl std::error::Error for LoweringError {} diff --git a/crates/lower/src/lib.rs b/crates/lower/src/lib.rs new file mode 100644 index 00000000..1fa233ce --- /dev/null +++ b/crates/lower/src/lib.rs @@ -0,0 +1,67 @@ +//! L1→L3 lowering passes for the ASAP controller core. +//! +//! Each query language has one pass that ends at the same intent-algebra +//! [`QueryExpr`](asap_control_core::intent_algebra::expr::QueryExpr): L1 parse +//! (delegated to a language parser crate), L2 per-language tree (the parser's +//! own AST), and L3 lowering to the language- and deployment-independent intent +//! algebra. PromQL lives in [`promql`]; the SQL path (PR #4) lives alongside it. + +pub mod error; +pub mod promql; +pub mod schema_pass; + +use asap_control_core::intent_algebra::expr::QueryExpr; +use asap_control_core::intent_algebra::schema::SchemaCatalog; +use asap_control_core::types::AccuracyTarget; +use asap_control_core::workload::{QueryLanguage, QueryWorkload}; + +pub use error::LoweringError; +pub use promql::PromqlLowerer; +pub use schema_pass::populate_schemas; + +/// Lower a single PromQL query string to an intent-algebra `QueryExpr`. +/// +/// The returned tree has empty schemas on every node; call [`populate_schemas`] +/// before inspecting node schemas or passing the tree to schema-aware stages. +pub fn lower_promql( + query: &str, + catalog: &SchemaCatalog, + accuracy: AccuracyTarget, +) -> Result { + PromqlLowerer::new(catalog, accuracy).lower(query) +} + +/// Lower every PromQL batch entry in `workload` to a `QueryExpr`. +/// +/// One `Result` per entry — errors are per-query, not fatal for the batch. +/// Returns an empty `Vec` if `workload.query_batch` is absent or empty, and a +/// `WrongLanguage` error for every entry if the workload language is not PromQL. +pub fn lower_promql_batch( + workload: &QueryWorkload, + catalog: &SchemaCatalog, +) -> Vec> { + let entries = match &workload.query_batch { + Some(e) if !e.is_empty() => e, + _ => return vec![], + }; + + if !matches!(workload.language, QueryLanguage::PromQL) { + let lang = format!("{:?}", workload.language); + return entries + .iter() + .map(|_| Err(LoweringError::WrongLanguage(lang.clone()))) + .collect(); + } + + entries + .iter() + .map(|entry| { + let accuracy = entry + .requirements + .as_ref() + .and_then(|r| r.accuracy.clone()) + .unwrap_or(AccuracyTarget::Exact); + lower_promql(&entry.query.0, catalog, accuracy) + }) + .collect() +} diff --git a/crates/lower/src/promql.rs b/crates/lower/src/promql.rs new file mode 100644 index 00000000..16332938 --- /dev/null +++ b/crates/lower/src/promql.rs @@ -0,0 +1,668 @@ +//! Layers 1→3 lowering: PromQL string → intent-algebra `QueryExpr`. +//! +//! - **L1 (parse)** is delegated to the `promql-parser` crate, which produces a +//! PromQL-specific AST (`promql_parser::parser::Expr`). +//! - **L2 (per-language tree)** is that same AST — `promql-parser` already hands +//! us a typed, language-flavored tree (instant vs range vectors, aggregate +//! operators, label matchers), so no separate L2 structure is materialised. +//! This mirrors the SQL path (PR #4), which uses DataFusion's `LogicalPlan` as +//! its free L2. +//! - **L3 (intent algebra)** is what this module emits: the language- and +//! deployment-independent [`QueryExpr`] with intent-only [`AggIntent`]s and a +//! `Source::TimeSeries` leaf. No sketch types, no sketch parameters. +//! +//! # PromQL → L3 mapping (summary) +//! +//! | PromQL | L3 shape | +//! |---|---| +//! | `quantile_over_time(φ, m{f}[w])` | `TimeWindow{w} → Aggregate{[Quantile{φ}]}` | +//! | `histogram_quantile(φ, rate(m{f}[w]))` | `TimeWindow{w} → Aggregate{[Quantile{φ}]}` | +//! | `avg/min/max/sum_over_time(m[w])` | `TimeWindow{w} → Aggregate{[Avg/Min/Max/Sum]}` | +//! | `stddev/stdvar_over_time(m[w])` | `TimeWindow{w} → Aggregate{[StdDev/Variance]}` | +//! | `count_over_time(m[w])` | `TimeWindow{w} → Aggregate{[Count]}` | +//! | `changes/resets(m[w])` | `TimeWindow{w} → Aggregate{[Count]}` | +//! | `rate/irate/increase(m[w])` | `Aggregate{[Rate{w}/Increase{w}]}` (window in intent) | +//! | `OUTER by (dims) (…)` | grouping `dims` flow onto the inner `Aggregate.by` | +//! | `count by (d) (… )` | `Aggregate{by:d, [Cardinality]}` | +//! | `topk(k, count_over_time(…))` | `Aggregate{[TopK{k}]}` (heavy-hitter, one pass) | +//! | `topk(k, )` / `bottomk(k, …)` | generic `Sort{value} → Limit{k}` | +//! | `m{f}` bare | `Scan{TimeSeries, predicates}` | +//! | `a OP b` | `BinaryOp{vector_match}` | +//! | `expr[r:res]` | `TimeWindow{Sliding, r, res}` | + +use std::sync::Arc; +use std::time::Duration; + +use promql_parser::label::{MatchOp, Matcher}; +use promql_parser::parser::{ + self, token, AggregateExpr, BinaryExpr, Call, Expr, LabelModifier, VectorMatchCardinality, + VectorSelector, +}; + +use asap_control_core::intent_algebra::expr::{ + AggIntent, BinaryOpKind, ColumnRef, GroupKey, GroupSide, L3Node, MetricRef, Predicate, + QueryExpr, SortKey, Source, TimeWindowKind, VectorGrouping, VectorMatch, +}; +use asap_control_core::intent_algebra::schema::{L3Schema, SchemaCatalog}; +use asap_control_core::intent_algebra::{CompareOp, L3Expr, L3Scalar}; +use asap_control_core::types::AccuracyTarget; + +use crate::error::LoweringError; + +type Result = std::result::Result; + +/// Lowers one PromQL query string into an intent-algebra `QueryExpr`. +/// +/// `catalog` is consulted only to resolve `without(...)` grouping into an +/// explicit `by` list (which needs the metric's full label set); everything +/// else lowers without it. `accuracy` is attached to every approximate +/// `AggIntent` (`Quantile`, `Count`, `Cardinality`, `TopK`). +pub struct PromqlLowerer<'a> { + catalog: &'a SchemaCatalog, + accuracy: AccuracyTarget, +} + +/// The aggregate "shape" an outer PromQL aggregator imposes on its argument. +#[derive(Debug, Clone)] +enum Outer { + /// No enclosing aggregator (top-level call / bare selector). + None, + /// Value aggregator (`sum`/`avg`/`min`/`max`/`group`/`stddev`/`stdvar`/ + /// `quantile`). When the argument is itself an `*_over_time` call the inner + /// function's intent wins; otherwise this op becomes the intent. + Plain(OuterIntent), + /// `count(...)` → set cardinality. + Count, + /// `topk` (`descending`) / `bottomk` (ascending) with limit `k`. + TopK { k: usize, descending: bool }, +} + +#[derive(Debug, Clone)] +enum OuterIntent { + Sum, + Avg, + Min, + Max, + StdDev, + Variance, + Quantile(f64), +} + +/// The aggregation a PromQL function over a range vector implies. +#[derive(Debug, Clone)] +enum InnerFunc { + Quantile(f64), + Avg, + Min, + Max, + Sum, + StdDev, + Variance, + Count, + Rate(Duration), + Increase(Duration), +} + +/// A lowered range/instant vector argument: the leaf metric, its label-matcher +/// predicates, an optional window (from a range vector), and the aggregation +/// implied by any enclosing function. +struct Inner { + metric: String, + predicates: Vec, + window: Option, + func: Option, +} + +impl<'a> PromqlLowerer<'a> { + pub fn new(catalog: &'a SchemaCatalog, accuracy: AccuracyTarget) -> Self { + Self { catalog, accuracy } + } + + /// Parse (L1) and lower (L2→L3) a PromQL query string. + pub fn lower(&self, query: &str) -> Result { + let ast = parser::parse(query).map_err(LoweringError::Parse)?; + self.walk(&ast) + } + + fn walk(&self, expr: &Expr) -> Result { + match expr { + Expr::Aggregate(agg) => self.walk_aggregate(agg), + Expr::Call(call) => { + let inner = self.lower_inner_call(call)?; + self.build(inner, vec![], Outer::None) + } + Expr::Binary(bin) => self.walk_binary(bin), + Expr::Paren(p) => self.walk(&p.expr), + Expr::Unary(u) => self.walk(&u.expr), + // `expr[range:resolution]` — a sliding evaluation window. + Expr::Subquery(sq) => { + let inner = self.walk(&sq.expr)?; + Ok(QueryExpr::TimeWindow { + child: node(inner), + kind: TimeWindowKind::Sliding, + size: sq.range, + slide: sq.step, + }) + } + // Bare instant selector → Scan with label-matcher predicates. + Expr::VectorSelector(vs) => { + let (metric, predicates) = vs_to_scan(vs); + Ok(scan(metric, predicates)) + } + // Bare range selector (no enclosing function) → windowed scan. + Expr::MatrixSelector(ms) => { + let (metric, predicates) = vs_to_scan(&ms.vs); + Ok(QueryExpr::TimeWindow { + child: node(scan(metric, predicates)), + kind: TimeWindowKind::Tumbling, + size: ms.range, + slide: None, + }) + } + Expr::NumberLiteral(_) | Expr::StringLiteral(_) => Err( + LoweringError::UnsupportedFeature("bare scalar/string at top level".into()), + ), + Expr::Extension(_) => Err(LoweringError::UnsupportedFeature( + "extension expression".into(), + )), + } + } + + fn walk_aggregate(&self, agg: &AggregateExpr) -> Result { + let group = self.resolve_group(agg)?; + let inner = self.lower_inner(&agg.expr)?; + let op = agg.op.id(); + + let outer = if op == token::T_TOPK { + Outer::TopK { + k: num_param(agg)? as usize, + descending: true, + } + } else if op == token::T_BOTTOMK { + Outer::TopK { + k: num_param(agg)? as usize, + descending: false, + } + } else if op == token::T_COUNT { + Outer::Count + } else if op == token::T_SUM || op == token::T_GROUP { + Outer::Plain(OuterIntent::Sum) + } else if op == token::T_AVG { + Outer::Plain(OuterIntent::Avg) + } else if op == token::T_MIN { + Outer::Plain(OuterIntent::Min) + } else if op == token::T_MAX { + Outer::Plain(OuterIntent::Max) + } else if op == token::T_STDDEV { + Outer::Plain(OuterIntent::StdDev) + } else if op == token::T_STDVAR { + Outer::Plain(OuterIntent::Variance) + } else if op == token::T_QUANTILE { + Outer::Plain(OuterIntent::Quantile(num_param(agg)?)) + } else { + return Err(LoweringError::UnsupportedAggregateOp(format!( + "aggregate token {op}" + ))); + }; + + self.build(inner, group, outer) + } + + fn walk_binary(&self, bin: &BinaryExpr) -> Result { + let lhs = self.walk(&bin.lhs)?; + let rhs = self.walk(&bin.rhs)?; + let op = binop(bin.op.id())?; + let vector_match = bin.modifier.as_ref().map(|m| { + let (on, labels) = match &m.matching { + Some(LabelModifier::Include(ls)) => (true, ls.labels.clone()), + Some(LabelModifier::Exclude(ls)) => (false, ls.labels.clone()), + None => (true, vec![]), + }; + let grouping = match &m.card { + VectorMatchCardinality::ManyToOne(ls) => Some(VectorGrouping { + side: GroupSide::Left, + labels: ls.labels.clone(), + }), + VectorMatchCardinality::OneToMany(ls) => Some(VectorGrouping { + side: GroupSide::Right, + labels: ls.labels.clone(), + }), + _ => None, + }; + VectorMatch { + on, + labels, + grouping, + } + }); + Ok(QueryExpr::BinaryOp { + op, + lhs: node(lhs), + rhs: node(rhs), + vector_match, + }) + } + + /// Lower a function/selector argument into an `Inner`. + fn lower_inner(&self, expr: &Expr) -> Result { + match expr { + Expr::VectorSelector(vs) => { + let (metric, predicates) = vs_to_scan(vs); + Ok(Inner { + metric, + predicates, + window: None, + func: None, + }) + } + Expr::MatrixSelector(ms) => { + let (metric, predicates) = vs_to_scan(&ms.vs); + Ok(Inner { + metric, + predicates, + window: Some(ms.range), + func: None, + }) + } + Expr::Paren(p) => self.lower_inner(&p.expr), + Expr::Call(call) => self.lower_inner_call(call), + other => Err(LoweringError::UnsupportedFeature(format!( + "aggregate argument: {:?}", + std::mem::discriminant(other) + ))), + } + } + + fn lower_inner_call(&self, call: &Call) -> Result { + let name = call.func.name; + // Functions whose range-vector argument is at index 0. + let at0 = |func: InnerFunc| -> Result { + let (metric, predicates, window) = extract_matrix(arg(call, 0)?)?; + Ok(Inner { + metric, + predicates, + window: Some(window), + func: Some(func), + }) + }; + match name { + "rate" | "irate" => { + let (metric, predicates, window) = extract_matrix(arg(call, 0)?)?; + Ok(Inner { + metric, + predicates, + window: Some(window), + func: Some(InnerFunc::Rate(window)), + }) + } + "increase" => { + let (metric, predicates, window) = extract_matrix(arg(call, 0)?)?; + Ok(Inner { + metric, + predicates, + window: Some(window), + func: Some(InnerFunc::Increase(window)), + }) + } + "quantile_over_time" => { + let phi = num_arg(call, 0)?; + let (metric, predicates, window) = extract_matrix(arg(call, 1)?)?; + Ok(Inner { + metric, + predicates, + window: Some(window), + func: Some(InnerFunc::Quantile(phi)), + }) + } + // Substitute histogram_quantile(φ, buckets) with a plain Quantile(φ) + // over the bucket stream; bucket-aware physical reduction is an L4/L5 + // concern, not an L3 IR variant. + "histogram_quantile" => { + let phi = num_arg(call, 0)?; + let (metric, predicates, window) = extract_matrix(arg(call, 1)?)?; + Ok(Inner { + metric, + predicates, + window: Some(window), + func: Some(InnerFunc::Quantile(phi)), + }) + } + "avg_over_time" => at0(InnerFunc::Avg), + "min_over_time" => at0(InnerFunc::Min), + "max_over_time" => at0(InnerFunc::Max), + "sum_over_time" => at0(InnerFunc::Sum), + "stddev_over_time" => at0(InnerFunc::StdDev), + "stdvar_over_time" => at0(InnerFunc::Variance), + "count_over_time" | "changes" | "resets" => at0(InnerFunc::Count), + other => Err(LoweringError::UnsupportedFunction(other.to_string())), + } + } + + /// Assemble the final `QueryExpr` from a lowered inner vector, the resolved + /// group keys, and the enclosing aggregator shape. + fn build(&self, inner: Inner, group: Vec, outer: Outer) -> Result { + match outer { + Outer::None => match &inner.func { + None => Ok(scan(inner.metric, inner.predicates)), + Some(f) => { + let intent = self.inner_intent(f); + Ok(self.windowed_aggregate(inner, group, vec![intent])) + } + }, + Outer::Plain(outer_intent) => { + let intent = match &inner.func { + Some(f) => self.inner_intent(f), + None => self.outer_intent(&outer_intent), + }; + Ok(self.windowed_aggregate(inner, group, vec![intent])) + } + Outer::Count => Ok(self.windowed_aggregate( + inner, + group, + vec![AggIntent::Cardinality { + accuracy: self.accuracy.clone(), + }], + )), + Outer::TopK { k, descending } => { + // Heavy-hitter only when ranking by frequency (`count`): a + // dedicated sketch (SpaceSaving, CMS-with-heap) serves it in one + // pass, so it earns the first-class `AggIntent::TopK`. Any other + // ranking (`topk` over avg/quantile, all `bottomk`) is a generic + // order-by-value + limit, which has no sketch primitive — per the + // L3 design rule that keeps `Sort + Limit` distinct from `TopK`. + let heavy_hitter = descending && matches!(inner.func, Some(InnerFunc::Count)); + if heavy_hitter { + let by: Vec = group.iter().map(|g| ColumnRef(g.0.clone())).collect(); + let agg = vec![AggIntent::TopK { + k, + by, + accuracy: self.accuracy.clone(), + }]; + // Window over Aggregate, same as `windowed_aggregate`, but + // with an empty group list (the heavy-hitter keys live in + // the `TopK` intent itself). + Ok(self.windowed_aggregate(inner, vec![], agg)) + } else { + let intent = match &inner.func { + Some(f) => self.inner_intent(f), + None => AggIntent::Sum, + }; + let base = self.windowed_aggregate(inner, group, vec![intent]); + let sorted = QueryExpr::Sort { + child: node(base), + keys: vec![SortKey { + expr: L3Expr::Column(ColumnRef("value".into())), + ascending: !descending, + nulls_first: false, + }], + }; + Ok(QueryExpr::Limit { + child: node(sorted), + n: Some(k as u64), + offset: 0, + }) + } + } + } + } + + /// `[TimeWindow →] Aggregate → Scan`. The canonical windowed-aggregate + /// shape is **Window over Aggregate** (the window defines the flush/reset + /// lifecycle of the aggregate in its sub-DAG). Rate/Increase carry their own + /// window in the intent, so no `TimeWindow` node is emitted for them. + fn windowed_aggregate( + &self, + inner: Inner, + group: Vec, + aggs: Vec, + ) -> QueryExpr { + let skip_window = matches!( + inner.func, + Some(InnerFunc::Rate(_)) | Some(InnerFunc::Increase(_)) + ); + let window = inner.window; + let base = scan(inner.metric, inner.predicates); + let agg = QueryExpr::Aggregate { + child: node(base), + by: group, + aggs, + having: None, + }; + match window { + Some(w) if !skip_window => QueryExpr::TimeWindow { + child: node(agg), + kind: TimeWindowKind::Tumbling, + size: w, + slide: None, + }, + _ => agg, + } + } + + fn inner_intent(&self, f: &InnerFunc) -> AggIntent { + match f { + InnerFunc::Quantile(q) => AggIntent::Quantile { + q: *q, + accuracy: self.accuracy.clone(), + }, + InnerFunc::Avg => AggIntent::Avg, + InnerFunc::Min => AggIntent::Min, + InnerFunc::Max => AggIntent::Max, + InnerFunc::Sum => AggIntent::Sum, + InnerFunc::StdDev => AggIntent::StdDev { population: false }, + InnerFunc::Variance => AggIntent::Variance { population: false }, + InnerFunc::Count => AggIntent::Count { + accuracy: self.accuracy.clone(), + }, + InnerFunc::Rate(w) => AggIntent::Rate { window: *w }, + InnerFunc::Increase(w) => AggIntent::Increase { window: *w }, + } + } + + fn outer_intent(&self, o: &OuterIntent) -> AggIntent { + match o { + OuterIntent::Sum => AggIntent::Sum, + OuterIntent::Avg => AggIntent::Avg, + OuterIntent::Min => AggIntent::Min, + OuterIntent::Max => AggIntent::Max, + OuterIntent::StdDev => AggIntent::StdDev { population: false }, + OuterIntent::Variance => AggIntent::Variance { population: false }, + OuterIntent::Quantile(q) => AggIntent::Quantile { + q: *q, + accuracy: self.accuracy.clone(), + }, + } + } + + /// Resolve `by(labels)` / `without(labels)` into an explicit group-key list. + /// `without` needs the metric's full label set, looked up in the catalog. + fn resolve_group(&self, agg: &AggregateExpr) -> Result> { + match &agg.modifier { + None => Ok(vec![]), + Some(LabelModifier::Include(ls)) => { + Ok(ls.labels.iter().cloned().map(GroupKey).collect()) + } + Some(LabelModifier::Exclude(ls)) => { + let metric = find_metric(&agg.expr).ok_or_else(|| { + LoweringError::UnsupportedFeature( + "`without` over an expression with no metric selector".into(), + ) + })?; + let meta = self.catalog.metrics.get(&metric).ok_or_else(|| { + LoweringError::UnsupportedFeature(format!( + "`without(...)` requires metric '{metric}' to be registered in the catalog \ + (its full label set is needed to compute the kept labels)" + )) + })?; + Ok(meta + .labels + .iter() + .filter(|l| !ls.labels.contains(l)) + .cloned() + .map(GroupKey) + .collect()) + } + } + } +} + +// ── Free helpers ────────────────────────────────────────────────────────────── + +/// Wrap a `QueryExpr` in an untyped `L3Node` (empty schema). The schema pass +/// (`crate::populate_schemas`) fills schemas in bottom-up afterwards. +fn node(expr: QueryExpr) -> Arc { + Arc::new(L3Node { + expr, + schema: L3Schema { + fields: vec![], + time_index: None, + }, + }) +} + +fn scan(metric: String, predicates: Vec) -> QueryExpr { + QueryExpr::Scan { + source: Source::TimeSeries { + metric: MetricRef(metric), + time: None, + }, + predicates, + } +} + +/// Extract `(metric_name, label_predicates)` from a vector selector. +fn vs_to_scan(vs: &VectorSelector) -> (String, Vec) { + let metric = vs.name.clone().unwrap_or_else(|| { + vs.matchers + .matchers + .iter() + .find(|m| m.name == "__name__") + .map(|m| m.value.clone()) + .unwrap_or_default() + }); + let predicates = vs + .matchers + .matchers + .iter() + .filter(|m| m.name != "__name__") + .map(matcher_to_predicate) + .collect(); + (metric, predicates) +} + +fn matcher_to_predicate(m: &Matcher) -> Predicate { + let op = match &m.op { + MatchOp::Equal => CompareOp::Eq, + MatchOp::NotEqual => CompareOp::Ne, + MatchOp::Re(_) => CompareOp::Regex, + MatchOp::NotRe(_) => CompareOp::NotRegex, + }; + Predicate(L3Expr::Compare { + left: Box::new(L3Expr::Column(ColumnRef(m.name.clone()))), + op, + right: Box::new(L3Expr::Literal(L3Scalar::Utf8(m.value.clone()))), + }) +} + +/// Descend through `Call` / `Paren` wrappers to the `MatrixSelector` and pull +/// out `(metric, predicates, window)`. +fn extract_matrix(expr: &Expr) -> Result<(String, Vec, Duration)> { + match expr { + Expr::MatrixSelector(ms) => { + let (metric, predicates) = vs_to_scan(&ms.vs); + Ok((metric, predicates, ms.range)) + } + Expr::Paren(p) => extract_matrix(&p.expr), + Expr::Call(c) => extract_matrix(arg(c, 0)?), + other => Err(LoweringError::UnsupportedFeature(format!( + "expected range-vector argument, got {:?}", + std::mem::discriminant(other) + ))), + } +} + +/// First `VectorSelector` metric name reachable from `expr`. +fn find_metric(expr: &Expr) -> Option { + match expr { + Expr::VectorSelector(vs) => Some(vs_to_scan(vs).0), + Expr::MatrixSelector(ms) => Some(vs_to_scan(&ms.vs).0), + Expr::Paren(p) => find_metric(&p.expr), + Expr::Unary(u) => find_metric(&u.expr), + Expr::Subquery(sq) => find_metric(&sq.expr), + Expr::Aggregate(a) => find_metric(&a.expr), + Expr::Call(c) => c.args.args.iter().find_map(|a| find_metric(a)), + Expr::Binary(b) => find_metric(&b.lhs).or_else(|| find_metric(&b.rhs)), + _ => None, + } +} + +fn arg(call: &Call, idx: usize) -> Result<&Expr> { + call.args + .args + .get(idx) + .map(|b| b.as_ref()) + .ok_or_else(|| LoweringError::MissingArgument(format!("{} arg #{idx}", call.func.name))) +} + +fn num_arg(call: &Call, idx: usize) -> Result { + num_expr(arg(call, idx)?) +} + +fn num_param(agg: &AggregateExpr) -> Result { + match &agg.param { + Some(e) => num_expr(e), + None => Err(LoweringError::MissingArgument( + "aggregate parameter (k / φ)".into(), + )), + } +} + +fn num_expr(expr: &Expr) -> Result { + match expr { + Expr::NumberLiteral(n) => Ok(n.val), + other => Err(LoweringError::InvalidParameter(format!( + "expected a numeric literal, got {:?}", + std::mem::discriminant(other) + ))), + } +} + +fn binop(id: token::TokenId) -> Result { + Ok(if id == token::T_ADD { + BinaryOpKind::Add + } else if id == token::T_SUB { + BinaryOpKind::Sub + } else if id == token::T_MUL { + BinaryOpKind::Mul + } else if id == token::T_DIV { + BinaryOpKind::Div + } else if id == token::T_MOD { + BinaryOpKind::Mod + } else if id == token::T_POW { + BinaryOpKind::Pow + } else if id == token::T_ATAN2 { + BinaryOpKind::Atan2 + } else if id == token::T_EQLC { + BinaryOpKind::Eq + } else if id == token::T_NEQ { + BinaryOpKind::NotEq + } else if id == token::T_LSS { + BinaryOpKind::Lt + } else if id == token::T_LTE { + BinaryOpKind::LtEq + } else if id == token::T_GTR { + BinaryOpKind::Gt + } else if id == token::T_GTE { + BinaryOpKind::GtEq + } else if id == token::T_LAND { + BinaryOpKind::And + } else if id == token::T_LOR { + BinaryOpKind::Or + } else if id == token::T_LUNLESS { + BinaryOpKind::Unless + } else { + return Err(LoweringError::UnsupportedFeature(format!( + "binary operator token {id}" + ))); + }) +} diff --git a/crates/lower/src/schema_pass.rs b/crates/lower/src/schema_pass.rs new file mode 100644 index 00000000..73159370 --- /dev/null +++ b/crates/lower/src/schema_pass.rs @@ -0,0 +1,159 @@ +use std::sync::Arc; + +use asap_control_core::intent_algebra::expr::QueryExpr; +use asap_control_core::intent_algebra::schema::{HasSchema, L3Schema, SchemaCatalog}; +use asap_control_core::intent_algebra::L3Node; + +/// Recursively populate the `schema` field on every node in a `QueryExpr` tree. +/// +/// The lowerer builds every node with an empty schema. This pass walks the tree +/// bottom-up, computing each node's output schema from its children's schemas +/// and the catalog, and returns a fully typed `Arc` tree. +pub fn populate_schemas(expr: QueryExpr, catalog: &SchemaCatalog) -> Arc { + let (rebuilt, child_schemas) = rebuild(expr, catalog); + let refs: Vec<&L3Schema> = child_schemas.iter().collect(); + let schema = rebuilt.output_schema(&refs, catalog); + Arc::new(L3Node { + expr: rebuilt, + schema, + }) +} + +/// Rebuild the expression tree with populated child nodes, returning +/// `(rebuilt_expr, child_output_schemas)`. +fn rebuild(expr: QueryExpr, catalog: &SchemaCatalog) -> (QueryExpr, Vec) { + use QueryExpr::*; + + let proc = |n: Arc| populate_schemas(n.expr.clone(), catalog); + + match expr { + // Leaf: schema comes from the catalog, no child schemas needed. + Scan { .. } => (expr, vec![]), + + Filter { child, pred } => { + let c = proc(child); + let cs = c.schema.clone(); + (Filter { child: c, pred }, vec![cs]) + } + Project { child, cols } => { + let c = proc(child); + let cs = c.schema.clone(); + (Project { child: c, cols }, vec![cs]) + } + Aggregate { + child, + by, + aggs, + having, + } => { + let c = proc(child); + let cs = c.schema.clone(); + ( + Aggregate { + child: c, + by, + aggs, + having, + }, + vec![cs], + ) + } + Sort { child, keys } => { + let c = proc(child); + let cs = c.schema.clone(); + (Sort { child: c, keys }, vec![cs]) + } + Limit { child, n, offset } => { + let c = proc(child); + let cs = c.schema.clone(); + ( + Limit { + child: c, + n, + offset, + }, + vec![cs], + ) + } + Distinct { child, cols } => { + let c = proc(child); + let cs = c.schema.clone(); + (Distinct { child: c, cols }, vec![cs]) + } + Partition { child, keys } => { + let c = proc(child); + let cs = c.schema.clone(); + (Partition { child: c, keys }, vec![cs]) + } + TimeWindow { + child, + kind, + size, + slide, + } => { + let c = proc(child); + let cs = c.schema.clone(); + ( + TimeWindow { + child: c, + kind, + size, + slide, + }, + vec![cs], + ) + } + BinaryOp { + op, + lhs, + rhs, + vector_match, + } => { + let l = proc(lhs); + let r = proc(rhs); + let ls = l.schema.clone(); + let rs = r.schema.clone(); + ( + BinaryOp { + op, + lhs: l, + rhs: r, + vector_match, + }, + vec![ls, rs], + ) + } + SetOp { + kind, + all, + left, + right, + } => { + let l = proc(left); + let r = proc(right); + let ls = l.schema.clone(); + let rs = r.schema.clone(); + ( + SetOp { + kind, + all, + left: l, + right: r, + }, + vec![ls, rs], + ) + } + Merge { children } => { + let new_children: Vec> = children.into_iter().map(proc).collect(); + let schemas: Vec = new_children.iter().map(|c| c.schema.clone()).collect(); + ( + Merge { + children: new_children, + }, + schemas, + ) + } + // Unimplemented variants: return as-is; output_schema will todo!() if called. + other => (other, vec![]), + } +} diff --git a/crates/lower/tests/promql_lowering.rs b/crates/lower/tests/promql_lowering.rs new file mode 100644 index 00000000..9d51965d --- /dev/null +++ b/crates/lower/tests/promql_lowering.rs @@ -0,0 +1,496 @@ +//! End-to-end tests for PromQL → L3 intent-algebra lowering. + +use std::time::Duration; + +use asap_control_core::intent_algebra::expr::{ + AggIntent, BinaryOpKind, QueryExpr, Source, TimeWindowKind, +}; +use asap_control_core::intent_algebra::schema::{L3DataType, MetricSchema, SchemaCatalog}; +use asap_control_core::intent_algebra::{CompareOp, L3Expr, L3Scalar}; +use asap_control_core::types::AccuracyTarget; +use asap_control_core::workload::{ + BatchEntry, Query, QueryLanguage, QueryRequirements, QueryWorkload, +}; + +use asap_control_lower::{lower_promql, lower_promql_batch, populate_schemas}; + +// ── Fixtures ─────────────────────────────────────────────────────────────────── + +fn empty_catalog() -> SchemaCatalog { + SchemaCatalog::default() +} + +fn catalog_with(metric: &str, labels: &[&str]) -> SchemaCatalog { + let mut c = SchemaCatalog::default(); + c.metrics.insert( + metric.to_string(), + MetricSchema { + labels: labels.iter().map(|s| s.to_string()).collect(), + value_type: L3DataType::Float64, + }, + ); + c +} + +fn lower(q: &str) -> QueryExpr { + lower_promql(q, &empty_catalog(), AccuracyTarget::Exact) + .unwrap_or_else(|e| panic!("lower failed for {q:?}: {e}")) +} + +fn lower_eps(q: &str) -> QueryExpr { + lower_promql(q, &empty_catalog(), AccuracyTarget::Epsilon(0.01)) + .unwrap_or_else(|e| panic!("lower failed for {q:?}: {e}")) +} + +// ── Bare selectors & label matchers ───────────────────────────────────────────── + +#[test] +fn bare_selector_is_scan_with_predicates() { + let qe = lower(r#"http_requests_total{env="prod",status!="500"}"#); + let QueryExpr::Scan { source, predicates } = &qe else { + panic!("expected Scan, got {qe:?}"); + }; + match source { + Source::TimeSeries { metric, time } => { + assert_eq!(metric.0, "http_requests_total"); + assert!(time.is_none(), "PromQL string carries no absolute range"); + } + other => panic!("expected TimeSeries source, got {other:?}"), + } + assert_eq!(predicates.len(), 2); +} + +#[test] +fn regex_matcher_lowers_to_regex_compareop() { + let qe = lower(r#"http_requests_total{path=~"/api/.*"}"#); + let QueryExpr::Scan { predicates, .. } = &qe else { + panic!("expected Scan, got {qe:?}"); + }; + let L3Expr::Compare { left, op, right } = &predicates[0].0 else { + panic!("expected Compare predicate, got {:?}", predicates[0].0); + }; + assert_eq!(*op, CompareOp::Regex); + assert!(matches!(left.as_ref(), L3Expr::Column(c) if c.0 == "path")); + assert!(matches!(right.as_ref(), L3Expr::Literal(L3Scalar::Utf8(v)) if v == "/api/.*")); +} + +// ── *_over_time → Window → Aggregate ───────────────────────────────────────────── + +#[test] +fn quantile_over_time_is_window_over_aggregate() { + let qe = lower(r#"quantile_over_time(0.99, http_request_duration{env="prod"}[5m])"#); + let QueryExpr::TimeWindow { + kind, size, child, .. + } = &qe + else { + panic!("expected TimeWindow, got {qe:?}"); + }; + assert_eq!(*kind, TimeWindowKind::Tumbling); + assert_eq!(*size, Duration::from_secs(300)); + match &child.expr { + QueryExpr::Aggregate { + by, aggs, child, .. + } => { + assert!(by.is_empty()); + assert!(matches!( + aggs.as_slice(), + [AggIntent::Quantile { q, .. }] if (*q - 0.99).abs() < 1e-9 + )); + // The label matcher rides on the Scan, not a separate Filter node. + assert!( + matches!(&child.expr, QueryExpr::Scan { predicates, .. } if predicates.len() == 1) + ); + } + other => panic!("expected Aggregate under TimeWindow, got {other:?}"), + } +} + +#[test] +fn outer_sum_by_pushes_group_keys_onto_inner_aggregate() { + let qe = lower(r#"sum by (host) (quantile_over_time(0.99, latency{service="web"}[5m]))"#); + let QueryExpr::TimeWindow { child, .. } = &qe else { + panic!("expected TimeWindow, got {qe:?}"); + }; + let QueryExpr::Aggregate { by, aggs, .. } = &child.expr else { + panic!("expected Aggregate, got {:?}", child.expr); + }; + assert_eq!(by.len(), 1); + assert_eq!(by[0].0, "host"); + // The inner quantile_over_time supplies the intent; the outer `sum` only groups. + assert!(matches!(aggs.as_slice(), [AggIntent::Quantile { .. }])); +} + +#[test] +fn avg_over_time_maps_to_avg_intent() { + let qe = lower("avg_over_time(cpu_seconds_total[10m])"); + let QueryExpr::TimeWindow { size, child, .. } = &qe else { + panic!("expected TimeWindow, got {qe:?}"); + }; + assert_eq!(*size, Duration::from_secs(600)); + assert!(matches!( + &child.expr, + QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Avg]) + )); +} + +#[test] +fn min_max_sum_over_time_map_directly() { + for (q, want) in [ + ("min_over_time(m[5m])", "min"), + ("max_over_time(m[5m])", "max"), + ("sum_over_time(m[5m])", "sum"), + ] { + let qe = lower(q); + let QueryExpr::TimeWindow { child, .. } = &qe else { + panic!("expected TimeWindow for {q}"); + }; + let QueryExpr::Aggregate { aggs, .. } = &child.expr else { + panic!("expected Aggregate for {q}"); + }; + let ok = matches!( + (want, &aggs[0]), + ("min", AggIntent::Min) | ("max", AggIntent::Max) | ("sum", AggIntent::Sum) + ); + assert!(ok, "{q} produced {:?}", aggs[0]); + } +} + +#[test] +fn stddev_and_stdvar_over_time() { + let qe = lower("stddev_over_time(m[5m])"); + let QueryExpr::TimeWindow { child, .. } = &qe else { + panic!("expected TimeWindow"); + }; + assert!(matches!( + &child.expr, + QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::StdDev { population: false }]) + )); + + let qe = lower("stdvar_over_time(m[5m])"); + let QueryExpr::TimeWindow { child, .. } = &qe else { + panic!("expected TimeWindow"); + }; + assert!(matches!( + &child.expr, + QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Variance { population: false }]) + )); +} + +// ── histogram_quantile ─────────────────────────────────────────────────────────── + +#[test] +fn histogram_quantile_substitutes_to_quantile() { + let qe = lower(r#"histogram_quantile(0.95, rate(http_duration_seconds_bucket{le="0.5"}[5m]))"#); + let QueryExpr::TimeWindow { size, child, .. } = &qe else { + panic!("expected TimeWindow, got {qe:?}"); + }; + assert_eq!(*size, Duration::from_secs(300)); + let QueryExpr::Aggregate { aggs, child, .. } = &child.expr else { + panic!("expected Aggregate"); + }; + assert!(matches!( + aggs.as_slice(), + [AggIntent::Quantile { q, .. }] if (*q - 0.95).abs() < 1e-9 + )); + // The `le` matcher is preserved on the bucket scan. + assert!(matches!(&child.expr, QueryExpr::Scan { predicates, .. } if predicates.len() == 1)); +} + +// ── rate / increase carry their own window ─────────────────────────────────────── + +#[test] +fn rate_has_no_timewindow_node() { + let qe = lower("rate(http_requests_total[5m])"); + let QueryExpr::Aggregate { aggs, child, .. } = &qe else { + panic!("expected Aggregate (no TimeWindow) for rate, got {qe:?}"); + }; + assert!(matches!( + aggs.as_slice(), + [AggIntent::Rate { window }] if *window == Duration::from_secs(300) + )); + assert!(matches!(&child.expr, QueryExpr::Scan { .. })); +} + +#[test] +fn increase_maps_to_increase_intent() { + let qe = lower("increase(errors_total[1h])"); + let QueryExpr::Aggregate { aggs, .. } = &qe else { + panic!("expected Aggregate for increase, got {qe:?}"); + }; + assert!(matches!( + aggs.as_slice(), + [AggIntent::Increase { window }] if *window == Duration::from_secs(3600) + )); +} + +// ── count / cardinality ─────────────────────────────────────────────────────────── + +#[test] +fn count_over_time_is_count_intent() { + let qe = lower("count_over_time(m[5m])"); + let QueryExpr::TimeWindow { child, .. } = &qe else { + panic!("expected TimeWindow"); + }; + assert!(matches!( + &child.expr, + QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Count { .. }]) + )); +} + +#[test] +fn outer_count_is_cardinality() { + let qe = lower("count by (symbol) (count_over_time(financial_last_trade_price[5m]))"); + let QueryExpr::TimeWindow { child, .. } = &qe else { + panic!("expected TimeWindow, got {qe:?}"); + }; + let QueryExpr::Aggregate { by, aggs, .. } = &child.expr else { + panic!("expected Aggregate"); + }; + assert_eq!(by.len(), 1); + assert_eq!(by[0].0, "symbol"); + assert!(matches!(aggs.as_slice(), [AggIntent::Cardinality { .. }])); +} + +// ── topk / bottomk ──────────────────────────────────────────────────────────────── + +#[test] +fn topk_over_count_is_heavy_hitter_topk() { + let qe = lower(r#"topk by (service) (10, count_over_time(requests{env="prod"}[1m]))"#); + // Window over Aggregate: the heavy-hitter top-k is computed per 1m window. + let QueryExpr::TimeWindow { size, child, .. } = &qe else { + panic!("expected TimeWindow, got {qe:?}"); + }; + assert_eq!(*size, Duration::from_secs(60)); + let QueryExpr::Aggregate { + by, aggs, child, .. + } = &child.expr + else { + panic!("expected Aggregate with TopK, got {:?}", child.expr); + }; + assert!(by.is_empty(), "heavy-hitter keys live in the TopK intent"); + match aggs.as_slice() { + [AggIntent::TopK { k, by, .. }] => { + assert_eq!(*k, 10); + assert_eq!(by.len(), 1); + assert_eq!(by[0].0, "service"); + } + other => panic!("expected TopK intent, got {other:?}"), + } + // The heavy-hitter sketch counts directly off the scan in one pass. + assert!(matches!(&child.expr, QueryExpr::Scan { .. })); +} + +#[test] +fn topk_over_avg_is_generic_sort_limit() { + // Ranking by avg value has no heavy-hitter sketch → generic Sort + Limit. + let qe = lower("topk by (host) (5, avg_over_time(cpu[5m]))"); + let QueryExpr::Limit { n, offset, child } = &qe else { + panic!("expected Limit, got {qe:?}"); + }; + assert_eq!(*n, Some(5)); + assert_eq!(*offset, 0); + let QueryExpr::Sort { keys, child } = &child.expr else { + panic!("expected Sort under Limit, got {:?}", child.expr); + }; + assert_eq!(keys.len(), 1); + assert!(!keys[0].ascending, "topk ranks descending"); + // Underneath: the windowed avg aggregate grouped by host. + let QueryExpr::TimeWindow { child, .. } = &child.expr else { + panic!("expected TimeWindow under Sort"); + }; + assert!(matches!( + &child.expr, + QueryExpr::Aggregate { by, aggs, .. } + if by.len() == 1 && by[0].0 == "host" && matches!(aggs.as_slice(), [AggIntent::Avg]) + )); +} + +#[test] +fn bottomk_is_always_generic_sort_ascending() { + let qe = lower("bottomk(3, count_over_time(m[5m]))"); + let QueryExpr::Limit { n, child, .. } = &qe else { + panic!("expected Limit, got {qe:?}"); + }; + assert_eq!(*n, Some(3)); + let QueryExpr::Sort { keys, .. } = &child.expr else { + panic!("expected Sort"); + }; + assert!(keys[0].ascending, "bottomk ranks ascending"); +} + +// ── binary ops ──────────────────────────────────────────────────────────────────── + +#[test] +fn binary_op_division() { + let qe = lower("rate(a[5m]) / rate(b[5m])"); + let QueryExpr::BinaryOp { op, lhs, rhs, .. } = &qe else { + panic!("expected BinaryOp, got {qe:?}"); + }; + assert_eq!(*op, BinaryOpKind::Div); + assert!( + matches!(&lhs.expr, QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Rate { .. }])) + ); + assert!( + matches!(&rhs.expr, QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Rate { .. }])) + ); +} + +#[test] +fn binary_op_with_on_grouping() { + let qe = lower("a / on(host) b"); + let QueryExpr::BinaryOp { vector_match, .. } = &qe else { + panic!("expected BinaryOp, got {qe:?}"); + }; + let vm = vector_match.as_ref().expect("vector_match present"); + assert!(vm.on); + assert_eq!(vm.labels, vec!["host".to_string()]); +} + +// ── without resolution ────────────────────────────────────────────────────────── + +#[test] +fn without_resolves_kept_labels_from_catalog() { + let catalog = catalog_with("m", &["host", "region", "instance"]); + let qe = lower_promql( + "sum without (instance) (rate(m[5m]))", + &catalog, + AccuracyTarget::Exact, + ) + .expect("lower with catalog"); + let QueryExpr::Aggregate { by, .. } = &qe else { + panic!("expected Aggregate, got {qe:?}"); + }; + let mut got: Vec<&str> = by.iter().map(|g| g.0.as_str()).collect(); + got.sort(); + assert_eq!(got, vec!["host", "region"]); +} + +#[test] +fn without_without_catalog_is_unsupported() { + let err = lower_promql( + "sum without (instance) (rate(m[5m]))", + &empty_catalog(), + AccuracyTarget::Exact, + ) + .unwrap_err(); + assert!(format!("{err}").contains("without"), "got {err}"); +} + +// ── accuracy propagation ────────────────────────────────────────────────────────── + +#[test] +fn accuracy_target_flows_into_quantile_intent() { + let qe = lower_eps("quantile_over_time(0.9, m[5m])"); + let QueryExpr::TimeWindow { child, .. } = &qe else { + panic!("expected TimeWindow"); + }; + let QueryExpr::Aggregate { aggs, .. } = &child.expr else { + panic!("expected Aggregate"); + }; + assert!(matches!( + &aggs[0], + AggIntent::Quantile { accuracy: AccuracyTarget::Epsilon(e), .. } if (*e - 0.01).abs() < 1e-12 + )); +} + +// ── schema population ─────────────────────────────────────────────────────────── + +#[test] +fn schema_population_for_quantile_over_time() { + let catalog = catalog_with("http_request_duration", &["env", "host"]); + let qe = lower_promql( + r#"quantile_over_time(0.99, http_request_duration{env="prod"}[5m])"#, + &catalog, + AccuracyTarget::Exact, + ) + .unwrap(); + let typed = populate_schemas(qe, &catalog); + // Root TimeWindow passes the aggregate's schema through: a single Float64 + // `value` column (group-by is empty). + assert_eq!(typed.schema.fields.len(), 1); + assert_eq!(typed.schema.fields[0].name, "value"); + assert_eq!(typed.schema.fields[0].dtype, L3DataType::Float64); + + // Walk to the Scan leaf: timestamp + value + the two catalog labels. + fn scan_schema(n: &asap_control_core::intent_algebra::L3Node) -> Vec { + match &n.expr { + QueryExpr::Scan { .. } => n.schema.fields.iter().map(|f| f.name.clone()).collect(), + QueryExpr::TimeWindow { child, .. } + | QueryExpr::Aggregate { child, .. } + | QueryExpr::Filter { child, .. } => scan_schema(child), + other => panic!("unexpected node {other:?}"), + } + } + let mut leaf = scan_schema(&typed); + leaf.sort(); + assert_eq!(leaf, vec!["env", "host", "timestamp", "value"]); +} + +#[test] +fn schema_population_for_heavy_hitter_topk() { + let catalog = catalog_with("requests", &["service", "env"]); + let qe = lower_promql( + "topk by (service) (10, count_over_time(requests[1m]))", + &catalog, + AccuracyTarget::Exact, + ) + .unwrap(); + let typed = populate_schemas(qe, &catalog); + // TopK output: the by-column(s) + a synthetic `count` column. + let names: Vec<&str> = typed + .schema + .fields + .iter() + .map(|f| f.name.as_str()) + .collect(); + assert_eq!(names, vec!["service", "count"]); + assert_eq!(typed.schema.fields[1].dtype, L3DataType::Int64); +} + +// ── batch entry point ───────────────────────────────────────────────────────────── + +#[test] +fn batch_lowers_each_entry_and_reads_per_query_accuracy() { + let workload = QueryWorkload { + language: QueryLanguage::PromQL, + query_batch: Some(vec![ + BatchEntry { + query: Query("rate(a[5m])".into()), + requirements: None, + }, + BatchEntry { + query: Query("quantile_over_time(0.9, b[5m])".into()), + requirements: Some(QueryRequirements { + accuracy: Some(AccuracyTarget::Epsilon(0.02)), + latency_ms: None, + }), + }, + ]), + repeating_queries: None, + data_characteristics: None, + }; + let results = lower_promql_batch(&workload, &empty_catalog()); + assert_eq!(results.len(), 2); + assert!(results[0].is_ok()); + assert!(results[1].is_ok()); +} + +#[test] +fn batch_rejects_non_promql_language() { + use asap_control_core::workload::SqlDialect; + let workload = QueryWorkload { + language: QueryLanguage::SQL(SqlDialect::DataFusionSQL), + query_batch: Some(vec![BatchEntry { + query: Query("SELECT 1".into()), + requirements: None, + }]), + repeating_queries: None, + data_characteristics: None, + }; + let results = lower_promql_batch(&workload, &empty_catalog()); + assert_eq!(results.len(), 1); + assert!(matches!( + results[0], + Err(asap_control_lower::LoweringError::WrongLanguage(_)) + )); +} From fb17ce95aed12eaf585446f9d4e19f0ef5a6053b Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 20 May 2026 11:42:32 -0600 Subject: [PATCH 02/40] refactor(promql): adopt asapquery-backend control-plane IR architecture Restructure the L1-L3 plumbing to mirror the ASAPQuery-backend control plane while keeping this PR's PromQL semantics and intent vocabulary: - Two IRs + Binder: the PromQL parser now emits a Layer-2 `relational::QueryExpr`; `lower::convert_root` runs the `Binder` (name -> positional ColumnId resolution against a self-contained Schema) and folds single-statistic aggregates into canonical shapes. - Positional Schema: `Schema { columns, time_index, unique_keys }` with `ColumnId = usize`; `Aggregate.by: Vec`; `cse_reuse_is_legal` gating shared-producer reuse on `unique_keys`. - `Source::TimeSeries { metric: String }` with the schema carried on the `Scan` node (Binder-built), not on an L3Node edge wrapper. - `Box` tree (DAG fan-in via LetBinding/Ref) replaces the Arc schema-per-edge model. - Full workload-level CSE (`cse::dedupe_subtrees` + LetBinding/Ref). Kept from the prior PromQL work: the AggIntent vocabulary (adds StdDev/Variance; no Frequency/archive intents), the heavy-hitter TopK vs generic Sort+Limit split, no redundant Window node for rate/increase, L3Expr label-matcher predicates with CompareOp::Regex, and the typed LoweringError. Co-Authored-By: Claude Opus 4.7 (1M context) --- Cargo.lock | 44 + crates/core/Cargo.toml | 7 + crates/core/src/intent_algebra/agg_intent.rs | 245 +++++ crates/core/src/intent_algebra/binder.rs | 189 ++++ .../src/intent_algebra/column_resolution.rs | 177 ++++ crates/core/src/intent_algebra/cse.rs | 224 +++++ crates/core/src/intent_algebra/expr.rs | 639 -------------- crates/core/src/intent_algebra/expr_ir.rs | 45 +- crates/core/src/intent_algebra/lower.rs | 288 ++++++ crates/core/src/intent_algebra/mod.rs | 42 +- crates/core/src/intent_algebra/names.rs | 27 + crates/core/src/intent_algebra/query_expr.rs | 445 ++++++++++ crates/core/src/intent_algebra/relational.rs | 212 +++++ crates/core/src/intent_algebra/schema.rs | 393 +++++++-- crates/core/src/sketch_algebra/expr.rs | 8 +- crates/core/src/sketch_algebra/schema.rs | 4 +- crates/core/src/types.rs | 4 +- crates/lower/src/error.rs | 9 + crates/lower/src/lib.rs | 40 +- crates/lower/src/promql.rs | 834 ++++++++---------- crates/lower/src/schema_pass.rs | 159 ---- crates/lower/tests/promql_lowering.rs | 397 ++++----- 22 files changed, 2779 insertions(+), 1653 deletions(-) create mode 100644 crates/core/src/intent_algebra/agg_intent.rs create mode 100644 crates/core/src/intent_algebra/binder.rs create mode 100644 crates/core/src/intent_algebra/column_resolution.rs create mode 100644 crates/core/src/intent_algebra/cse.rs delete mode 100644 crates/core/src/intent_algebra/expr.rs create mode 100644 crates/core/src/intent_algebra/lower.rs create mode 100644 crates/core/src/intent_algebra/names.rs create mode 100644 crates/core/src/intent_algebra/query_expr.rs create mode 100644 crates/core/src/intent_algebra/relational.rs delete mode 100644 crates/lower/src/schema_pass.rs diff --git a/Cargo.lock b/Cargo.lock index d2c6cc16..e61301ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,6 +29,11 @@ checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "asap-control-core" version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "thiserror", +] [[package]] name = "asap-control-lower" @@ -469,6 +474,19 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + [[package]] name = "shlex" version = "1.3.0" @@ -510,6 +528,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "time" version = "0.3.47" @@ -679,3 +717,9 @@ checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" dependencies = [ "windows-link", ] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 99972465..beadf2e0 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -2,3 +2,10 @@ name = "asap-control-core" version = "0.1.0" edition = "2021" + +[dependencies] +serde = { version = "1", features = ["derive"] } +thiserror = "2" + +[dev-dependencies] +serde_json = "1" diff --git a/crates/core/src/intent_algebra/agg_intent.rs b/crates/core/src/intent_algebra/agg_intent.rs new file mode 100644 index 00000000..6c8d90b5 --- /dev/null +++ b/crates/core/src/intent_algebra/agg_intent.rs @@ -0,0 +1,245 @@ +//! Layer 3 aggregation-intent vocabulary — "what to compute, not how". +//! +//! L3 carries intent ("compute a quantile to ε=0.01 accuracy"); the choice +//! between `HashAgg` / `SortAgg` / `SketchAgg(KLL{k=200})` is an L4 cost-aware +//! decision, not encoded here. +//! +//! `AggIntent::TopK` is a first-class *intent* — a dedicated heavy-hitter +//! sketch (SpaceSaving, CMS-with-heap) computes it in one pass. Generic +//! `ORDER BY value LIMIT k` stays as the `QueryExpr::Sort + Limit` operator +//! pair. L1→L2→L3 lowering picks one or the other deterministically. + +use std::time::Duration; + +use serde::{Deserialize, Serialize}; + +use crate::intent_algebra::query_expr::DataModel; +use crate::intent_algebra::schema::{Column, DataType}; +use crate::types::AccuracyTarget; + +/// "What to compute" at L3 — the vocabulary the planner pivots on. +/// +/// Grouping for `TopK` rides on the enclosing `QueryExpr::Aggregate.by` +/// (positional `ColumnId`s), like every other aggregate; the intent itself +/// carries only `k` + the accuracy target. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum AggIntent { + // ── Data-model-agnostic ────────────────────────────────────────────── + Count { + accuracy: AccuracyTarget, + }, + Sum, + Min, + Max, + Avg, + /// Sample standard deviation when `population == false`; population stddev + /// otherwise. PromQL `stddev` / `stddev_over_time`. + StdDev { + population: bool, + }, + /// Variance — PromQL `stdvar` / `stdvar_over_time`. + Variance { + population: bool, + }, + Quantile { + q: f64, + accuracy: AccuracyTarget, + }, + /// Heavy-hitter top-k — served by a dedicated sketch in one pass. The + /// group-by keys live on the enclosing `Aggregate.by`. + TopK { + k: usize, + accuracy: AccuracyTarget, + }, + Cardinality { + accuracy: AccuracyTarget, + }, + + // ── Time-series streaming derivatives ──────────────────────────────── + // Carry PromQL's counter-reset adjustment; not equivalent to Sum/Count + // over a Window. Kept distinct so delta-set aggregators bind directly. + Rate { + window: Duration, + }, + Increase { + window: Duration, + }, +} + +impl AggIntent { + /// Which data model this intent semantically requires. L4 rules consult + /// this to skip non-applicable intents (e.g. `Rate` over a tabular source). + pub fn requires(&self) -> DataModel { + match self { + Self::Rate { .. } | Self::Increase { .. } => DataModel::TimeSeries, + _ => DataModel::Any, + } + } + + /// Output column name + type produced by this intent over `input`. + /// Used by `QueryExpr::Aggregate`'s schema-derivation rule. The PromQL + /// convention names the column after the intent kind so consumers can + /// locate it without an alias lookup. + pub fn output_column(&self, input: &Column) -> Column { + match self { + AggIntent::Count { .. } => col("count", DataType::Int64, false), + AggIntent::Sum => col("sum", input.dtype.clone(), false), + AggIntent::Min => col("min", input.dtype.clone(), input.nullable), + AggIntent::Max => col("max", input.dtype.clone(), input.nullable), + AggIntent::Avg => col("avg", DataType::Float64, false), + AggIntent::StdDev { .. } => col("stddev", DataType::Float64, false), + AggIntent::Variance { .. } => col("variance", DataType::Float64, false), + AggIntent::Quantile { q, .. } => col( + &format!("quantile_{}", quantile_suffix(*q)), + DataType::Float64, + false, + ), + // TopK output is a per-row struct/list; modeled as Utf8 at L3 + // (the L4 sketch-bound IR upgrades the dtype). + AggIntent::TopK { k, .. } => col(&format!("topk_{k}"), DataType::Utf8, false), + AggIntent::Cardinality { .. } => col("cardinality", DataType::Int64, false), + AggIntent::Rate { .. } => col("rate", DataType::Float64, false), + AggIntent::Increase { .. } => col("increase", DataType::Float64, false), + } + } +} + +fn col(name: &str, dtype: DataType, nullable: bool) -> Column { + Column { + name: name.into(), + dtype, + nullable, + } +} + +/// `0.99` → `"0_99"`, `0.5` → `"0_5"`. Used by `Quantile` output naming so +/// `quantile_0_99` is a valid identifier downstream. +fn quantile_suffix(q: f64) -> String { + let mut s = format!("{q}"); + if let Some(stripped) = s.strip_prefix('-') { + s = format!("neg_{stripped}"); + } + s.replace('.', "_") +} + +// ── AggIntent helpers ──────────────────────────────────────────────────────── + +/// Two instances of this aggregation can be merged +/// (`agg(A ∪ B) = combine(agg(A), agg(B))`). `Avg` / `StdDev` / `Variance` +/// need richer partial state than a single value, so they are not mergeable. +pub fn agg_is_mergeable(op: &AggIntent) -> bool { + !matches!( + op, + AggIntent::Avg | AggIntent::StdDev { .. } | AggIntent::Variance { .. } + ) +} + +/// Whether this op implies `exact_required` — no sketch benefit. The exact +/// intents are `Sum / Count / Avg / Min / Max`. +pub fn agg_is_exact(op: &AggIntent) -> bool { + matches!( + op, + AggIntent::Sum | AggIntent::Count { .. } | AggIntent::Avg | AggIntent::Min | AggIntent::Max + ) +} + +/// Accuracy parameter as a fractional ε (`0.0` for exact ops), unpacked from +/// the typed `AccuracyTarget` on Quantile / Cardinality / Count / TopK. +pub fn agg_accuracy(op: &AggIntent) -> f64 { + match op { + AggIntent::Quantile { accuracy, .. } + | AggIntent::Cardinality { accuracy } + | AggIntent::Count { accuracy } + | AggIntent::TopK { accuracy, .. } => accuracy_target_to_f64(accuracy), + _ => 0.0, + } +} + +fn accuracy_target_to_f64(t: &AccuracyTarget) -> f64 { + match t { + AccuracyTarget::Exact => 0.0, + AccuracyTarget::Epsilon(eps) => *eps, + AccuracyTarget::EpsilonDelta { epsilon, .. } => *epsilon, + } +} + +/// Default `Cardinality` intent — HLL standard error at precision p=14. +pub fn default_cardinality() -> AggIntent { + AggIntent::Cardinality { + accuracy: AccuracyTarget::Epsilon(1.04 / ((1u64 << 14) as f64).sqrt()), + } +} + +/// Default `Quantile` intent at φ = `q`, `accuracy = ε 0.01`. +pub fn default_quantile(q: f64) -> AggIntent { + AggIntent::Quantile { + q, + accuracy: AccuracyTarget::Epsilon(0.01), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::intent_algebra::schema::{Column, DataType}; + + fn c(name: &str, dtype: DataType) -> Column { + Column { + name: name.into(), + dtype, + nullable: false, + } + } + + #[test] + fn output_column_names_are_intent_keyed() { + let v = c("value", DataType::Float64); + assert_eq!( + AggIntent::Count { + accuracy: AccuracyTarget::Exact + } + .output_column(&v) + .name, + "count" + ); + assert_eq!(AggIntent::Sum.output_column(&v).name, "sum"); + assert_eq!( + AggIntent::Quantile { + q: 0.99, + accuracy: AccuracyTarget::Epsilon(0.01) + } + .output_column(&v) + .name, + "quantile_0_99" + ); + } + + #[test] + fn sum_preserves_input_dtype() { + assert!(matches!( + AggIntent::Sum.output_column(&c("c", DataType::Int64)).dtype, + DataType::Int64 + )); + } + + #[test] + fn mergeability_and_exactness() { + assert!(agg_is_mergeable(&AggIntent::Sum)); + assert!(!agg_is_mergeable(&AggIntent::Avg)); + assert!(!agg_is_mergeable(&AggIntent::StdDev { population: false })); + assert!(agg_is_exact(&AggIntent::Min)); + assert!(!agg_is_exact(&default_cardinality())); + } + + #[test] + fn agg_intent_serde_roundtrip() { + let v = AggIntent::Quantile { + q: 0.99, + accuracy: AccuracyTarget::Epsilon(0.01), + }; + let json = serde_json::to_string(&v).unwrap(); + let back: AggIntent = serde_json::from_str(&json).unwrap(); + assert_eq!(v, back); + } +} diff --git a/crates/core/src/intent_algebra/binder.rs b/crates/core/src/intent_algebra/binder.rs new file mode 100644 index 00000000..4d0d2bc4 --- /dev/null +++ b/crates/core/src/intent_algebra/binder.rs @@ -0,0 +1,189 @@ +//! The L3 **Binder** — name resolution as an explicit pass. +//! +//! [`Binder::bind`] produces the complete, self-contained [`Schema`] every +//! `ColumnId` in the converted canonical tree indexes into. The converter +//! ([`super::lower::convert`]) then becomes purely structural: it threads the +//! Binder's schema and positional resolution downstream is **total**. +//! +//! The default [`UsageDerivedCatalog`] knows nothing — every schema is derived +//! purely from the query's own usage. That is the honest state for the +//! observability domain (metric label sets are open-ended). A registry-backed +//! `SchemaCatalog` is future work; the `Binder` pass does not change when it +//! lands, only the catalog impl swaps. + +use crate::intent_algebra::relational::QueryExpr as LQueryExpr; +use crate::intent_algebra::schema::{Column, DataType, Schema}; + +/// The DB / source-schema metadata source — resolves a source (metric / +/// table) name to its known columns. +pub trait SchemaCatalog { + /// Columns known for `source`. `None` when unknown — the [`Binder`] then + /// falls back to a usage-derived column set. + fn columns_for(&self, source: &str) -> Option>; +} + +/// The default catalog: knows nothing. Every schema the [`Binder`] produces +/// is derived purely from the query's own usage. +pub struct UsageDerivedCatalog; + +impl SchemaCatalog for UsageDerivedCatalog { + fn columns_for(&self, _source: &str) -> Option> { + None + } +} + +/// The L3 Binder — the explicit name-resolution pass. +pub struct Binder { + catalog: C, +} + +impl Default for Binder { + fn default() -> Self { + Self::new() + } +} + +impl Binder { + pub fn new() -> Self { + Self { + catalog: UsageDerivedCatalog, + } + } +} + +impl Binder { + pub fn with_catalog(catalog: C) -> Self { + Self { catalog } + } + + /// Resolve the complete [`Schema`] in scope for a query rooted at `tree`. + /// + /// Contains the time axis, the synthetic `value` column, and one column + /// per distinct name referenced anywhere in the tree — so positional + /// `ColumnId` resolution downstream is total. + pub fn bind(&self, tree: &LQueryExpr) -> Schema { + let mut columns: Vec = tree + .source_name() + .and_then(|name| self.catalog.columns_for(name)) + .unwrap_or_else(default_leaf_columns); + + // Ensure the (ts, value) floor is present. + for floor in default_leaf_columns() { + if !columns.iter().any(|c| c.name == floor.name) { + columns.push(floor); + } + } + + // Append one column per referenced-but-unknown name (group keys etc.). + for name in collect_referenced_columns(tree) { + if !columns.iter().any(|c| c.name == name) { + columns.push(Column { + name, + dtype: DataType::Utf8, + nullable: true, + }); + } + } + + let time_index = columns.iter().position(|c| c.name == "ts"); + Schema { + columns, + time_index, + unique_keys: Vec::new(), + } + } +} + +/// The conventional PromQL leaf shape: `(ts: Timestamp, value: Float64)`. +fn default_leaf_columns() -> Vec { + vec![ + Column { + name: "ts".into(), + dtype: DataType::Timestamp, + nullable: false, + }, + Column { + name: "value".into(), + dtype: DataType::Float64, + nullable: false, + }, + ] +} + +/// Collect every distinct group-key name the converter resolves positionally: +/// `Aggregate.keys`, `TopK.by`, and `Partition.keys`. +fn collect_referenced_columns(tree: &LQueryExpr) -> Vec { + let mut out: Vec = Vec::new(); + tree.walk(&mut |node| match node { + LQueryExpr::Aggregate { keys, .. } => out.extend(keys.iter().cloned()), + LQueryExpr::TopK { by, .. } => out.extend(by.iter().cloned()), + LQueryExpr::Partition { keys, .. } => out.extend(keys.keys().iter().cloned()), + _ => {} + }); + out.sort(); + out.dedup(); + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::intent_algebra::query_expr::PartitionKeys; + use crate::intent_algebra::relational::{QueryExpr as LQueryExpr, SourceSpec}; + + fn src(name: &str) -> LQueryExpr { + LQueryExpr::Source(SourceSpec { name: name.into() }) + } + + #[test] + fn bare_source_yields_ts_value_floor() { + let schema = Binder::new().bind(&src("m")); + assert_eq!(schema.columns.len(), 2); + assert_eq!(schema.columns[0].name, "ts"); + assert_eq!(schema.columns[1].name, "value"); + assert_eq!(schema.time_index, Some(0)); + } + + #[test] + fn partition_keys_land_in_schema() { + let tree = LQueryExpr::Partition { + keys: PartitionKeys::By(vec!["host".into()]), + input: Box::new(src("hits")), + }; + let schema = Binder::new().bind(&tree); + assert!(schema.column_id("host").is_some()); + } + + #[test] + fn custom_catalog_supplies_base_columns() { + struct FixedCatalog; + impl SchemaCatalog for FixedCatalog { + fn columns_for(&self, source: &str) -> Option> { + (source == "known").then(|| { + vec![ + Column { + name: "ts".into(), + dtype: DataType::Timestamp, + nullable: false, + }, + Column { + name: "value".into(), + dtype: DataType::Float64, + nullable: false, + }, + Column { + name: "datacenter".into(), + dtype: DataType::Utf8, + nullable: false, + }, + ] + }) + } + } + let schema = Binder::with_catalog(FixedCatalog).bind(&src("known")); + let dc = schema + .column_id("datacenter") + .and_then(|id| schema.columns.get(id)); + assert!(matches!(dc, Some(c) if !c.nullable)); + } +} diff --git a/crates/core/src/intent_algebra/column_resolution.rs b/crates/core/src/intent_algebra/column_resolution.rs new file mode 100644 index 00000000..580b994b --- /dev/null +++ b/crates/core/src/intent_algebra/column_resolution.rs @@ -0,0 +1,177 @@ +//! Schema-driven column resolution for the Layer-2 `relational` IR. +//! +//! The Layer-2 IR uses `ColumnRef::Named(String)` / `Aggregate.keys: +//! Vec`; the canonical IR uses positional [`ColumnId`] resolved +//! against a per-node [`Schema`]. These helpers bridge the two — the +//! [`Binder`](super::binder) builds the schema, and [`resolve_named_keys`] +//! turns the L2 names into positional ids. + +use thiserror::Error; + +use crate::intent_algebra::agg_intent::AggIntent; +use crate::intent_algebra::query_expr::ColumnRef; +use crate::intent_algebra::relational::QueryExpr; +use crate::intent_algebra::schema::{Column, ColumnId, DataType, Schema}; + +/// Errors returned by the resolution helpers. +#[derive(Debug, Error, PartialEq, Eq)] +pub enum ResolveError { + #[error("column `{name}` not found in schema (have: {available:?})")] + NotFound { + name: String, + available: Vec, + }, + #[error("ColumnRef::SampleValue has no `value` column in schema (have: {available:?})")] + NoSampleValue { available: Vec }, + #[error("ColumnRef::Wildcard cannot be resolved to a single ColumnId")] + WildcardNotPositional, +} + +/// Synthesize the conventional PromQL leaf schema `(ts, value)` for a metric. +pub fn infer_source_schema(_metric_or_table: &str) -> Schema { + Schema::with_time_index( + vec![ + Column { + name: "ts".into(), + dtype: DataType::Timestamp, + nullable: false, + }, + Column { + name: "value".into(), + dtype: DataType::Float64, + nullable: false, + }, + ], + 0, + Vec::new(), + ) +} + +/// Synthesise the root schema by walking to the outermost `Source` leaf. +pub fn infer_schema_for_root(expr: &QueryExpr) -> Schema { + match expr.source_name() { + Some(name) => infer_source_schema(name), + None => Schema::default(), + } +} + +/// Resolve a single [`ColumnRef`] to a positional [`ColumnId`]. +pub fn resolve_column_ref(col: &ColumnRef, schema: &Schema) -> Result { + match col { + ColumnRef::Named(name) => schema + .column_id(name) + .ok_or_else(|| ResolveError::NotFound { + name: name.clone(), + available: schema.columns.iter().map(|c| c.name.clone()).collect(), + }), + ColumnRef::SampleValue => { + schema + .column_id("value") + .ok_or_else(|| ResolveError::NoSampleValue { + available: schema.columns.iter().map(|c| c.name.clone()).collect(), + }) + } + ColumnRef::Wildcard => Err(ResolveError::WildcardNotPositional), + } +} + +/// Resolve every entry, short-circuiting on the first error. +pub fn resolve_column_refs( + cols: &[ColumnRef], + schema: &Schema, +) -> Result, ResolveError> { + cols.iter().map(|c| resolve_column_ref(c, schema)).collect() +} + +/// Resolve a list of named GROUP BY keys (`Aggregate.keys`) to `ColumnId`s. +pub fn resolve_named_keys(keys: &[String], schema: &Schema) -> Result, ResolveError> { + keys.iter() + .map(|name| { + schema + .column_id(name) + .ok_or_else(|| ResolveError::NotFound { + name: name.clone(), + available: schema.columns.iter().map(|c| c.name.clone()).collect(), + }) + }) + .collect() +} + +/// Output schema produced by an `Aggregate { by, aggs }` over `input`. +/// Mirrors `QueryExpr::output_schema_in`'s `Aggregate` arm; out-of-range `by` +/// ids are silently dropped (callers needing the strict check resolve `by` +/// via [`resolve_named_keys`], which surfaces `NotFound`). +pub fn output_schema_for_aggregate(input: &Schema, by: &[ColumnId], aggs: &[AggIntent]) -> Schema { + let mut out_cols: Vec = Vec::with_capacity(by.len() + aggs.len()); + for &id in by { + if let Some(c) = input.columns.get(id) { + out_cols.push(c.clone()); + } + } + let value_col_idx = input + .column_id("value") + .or_else(|| (0..input.columns.len()).find(|i| !by.contains(i))); + let probe = value_col_idx + .and_then(|i| input.columns.get(i)) + .cloned() + .unwrap_or(Column { + name: "value".into(), + dtype: DataType::Float64, + nullable: false, + }); + for intent in aggs { + out_cols.push(intent.output_column(&probe)); + } + let unique_keys = if by.is_empty() { + Vec::new() + } else { + vec![(0..by.len()).collect()] + }; + Schema { + columns: out_cols, + time_index: None, + unique_keys, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn source_schema_has_ts_and_value() { + let s = infer_source_schema("m"); + assert_eq!(s.columns.len(), 2); + assert_eq!(s.time_index, Some(0)); + assert!(!s.has_unique_key()); + } + + #[test] + fn resolve_sample_value() { + let s = infer_source_schema("m"); + assert_eq!(resolve_column_ref(&ColumnRef::SampleValue, &s), Ok(1)); + } + + #[test] + fn resolve_unknown_name_errors() { + let s = infer_source_schema("m"); + let err = resolve_column_ref(&ColumnRef::Named("host".into()), &s).unwrap_err(); + assert!(matches!(err, ResolveError::NotFound { .. })); + } + + #[test] + fn aggregate_strips_time_and_keeps_unique_keys() { + let mut input = infer_source_schema("m"); + input.columns.push(Column { + name: "host".into(), + dtype: DataType::Utf8, + nullable: false, + }); + let out = output_schema_for_aggregate(&input, &[2usize], &[AggIntent::Sum]); + assert_eq!(out.columns.len(), 2); // host, sum + assert_eq!(out.columns[0].name, "host"); + assert_eq!(out.columns[1].name, "sum"); + assert!(out.time_index.is_none()); + assert_eq!(out.unique_keys, vec![vec![0]]); + } +} diff --git a/crates/core/src/intent_algebra/cse.rs b/crates/core/src/intent_algebra/cse.rs new file mode 100644 index 00000000..b88ffe26 --- /dev/null +++ b/crates/core/src/intent_algebra/cse.rs @@ -0,0 +1,224 @@ +//! Workload-level Common Sub-Expression Elimination. +//! +//! Multi-root planning hoists shared sub-DAGs into `LetBinding`s so the cost +//! model can credit the producer once. Legality is gated by +//! [`cse_reuse_is_legal`](super::schema::cse_reuse_is_legal): a candidate +//! sub-DAG only becomes a binding when its output schema has at least one +//! `unique_keys` set — the load-bearing field for this pass. +//! +//! Scope: the basic "≥2 roots with identical `Aggregate`-child sub-trees" +//! case. The fully-general algorithm (alpha-equivalence, schema-merge, +//! nested CSE) is a downstream optimisation, not part of the IR contract. + +use std::collections::HashMap; + +use crate::intent_algebra::names::{BindingName, QueryId}; +use crate::intent_algebra::query_expr::QueryExpr; +use crate::intent_algebra::schema::cse_reuse_is_legal; + +/// Multi-root container produced by the CSE pass. +#[derive(Debug, Clone, PartialEq)] +pub struct CseWorkloadPlan { + /// Named shared producers, each referenced by ≥2 roots via `QueryExpr::Ref`. + pub bindings: Vec<(BindingName, QueryExpr)>, + /// One root per input query, in input order. + pub roots: Vec<(QueryId, QueryExpr)>, +} + +/// Hoist sub-expressions structurally identical across ≥2 roots into shared +/// `LetBinding`s, leaving each root with a `Ref` where the duplicate lived. +/// +/// A candidate is hoisted only when +/// `cse_reuse_is_legal(&candidate.output_schema(), consumers)` returns `Ok`. +pub fn dedupe_subtrees(roots: Vec<(QueryId, QueryExpr)>) -> CseWorkloadPlan { + if roots.len() < 2 { + return CseWorkloadPlan { + bindings: vec![], + roots, + }; + } + + // Count `Aggregate`-child sub-trees that appear in ≥2 roots. + let mut candidate_counts: HashMap = HashMap::new(); + for (_, root) in &roots { + if let QueryExpr::Aggregate { child, .. } = root { + if matches!(**child, QueryExpr::Ref { .. }) { + continue; + } + let key = format!("{child:?}"); + let entry = candidate_counts + .entry(key) + .or_insert_with(|| ((**child).clone(), 0)); + entry.1 += 1; + } + } + + // Pick the most-shared legal candidate (biggest reuse first). + let mut chosen: Option<(QueryExpr, usize)> = None; + for (_key, (expr, count)) in candidate_counts.into_iter() { + if count < 2 { + continue; + } + let Ok(out_schema) = expr.output_schema() else { + continue; + }; + if cse_reuse_is_legal(&out_schema, count).is_err() { + continue; + } + match &chosen { + Some((_, best)) if *best >= count => {} + _ => chosen = Some((expr, count)), + } + } + + let Some((shared_expr, _)) = chosen else { + return CseWorkloadPlan { + bindings: vec![], + roots, + }; + }; + + let binding_name = BindingName::new("shared_0"); + let mut rewritten: Vec<(QueryId, QueryExpr)> = Vec::with_capacity(roots.len()); + for (qid, root) in roots { + let new_root = match root { + QueryExpr::Aggregate { + by, + aggs, + having, + child, + } if *child == shared_expr => QueryExpr::Aggregate { + by, + aggs, + having, + child: Box::new(QueryExpr::Ref { + name: binding_name.clone(), + }), + }, + other => other, + }; + rewritten.push((qid, new_root)); + } + + CseWorkloadPlan { + bindings: vec![(binding_name, shared_expr)], + roots: rewritten, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::intent_algebra::agg_intent::AggIntent; + use crate::intent_algebra::query_expr::{Source, WindowKind}; + use crate::intent_algebra::schema::{Column, DataType, Schema}; + use crate::types::AccuracyTarget; + use std::time::Duration; + + fn col(name: &str, dtype: DataType) -> Column { + Column { + name: name.into(), + dtype, + nullable: false, + } + } + + fn ts_scan() -> QueryExpr { + QueryExpr::Scan { + source: Source::TimeSeries { + metric: "http_request_duration_seconds".into(), + }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + col("ts", DataType::Timestamp), + col("service", DataType::Utf8), + col("value", DataType::Float64), + ], + 0, + vec![vec![0, 1]], + ), + } + } + + fn windowed_scan() -> QueryExpr { + QueryExpr::Window { + kind: WindowKind::Sliding, + size: Duration::from_secs(300), + slide: None, + child: Box::new(ts_scan()), + } + } + + #[test] + fn dedupe_subtrees_single_root_passthrough() { + let q = QueryExpr::Aggregate { + by: vec![1], + aggs: vec![AggIntent::Quantile { + q: 0.99, + accuracy: AccuracyTarget::Epsilon(0.01), + }], + having: None, + child: Box::new(windowed_scan()), + }; + let out = dedupe_subtrees(vec![(QueryId::new("q1"), q.clone())]); + assert!(out.bindings.is_empty()); + assert_eq!(out.roots[0].1, q); + } + + #[test] + fn dedupe_subtrees_basic() { + let mk = |q: f64| QueryExpr::Aggregate { + by: vec![1], + aggs: vec![AggIntent::Quantile { + q, + accuracy: AccuracyTarget::Epsilon(0.01), + }], + having: None, + child: Box::new(windowed_scan()), + }; + let out = dedupe_subtrees(vec![ + (QueryId::new("q1"), mk(0.99)), + (QueryId::new("q2"), mk(0.95)), + ]); + assert_eq!(out.bindings.len(), 1); + assert_eq!(out.bindings[0].0, BindingName::new("shared_0")); + assert_eq!(out.bindings[0].1, windowed_scan()); + for (_, root) in &out.roots { + match root { + QueryExpr::Aggregate { child, .. } => assert_eq!( + **child, + QueryExpr::Ref { + name: BindingName::new("shared_0") + } + ), + other => panic!("expected Aggregate root, got {other:?}"), + } + } + } + + #[test] + fn dedupe_subtrees_no_shared_subexpr_when_unique_keys_absent() { + // Schema without unique_keys → CSE refuses to share even if identical. + let scan_no_uk = QueryExpr::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + col("ts", DataType::Timestamp), + col("value", DataType::Float64), + ], + 0, + vec![], + ), + }; + let mk = || QueryExpr::Aggregate { + by: vec![], + aggs: vec![AggIntent::Sum], + having: None, + child: Box::new(scan_no_uk.clone()), + }; + let out = dedupe_subtrees(vec![(QueryId::new("q1"), mk()), (QueryId::new("q2"), mk())]); + assert!(out.bindings.is_empty(), "no unique_keys → no hoisting"); + } +} diff --git a/crates/core/src/intent_algebra/expr.rs b/crates/core/src/intent_algebra/expr.rs deleted file mode 100644 index a39248c0..00000000 --- a/crates/core/src/intent_algebra/expr.rs +++ /dev/null @@ -1,639 +0,0 @@ -use std::sync::Arc; -use std::time::Duration; - -use super::expr_ir::L3Expr; -use super::schema::{HasSchema, L3Schema, SchemaCatalog}; -use crate::types::AccuracyTarget; - -// ── Leaf / supporting types ─────────────────────────────────────────────────── - -/// A row-level filter predicate (WHERE clause / PromQL label matcher). -#[derive(Debug, Clone)] -pub struct Predicate(pub L3Expr); - -/// One item in a SELECT projection list. -#[derive(Debug, Clone)] -pub struct ProjectItem { - pub expr: L3Expr, - pub alias: Option, -} - -/// A GROUP BY key reference (column / label name). -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct GroupKey(pub String); - -/// A reference to a column / label by name. For time-series sources this is -/// a label name or the synthetic sample-value / timestamp column. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct ColumnRef(pub String); - -/// A set of partitioning keys (sharding hint for L5 stage allocator). -#[derive(Debug, Clone)] -pub struct PartitionKeys; - -/// One key in an ORDER BY clause. -#[derive(Debug, Clone)] -pub struct SortKey { - pub expr: L3Expr, - pub ascending: bool, - pub nulls_first: bool, -} - -/// An analytic window frame (ROWS / RANGE BETWEEN …). -#[derive(Debug, Clone)] -pub struct WindowFrame; - -/// PromQL vector-match modifiers: `on(...)` / `ignoring(...)` for label-set -/// matching, plus `group_left(...)` / `group_right(...)` for many-to-one and -/// one-to-many cardinality. Carried by `QueryExpr::BinaryOp`. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct VectorMatch { - /// `true` for `on(labels)`, `false` for `ignoring(labels)`. - pub on: bool, - /// The labels named in the `on` / `ignoring` clause. - pub labels: Vec, - /// Grouping side for many-to-one / one-to-many matches, if any. - pub grouping: Option, -} - -/// `group_left(labels)` / `group_right(labels)` modifier on a vector match. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct VectorGrouping { - pub side: GroupSide, - pub labels: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum GroupSide { - Left, - Right, -} - -/// Reference to a metric by name (PromQL / OTLP). -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct MetricRef(pub String); - -/// Closed time interval `[start_ms, end_ms]` in milliseconds since the Unix -/// epoch. Either bound may be `None`, meaning unbounded on that side. -/// -/// For PromQL this is the query's absolute evaluation range, which is supplied -/// by the query API (`/query_range` `start`/`end`), **not** by the query -/// string — so a PromQL `Scan` carries `time = None` until that context is -/// threaded in. The range-vector duration (`m[5m]`) is a *window* and lives on -/// `QueryExpr::TimeWindow`, not here. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct TimeRange { - pub start_ms: Option, - pub end_ms: Option, -} - -/// Reference to a relational table by name. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct TableRef(pub String); - -/// Join key specification (USING / ON column reference). -#[derive(Debug, Clone)] -pub struct JoinKey; - -// ── Enum supporting types ───────────────────────────────────────────────────── - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum JoinKind { - Inner, - Left, - Right, - Full, - Cross, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum SetOpKind { - Union, - Intersect, - Except, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum BinaryOpKind { - // Arithmetic - Add, - Sub, - Mul, - Div, - Mod, - Pow, - Atan2, - // Comparison - Eq, - NotEq, - Lt, - LtEq, - Gt, - GtEq, - // Boolean / PromQL set operators - And, - Or, - Unless, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum WindowFuncKind { - RowNumber, - Rank, - DenseRank, - Lag, - Lead, - FirstValue, - LastValue, - NthValue(u64), - Sum, - Avg, - Count, - Min, - Max, -} - -/// Which data model a `Source` or `AggIntent` operates over. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum DataModel { - TimeSeries, - Tabular, - /// Agnostic — works over either data model. - Any, -} - -// ── Time window kind ────────────────────────────────────────────────────────── - -/// The lifecycle / flush semantics of a streaming time window. -/// Used by `QueryExpr::TimeWindow`; distinct from SQL analytic frames -/// (`QueryExpr::WindowFunc`). -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum TimeWindowKind { - /// Non-overlapping fixed-size windows. - Tumbling, - /// Overlapping windows advancing by `slide` interval. - Sliding, - /// Windows that open on activity and close after a gap of inactivity. - Session, -} - -// ── Leaf data source ────────────────────────────────────────────────────────── - -/// The leaf data source of a query. Carried by `QueryExpr::Scan` to keep -/// L3 data-model-agnostic: everything above `Scan` (`Filter`, `Aggregate`, -/// etc.) works identically regardless of `Source` variant. -#[derive(Debug, Clone)] -pub enum Source { - /// Time-series input — deployment-model-asapquery / asaplifecycle shape. - /// - /// Label matchers are **not** carried here: they are language-independent - /// row filters and live in `QueryExpr::Scan.predicates` (the same place SQL - /// `WHERE` conjuncts on a table scan would go), so every node above the leaf - /// stays data-model-agnostic. - TimeSeries { - metric: MetricRef, - /// Absolute evaluation range; `None` when not yet bound (see `TimeRange`). - time: Option, - }, - /// Tabular input — deployment-model-asapfusion / future-OLAP shape. - Table { - table_ref: TableRef, - columns: Vec, - }, - /// Recursive join over sources (multi-table tabular queries). - Join { - left: Box, - right: Box, - on: JoinKey, - }, -} - -impl Source { - pub fn data_model(&self) -> DataModel { - match self { - Source::TimeSeries { .. } => DataModel::TimeSeries, - Source::Table { .. } | Source::Join { .. } => DataModel::Tabular, - } - } -} - -// ── Aggregation intent ──────────────────────────────────────────────────────── - -/// What to compute, not how. Sketch type and parameters are chosen by L4 -/// rules; `AggIntent` is the L3 statement of intent only. -/// -/// Heavy-hitter top-k (`TopK`) is a first-class intent because dedicated -/// sketch primitives (SpaceSaving, CMS-with-heap) compute it in one pass. -/// Generic ordering+limit stays as `QueryExpr::Sort + QueryExpr::Limit`. -#[derive(Debug, Clone)] -pub enum AggIntent { - // ── Data-model-agnostic ─────────────────────────────────────────────────── - Count { - accuracy: AccuracyTarget, - }, - Sum, - Min, - Max, - Avg, - /// Sample standard deviation when `population == false`; population stddev - /// otherwise. PromQL `stddev` / `stddev_over_time`. - StdDev { - population: bool, - }, - /// Variance — PromQL `stdvar` / `stdvar_over_time`. - Variance { - population: bool, - }, - Quantile { - q: f64, - accuracy: AccuracyTarget, - }, - /// Heavy-hitter top-k. Distinct from generic `Sort + Limit` — a - /// dedicated sketch (SpaceSaving, CMS-with-heap) computes it as a single - /// primitive. Recognised by L1→L2→L3 lowering on `ORDER BY count DESC - /// LIMIT k` / PromQL `topk(k, …)`. - TopK { - k: usize, - by: Vec, - accuracy: AccuracyTarget, - }, - Cardinality { - accuracy: AccuracyTarget, - }, - - // ── Time-series streaming derivatives ──────────────────────────────────── - // Include PromQL counter-reset adjustment; not equivalent to Sum/Count - // over a Window. Kept distinct so delta-set aggregators bind directly. - Rate { - window: Duration, - }, - Increase { - window: Duration, - }, -} - -impl AggIntent { - /// Which data model this intent semantically requires. L4 rules consult - /// this to skip non-applicable intents (e.g. `Rate` over a `Source::Table`). - pub fn requires(&self) -> DataModel { - match self { - Self::Rate { .. } | Self::Increase { .. } => DataModel::TimeSeries, - _ => DataModel::Any, - } - } - - /// Output column type for a single-column aggregate result. - /// - /// `input` is the field this intent reduces (the sample-value field for a - /// time-series window). Returns `None` for `TopK`, which produces multiple - /// output columns — use `QueryExpr::output_schema` for its schema. - pub fn output_type(&self, input: &super::schema::L3Field) -> Option { - use super::schema::L3DataType; - match self { - Self::Count { .. } | Self::Cardinality { .. } => Some(L3DataType::Int64), - Self::Min | Self::Max => Some(input.dtype.clone()), - Self::Sum - | Self::Avg - | Self::StdDev { .. } - | Self::Variance { .. } - | Self::Quantile { .. } - | Self::Rate { .. } - | Self::Increase { .. } => Some(L3DataType::Float64), - Self::TopK { .. } => None, - } - } -} - -// ── L3 DAG node ─────────────────────────────────────────────────────────────── - -/// A node in the L3 DAG. Wraps the expression and its derived output schema -/// so that every edge implicitly carries a typed schema: holding an -/// `Arc` gives you both the child expression and the schema of the -/// data flowing on that edge. -#[derive(Debug, Clone)] -pub struct L3Node { - pub expr: QueryExpr, - /// Output schema of `expr` — the schema of the data flowing on the edge - /// leading *from* this node to its parent(s). - pub schema: L3Schema, -} - -// ── L3 intent algebra IR ────────────────────────────────────────────────────── - -/// Language- and deployment-independent intent-only IR. No sketch types, -/// no sketch parameters, no language-specific operators. Traversing from -/// the root node yields a DAG; shared sub-expressions appear as multiple -/// `Arc` references to the same `L3Node`. -#[derive(Debug, Clone)] -pub enum QueryExpr { - // ── Base relations ──────────────────────────────────────────────────────── - /// Outermost leaf. `source` carries the data-model-specific leaf shape; - /// `predicates` are leaf-level row filters (PromQL label matchers, SQL - /// pushed-down `WHERE` conjuncts). - Scan { - source: Source, - predicates: Vec, - }, - /// Reference to a named `LetBinding` sub-expression; resolved at plan time. - Ref(String), - - // ── Filtering & projection ──────────────────────────────────────────────── - /// σ — row-level filter. Output schema = child schema (unchanged). - Filter { - child: Arc, - pred: Predicate, - }, - /// π — column projection. Output schema = child schema projected to `cols`. - Project { - child: Arc, - cols: Vec, - }, - - // ── Aggregation ─────────────────────────────────────────────────────────── - /// γ + α — GROUP BY + aggregate intents. Concrete operator (HashAgg / - /// SortAgg / SketchAgg) chosen by L4; `aggs` carry intent only. - Aggregate { - child: Arc, - by: Vec, - aggs: Vec, - having: Option, - }, - - // ── Time / streaming windows ────────────────────────────────────────────── - /// ψ — tumbling / sliding / session window over the time axis. Defines - /// the flush / reset lifecycle for aggregates in its sub-DAG. PromQL range - /// vectors (`m[5m]`) and subqueries (`m[5m:1m]`) lower here. SQL analytic - /// frames are a different node (`WindowFunc`). - TimeWindow { - child: Arc, - kind: TimeWindowKind, - size: Duration, - slide: Option, - }, - - // ── Distributed-execution structure ─────────────────────────────────────── - /// Logical-only partitioning marker. Output schema = child schema. - /// Carries a sharding hint for the L5 stage allocator. - Partition { - child: Arc, - keys: PartitionKeys, - }, - /// δ — SQL `DISTINCT` / row deduplication. - Distinct { - child: Arc, - cols: Vec, - }, - /// ⊕ — exact union of sub-results from independent stages or shards. - /// Sketch unions are a separate node in `SummaryExpr` because they carry - /// sketch-family / params type constraints. - Merge { - children: Vec>, - }, - - // ── Joins ───────────────────────────────────────────────────────────────── - /// Logical join. L4 picks the physical alternative (HashJoin / - /// SortMergeJoin / SketchJoin) based on selectivity, memory budget, and - /// accuracy target. - Join { - kind: JoinKind, - left: Arc, - right: Arc, - pred: Option, - }, - - // ── Set operators ───────────────────────────────────────────────────────── - SetOp { - kind: SetOpKind, - all: bool, - left: Arc, - right: Arc, - }, - - // ── Ordering & limiting ─────────────────────────────────────────────────── - /// Generic order-by for non-heavy-hitter cases (`ORDER BY name LIMIT 10`). - /// Heavy-hitter shapes lower to `AggIntent::TopK` instead. - Sort { - child: Arc, - keys: Vec, - }, - Limit { - child: Arc, - /// `None` means no upper bound (only an offset applies). - n: Option, - offset: u64, - }, - - // ── Subquery / CTE ──────────────────────────────────────────────────────── - Subquery { - child: Arc, - alias: String, - }, - /// SQL `WITH name AS (expr) … body`; lowering target for PromQL - /// recording-rule bindings. The `expr` sub-DAG may be referenced N times - /// via `Ref(name)` in `body`, giving the DAG its fan-in. - LetBinding { - name: String, - expr: Arc, - body: Arc, - }, - - // ── Analytic window functions ───────────────────────────────────────────── - /// SQL `OVER (PARTITION BY … ORDER BY … ROWS BETWEEN …)`. - /// Distinct from `TimeWindow` — that is a streaming window over the time - /// axis; this is an analytic frame over already-grouped rows. - WindowFunc { - child: Arc, - func: WindowFuncKind, - partition_by: Vec, - order_by: Vec, - frame: Option, - }, - - // ── Binary composition ──────────────────────────────────────────────────── - /// Arithmetic / comparison / boolean composition (PromQL binary ops - /// including `and` / `or` / `unless`, SQL boolean composition). - BinaryOp { - op: BinaryOpKind, - lhs: Arc, - rhs: Arc, - vector_match: Option, - }, -} - -impl HasSchema for QueryExpr { - fn output_schema(&self, input_schemas: &[&L3Schema], catalog: &SchemaCatalog) -> L3Schema { - use super::schema::{L3DataType, L3Field}; - - // Shorthand: first child's schema (most nodes have exactly one child). - let child = || input_schemas[0]; - - match self { - // ── Leaf: schema comes from the catalog ─────────────────────────── - QueryExpr::Scan { source, predicates } => match source { - // Time-series leaf: synthesize `timestamp` (the time axis) and - // the sample-`value` column, then one `Utf8` label column per - // label known from the catalog, unioned with any label - // referenced by the scan's predicates (so a filter on an - // unregistered label still type-checks). - Source::TimeSeries { metric, .. } => { - let meta = catalog.metrics.get(&metric.0); - let value_dtype = meta - .map(|m| m.value_type.clone()) - .unwrap_or(L3DataType::Float64); - - let mut fields = vec![ - L3Field { - name: "timestamp".to_string(), - dtype: L3DataType::Timestamp, - nullable: false, - }, - L3Field { - name: "value".to_string(), - dtype: value_dtype, - nullable: false, - }, - ]; - - let mut label_names: Vec = - meta.map(|m| m.labels.clone()).unwrap_or_default(); - for p in predicates { - for c in p.0.columns_referenced() { - if c.0 != "value" && c.0 != "timestamp" && !label_names.contains(&c.0) { - label_names.push(c.0.clone()); - } - } - } - for name in label_names { - fields.push(L3Field { - name, - dtype: L3DataType::Utf8, - nullable: true, - }); - } - L3Schema { - fields, - time_index: Some(0), - } - } - Source::Table { table_ref, .. } => { - let table = catalog - .tables - .get(&table_ref.0) - .unwrap_or_else(|| panic!("table '{}' not in catalog", table_ref.0)); - let fields: Vec = table - .columns - .iter() - .map(|c| L3Field { - name: c.name.clone(), - dtype: c.data_type.clone(), - nullable: c.nullable, - }) - .collect(); - let time_index = table - .time_column - .as_ref() - .and_then(|tc| fields.iter().position(|f| &f.name == tc)); - L3Schema { fields, time_index } - } - Source::Join { .. } => { - todo!("schema derivation for Join sources") - } - }, - - // ── Pass-through: output schema == child schema ─────────────────── - QueryExpr::Filter { .. } - | QueryExpr::Sort { .. } - | QueryExpr::Limit { .. } - | QueryExpr::Distinct { .. } - | QueryExpr::Partition { .. } - | QueryExpr::TimeWindow { .. } => child().clone(), - - // ── Aggregate: GROUP BY cols + one output col per AggIntent ─────── - QueryExpr::Aggregate { by, aggs, .. } => { - let cs = child(); - - // TopK is the only multi-column AggIntent: produces the TopK - // by-columns looked up from the child schema, followed by a - // synthetic "count" Int64 column. - if let [AggIntent::TopK { by: topk_by, .. }] = aggs.as_slice() { - let mut fields: Vec = topk_by - .iter() - .map(|col| { - cs.fields - .iter() - .find(|f| f.name == col.0) - .cloned() - .unwrap_or(L3Field { - name: col.0.clone(), - dtype: L3DataType::Utf8, - nullable: true, - }) - }) - .collect(); - fields.push(L3Field { - name: "count".to_string(), - dtype: L3DataType::Int64, - nullable: false, - }); - return L3Schema { - fields, - time_index: None, - }; - } - - // General case: GROUP BY fields (preserving child type) followed - // by one output field per AggIntent. The sample-`value` field is - // the canonical reduction input for a time-series window. - let value_field = cs - .fields - .iter() - .find(|f| f.name == "value") - .cloned() - .unwrap_or(L3Field { - name: "value".to_string(), - dtype: L3DataType::Float64, - nullable: true, - }); - let by_fields: Vec = by - .iter() - .filter_map(|key| cs.fields.iter().find(|f| f.name == key.0).cloned()) - .collect(); - let agg_fields: Vec = aggs - .iter() - .enumerate() - .map(|(i, agg)| { - let name = if aggs.len() == 1 { - "value".to_string() - } else { - format!("value_{i}") - }; - L3Field { - name, - dtype: agg.output_type(&value_field).unwrap_or(L3DataType::Float64), - nullable: true, - } - }) - .collect(); - let all_fields: Vec = by_fields.into_iter().chain(agg_fields).collect(); - L3Schema { - fields: all_fields, - time_index: None, - } - } - - // ── BinaryOp: left operand shape, value column re-typed ─────────── - // Arithmetic/comparison between two vectors yields the left vector's - // shape (label set + value); set ops (`and`/`or`/`unless`) likewise. - QueryExpr::BinaryOp { .. } => input_schemas[0].clone(), - - // ── Merge / SetOp: union-compatible; representative is the first ── - QueryExpr::Merge { .. } | QueryExpr::SetOp { .. } => input_schemas[0].clone(), - - // ── Everything else: not yet implemented ────────────────────────── - _ => todo!( - "output_schema not yet implemented for {:?}", - std::mem::discriminant(self) - ), - } - } -} diff --git a/crates/core/src/intent_algebra/expr_ir.rs b/crates/core/src/intent_algebra/expr_ir.rs index 03b01ba8..464ddcc4 100644 --- a/crates/core/src/intent_algebra/expr_ir.rs +++ b/crates/core/src/intent_algebra/expr_ir.rs @@ -1,9 +1,16 @@ -use super::expr::ColumnRef; +//! Language-independent scalar expression IR. +//! +//! Used for filter predicates (PromQL label matchers, SQL `WHERE` conjuncts) +//! and projection / sort-key expressions. Carried by both the Layer-2 +//! `relational` IR and the canonical L3 `query_expr` IR so the predicate +//! representation is identical across the lowering boundary. -// ── Scalar literals ─────────────────────────────────────────────────────────── +use serde::{Deserialize, Serialize}; -/// A typed scalar constant. Used in `L3Expr::Literal`. -#[derive(Debug, Clone, PartialEq)] +use super::query_expr::ColumnRef; + +/// A typed scalar constant. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum L3Scalar { Int64(i64), Float64(f64), @@ -12,15 +19,11 @@ pub enum L3Scalar { Null, } -// ── Comparison operators ────────────────────────────────────────────────────── - /// Binary comparison operators for `L3Expr::Compare`. /// /// `Regex` / `NotRegex` carry PromQL/RE2 regex-match semantics (`=~` / `!~`): /// the right-hand side is a regular-expression pattern, not a literal value. -/// SQL `LIKE` / `ILIKE` are kept as separate operators because their -/// wildcard grammar (`%` / `_`) differs from regex. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum CompareOp { Eq, Ne, @@ -34,19 +37,12 @@ pub enum CompareOp { NotRegex, } -// ── Expression IR ───────────────────────────────────────────────────────────── - -/// A scalar expression used in filter predicates and (eventually) projection -/// lists and sort keys. Language-independent: PromQL label matchers, SQL -/// `WHERE` conjuncts, and Elastic term filters all lower to the same shape. -/// -/// Flat conjunctions (`BoolAnd`) / disjunctions (`BoolOr`) make per-conjunct -/// selectivity estimation and label-matcher lowering straightforward without -/// recursive descent. -#[derive(Debug, Clone, PartialEq)] +/// A scalar expression. Flat conjunctions (`BoolAnd`) / disjunctions +/// (`BoolOr`) make per-conjunct selectivity estimation and label-matcher +/// lowering straightforward without recursive descent. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum L3Expr { - /// Reference to a named column. For time-series sources a label name - /// (e.g. `Column("env")`) or the synthetic sample-value column. + /// Reference to a named column / label. Column(ColumnRef), /// A constant literal value. Literal(L3Scalar), @@ -66,8 +62,7 @@ pub enum L3Expr { impl L3Expr { /// If this expression is a `BoolAnd`, return its elements; otherwise a - /// single-element slice containing `self`. Lets callers iterate all - /// top-level conjuncts without cloning. + /// single-element slice containing `self`. pub fn conjuncts(&self) -> &[L3Expr] { match self { L3Expr::BoolAnd(v) => v.as_slice(), @@ -76,9 +71,7 @@ impl L3Expr { } /// Recursively collect every `ColumnRef` referenced anywhere in this - /// expression. Used by L4 for column-lineage and selectivity estimation, - /// and by schema derivation to discover label columns referenced by a - /// time-series scan's predicates. + /// expression. pub fn columns_referenced(&self) -> Vec<&ColumnRef> { match self { L3Expr::Column(c) => vec![c], diff --git a/crates/core/src/intent_algebra/lower.rs b/crates/core/src/intent_algebra/lower.rs new file mode 100644 index 00000000..3716fb50 --- /dev/null +++ b/crates/core/src/intent_algebra/lower.rs @@ -0,0 +1,288 @@ +//! Layer-2 → canonical L3 IR converter. +//! +//! Recursively converts a whole [`relational::QueryExpr`] tree into a whole +//! [`query_expr::QueryExpr`] tree. The single-statistic sketchable `Aggregate` +//! fuses directly in canonical terms (window-swap, `Partition` wrap); see the +//! `Aggregate` arm. +//! +//! Name resolution is an explicit pass: [`convert_root`] runs the +//! [`Binder`](super::binder) first to build the complete, self-contained +//! schema every `ColumnId` indexes into, so positional resolution downstream +//! is total. + +use thiserror::Error; + +use crate::intent_algebra::agg_intent::AggIntent; +use crate::intent_algebra::binder::Binder; +use crate::intent_algebra::column_resolution::{resolve_named_keys, ResolveError}; +use crate::intent_algebra::names::BindingName; +use crate::intent_algebra::query_expr::{ + PartitionKeys as CPartitionKeys, Predicate, QueryExpr as CQueryExpr, Source, WindowKind, +}; +use crate::intent_algebra::relational::{AggFunc, QueryExpr as LQueryExpr}; +use crate::intent_algebra::schema::Schema; +use crate::types::AccuracyTarget; + +/// Errors produced while converting a Layer-2 tree to canonical. +#[derive(Debug, Error)] +pub enum ConvertError { + /// A column reference (`Aggregate` key, `Partition` / `TopK` key) did not + /// resolve against the inherited schema. + #[error("column resolution failed: {0}")] + Resolve(#[from] ResolveError), +} + +/// Lower a Layer-2 tree to canonical L3, threading `accuracy` onto every +/// approximate intent (`Count`, `Quantile`, `Cardinality`, `TopK`). +pub fn convert_root( + legacy: &LQueryExpr, + accuracy: &AccuracyTarget, +) -> Result { + let schema = Binder::new().bind(legacy); + convert(legacy, &schema, accuracy) +} + +/// Convert a Layer-2 tree to canonical against an explicit inherited schema. +pub fn convert( + legacy: &LQueryExpr, + schema: &Schema, + acc: &AccuracyTarget, +) -> Result { + Ok(match legacy { + LQueryExpr::Source(spec) => scan(spec.name.clone(), schema, Vec::new()), + + LQueryExpr::Ref(name) => CQueryExpr::Ref { + name: BindingName::new(name.clone()), + }, + + // Fold label matchers / pushed-down predicates directly onto the Scan + // when the immediate child is a `Source`; otherwise emit a `Filter`. + LQueryExpr::Filter { pred, input } => match input.as_ref() { + LQueryExpr::Source(spec) => { + let predicates = pred.conjuncts().iter().cloned().map(Predicate).collect(); + scan(spec.name.clone(), schema, predicates) + } + other => CQueryExpr::Filter { + pred: Predicate(pred.clone()), + child: Box::new(convert(other, schema, acc)?), + }, + }, + + LQueryExpr::Aggregate { + keys, + aggs, + having, + input, + } => { + // Single-statistic aggregate (no HAVING) fuses: a `Window` input + // becomes `Window { Aggregate { by: [] } }`; GROUP BY keys wrap the + // result in a `Partition`. + if aggs.len() == 1 && having.is_none() { + let intent = agg_func_to_intent(&aggs[0].func, acc); + let sketch = match input.as_ref() { + LQueryExpr::Window { + duration, + slide, + input: win_input, + } => CQueryExpr::Window { + kind: if slide.is_some() { + WindowKind::Sliding + } else { + WindowKind::Tumbling + }, + size: *duration, + slide: *slide, + child: Box::new(CQueryExpr::Aggregate { + by: Vec::new(), + aggs: vec![intent], + having: None, + child: Box::new(convert(win_input, schema, acc)?), + }), + }, + other => CQueryExpr::Aggregate { + by: Vec::new(), + aggs: vec![intent], + having: None, + child: Box::new(convert(other, schema, acc)?), + }, + }; + return Ok(if keys.is_empty() { + sketch + } else { + CQueryExpr::Partition { + keys: CPartitionKeys::By(keys.clone()), + child: Box::new(sketch), + } + }); + } + + // Plain canonical `Aggregate`: multi-agg or HAVING-bearing. + let by = resolve_named_keys(keys, schema)?; + let intents = aggs + .iter() + .map(|item| agg_func_to_intent(&item.func, acc)) + .collect(); + CQueryExpr::Aggregate { + by, + aggs: intents, + having: having.clone().map(Predicate), + child: Box::new(convert(input, schema, acc)?), + } + } + + LQueryExpr::Window { + duration, + slide, + input, + } => CQueryExpr::Window { + kind: if slide.is_some() { + WindowKind::Sliding + } else { + WindowKind::Tumbling + }, + size: *duration, + slide: *slide, + child: Box::new(convert(input, schema, acc)?), + }, + + LQueryExpr::Partition { keys, input } => CQueryExpr::Partition { + keys: keys.clone(), + child: Box::new(convert(input, schema, acc)?), + }, + + LQueryExpr::Distinct { cols, input } => CQueryExpr::Distinct { + cols: cols.clone(), + child: Box::new(convert(input, schema, acc)?), + }, + + LQueryExpr::TopK { k, by, input } => { + let by = resolve_named_keys(by, schema)?; + CQueryExpr::Aggregate { + by, + aggs: vec![AggIntent::TopK { + k: *k as usize, + accuracy: acc.clone(), + }], + having: None, + child: Box::new(convert(input, schema, acc)?), + } + } + + LQueryExpr::Merge { inputs } => CQueryExpr::Merge { + children: inputs + .iter() + .map(|i| convert(i, schema, acc)) + .collect::, _>>()?, + }, + + LQueryExpr::Join { + kind, + pred, + left, + right, + } => CQueryExpr::Join { + kind: kind.clone(), + pred: Predicate(pred.clone().unwrap_or( + crate::intent_algebra::expr_ir::L3Expr::Literal( + crate::intent_algebra::expr_ir::L3Scalar::Boolean(true), + ), + )), + left: Box::new(convert(left, schema, acc)?), + right: Box::new(convert(right, schema, acc)?), + }, + + LQueryExpr::SetOp { + kind, + all, + left, + right, + } => CQueryExpr::SetOp { + kind: kind.clone(), + all: *all, + left: Box::new(convert(left, schema, acc)?), + right: Box::new(convert(right, schema, acc)?), + }, + + LQueryExpr::Sort { keys, input } => CQueryExpr::Sort { + keys: keys.clone(), + child: Box::new(convert(input, schema, acc)?), + }, + + LQueryExpr::Limit { n, offset, input } => CQueryExpr::Limit { + n: *n as usize, + offset: *offset as usize, + child: Box::new(convert(input, schema, acc)?), + }, + + LQueryExpr::LetBinding { name, expr, body } => CQueryExpr::LetBinding { + name: BindingName::new(name.clone()), + expr: Box::new(convert(expr, schema, acc)?), + child: Box::new(convert(body, schema, acc)?), + }, + + LQueryExpr::PromQLSubquery { + range, + resolution, + input, + } => CQueryExpr::Subquery { + range: *range, + resolution: *resolution, + child: Box::new(convert(input, schema, acc)?), + }, + + LQueryExpr::BinaryOp { + op, + lhs, + rhs, + vector_match, + } => CQueryExpr::BinaryOp { + op: op.clone(), + lhs: Box::new(convert(lhs, schema, acc)?), + rhs: Box::new(convert(rhs, schema, acc)?), + vector_match: vector_match.clone(), + }, + }) +} + +/// Build a canonical `Scan` over a time-series source carrying the Binder's +/// self-contained schema. +fn scan(metric: String, schema: &Schema, predicates: Vec) -> CQueryExpr { + CQueryExpr::Scan { + source: Source::TimeSeries { metric }, + predicates, + schema: schema.clone(), + } +} + +/// Map a Layer-2 [`AggFunc`] to its canonical [`AggIntent`], threading the +/// workload's accuracy target onto the approximate intents. +fn agg_func_to_intent(func: &AggFunc, acc: &AccuracyTarget) -> AggIntent { + match func { + AggFunc::Count => AggIntent::Count { + accuracy: acc.clone(), + }, + AggFunc::Sum => AggIntent::Sum, + AggFunc::Avg => AggIntent::Avg, + AggFunc::Min => AggIntent::Min, + AggFunc::Max => AggIntent::Max, + AggFunc::StdDev { population } => AggIntent::StdDev { + population: *population, + }, + AggFunc::Variance { population } => AggIntent::Variance { + population: *population, + }, + AggFunc::Quantile(q) => AggIntent::Quantile { + q: *q, + accuracy: acc.clone(), + }, + AggFunc::CountDistinct => AggIntent::Cardinality { + accuracy: acc.clone(), + }, + AggFunc::HeavyHitters { k } => AggIntent::TopK { + k: *k as usize, + accuracy: acc.clone(), + }, + AggFunc::Rate { window } => AggIntent::Rate { window: *window }, + AggFunc::Increase { window } => AggIntent::Increase { window: *window }, + } +} diff --git a/crates/core/src/intent_algebra/mod.rs b/crates/core/src/intent_algebra/mod.rs index 85521293..0b688544 100644 --- a/crates/core/src/intent_algebra/mod.rs +++ b/crates/core/src/intent_algebra/mod.rs @@ -1,13 +1,41 @@ -pub mod expr; +//! Layers 2–3 of the controller pipeline. +//! +//! - [`relational`] — the Layer-2 per-language algebra tree the parser front +//! ends emit (PromQL / SQL). +//! - [`lower`] — the L2→L3 converter ([`convert_root`]), which runs the +//! [`Binder`] for name resolution and folds single-statistic sketchable +//! aggregates into canonical shapes. +//! - [`query_expr`] — the canonical, language- and deployment-independent L3 +//! intent algebra ([`QueryExpr`] + [`AggIntent`]), with positional +//! [`ColumnId`] schema flow. +//! - [`cse`] — workload-level common-sub-expression elimination over L3. + +pub mod agg_intent; +pub mod binder; +pub mod column_resolution; +pub mod cse; pub mod expr_ir; +pub mod lower; +pub mod names; +pub mod query_expr; +pub mod relational; pub mod schema; -pub use expr::{ - AggIntent, BinaryOpKind, ColumnRef, DataModel, GroupKey, GroupSide, JoinKey, JoinKind, L3Node, - MetricRef, PartitionKeys, Predicate, ProjectItem, QueryExpr, SetOpKind, SortKey, Source, - TableRef, TimeRange, TimeWindowKind, VectorGrouping, VectorMatch, WindowFrame, WindowFuncKind, +pub use agg_intent::{ + agg_accuracy, agg_is_exact, agg_is_mergeable, default_cardinality, default_quantile, AggIntent, +}; +pub use binder::{Binder, SchemaCatalog, UsageDerivedCatalog}; +pub use column_resolution::{ + infer_schema_for_root, infer_source_schema, output_schema_for_aggregate, resolve_column_ref, + resolve_column_refs, resolve_named_keys, ResolveError, }; +pub use cse::{dedupe_subtrees, CseWorkloadPlan}; pub use expr_ir::{CompareOp, L3Expr, L3Scalar}; -pub use schema::{ - ColumnDef, HasSchema, L3DataType, L3Field, L3Schema, MetricSchema, SchemaCatalog, TableSchema, +pub use lower::{convert, convert_root, ConvertError}; +pub use names::{BindingName, QueryId}; +pub use query_expr::{ + BinaryOpKind, BindingScope, ColumnRef, DataModel, GroupSide, JoinKind, PartitionKeys, + Predicate, ProjectItem, QueryExpr, QueryExprError, SetOpKind, SortKey, Source, VectorGrouping, + VectorMatch, VectorMatchKind, WindowKind, }; +pub use schema::{cse_reuse_is_legal, Column, ColumnId, CseError, DataType, Schema}; diff --git a/crates/core/src/intent_algebra/names.rs b/crates/core/src/intent_algebra/names.rs new file mode 100644 index 00000000..071bf905 --- /dev/null +++ b/crates/core/src/intent_algebra/names.rs @@ -0,0 +1,27 @@ +use serde::{Deserialize, Serialize}; + +/// Name of a `LetBinding` / `Ref` sub-expression (CSE producer alias). +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct BindingName(pub String); + +impl BindingName { + pub fn new(s: impl Into) -> Self { + Self(s.into()) + } + pub fn as_str(&self) -> &str { + &self.0 + } +} + +/// Stable identifier for one query within a workload (CSE root key). +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct QueryId(pub String); + +impl QueryId { + pub fn new(s: impl Into) -> Self { + Self(s.into()) + } + pub fn as_str(&self) -> &str { + &self.0 + } +} diff --git a/crates/core/src/intent_algebra/query_expr.rs b/crates/core/src/intent_algebra/query_expr.rs new file mode 100644 index 00000000..c94dced3 --- /dev/null +++ b/crates/core/src/intent_algebra/query_expr.rs @@ -0,0 +1,445 @@ +//! The canonical Layer-3 intent algebra IR. +//! +//! Language- and deployment-independent. Box-owned tree (DAG fan-in is +//! expressed via `LetBinding` / `Ref`); column identity is **positional** +//! (`Aggregate.by: Vec`), resolved by the [`Binder`](super::binder) +//! against the self-contained [`Schema`] carried on each `Scan`. + +use std::collections::HashMap; +use std::time::Duration; + +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +use super::agg_intent::AggIntent; +use super::expr_ir::L3Expr; +use super::names::BindingName; +use super::schema::{Column, ColumnId, DataType, Schema}; + +/// Errors from schema derivation over a canonical tree. +#[derive(Debug, Error)] +pub enum QueryExprError { + #[error("unresolved ref: {0}")] + UnresolvedRef(String), + #[error("by-column id {0} out of range (input has {1} columns)")] + InvalidGroupByColumn(ColumnId, usize), + #[error("Window requires a time_index on input schema")] + WindowMissingTimeIndex, + #[error("Merge requires at least one child")] + EmptyMerge, +} + +// ── Leaf / supporting types ─────────────────────────────────────────────────── + +/// Lifecycle / flush semantics of a streaming time window. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum WindowKind { + Tumbling, + Sliding, + Session, +} + +/// Which data model a `Source` / `AggIntent` operates over. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum DataModel { + TimeSeries, + Tabular, + Any, +} + +/// The leaf data source of a `Scan`. The schema itself rides on the +/// `Scan.schema` field (Binder-built); `Source` carries only the leaf's +/// identity. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum Source { + /// Time-series leaf — PromQL / DC lifecycle. Produces `(ts, value, *labels)`. + TimeSeries { metric: String }, + /// Tabular leaf — asap-fusion / future OLAP. Columns ride on `Scan.schema`. + Table { table_ref: String }, +} + +impl Source { + pub fn data_model(&self) -> DataModel { + match self { + Source::TimeSeries { .. } => DataModel::TimeSeries, + Source::Table { .. } => DataModel::Tabular, + } + } +} + +/// A column reference by name, or one of the two PromQL-conventional +/// synthetic columns. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum ColumnRef { + Named(String), + /// The implicit metric sample value (PromQL — always the series value). + SampleValue, + /// All rows / COUNT(*). + Wildcard, +} + +/// Grouping key set (`by (...)` / `without (...)`). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum PartitionKeys { + By(Vec), + Without(Vec), +} + +impl PartitionKeys { + pub fn keys(&self) -> &[String] { + match self { + PartitionKeys::By(k) | PartitionKeys::Without(k) => k, + } + } + pub fn is_empty(&self) -> bool { + self.keys().is_empty() + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum BinaryOpKind { + Add, + Sub, + Mul, + Div, + Mod, + Pow, + Atan2, + Eq, + Ne, + Lt, + Le, + Gt, + Ge, + And, + Or, + Unless, + Like, + NotLike, + Regex, + NotRegex, +} + +impl std::fmt::Display for BinaryOpKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s = match self { + BinaryOpKind::Add => "+", + BinaryOpKind::Sub => "-", + BinaryOpKind::Mul => "*", + BinaryOpKind::Div => "/", + BinaryOpKind::Mod => "%", + BinaryOpKind::Pow => "^", + BinaryOpKind::Atan2 => "atan2", + BinaryOpKind::Eq => "==", + BinaryOpKind::Ne => "!=", + BinaryOpKind::Lt => "<", + BinaryOpKind::Le => "<=", + BinaryOpKind::Gt => ">", + BinaryOpKind::Ge => ">=", + BinaryOpKind::And => "AND", + BinaryOpKind::Or => "OR", + BinaryOpKind::Unless => "unless", + BinaryOpKind::Like => "LIKE", + BinaryOpKind::NotLike => "NOT LIKE", + BinaryOpKind::Regex => "=~", + BinaryOpKind::NotRegex => "!~", + }; + f.write_str(s) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum JoinKind { + Inner, + Left, + Right, + Full, + Cross, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum SetOpKind { + Union, + Intersect, + Except, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct SortKey { + pub expr: L3Expr, + pub ascending: bool, + pub nulls_first: bool, +} + +/// PromQL vector-match modifier (`on`/`ignoring` + `group_left`/`group_right`). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct VectorMatch { + pub kind: VectorMatchKind, + pub labels: Vec, + pub grouping: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum VectorMatchKind { + On, + Ignoring, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct VectorGrouping { + pub side: GroupSide, + pub labels: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum GroupSide { + Left, + Right, +} + +/// A row-level filter predicate (WHERE clause / PromQL label matcher). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Predicate(pub L3Expr); + +/// One item in a SELECT projection list. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ProjectItem { + pub alias: Option, + pub expr: L3Expr, +} + +// ── L3 intent algebra IR ────────────────────────────────────────────────────── + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum QueryExpr { + /// Outermost leaf. `schema` is the authoritative, self-contained output + /// schema (Binder-built); `predicates` are leaf-level row filters + /// (PromQL label matchers, pushed-down `WHERE` conjuncts). + Scan { + source: Source, + #[serde(default)] + predicates: Vec, + schema: Schema, + }, + /// Reference to a `LetBinding` by name; resolved at plan time. + Ref { name: BindingName }, + + /// σ — row-level filter. Output schema = child schema. + Filter { + pred: Predicate, + child: Box, + }, + /// π — column projection. + Project { + cols: Vec, + child: Box, + }, + + /// γ + α — GROUP BY (positional) + aggregate intents. + Aggregate { + by: Vec, + aggs: Vec, + #[serde(default)] + having: Option, + child: Box, + }, + + /// ψ — tumbling / sliding / session window over the time axis. Window + /// over Aggregate is the canonical windowed-aggregate shape. + Window { + kind: WindowKind, + size: Duration, + #[serde(default)] + slide: Option, + child: Box, + }, + + /// Logical-only partitioning marker (sharding hint for L5). + Partition { + keys: PartitionKeys, + child: Box, + }, + /// δ — SQL `DISTINCT` / row deduplication. + Distinct { + cols: Vec, + child: Box, + }, + /// ⊕ — exact union of sub-results from independent stages / shards. + Merge { children: Vec }, + + /// Logical join. L4 picks the physical alternative. + Join { + kind: JoinKind, + pred: Predicate, + left: Box, + right: Box, + }, + SetOp { + kind: SetOpKind, + all: bool, + left: Box, + right: Box, + }, + + /// Generic order-by for non-heavy-hitter cases. + Sort { + keys: Vec, + child: Box, + }, + Limit { + n: usize, + offset: usize, + child: Box, + }, + + /// SQL `WITH name AS (expr) … child`; PromQL recording-rule binding. + LetBinding { + name: BindingName, + expr: Box, + child: Box, + }, + + /// PromQL sub-query (`[range:resolution]`). Logical pass-through. + Subquery { + range: Duration, + #[serde(default)] + resolution: Option, + child: Box, + }, + + /// Arithmetic / comparison / boolean composition (PromQL binary ops). + BinaryOp { + op: BinaryOpKind, + lhs: Box, + rhs: Box, + #[serde(default)] + vector_match: Option, + }, +} + +impl QueryExpr { + /// Output schema of the root of a single query (empty binding scope). + pub fn output_schema(&self) -> Result { + self.output_schema_in(&BindingScope::default()) + } + + /// Output schema given an explicit `LetBinding` scope. + pub fn output_schema_in(&self, scope: &BindingScope) -> Result { + match self { + QueryExpr::Scan { schema, .. } => Ok(schema.clone()), + + QueryExpr::Window { child, .. } => { + let in_schema = child.output_schema_in(scope)?; + if in_schema.time_index.is_none() { + return Err(QueryExprError::WindowMissingTimeIndex); + } + Ok(in_schema) + } + + QueryExpr::Aggregate { + by, aggs, child, .. + } => { + let in_schema = child.output_schema_in(scope)?; + let mut out_cols: Vec = Vec::with_capacity(by.len() + aggs.len()); + for &id in by { + let c = + in_schema + .columns + .get(id) + .ok_or(QueryExprError::InvalidGroupByColumn( + id, + in_schema.columns.len(), + ))?; + out_cols.push(c.clone()); + } + let value_col_idx = in_schema + .column_id("value") + .or_else(|| (0..in_schema.columns.len()).find(|i| !by.contains(i))); + let probe = value_col_idx + .and_then(|i| in_schema.columns.get(i)) + .cloned() + .unwrap_or(Column { + name: "value".into(), + dtype: DataType::Float64, + nullable: false, + }); + for intent in aggs { + out_cols.push(intent.output_column(&probe)); + } + let unique_keys = if by.is_empty() { + Vec::new() + } else { + vec![(0..by.len()).collect()] + }; + Ok(Schema { + columns: out_cols, + time_index: None, + unique_keys, + }) + } + + QueryExpr::LetBinding { name, expr, child } => { + let bound = expr.output_schema_in(scope)?; + let extended = scope.with(name.clone(), bound); + child.output_schema_in(&extended) + } + QueryExpr::Ref { name } => scope + .lookup(name) + .cloned() + .ok_or_else(|| QueryExprError::UnresolvedRef(name.as_str().into())), + + QueryExpr::Filter { child, .. } + | QueryExpr::Partition { child, .. } + | QueryExpr::Sort { child, .. } + | QueryExpr::Limit { child, .. } + | QueryExpr::Subquery { child, .. } + | QueryExpr::Project { child, .. } => child.output_schema_in(scope), + + QueryExpr::Distinct { cols, child } => { + let in_schema = child.output_schema_in(scope)?; + let mut out = in_schema.clone(); + let mut key_ids: Vec = Vec::with_capacity(cols.len()); + for c in cols { + if let ColumnRef::Named(name) = c { + if let Some(id) = in_schema.column_id(name) { + key_ids.push(id); + } + } + } + if !key_ids.is_empty() { + out.add_unique_key(key_ids); + } + Ok(out) + } + + QueryExpr::Merge { children } => children + .first() + .ok_or(QueryExprError::EmptyMerge) + .and_then(|c| c.output_schema_in(scope)), + QueryExpr::SetOp { left, .. } | QueryExpr::Join { left, .. } => { + left.output_schema_in(scope) + } + QueryExpr::BinaryOp { lhs, .. } => lhs.output_schema_in(scope), + } + } +} + +/// Lexical scope for `LetBinding` / `Ref` resolution. +#[derive(Debug, Default, Clone)] +pub struct BindingScope { + bindings: HashMap, +} + +impl BindingScope { + pub fn new() -> Self { + Self::default() + } + pub fn with(&self, name: BindingName, schema: Schema) -> Self { + let mut bindings = self.bindings.clone(); + bindings.insert(name.as_str().into(), schema); + Self { bindings } + } + pub fn lookup(&self, name: &BindingName) -> Option<&Schema> { + self.bindings.get(name.as_str()) + } +} diff --git a/crates/core/src/intent_algebra/relational.rs b/crates/core/src/intent_algebra/relational.rs new file mode 100644 index 00000000..592ef590 --- /dev/null +++ b/crates/core/src/intent_algebra/relational.rs @@ -0,0 +1,212 @@ +//! The Layer-2 relational IR — the per-language query algebra the parser +//! front ends emit, before [`convert_root`](super::lower::convert_root) lowers +//! it to the canonical L3 [`query_expr::QueryExpr`](super::query_expr::QueryExpr). +//! +//! Leaf / scalar types (`ColumnRef`, `PartitionKeys`, `SortKey`, +//! `BinaryOpKind`, `VectorMatch`) are owned by `query_expr` and re-used here so +//! there is one canonical spelling. Filter / having / project expressions use +//! the shared language-independent [`L3Expr`](super::expr_ir::L3Expr). + +use std::time::Duration; + +use super::expr_ir::L3Expr; +pub use super::query_expr::{BinaryOpKind, ColumnRef, PartitionKeys, SortKey, VectorMatch}; + +/// Base relation / metric stream source. +#[derive(Debug, Clone, PartialEq)] +pub struct SourceSpec { + /// Metric name (PromQL) or table name (SQL). + pub name: String, +} + +/// One aggregate function in a GROUP BY / AGGREGATE node. +#[derive(Debug, Clone, PartialEq)] +pub struct AggItem { + pub alias: String, + pub func: AggFunc, + pub col: ColumnRef, + pub distinct: bool, +} + +/// Layer-2 aggregate functions. Mapped to canonical [`AggIntent`] by +/// [`super::lower::convert`]. +#[derive(Debug, Clone, PartialEq)] +pub enum AggFunc { + Count, + Sum, + Avg, + Min, + Max, + StdDev { + population: bool, + }, + Variance { + population: bool, + }, + Quantile(f64), + /// COUNT DISTINCT — maps to `Cardinality`. + CountDistinct, + /// Heavy-hitter top-k — maps to `AggIntent::TopK`. + HeavyHitters { + k: u64, + }, + /// PromQL `rate()` / `irate()` — carries the range-vector window so the + /// canonical `Rate` intent owns it (no separate `Window` node). + Rate { + window: Duration, + }, + /// PromQL `increase()` — see `Rate`. + Increase { + window: Duration, + }, +} + +/// The Layer-2 relational query IR. +#[derive(Debug, Clone, PartialEq)] +pub enum QueryExpr { + /// A named metric stream or table — the outermost leaf. + Source(SourceSpec), + /// Reference to a CTE / let-binding by name. + Ref(String), + + /// σ — row-level filter (WHERE / PromQL label matchers). + Filter { pred: L3Expr, input: Box }, + + /// γ + α — GROUP BY (`keys`) followed by aggregate functions. + Aggregate { + keys: Vec, + aggs: Vec, + having: Option, + input: Box, + }, + + /// ψ — time window (PromQL `[5m]`). + Window { + duration: Duration, + slide: Option, + input: Box, + }, + + /// Partition the stream by key-tuple (`by (dims)` / `without (dims)`). + Partition { + keys: PartitionKeys, + input: Box, + }, + /// δ — deduplicate on `cols`. + Distinct { + cols: Vec, + input: Box, + }, + /// τ — heavy-hitter top-k. `by` are the grouping keys. + TopK { + k: u64, + by: Vec, + input: Box, + }, + /// ⊕ — merge sub-results from independent branches. + Merge { inputs: Vec }, + + Join { + kind: super::query_expr::JoinKind, + pred: Option, + left: Box, + right: Box, + }, + SetOp { + kind: super::query_expr::SetOpKind, + all: bool, + left: Box, + right: Box, + }, + + Sort { + keys: Vec, + input: Box, + }, + Limit { + n: u64, + offset: u64, + input: Box, + }, + + LetBinding { + name: String, + expr: Box, + body: Box, + }, + + /// PromQL sub-query syntax: `[range:resolution]`. + PromQLSubquery { + range: Duration, + resolution: Option, + input: Box, + }, + + /// Binary op between two instant-vector expressions (PromQL `+`, `/`, …). + BinaryOp { + op: BinaryOpKind, + lhs: Box, + rhs: Box, + vector_match: Option, + }, +} + +impl QueryExpr { + /// Walk the tree depth-first, calling `f` on every node. + pub fn walk(&self, f: &mut F) { + f(self); + match self { + QueryExpr::Source(_) | QueryExpr::Ref(_) => {} + QueryExpr::Filter { input, .. } + | QueryExpr::Aggregate { input, .. } + | QueryExpr::Window { input, .. } + | QueryExpr::Partition { input, .. } + | QueryExpr::Distinct { input, .. } + | QueryExpr::TopK { input, .. } + | QueryExpr::Sort { input, .. } + | QueryExpr::Limit { input, .. } + | QueryExpr::PromQLSubquery { input, .. } => input.walk(f), + QueryExpr::Merge { inputs } => { + for i in inputs { + i.walk(f); + } + } + QueryExpr::Join { left, right, .. } + | QueryExpr::SetOp { left, right, .. } + | QueryExpr::BinaryOp { + lhs: left, + rhs: right, + .. + } => { + left.walk(f); + right.walk(f); + } + QueryExpr::LetBinding { expr, body, .. } => { + expr.walk(f); + body.walk(f); + } + } + } + + /// Outermost metric/table name from the first `Source` leaf. + pub fn source_name(&self) -> Option<&str> { + match self { + QueryExpr::Source(s) => Some(&s.name), + QueryExpr::Filter { input, .. } + | QueryExpr::Aggregate { input, .. } + | QueryExpr::Window { input, .. } + | QueryExpr::Partition { input, .. } + | QueryExpr::Distinct { input, .. } + | QueryExpr::TopK { input, .. } + | QueryExpr::Sort { input, .. } + | QueryExpr::Limit { input, .. } + | QueryExpr::PromQLSubquery { input, .. } => input.source_name(), + QueryExpr::Merge { inputs } => inputs.first()?.source_name(), + QueryExpr::Join { left, .. } + | QueryExpr::SetOp { left, .. } + | QueryExpr::BinaryOp { lhs: left, .. } => left.source_name(), + QueryExpr::LetBinding { body, .. } => body.source_name(), + QueryExpr::Ref(_) => None, + } + } +} diff --git a/crates/core/src/intent_algebra/schema.rs b/crates/core/src/intent_algebra/schema.rs index 0220a45a..0a895cce 100644 --- a/crates/core/src/intent_algebra/schema.rs +++ b/crates/core/src/intent_algebra/schema.rs @@ -1,107 +1,332 @@ -use std::collections::HashMap; +//! Layer 3 schema flow — every L3 edge carries a typed `Schema`. +//! +//! Per `control_plane/docs/design.md` §6 "Schema flow — every L3 edge carries +//! a typed schema". The DAG is type-checked: a node's output schema is a +//! function of its inputs and parameters and is verifiable independently +//! of the surrounding context. +//! +//! `Schema::unique_keys` is the load-bearing field for the workload-level +//! CSE pass (`design.md` §6 "DAG, not tree" + the batched-queries example +//! around line ~1284). Two `QueryExpr::Ref` consumers can share a producer +//! only when its output schema is provably stable across reads — the +//! unique-key metadata is what lets the deduper assert that. +//! +//! Single-query plans don't read this field; it lives here so the metadata +//! is available the moment workload-aware planning lands without requiring +//! an L3-wide schema change. -/// Catalog of external data-source metadata. Used by L1→L3 lowering and by -/// `Scan` schema derivation to resolve leaf column / label types; all other -/// nodes derive their output schemas purely from their input schemas. -/// -/// Holds both relational tables (SQL / DataFusion sources) and time-series -/// metrics (PromQL / OTLP sources). A deployment model populates only the -/// half it needs. -#[derive(Debug, Clone, Default)] -pub struct SchemaCatalog { - /// Relational tables, keyed by table name. - pub tables: HashMap, - /// Time-series metrics, keyed by metric name. - pub metrics: HashMap, -} +#![allow(dead_code)] -/// Schema for a single relational table. -#[derive(Debug, Clone)] -pub struct TableSchema { - pub columns: Vec, - /// Name of the column that holds the row timestamp, if any. - pub time_column: Option, -} +use serde::{Deserialize, Serialize}; -/// One column in a `TableSchema`. -#[derive(Debug, Clone)] -pub struct ColumnDef { +/// Index into [`Schema::columns`] used everywhere a column position is +/// referenced (group-by keys, unique-key sets, the time axis index). +/// +/// Aliased to `usize` to match `design.md`'s `Vec>` for +/// `unique_keys`. Kept as a named type so downstream code can pattern on +/// the intent ("this is a column position, not just any number"). +pub type ColumnId = usize; + +/// One column in a [`Schema`]. Mirrors `design.md` §6 `Field` — +/// `name + dtype + nullable`. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct Column { + /// Column name as it appears in the producer's output. PromQL leaves + /// produce label-name + the synthetic `value` / `timestamp` columns; + /// SQL leaves carry their `information_schema` names. pub name: String, - pub data_type: L3DataType, + /// Column data type. Kept narrow at L3 (`Int64` / `Float64` / `Utf8` + /// / `Bool` / `Timestamp`); `Sketch(...)` is an L4-only addition per + /// design.md §6.4 and is intentionally absent here. + pub dtype: DataType, + /// Whether NULL values are allowed in this column. PromQL value + /// columns are non-nullable; SQL columns inherit their DDL nullability. pub nullable: bool, } -/// Schema for a single time-series metric. +/// L3 column data types. Deliberately narrow: no sketch state at this +/// layer (see `design.md` §6.4 for the L4 `DataType::Sketch(...)` +/// extension). +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DataType { + /// 64-bit signed integer. Counter columns, group-cardinality outputs. + Int64, + /// 64-bit IEEE-754 float. Quantile / Avg / Sum-over-floats output. + Float64, + /// UTF-8 string. PromQL label values, SQL `VARCHAR` / `TEXT`. + Utf8, + /// Boolean — predicate output, `unless` / `and` / `or` PromQL ops. + Bool, + /// Wall-clock timestamp. PromQL leaves carry exactly one of these + /// (the `time_index` column); SQL leaves may or may not. + Timestamp, +} + +/// Per-edge L3 schema. Flowing between any two L3 operators, on every +/// node's input and output. /// -/// A metric scan always produces a `timestamp` (the time axis) and a sample -/// `value` column; this metadata names the label set carried alongside and the -/// value's type. When a metric is absent from the catalog the lowerer falls -/// back to `value: Float64` with labels discovered from the query's matchers. -#[derive(Debug, Clone)] -pub struct MetricSchema { - /// Label names exposed by this metric (e.g. `["service", "host", "env"]`). - pub labels: Vec, - /// Type of the sample value column. Usually `Float64`. - pub value_type: L3DataType, +/// `unique_keys` is metadata for reuse-aware planning: each inner `Vec` +/// is a set of column indices that together uniquely identify rows. The +/// outer `Vec` allows multiple unique-key sets (primary key + another +/// unique constraint). Populated by per-node input/output spec — +/// `Aggregate { by, .. }` emits `unique_keys = [by]`; `Distinct { cols }` +/// adds `cols`; most other nodes pass through. +/// +/// **Consumed by**: workload-level CSE (`CostModel::workload_cost` in the +/// design, not yet shipped). The single-query path, the `Bind*` rules, +/// push-down, and L5 emitters do not read this field. +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct Schema { + /// Columns flowing on this edge, in positional order. + pub columns: Vec, + /// Index into `columns` for the time axis, if any. PromQL leaves + /// always carry one; SQL leaves may or may not. + #[serde(default)] + pub time_index: Option, + /// Unique-key sets — each inner vec is a tuple of column indices + /// that together uniquely identifies a row. Empty `Vec` means + /// "no provable unique constraint" (the conservative default). + #[serde(default)] + pub unique_keys: Vec>, } -impl Default for MetricSchema { - fn default() -> Self { +impl Schema { + /// Construct a `Schema` from columns alone — no time index, no + /// unique-key constraint. Used by `Scan` over a tabular source + /// when the catalog supplies no primary-key metadata. + pub fn new(columns: Vec) -> Self { Self { - labels: Vec::new(), - value_type: L3DataType::Float64, + columns, + time_index: None, + unique_keys: Vec::new(), } } -} -// ── Data types ──────────────────────────────────────────────────────────────── + /// Construct a `Scan`-style schema with explicit `time_index` + + /// inferred unique keys (e.g. PromQL leaves: `[time_index, label_set]`). + pub fn with_time_index( + columns: Vec, + time_index: ColumnId, + unique_keys: Vec>, + ) -> Self { + Self { + columns, + time_index: Some(time_index), + unique_keys, + } + } -/// Column types that may appear on an L3 DAG edge. -/// L4 extends this set with `L4DataType::Sketch`; L3 edges never carry -/// sketch-state columns. -#[derive(Debug, Clone, PartialEq)] -pub enum L3DataType { - Int64, - Float64, - Utf8, - Boolean, - Timestamp, - Duration, - /// Key→Value map (e.g. PromQL label set encoded as a column). - Map(Box, Box), - List(Box), + /// Look up a column by name. `None` if not present. + pub fn column_id(&self, name: &str) -> Option { + self.columns.iter().position(|c| c.name == name) + } + + /// Whether this schema has *any* provable unique key. The CSE pass + /// reads this to decide whether two `Ref` consumers can safely share + /// a producer (see `design.md` §6 line ~1284 + the unit test in + /// `tests::cse_substitution_legal_only_with_unique_keys`). + pub fn has_unique_key(&self) -> bool { + !self.unique_keys.is_empty() + } + + /// Append `cols` as an additional unique-key set if not already present. + /// Used by `Distinct { cols }` per design.md §6 schema-flow table: + /// "the input schema with `unique_keys` tightened to include `cols`". + pub fn add_unique_key(&mut self, cols: Vec) { + if !self.unique_keys.contains(&cols) { + self.unique_keys.push(cols); + } + } } -// ── Schema ──────────────────────────────────────────────────────────────────── +// ── CSE legality (the load-bearing consumer of `unique_keys`) ──────────────── +// +// Phase F per `control_plane/docs/design.md` §6 Schema flow + the batched- +// queries example (§6 line ~1320): +// +// "CSE legality leans on `Schema::unique_keys` (§6 Schema flow): two +// `QueryExpr::Ref` consumers can share a producer only when its +// output schema is provably stable across reads — the unique-key +// metadata is what lets the deduper assert that without re-running +// the producer's logic." +// +// `cse_reuse_is_legal` is the gatekeeper. The workload-level CSE pass +// (`intent_algebra::cse::dedupe_subtrees`) consults it before emitting +// a `LetBinding` to share a producer between ≥2 `Ref` consumers. -#[derive(Debug, Clone, PartialEq)] -pub struct L3Field { - pub name: String, - pub dtype: L3DataType, - pub nullable: bool, +use thiserror::Error; + +/// Errors returned by [`cse_reuse_is_legal`] when shared-producer reuse +/// would violate the design's stability invariant. +#[derive(Debug, Error, PartialEq, Eq)] +pub enum CseError { + /// Producer schema lacks any `unique_keys` set — row identity is + /// not provably stable across reads, so two `Ref` consumers cannot + /// safely share it. The deduper falls back to per-consumer + /// recomputation. Per design.md §6 line ~1356. + #[error( + "shared-producer CSE refused: producer schema has no unique_keys \ + (design.md §6 schema-flow — without a provable unique key the \ + deduper cannot assert row identity across reads)" + )] + NoUniqueKeys, + /// Trivially-callable case: only one consumer means no reuse to + /// gate. Returned so the caller can short-circuit instead of + /// emitting a degenerate `LetBinding`. + #[error("CSE not applicable: {0} consumer(s) — need ≥ 2 for shared-producer reuse")] + InsufficientConsumers(usize), } -/// Schema carried on every edge of the L3 DAG. Describes the columns -/// flowing between two operators. Type-checked at plan construction time: -/// a node whose predicate references a column absent from its child's -/// `L3Schema` is a plan-time error. -#[derive(Debug, Clone, PartialEq)] -pub struct L3Schema { - pub fields: Vec, - /// Index into `fields` for the time axis, if any. - /// PromQL `Scan` leaves always carry one; SQL leaves may or may not. - pub time_index: Option, +/// Two `QueryExpr::Ref` nodes can share a producer (same `LetBinding`) +/// only when the producer's output schema has stable per-row identity — +/// i.e. `Schema::unique_keys` is non-empty. This is the gatekeeper: +/// returns `Ok(())` if shared-producer reuse is legal, otherwise `Err`. +/// +/// Per design.md §6 line ~1356 — `unique_keys` is what makes CSE +/// provably correct. The deduper consults this before emitting a +/// `LetBinding`, and `CostModel::workload_cost` only credits a shared +/// binding when this gate has fired green. +/// +/// `consumer_count` is the number of `QueryExpr::Ref { name }` sites the +/// deduper has identified for the candidate binding. Single-consumer +/// cases short-circuit with `InsufficientConsumers` — a `LetBinding` +/// with one `Ref` is just a no-op alias and shouldn't be hoisted. +pub fn cse_reuse_is_legal(producer_schema: &Schema, consumer_count: usize) -> Result<(), CseError> { + if consumer_count < 2 { + return Err(CseError::InsufficientConsumers(consumer_count)); + } + if !producer_schema.has_unique_key() { + return Err(CseError::NoUniqueKeys); + } + Ok(()) } -// ── Schema derivation trait ─────────────────────────────────────────────────── - -/// Implemented by `QueryExpr` to compute the output schema of a node given -/// its children's output schemas. The `L3Node` wrapper stores the derived -/// schema so derivation runs once at construction, not on every traversal. -pub trait HasSchema { - /// # Panics - /// Panics if a `Scan` over a `Source::Table` references a table absent from - /// `catalog`. Time-series scans never panic — an unregistered metric falls - /// back to a `value: Float64` default schema. - fn output_schema(&self, input_schemas: &[&L3Schema], catalog: &SchemaCatalog) -> L3Schema; +// ── Tests ───────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + fn col(name: &str, dtype: DataType) -> Column { + Column { + name: name.into(), + dtype, + nullable: false, + } + } + + /// `cse_reuse_is_legal` accepts a producer schema with at least one + /// `unique_keys` set + ≥2 consumers. This is the design.md §6 + /// "load-bearing" green path. + #[test] + fn cse_reuse_legal_when_unique_keys_set() { + let producer = Schema::with_time_index( + vec![ + col("ts", DataType::Timestamp), + col("service", DataType::Utf8), + col("value", DataType::Float64), + ], + 0, + vec![vec![0, 1]], + ); + assert_eq!(cse_reuse_is_legal(&producer, 2), Ok(())); + assert_eq!(cse_reuse_is_legal(&producer, 5), Ok(())); + } + + /// Schema without `unique_keys` is the conservative-default case — + /// the deduper must refuse to share it. Pins design.md §6 line + /// ~1356 ("Without it, the deduper has to be conservative and reuse + /// drops on the floor"). + #[test] + fn cse_reuse_illegal_when_unique_keys_empty() { + let producer = Schema::new(vec![col("a", DataType::Int64), col("b", DataType::Float64)]); + assert_eq!( + cse_reuse_is_legal(&producer, 2), + Err(CseError::NoUniqueKeys) + ); + } + + /// Single-consumer case is short-circuited — no `LetBinding` should + /// be emitted for one `Ref` because there's no reuse to credit. + #[test] + fn cse_reuse_rejects_single_consumer() { + let producer = Schema::with_time_index( + vec![col("ts", DataType::Timestamp), col("v", DataType::Float64)], + 0, + vec![vec![0]], + ); + assert_eq!( + cse_reuse_is_legal(&producer, 1), + Err(CseError::InsufficientConsumers(1)) + ); + assert_eq!( + cse_reuse_is_legal(&producer, 0), + Err(CseError::InsufficientConsumers(0)) + ); + } + + /// Empty `unique_keys` rejection takes precedence over the consumer + /// count check only when both pass — but here we verify the + /// insufficient-consumers branch fires first (a defensive ordering + /// so callers see the clearer error when they get the call wrong). + #[test] + fn cse_reuse_consumer_check_precedes_unique_key_check() { + let producer = Schema::new(vec![col("a", DataType::Int64)]); + // Both conditions fail; consumer check is reported. + assert_eq!( + cse_reuse_is_legal(&producer, 1), + Err(CseError::InsufficientConsumers(1)) + ); + } + + #[test] + fn schema_new_has_no_time_or_unique_key() { + let s = Schema::new(vec![col("k", DataType::Utf8), col("v", DataType::Float64)]); + assert!(s.time_index.is_none()); + assert!(!s.has_unique_key()); + assert_eq!(s.column_id("k"), Some(0)); + assert_eq!(s.column_id("v"), Some(1)); + assert_eq!(s.column_id("missing"), None); + } + + #[test] + fn schema_with_time_index_populates_metadata() { + let s = Schema::with_time_index( + vec![ + col("ts", DataType::Timestamp), + col("service", DataType::Utf8), + col("value", DataType::Float64), + ], + 0, + vec![vec![0, 1]], + ); + assert_eq!(s.time_index, Some(0)); + assert!(s.has_unique_key()); + assert_eq!(s.unique_keys, vec![vec![0, 1]]); + } + + #[test] + fn add_unique_key_dedupes() { + let mut s = Schema::new(vec![col("a", DataType::Utf8), col("b", DataType::Utf8)]); + s.add_unique_key(vec![0]); + s.add_unique_key(vec![0]); + s.add_unique_key(vec![0, 1]); + assert_eq!(s.unique_keys, vec![vec![0], vec![0, 1]]); + } + + #[test] + fn schema_serde_roundtrip() { + let s = Schema::with_time_index( + vec![ + col("ts", DataType::Timestamp), + col("value", DataType::Float64), + ], + 0, + vec![vec![0]], + ); + let json = serde_json::to_string(&s).unwrap(); + let back: Schema = serde_json::from_str(&json).unwrap(); + assert_eq!(s, back); + } } diff --git a/crates/core/src/sketch_algebra/expr.rs b/crates/core/src/sketch_algebra/expr.rs index 3f1bd904..0cc343f1 100644 --- a/crates/core/src/sketch_algebra/expr.rs +++ b/crates/core/src/sketch_algebra/expr.rs @@ -2,7 +2,7 @@ use std::rc::Rc; use super::schema::L4Schema; use super::sketch::{SketchQuery, SummaryKind, SummaryParams}; -use crate::intent_algebra::{ColumnRef, GroupKey, L3Node}; +use crate::intent_algebra::{ColumnId, ColumnRef, QueryExpr}; // ── L4 DAG node ─────────────────────────────────────────────────────────────── @@ -31,7 +31,7 @@ pub enum SummaryExpr { /// Any L3 node that no L4 rule rewrote (e.g. `Filter`, `Project`, `Sort`). /// Output schema is the inner L3 node's schema, lifted to `L4Schema` /// with all fields as `L4DataType::Primitive`. - Logical(Rc), + Logical(Box), /// Sketch aggregation. L4 chose `sketch` + `params` from the catalog /// for `AggIntent` under `DeploymentConstraints`. @@ -43,8 +43,8 @@ pub enum SummaryExpr { params: SummaryParams, /// The column being summarised (fed into the sketch). col: ColumnRef, - /// GROUP BY keys carried through to the output schema. - by: Vec, + /// GROUP BY keys (positional) carried through to the output schema. + by: Vec, }, /// Sketch-aware join (KMV / theta for join-cardinality; join-sample for diff --git a/crates/core/src/sketch_algebra/schema.rs b/crates/core/src/sketch_algebra/schema.rs index 1ee536e7..c2fdcaac 100644 --- a/crates/core/src/sketch_algebra/schema.rs +++ b/crates/core/src/sketch_algebra/schema.rs @@ -1,5 +1,5 @@ use super::sketch::{SummaryKind, SummaryParams}; -use crate::intent_algebra::L3DataType; +use crate::intent_algebra::DataType; // ── L4 data types ───────────────────────────────────────────────────────────── @@ -13,7 +13,7 @@ use crate::intent_algebra::L3DataType; #[derive(Debug, Clone, PartialEq)] pub enum L4DataType { /// Any base L3 column type — passed through unchanged from L3 edges. - Primitive(L3DataType), + Primitive(DataType), /// Opaque summary state (exact accumulator or approximate sketch). /// The `(kind, params)` pair is the type identity: two summary columns /// are compatible only if both match exactly. diff --git a/crates/core/src/types.rs b/crates/core/src/types.rs index 4dfad52c..089ea9fa 100644 --- a/crates/core/src/types.rs +++ b/crates/core/src/types.rs @@ -1,5 +1,7 @@ +use serde::{Deserialize, Serialize}; + /// Accuracy requirement that a query result must satisfy. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum AccuracyTarget { /// Additive error bound ε: |estimate − true| ≤ ε · (domain size). Epsilon(f64), diff --git a/crates/lower/src/error.rs b/crates/lower/src/error.rs index 8ce9857a..9f377a2c 100644 --- a/crates/lower/src/error.rs +++ b/crates/lower/src/error.rs @@ -16,6 +16,8 @@ pub enum LoweringError { InvalidParameter(String), /// The workload's query language is not handled by this lowerer. WrongLanguage(String), + /// The L2→L3 converter failed (name resolution against the bound schema). + Convert(asap_control_core::intent_algebra::ConvertError), } impl fmt::Display for LoweringError { @@ -28,8 +30,15 @@ impl fmt::Display for LoweringError { Self::MissingArgument(m) => write!(f, "missing argument: {m}"), Self::InvalidParameter(m) => write!(f, "invalid parameter: {m}"), Self::WrongLanguage(l) => write!(f, "unsupported query language: {l}"), + Self::Convert(e) => write!(f, "L2→L3 conversion failed: {e}"), } } } impl std::error::Error for LoweringError {} + +impl From for LoweringError { + fn from(e: asap_control_core::intent_algebra::ConvertError) -> Self { + Self::Convert(e) + } +} diff --git a/crates/lower/src/lib.rs b/crates/lower/src/lib.rs index 1fa233ce..c369f03c 100644 --- a/crates/lower/src/lib.rs +++ b/crates/lower/src/lib.rs @@ -1,34 +1,31 @@ //! L1→L3 lowering passes for the ASAP controller core. //! -//! Each query language has one pass that ends at the same intent-algebra -//! [`QueryExpr`](asap_control_core::intent_algebra::expr::QueryExpr): L1 parse -//! (delegated to a language parser crate), L2 per-language tree (the parser's -//! own AST), and L3 lowering to the language- and deployment-independent intent -//! algebra. PromQL lives in [`promql`]; the SQL path (PR #4) lives alongside it. +//! PromQL flows through three layers, all ending at the canonical intent +//! algebra: L1 parse (`promql-parser`), L2 per-language tree +//! ([`relational::QueryExpr`](asap_control_core::intent_algebra::relational)), +//! and the L2→L3 conversion ([`convert_root`]) that runs the +//! [`Binder`](asap_control_core::intent_algebra::Binder) and folds the +//! single-statistic sketchable aggregate into canonical shapes. pub mod error; pub mod promql; -pub mod schema_pass; -use asap_control_core::intent_algebra::expr::QueryExpr; -use asap_control_core::intent_algebra::schema::SchemaCatalog; +use asap_control_core::intent_algebra::{convert_root, QueryExpr}; use asap_control_core::types::AccuracyTarget; use asap_control_core::workload::{QueryLanguage, QueryWorkload}; pub use error::LoweringError; pub use promql::PromqlLowerer; -pub use schema_pass::populate_schemas; -/// Lower a single PromQL query string to an intent-algebra `QueryExpr`. +/// Lower a single PromQL query string to the canonical L3 `QueryExpr`. /// -/// The returned tree has empty schemas on every node; call [`populate_schemas`] -/// before inspecting node schemas or passing the tree to schema-aware stages. -pub fn lower_promql( - query: &str, - catalog: &SchemaCatalog, - accuracy: AccuracyTarget, -) -> Result { - PromqlLowerer::new(catalog, accuracy).lower(query) +/// `accuracy` is threaded onto every approximate intent (`Count`, `Quantile`, +/// `Cardinality`, `TopK`). The returned tree carries a self-contained `Schema` +/// on its `Scan`; call [`QueryExpr::output_schema`] for any node's schema. +pub fn lower_promql(query: &str, accuracy: AccuracyTarget) -> Result { + let l2 = PromqlLowerer::lower(query)?; + let l3 = convert_root(&l2, &accuracy)?; + Ok(l3) } /// Lower every PromQL batch entry in `workload` to a `QueryExpr`. @@ -36,10 +33,7 @@ pub fn lower_promql( /// One `Result` per entry — errors are per-query, not fatal for the batch. /// Returns an empty `Vec` if `workload.query_batch` is absent or empty, and a /// `WrongLanguage` error for every entry if the workload language is not PromQL. -pub fn lower_promql_batch( - workload: &QueryWorkload, - catalog: &SchemaCatalog, -) -> Vec> { +pub fn lower_promql_batch(workload: &QueryWorkload) -> Vec> { let entries = match &workload.query_batch { Some(e) if !e.is_empty() => e, _ => return vec![], @@ -61,7 +55,7 @@ pub fn lower_promql_batch( .as_ref() .and_then(|r| r.accuracy.clone()) .unwrap_or(AccuracyTarget::Exact); - lower_promql(&entry.query.0, catalog, accuracy) + lower_promql(&entry.query.0, accuracy) }) .collect() } diff --git a/crates/lower/src/promql.rs b/crates/lower/src/promql.rs index 16332938..c7ed56ba 100644 --- a/crates/lower/src/promql.rs +++ b/crates/lower/src/promql.rs @@ -1,36 +1,31 @@ -//! Layers 1→3 lowering: PromQL string → intent-algebra `QueryExpr`. +//! Layers 1→2 lowering: PromQL string → Layer-2 `relational::QueryExpr`. //! -//! - **L1 (parse)** is delegated to the `promql-parser` crate, which produces a -//! PromQL-specific AST (`promql_parser::parser::Expr`). -//! - **L2 (per-language tree)** is that same AST — `promql-parser` already hands -//! us a typed, language-flavored tree (instant vs range vectors, aggregate -//! operators, label matchers), so no separate L2 structure is materialised. -//! This mirrors the SQL path (PR #4), which uses DataFusion's `LogicalPlan` as -//! its free L2. -//! - **L3 (intent algebra)** is what this module emits: the language- and -//! deployment-independent [`QueryExpr`] with intent-only [`AggIntent`]s and a -//! `Source::TimeSeries` leaf. No sketch types, no sketch parameters. +//! - **L1 (parse)** is delegated to `promql-parser` 0.8. +//! - **L2 (per-language tree)** is built here: the walk interprets PromQL +//! semantics (range vectors, aggregate operators, label matchers) and emits +//! the language-flavored [`relational::QueryExpr`] the controller's L2→L3 +//! converter ([`convert_root`](asap_control_core::intent_algebra::convert_root)) +//! consumes. Canonicalisation (window-over-aggregate fold, GROUP-BY → +//! `Partition`, positional name binding) happens in that converter, not here. //! -//! # PromQL → L3 mapping (summary) +//! # PromQL → L2 mapping (summary) //! -//! | PromQL | L3 shape | +//! | PromQL | L2 shape (→ canonical via `convert_root`) | //! |---|---| -//! | `quantile_over_time(φ, m{f}[w])` | `TimeWindow{w} → Aggregate{[Quantile{φ}]}` | -//! | `histogram_quantile(φ, rate(m{f}[w]))` | `TimeWindow{w} → Aggregate{[Quantile{φ}]}` | -//! | `avg/min/max/sum_over_time(m[w])` | `TimeWindow{w} → Aggregate{[Avg/Min/Max/Sum]}` | -//! | `stddev/stdvar_over_time(m[w])` | `TimeWindow{w} → Aggregate{[StdDev/Variance]}` | -//! | `count_over_time(m[w])` | `TimeWindow{w} → Aggregate{[Count]}` | -//! | `changes/resets(m[w])` | `TimeWindow{w} → Aggregate{[Count]}` | -//! | `rate/irate/increase(m[w])` | `Aggregate{[Rate{w}/Increase{w}]}` (window in intent) | -//! | `OUTER by (dims) (…)` | grouping `dims` flow onto the inner `Aggregate.by` | -//! | `count by (d) (… )` | `Aggregate{by:d, [Cardinality]}` | -//! | `topk(k, count_over_time(…))` | `Aggregate{[TopK{k}]}` (heavy-hitter, one pass) | -//! | `topk(k, )` / `bottomk(k, …)` | generic `Sort{value} → Limit{k}` | -//! | `m{f}` bare | `Scan{TimeSeries, predicates}` | +//! | `quantile_over_time(φ, m{f}[w])` | `Aggregate{[Quantile(φ)], Window{w, Filter(Source)}}` | +//! | `histogram_quantile(φ, rate(m[w]))` | `Aggregate{[Quantile(φ)], Window{w, …}}` | +//! | `avg/min/max/sum_over_time(m[w])` | `Aggregate{[Avg/Min/Max/Sum], Window{w}}` | +//! | `stddev/stdvar_over_time(m[w])` | `Aggregate{[StdDev/Variance], Window{w}}` | +//! | `count_over_time(m[w])` | `Aggregate{[Count], Window{w}}` | +//! | `rate/irate/increase(m[w])` | `Aggregate{[Rate{w}/Increase{w}]}` (no Window) | +//! | `OUTER by (dims) (…)` | `Aggregate.keys = dims` (→ `Partition` in L3) | +//! | `count by (d) (…)` | `Aggregate{[CountDistinct], …}` (→ `Cardinality`) | +//! | `topk(k, count_over_time(…))` | `TopK{k, by}` (heavy-hitter, one pass) | +//! | `topk(k, )` / `bottomk(k, …)` | `Sort{value} → Limit{k}` | +//! | `m{f}` | `Filter(Source)` | //! | `a OP b` | `BinaryOp{vector_match}` | -//! | `expr[r:res]` | `TimeWindow{Sliding, r, res}` | +//! | `expr[r:res]` | `PromQLSubquery{r, res}` | -use std::sync::Arc; use std::time::Duration; use promql_parser::label::{MatchOp, Matcher}; @@ -39,42 +34,27 @@ use promql_parser::parser::{ VectorSelector, }; -use asap_control_core::intent_algebra::expr::{ - AggIntent, BinaryOpKind, ColumnRef, GroupKey, GroupSide, L3Node, MetricRef, Predicate, - QueryExpr, SortKey, Source, TimeWindowKind, VectorGrouping, VectorMatch, +use asap_control_core::intent_algebra::query_expr::{ + BinaryOpKind, ColumnRef, GroupSide, SortKey, VectorGrouping, VectorMatch, VectorMatchKind, +}; +use asap_control_core::intent_algebra::relational::{ + AggFunc, AggItem, QueryExpr as L2, SourceSpec, }; -use asap_control_core::intent_algebra::schema::{L3Schema, SchemaCatalog}; use asap_control_core::intent_algebra::{CompareOp, L3Expr, L3Scalar}; -use asap_control_core::types::AccuracyTarget; use crate::error::LoweringError; type Result = std::result::Result; -/// Lowers one PromQL query string into an intent-algebra `QueryExpr`. -/// -/// `catalog` is consulted only to resolve `without(...)` grouping into an -/// explicit `by` list (which needs the metric's full label set); everything -/// else lowers without it. `accuracy` is attached to every approximate -/// `AggIntent` (`Quantile`, `Count`, `Cardinality`, `TopK`). -pub struct PromqlLowerer<'a> { - catalog: &'a SchemaCatalog, - accuracy: AccuracyTarget, -} +/// Parses (L1) and lowers (→ L2 relational) a PromQL query string. +pub struct PromqlLowerer; -/// The aggregate "shape" an outer PromQL aggregator imposes on its argument. #[derive(Debug, Clone)] enum Outer { - /// No enclosing aggregator (top-level call / bare selector). None, - /// Value aggregator (`sum`/`avg`/`min`/`max`/`group`/`stddev`/`stdvar`/ - /// `quantile`). When the argument is itself an `*_over_time` call the inner - /// function's intent wins; otherwise this op becomes the intent. Plain(OuterIntent), - /// `count(...)` → set cardinality. Count, - /// `topk` (`descending`) / `bottomk` (ascending) with limit `k`. - TopK { k: usize, descending: bool }, + TopK { k: u64, descending: bool }, } #[derive(Debug, Clone)] @@ -88,7 +68,6 @@ enum OuterIntent { Quantile(f64), } -/// The aggregation a PromQL function over a range vector implies. #[derive(Debug, Clone)] enum InnerFunc { Quantile(f64), @@ -103,435 +82,379 @@ enum InnerFunc { Increase(Duration), } -/// A lowered range/instant vector argument: the leaf metric, its label-matcher -/// predicates, an optional window (from a range vector), and the aggregation -/// implied by any enclosing function. struct Inner { metric: String, - predicates: Vec, + matchers: Vec, window: Option, func: Option, } -impl<'a> PromqlLowerer<'a> { - pub fn new(catalog: &'a SchemaCatalog, accuracy: AccuracyTarget) -> Self { - Self { catalog, accuracy } - } - - /// Parse (L1) and lower (L2→L3) a PromQL query string. - pub fn lower(&self, query: &str) -> Result { +impl PromqlLowerer { + pub fn lower(query: &str) -> Result { let ast = parser::parse(query).map_err(LoweringError::Parse)?; - self.walk(&ast) + walk(&ast) } +} - fn walk(&self, expr: &Expr) -> Result { - match expr { - Expr::Aggregate(agg) => self.walk_aggregate(agg), - Expr::Call(call) => { - let inner = self.lower_inner_call(call)?; - self.build(inner, vec![], Outer::None) - } - Expr::Binary(bin) => self.walk_binary(bin), - Expr::Paren(p) => self.walk(&p.expr), - Expr::Unary(u) => self.walk(&u.expr), - // `expr[range:resolution]` — a sliding evaluation window. - Expr::Subquery(sq) => { - let inner = self.walk(&sq.expr)?; - Ok(QueryExpr::TimeWindow { - child: node(inner), - kind: TimeWindowKind::Sliding, - size: sq.range, - slide: sq.step, - }) - } - // Bare instant selector → Scan with label-matcher predicates. - Expr::VectorSelector(vs) => { - let (metric, predicates) = vs_to_scan(vs); - Ok(scan(metric, predicates)) - } - // Bare range selector (no enclosing function) → windowed scan. - Expr::MatrixSelector(ms) => { - let (metric, predicates) = vs_to_scan(&ms.vs); - Ok(QueryExpr::TimeWindow { - child: node(scan(metric, predicates)), - kind: TimeWindowKind::Tumbling, - size: ms.range, - slide: None, - }) - } - Expr::NumberLiteral(_) | Expr::StringLiteral(_) => Err( - LoweringError::UnsupportedFeature("bare scalar/string at top level".into()), - ), - Expr::Extension(_) => Err(LoweringError::UnsupportedFeature( - "extension expression".into(), - )), +fn walk(expr: &Expr) -> Result { + match expr { + Expr::Aggregate(agg) => walk_aggregate(agg), + Expr::Call(call) => build(lower_inner_call(call)?, vec![], Outer::None), + Expr::Binary(bin) => walk_binary(bin), + Expr::Paren(p) => walk(&p.expr), + Expr::Unary(u) => walk(&u.expr), + Expr::Subquery(sq) => Ok(L2::PromQLSubquery { + range: sq.range, + resolution: sq.step, + input: Box::new(walk(&sq.expr)?), + }), + Expr::VectorSelector(vs) => { + let (metric, matchers) = vs_parts(vs); + Ok(filtered_source(metric, matchers)) + } + Expr::MatrixSelector(ms) => { + let (metric, matchers) = vs_parts(&ms.vs); + Ok(L2::Window { + duration: ms.range, + slide: None, + input: Box::new(filtered_source(metric, matchers)), + }) } + Expr::NumberLiteral(_) | Expr::StringLiteral(_) => Err(LoweringError::UnsupportedFeature( + "bare scalar/string at top level".into(), + )), + Expr::Extension(_) => Err(LoweringError::UnsupportedFeature( + "extension expression".into(), + )), } +} - fn walk_aggregate(&self, agg: &AggregateExpr) -> Result { - let group = self.resolve_group(agg)?; - let inner = self.lower_inner(&agg.expr)?; - let op = agg.op.id(); +fn walk_aggregate(agg: &AggregateExpr) -> Result { + let keys = resolve_group(agg)?; + let inner = lower_inner(&agg.expr)?; + let op = agg.op.id(); - let outer = if op == token::T_TOPK { - Outer::TopK { - k: num_param(agg)? as usize, - descending: true, - } - } else if op == token::T_BOTTOMK { - Outer::TopK { - k: num_param(agg)? as usize, - descending: false, - } - } else if op == token::T_COUNT { - Outer::Count - } else if op == token::T_SUM || op == token::T_GROUP { - Outer::Plain(OuterIntent::Sum) - } else if op == token::T_AVG { - Outer::Plain(OuterIntent::Avg) - } else if op == token::T_MIN { - Outer::Plain(OuterIntent::Min) - } else if op == token::T_MAX { - Outer::Plain(OuterIntent::Max) - } else if op == token::T_STDDEV { - Outer::Plain(OuterIntent::StdDev) - } else if op == token::T_STDVAR { - Outer::Plain(OuterIntent::Variance) - } else if op == token::T_QUANTILE { - Outer::Plain(OuterIntent::Quantile(num_param(agg)?)) - } else { - return Err(LoweringError::UnsupportedAggregateOp(format!( - "aggregate token {op}" - ))); - }; + let outer = if op == token::T_TOPK { + Outer::TopK { + k: num_param(agg)? as u64, + descending: true, + } + } else if op == token::T_BOTTOMK { + Outer::TopK { + k: num_param(agg)? as u64, + descending: false, + } + } else if op == token::T_COUNT { + Outer::Count + } else if op == token::T_SUM || op == token::T_GROUP { + Outer::Plain(OuterIntent::Sum) + } else if op == token::T_AVG { + Outer::Plain(OuterIntent::Avg) + } else if op == token::T_MIN { + Outer::Plain(OuterIntent::Min) + } else if op == token::T_MAX { + Outer::Plain(OuterIntent::Max) + } else if op == token::T_STDDEV { + Outer::Plain(OuterIntent::StdDev) + } else if op == token::T_STDVAR { + Outer::Plain(OuterIntent::Variance) + } else if op == token::T_QUANTILE { + Outer::Plain(OuterIntent::Quantile(num_param(agg)?)) + } else { + return Err(LoweringError::UnsupportedAggregateOp(format!( + "aggregate token {op}" + ))); + }; - self.build(inner, group, outer) - } + build(inner, keys, outer) +} - fn walk_binary(&self, bin: &BinaryExpr) -> Result { - let lhs = self.walk(&bin.lhs)?; - let rhs = self.walk(&bin.rhs)?; - let op = binop(bin.op.id())?; - let vector_match = bin.modifier.as_ref().map(|m| { - let (on, labels) = match &m.matching { - Some(LabelModifier::Include(ls)) => (true, ls.labels.clone()), - Some(LabelModifier::Exclude(ls)) => (false, ls.labels.clone()), - None => (true, vec![]), - }; - let grouping = match &m.card { - VectorMatchCardinality::ManyToOne(ls) => Some(VectorGrouping { - side: GroupSide::Left, - labels: ls.labels.clone(), - }), - VectorMatchCardinality::OneToMany(ls) => Some(VectorGrouping { - side: GroupSide::Right, - labels: ls.labels.clone(), - }), - _ => None, - }; - VectorMatch { - on, - labels, - grouping, - } - }); - Ok(QueryExpr::BinaryOp { - op, - lhs: node(lhs), - rhs: node(rhs), - vector_match, - }) - } +fn walk_binary(bin: &BinaryExpr) -> Result { + let lhs = walk(&bin.lhs)?; + let rhs = walk(&bin.rhs)?; + let op = binop(bin.op.id())?; + let vector_match = bin.modifier.as_ref().map(|m| { + let (kind, labels) = match &m.matching { + Some(LabelModifier::Include(ls)) => (VectorMatchKind::On, ls.labels.clone()), + Some(LabelModifier::Exclude(ls)) => (VectorMatchKind::Ignoring, ls.labels.clone()), + None => (VectorMatchKind::On, vec![]), + }; + let grouping = match &m.card { + VectorMatchCardinality::ManyToOne(ls) => Some(VectorGrouping { + side: GroupSide::Left, + labels: ls.labels.clone(), + }), + VectorMatchCardinality::OneToMany(ls) => Some(VectorGrouping { + side: GroupSide::Right, + labels: ls.labels.clone(), + }), + _ => None, + }; + VectorMatch { + kind, + labels, + grouping, + } + }); + Ok(L2::BinaryOp { + op, + lhs: Box::new(lhs), + rhs: Box::new(rhs), + vector_match, + }) +} - /// Lower a function/selector argument into an `Inner`. - fn lower_inner(&self, expr: &Expr) -> Result { - match expr { - Expr::VectorSelector(vs) => { - let (metric, predicates) = vs_to_scan(vs); - Ok(Inner { - metric, - predicates, - window: None, - func: None, - }) - } - Expr::MatrixSelector(ms) => { - let (metric, predicates) = vs_to_scan(&ms.vs); - Ok(Inner { - metric, - predicates, - window: Some(ms.range), - func: None, - }) - } - Expr::Paren(p) => self.lower_inner(&p.expr), - Expr::Call(call) => self.lower_inner_call(call), - other => Err(LoweringError::UnsupportedFeature(format!( - "aggregate argument: {:?}", - std::mem::discriminant(other) - ))), +fn lower_inner(expr: &Expr) -> Result { + match expr { + Expr::VectorSelector(vs) => { + let (metric, matchers) = vs_parts(vs); + Ok(Inner { + metric, + matchers, + window: None, + func: None, + }) + } + Expr::MatrixSelector(ms) => { + let (metric, matchers) = vs_parts(&ms.vs); + Ok(Inner { + metric, + matchers, + window: Some(ms.range), + func: None, + }) } + Expr::Paren(p) => lower_inner(&p.expr), + Expr::Call(call) => lower_inner_call(call), + other => Err(LoweringError::UnsupportedFeature(format!( + "aggregate argument: {:?}", + std::mem::discriminant(other) + ))), } +} - fn lower_inner_call(&self, call: &Call) -> Result { - let name = call.func.name; - // Functions whose range-vector argument is at index 0. - let at0 = |func: InnerFunc| -> Result { - let (metric, predicates, window) = extract_matrix(arg(call, 0)?)?; +fn lower_inner_call(call: &Call) -> Result { + let name = call.func.name; + let at0 = |func: InnerFunc| -> Result { + let (metric, matchers, window) = extract_matrix(arg(call, 0)?)?; + Ok(Inner { + metric, + matchers, + window: Some(window), + func: Some(func), + }) + }; + match name { + "rate" | "irate" => { + let (metric, matchers, window) = extract_matrix(arg(call, 0)?)?; Ok(Inner { metric, - predicates, + matchers, window: Some(window), - func: Some(func), + func: Some(InnerFunc::Rate(window)), + }) + } + "increase" => { + let (metric, matchers, window) = extract_matrix(arg(call, 0)?)?; + Ok(Inner { + metric, + matchers, + window: Some(window), + func: Some(InnerFunc::Increase(window)), + }) + } + "quantile_over_time" => { + let phi = num_arg(call, 0)?; + let (metric, matchers, window) = extract_matrix(arg(call, 1)?)?; + Ok(Inner { + metric, + matchers, + window: Some(window), + func: Some(InnerFunc::Quantile(phi)), + }) + } + // Substitute histogram_quantile(φ, buckets) with a plain Quantile(φ). + "histogram_quantile" => { + let phi = num_arg(call, 0)?; + let (metric, matchers, window) = extract_matrix(arg(call, 1)?)?; + Ok(Inner { + metric, + matchers, + window: Some(window), + func: Some(InnerFunc::Quantile(phi)), }) - }; - match name { - "rate" | "irate" => { - let (metric, predicates, window) = extract_matrix(arg(call, 0)?)?; - Ok(Inner { - metric, - predicates, - window: Some(window), - func: Some(InnerFunc::Rate(window)), - }) - } - "increase" => { - let (metric, predicates, window) = extract_matrix(arg(call, 0)?)?; - Ok(Inner { - metric, - predicates, - window: Some(window), - func: Some(InnerFunc::Increase(window)), - }) - } - "quantile_over_time" => { - let phi = num_arg(call, 0)?; - let (metric, predicates, window) = extract_matrix(arg(call, 1)?)?; - Ok(Inner { - metric, - predicates, - window: Some(window), - func: Some(InnerFunc::Quantile(phi)), - }) - } - // Substitute histogram_quantile(φ, buckets) with a plain Quantile(φ) - // over the bucket stream; bucket-aware physical reduction is an L4/L5 - // concern, not an L3 IR variant. - "histogram_quantile" => { - let phi = num_arg(call, 0)?; - let (metric, predicates, window) = extract_matrix(arg(call, 1)?)?; - Ok(Inner { - metric, - predicates, - window: Some(window), - func: Some(InnerFunc::Quantile(phi)), - }) - } - "avg_over_time" => at0(InnerFunc::Avg), - "min_over_time" => at0(InnerFunc::Min), - "max_over_time" => at0(InnerFunc::Max), - "sum_over_time" => at0(InnerFunc::Sum), - "stddev_over_time" => at0(InnerFunc::StdDev), - "stdvar_over_time" => at0(InnerFunc::Variance), - "count_over_time" | "changes" | "resets" => at0(InnerFunc::Count), - other => Err(LoweringError::UnsupportedFunction(other.to_string())), } + "avg_over_time" => at0(InnerFunc::Avg), + "min_over_time" => at0(InnerFunc::Min), + "max_over_time" => at0(InnerFunc::Max), + "sum_over_time" => at0(InnerFunc::Sum), + "stddev_over_time" => at0(InnerFunc::StdDev), + "stdvar_over_time" => at0(InnerFunc::Variance), + "count_over_time" | "changes" | "resets" => at0(InnerFunc::Count), + other => Err(LoweringError::UnsupportedFunction(other.to_string())), } +} - /// Assemble the final `QueryExpr` from a lowered inner vector, the resolved - /// group keys, and the enclosing aggregator shape. - fn build(&self, inner: Inner, group: Vec, outer: Outer) -> Result { - match outer { - Outer::None => match &inner.func { - None => Ok(scan(inner.metric, inner.predicates)), - Some(f) => { - let intent = self.inner_intent(f); - Ok(self.windowed_aggregate(inner, group, vec![intent])) - } - }, - Outer::Plain(outer_intent) => { - let intent = match &inner.func { - Some(f) => self.inner_intent(f), - None => self.outer_intent(&outer_intent), - }; - Ok(self.windowed_aggregate(inner, group, vec![intent])) +/// Assemble the Layer-2 tree from a lowered inner vector, the resolved group +/// keys, and the enclosing aggregator shape. +fn build(inner: Inner, keys: Vec, outer: Outer) -> Result { + match outer { + Outer::None => match &inner.func { + None => Ok(filtered_source(inner.metric, inner.matchers)), + Some(f) => { + let func = inner_func(f); + Ok(windowed_aggregate(inner, keys, func)) } - Outer::Count => Ok(self.windowed_aggregate( - inner, - group, - vec![AggIntent::Cardinality { - accuracy: self.accuracy.clone(), - }], - )), - Outer::TopK { k, descending } => { - // Heavy-hitter only when ranking by frequency (`count`): a - // dedicated sketch (SpaceSaving, CMS-with-heap) serves it in one - // pass, so it earns the first-class `AggIntent::TopK`. Any other - // ranking (`topk` over avg/quantile, all `bottomk`) is a generic - // order-by-value + limit, which has no sketch primitive — per the - // L3 design rule that keeps `Sort + Limit` distinct from `TopK`. - let heavy_hitter = descending && matches!(inner.func, Some(InnerFunc::Count)); - if heavy_hitter { - let by: Vec = group.iter().map(|g| ColumnRef(g.0.clone())).collect(); - let agg = vec![AggIntent::TopK { - k, - by, - accuracy: self.accuracy.clone(), - }]; - // Window over Aggregate, same as `windowed_aggregate`, but - // with an empty group list (the heavy-hitter keys live in - // the `TopK` intent itself). - Ok(self.windowed_aggregate(inner, vec![], agg)) - } else { - let intent = match &inner.func { - Some(f) => self.inner_intent(f), - None => AggIntent::Sum, - }; - let base = self.windowed_aggregate(inner, group, vec![intent]); - let sorted = QueryExpr::Sort { - child: node(base), - keys: vec![SortKey { - expr: L3Expr::Column(ColumnRef("value".into())), - ascending: !descending, - nulls_first: false, - }], - }; - Ok(QueryExpr::Limit { - child: node(sorted), - n: Some(k as u64), - offset: 0, - }) - } + }, + Outer::Plain(outer_intent) => { + let func = match &inner.func { + Some(f) => inner_func(f), + None => outer_func(&outer_intent), + }; + Ok(windowed_aggregate(inner, keys, func)) + } + Outer::Count => Ok(windowed_aggregate(inner, keys, AggFunc::CountDistinct)), + Outer::TopK { k, descending } => { + // Heavy-hitter only when ranking by frequency (`count`): a dedicated + // sketch serves it in one pass → first-class `TopK`. Any other + // ranking (topk over avg/quantile, all bottomk) is generic + // order-by-value + limit. + let heavy_hitter = descending && matches!(inner.func, Some(InnerFunc::Count)); + if heavy_hitter { + let scan = window_scan(inner); + Ok(L2::TopK { + k, + by: keys, + input: Box::new(scan), + }) + } else { + let func = match &inner.func { + Some(f) => inner_func(f), + None => AggFunc::Sum, + }; + let base = windowed_aggregate(inner, keys, func); + let sorted = L2::Sort { + keys: vec![SortKey { + expr: L3Expr::Column(ColumnRef::SampleValue), + ascending: !descending, + nulls_first: false, + }], + input: Box::new(base), + }; + Ok(L2::Limit { + n: k, + offset: 0, + input: Box::new(sorted), + }) } } } +} - /// `[TimeWindow →] Aggregate → Scan`. The canonical windowed-aggregate - /// shape is **Window over Aggregate** (the window defines the flush/reset - /// lifecycle of the aggregate in its sub-DAG). Rate/Increase carry their own - /// window in the intent, so no `TimeWindow` node is emitted for them. - fn windowed_aggregate( - &self, - inner: Inner, - group: Vec, - aggs: Vec, - ) -> QueryExpr { - let skip_window = matches!( - inner.func, - Some(InnerFunc::Rate(_)) | Some(InnerFunc::Increase(_)) - ); - let window = inner.window; - let base = scan(inner.metric, inner.predicates); - let agg = QueryExpr::Aggregate { - child: node(base), - by: group, - aggs, - having: None, - }; - match window { - Some(w) if !skip_window => QueryExpr::TimeWindow { - child: node(agg), - kind: TimeWindowKind::Tumbling, - size: w, - slide: None, - }, - _ => agg, - } +/// `Aggregate{keys, [func]}` over `[Window{w}] → Filter(Source)`. Rate/Increase +/// carry their own window in the func, so no `Window` node is emitted. +fn windowed_aggregate(inner: Inner, keys: Vec, func: AggFunc) -> L2 { + let skip_window = matches!(func, AggFunc::Rate { .. } | AggFunc::Increase { .. }); + let window = inner.window; + let base = filtered_source(inner.metric, inner.matchers); + let input = match window { + Some(w) if !skip_window => L2::Window { + duration: w, + slide: None, + input: Box::new(base), + }, + _ => base, + }; + L2::Aggregate { + keys, + aggs: vec![AggItem { + alias: "value".into(), + func, + col: ColumnRef::SampleValue, + distinct: false, + }], + having: None, + input: Box::new(input), } +} - fn inner_intent(&self, f: &InnerFunc) -> AggIntent { - match f { - InnerFunc::Quantile(q) => AggIntent::Quantile { - q: *q, - accuracy: self.accuracy.clone(), - }, - InnerFunc::Avg => AggIntent::Avg, - InnerFunc::Min => AggIntent::Min, - InnerFunc::Max => AggIntent::Max, - InnerFunc::Sum => AggIntent::Sum, - InnerFunc::StdDev => AggIntent::StdDev { population: false }, - InnerFunc::Variance => AggIntent::Variance { population: false }, - InnerFunc::Count => AggIntent::Count { - accuracy: self.accuracy.clone(), - }, - InnerFunc::Rate(w) => AggIntent::Rate { window: *w }, - InnerFunc::Increase(w) => AggIntent::Increase { window: *w }, - } +/// `[Window{w}] → Filter(Source)` with no aggregate (the heavy-hitter TopK +/// child — the sketch counts directly off the scan). +fn window_scan(inner: Inner) -> L2 { + let base = filtered_source(inner.metric, inner.matchers); + match inner.window { + Some(w) => L2::Window { + duration: w, + slide: None, + input: Box::new(base), + }, + None => base, } +} - fn outer_intent(&self, o: &OuterIntent) -> AggIntent { - match o { - OuterIntent::Sum => AggIntent::Sum, - OuterIntent::Avg => AggIntent::Avg, - OuterIntent::Min => AggIntent::Min, - OuterIntent::Max => AggIntent::Max, - OuterIntent::StdDev => AggIntent::StdDev { population: false }, - OuterIntent::Variance => AggIntent::Variance { population: false }, - OuterIntent::Quantile(q) => AggIntent::Quantile { - q: *q, - accuracy: self.accuracy.clone(), - }, +fn filtered_source(metric: String, matchers: Vec) -> L2 { + let source = L2::Source(SourceSpec { name: metric }); + if matchers.is_empty() { + source + } else { + let pred = if matchers.len() == 1 { + matchers.into_iter().next().unwrap() + } else { + L3Expr::BoolAnd(matchers) + }; + L2::Filter { + pred, + input: Box::new(source), } } +} - /// Resolve `by(labels)` / `without(labels)` into an explicit group-key list. - /// `without` needs the metric's full label set, looked up in the catalog. - fn resolve_group(&self, agg: &AggregateExpr) -> Result> { - match &agg.modifier { - None => Ok(vec![]), - Some(LabelModifier::Include(ls)) => { - Ok(ls.labels.iter().cloned().map(GroupKey).collect()) - } - Some(LabelModifier::Exclude(ls)) => { - let metric = find_metric(&agg.expr).ok_or_else(|| { - LoweringError::UnsupportedFeature( - "`without` over an expression with no metric selector".into(), - ) - })?; - let meta = self.catalog.metrics.get(&metric).ok_or_else(|| { - LoweringError::UnsupportedFeature(format!( - "`without(...)` requires metric '{metric}' to be registered in the catalog \ - (its full label set is needed to compute the kept labels)" - )) - })?; - Ok(meta - .labels - .iter() - .filter(|l| !ls.labels.contains(l)) - .cloned() - .map(GroupKey) - .collect()) - } - } +fn inner_func(f: &InnerFunc) -> AggFunc { + match f { + InnerFunc::Quantile(q) => AggFunc::Quantile(*q), + InnerFunc::Avg => AggFunc::Avg, + InnerFunc::Min => AggFunc::Min, + InnerFunc::Max => AggFunc::Max, + InnerFunc::Sum => AggFunc::Sum, + InnerFunc::StdDev => AggFunc::StdDev { population: false }, + InnerFunc::Variance => AggFunc::Variance { population: false }, + InnerFunc::Count => AggFunc::Count, + InnerFunc::Rate(w) => AggFunc::Rate { window: *w }, + InnerFunc::Increase(w) => AggFunc::Increase { window: *w }, } } -// ── Free helpers ────────────────────────────────────────────────────────────── - -/// Wrap a `QueryExpr` in an untyped `L3Node` (empty schema). The schema pass -/// (`crate::populate_schemas`) fills schemas in bottom-up afterwards. -fn node(expr: QueryExpr) -> Arc { - Arc::new(L3Node { - expr, - schema: L3Schema { - fields: vec![], - time_index: None, - }, - }) +fn outer_func(o: &OuterIntent) -> AggFunc { + match o { + OuterIntent::Sum => AggFunc::Sum, + OuterIntent::Avg => AggFunc::Avg, + OuterIntent::Min => AggFunc::Min, + OuterIntent::Max => AggFunc::Max, + OuterIntent::StdDev => AggFunc::StdDev { population: false }, + OuterIntent::Variance => AggFunc::Variance { population: false }, + OuterIntent::Quantile(q) => AggFunc::Quantile(*q), + } } -fn scan(metric: String, predicates: Vec) -> QueryExpr { - QueryExpr::Scan { - source: Source::TimeSeries { - metric: MetricRef(metric), - time: None, - }, - predicates, +/// Resolve `by(labels)` into a key list. `without(...)` needs the metric's +/// full label set, which the usage-derived schema model doesn't carry, so it +/// is rejected (a registry-backed `SchemaCatalog` would lift this). +fn resolve_group(agg: &AggregateExpr) -> Result> { + match &agg.modifier { + None => Ok(vec![]), + Some(LabelModifier::Include(ls)) => Ok(ls.labels.clone()), + Some(LabelModifier::Exclude(_)) => Err(LoweringError::UnsupportedFeature( + "`without(...)` grouping requires a registry-backed catalog of the \ + metric's label set (the usage-derived schema can't enumerate the \ + complement)" + .into(), + )), } } -/// Extract `(metric_name, label_predicates)` from a vector selector. -fn vs_to_scan(vs: &VectorSelector) -> (String, Vec) { +// ── Free helpers ────────────────────────────────────────────────────────────── + +fn vs_parts(vs: &VectorSelector) -> (String, Vec) { let metric = vs.name.clone().unwrap_or_else(|| { vs.matchers .matchers @@ -540,37 +463,35 @@ fn vs_to_scan(vs: &VectorSelector) -> (String, Vec) { .map(|m| m.value.clone()) .unwrap_or_default() }); - let predicates = vs + let matchers = vs .matchers .matchers .iter() .filter(|m| m.name != "__name__") - .map(matcher_to_predicate) + .map(matcher_to_l3expr) .collect(); - (metric, predicates) + (metric, matchers) } -fn matcher_to_predicate(m: &Matcher) -> Predicate { +fn matcher_to_l3expr(m: &Matcher) -> L3Expr { let op = match &m.op { MatchOp::Equal => CompareOp::Eq, MatchOp::NotEqual => CompareOp::Ne, MatchOp::Re(_) => CompareOp::Regex, MatchOp::NotRe(_) => CompareOp::NotRegex, }; - Predicate(L3Expr::Compare { - left: Box::new(L3Expr::Column(ColumnRef(m.name.clone()))), + L3Expr::Compare { + left: Box::new(L3Expr::Column(ColumnRef::Named(m.name.clone()))), op, right: Box::new(L3Expr::Literal(L3Scalar::Utf8(m.value.clone()))), - }) + } } -/// Descend through `Call` / `Paren` wrappers to the `MatrixSelector` and pull -/// out `(metric, predicates, window)`. -fn extract_matrix(expr: &Expr) -> Result<(String, Vec, Duration)> { +fn extract_matrix(expr: &Expr) -> Result<(String, Vec, Duration)> { match expr { Expr::MatrixSelector(ms) => { - let (metric, predicates) = vs_to_scan(&ms.vs); - Ok((metric, predicates, ms.range)) + let (metric, matchers) = vs_parts(&ms.vs); + Ok((metric, matchers, ms.range)) } Expr::Paren(p) => extract_matrix(&p.expr), Expr::Call(c) => extract_matrix(arg(c, 0)?), @@ -581,21 +502,6 @@ fn extract_matrix(expr: &Expr) -> Result<(String, Vec, Duration)> { } } -/// First `VectorSelector` metric name reachable from `expr`. -fn find_metric(expr: &Expr) -> Option { - match expr { - Expr::VectorSelector(vs) => Some(vs_to_scan(vs).0), - Expr::MatrixSelector(ms) => Some(vs_to_scan(&ms.vs).0), - Expr::Paren(p) => find_metric(&p.expr), - Expr::Unary(u) => find_metric(&u.expr), - Expr::Subquery(sq) => find_metric(&sq.expr), - Expr::Aggregate(a) => find_metric(&a.expr), - Expr::Call(c) => c.args.args.iter().find_map(|a| find_metric(a)), - Expr::Binary(b) => find_metric(&b.lhs).or_else(|| find_metric(&b.rhs)), - _ => None, - } -} - fn arg(call: &Call, idx: usize) -> Result<&Expr> { call.args .args @@ -645,15 +551,15 @@ fn binop(id: token::TokenId) -> Result { } else if id == token::T_EQLC { BinaryOpKind::Eq } else if id == token::T_NEQ { - BinaryOpKind::NotEq + BinaryOpKind::Ne } else if id == token::T_LSS { BinaryOpKind::Lt } else if id == token::T_LTE { - BinaryOpKind::LtEq + BinaryOpKind::Le } else if id == token::T_GTR { BinaryOpKind::Gt } else if id == token::T_GTE { - BinaryOpKind::GtEq + BinaryOpKind::Ge } else if id == token::T_LAND { BinaryOpKind::And } else if id == token::T_LOR { diff --git a/crates/lower/src/schema_pass.rs b/crates/lower/src/schema_pass.rs deleted file mode 100644 index 73159370..00000000 --- a/crates/lower/src/schema_pass.rs +++ /dev/null @@ -1,159 +0,0 @@ -use std::sync::Arc; - -use asap_control_core::intent_algebra::expr::QueryExpr; -use asap_control_core::intent_algebra::schema::{HasSchema, L3Schema, SchemaCatalog}; -use asap_control_core::intent_algebra::L3Node; - -/// Recursively populate the `schema` field on every node in a `QueryExpr` tree. -/// -/// The lowerer builds every node with an empty schema. This pass walks the tree -/// bottom-up, computing each node's output schema from its children's schemas -/// and the catalog, and returns a fully typed `Arc` tree. -pub fn populate_schemas(expr: QueryExpr, catalog: &SchemaCatalog) -> Arc { - let (rebuilt, child_schemas) = rebuild(expr, catalog); - let refs: Vec<&L3Schema> = child_schemas.iter().collect(); - let schema = rebuilt.output_schema(&refs, catalog); - Arc::new(L3Node { - expr: rebuilt, - schema, - }) -} - -/// Rebuild the expression tree with populated child nodes, returning -/// `(rebuilt_expr, child_output_schemas)`. -fn rebuild(expr: QueryExpr, catalog: &SchemaCatalog) -> (QueryExpr, Vec) { - use QueryExpr::*; - - let proc = |n: Arc| populate_schemas(n.expr.clone(), catalog); - - match expr { - // Leaf: schema comes from the catalog, no child schemas needed. - Scan { .. } => (expr, vec![]), - - Filter { child, pred } => { - let c = proc(child); - let cs = c.schema.clone(); - (Filter { child: c, pred }, vec![cs]) - } - Project { child, cols } => { - let c = proc(child); - let cs = c.schema.clone(); - (Project { child: c, cols }, vec![cs]) - } - Aggregate { - child, - by, - aggs, - having, - } => { - let c = proc(child); - let cs = c.schema.clone(); - ( - Aggregate { - child: c, - by, - aggs, - having, - }, - vec![cs], - ) - } - Sort { child, keys } => { - let c = proc(child); - let cs = c.schema.clone(); - (Sort { child: c, keys }, vec![cs]) - } - Limit { child, n, offset } => { - let c = proc(child); - let cs = c.schema.clone(); - ( - Limit { - child: c, - n, - offset, - }, - vec![cs], - ) - } - Distinct { child, cols } => { - let c = proc(child); - let cs = c.schema.clone(); - (Distinct { child: c, cols }, vec![cs]) - } - Partition { child, keys } => { - let c = proc(child); - let cs = c.schema.clone(); - (Partition { child: c, keys }, vec![cs]) - } - TimeWindow { - child, - kind, - size, - slide, - } => { - let c = proc(child); - let cs = c.schema.clone(); - ( - TimeWindow { - child: c, - kind, - size, - slide, - }, - vec![cs], - ) - } - BinaryOp { - op, - lhs, - rhs, - vector_match, - } => { - let l = proc(lhs); - let r = proc(rhs); - let ls = l.schema.clone(); - let rs = r.schema.clone(); - ( - BinaryOp { - op, - lhs: l, - rhs: r, - vector_match, - }, - vec![ls, rs], - ) - } - SetOp { - kind, - all, - left, - right, - } => { - let l = proc(left); - let r = proc(right); - let ls = l.schema.clone(); - let rs = r.schema.clone(); - ( - SetOp { - kind, - all, - left: l, - right: r, - }, - vec![ls, rs], - ) - } - Merge { children } => { - let new_children: Vec> = children.into_iter().map(proc).collect(); - let schemas: Vec = new_children.iter().map(|c| c.schema.clone()).collect(); - ( - Merge { - children: new_children, - }, - schemas, - ) - } - // Unimplemented variants: return as-is; output_schema will todo!() if called. - other => (other, vec![]), - } -} diff --git a/crates/lower/tests/promql_lowering.rs b/crates/lower/tests/promql_lowering.rs index 9d51965d..72bf0a47 100644 --- a/crates/lower/tests/promql_lowering.rs +++ b/crates/lower/tests/promql_lowering.rs @@ -1,63 +1,40 @@ -//! End-to-end tests for PromQL → L3 intent-algebra lowering. +//! End-to-end tests for PromQL → L2 → canonical L3 lowering. use std::time::Duration; -use asap_control_core::intent_algebra::expr::{ - AggIntent, BinaryOpKind, QueryExpr, Source, TimeWindowKind, +use asap_control_core::intent_algebra::{ + AggIntent, BinaryOpKind, ColumnRef, CompareOp, L3Expr, L3Scalar, PartitionKeys, QueryExpr, + Source, WindowKind, }; -use asap_control_core::intent_algebra::schema::{L3DataType, MetricSchema, SchemaCatalog}; -use asap_control_core::intent_algebra::{CompareOp, L3Expr, L3Scalar}; use asap_control_core::types::AccuracyTarget; use asap_control_core::workload::{ BatchEntry, Query, QueryLanguage, QueryRequirements, QueryWorkload, }; -use asap_control_lower::{lower_promql, lower_promql_batch, populate_schemas}; - -// ── Fixtures ─────────────────────────────────────────────────────────────────── - -fn empty_catalog() -> SchemaCatalog { - SchemaCatalog::default() -} - -fn catalog_with(metric: &str, labels: &[&str]) -> SchemaCatalog { - let mut c = SchemaCatalog::default(); - c.metrics.insert( - metric.to_string(), - MetricSchema { - labels: labels.iter().map(|s| s.to_string()).collect(), - value_type: L3DataType::Float64, - }, - ); - c -} +use asap_control_lower::{lower_promql, lower_promql_batch, LoweringError}; fn lower(q: &str) -> QueryExpr { - lower_promql(q, &empty_catalog(), AccuracyTarget::Exact) - .unwrap_or_else(|e| panic!("lower failed for {q:?}: {e}")) -} - -fn lower_eps(q: &str) -> QueryExpr { - lower_promql(q, &empty_catalog(), AccuracyTarget::Epsilon(0.01)) - .unwrap_or_else(|e| panic!("lower failed for {q:?}: {e}")) + lower_promql(q, AccuracyTarget::Exact).unwrap_or_else(|e| panic!("lower failed for {q:?}: {e}")) } -// ── Bare selectors & label matchers ───────────────────────────────────────────── +// ── Bare selectors & label matchers (folded onto Scan.predicates) ─────────────── #[test] fn bare_selector_is_scan_with_predicates() { let qe = lower(r#"http_requests_total{env="prod",status!="500"}"#); - let QueryExpr::Scan { source, predicates } = &qe else { + let QueryExpr::Scan { + source, predicates, .. + } = &qe + else { panic!("expected Scan, got {qe:?}"); }; - match source { - Source::TimeSeries { metric, time } => { - assert_eq!(metric.0, "http_requests_total"); - assert!(time.is_none(), "PromQL string carries no absolute range"); - } - other => panic!("expected TimeSeries source, got {other:?}"), - } + assert!(matches!(source, Source::TimeSeries { metric } if metric == "http_requests_total")); + // The converter splits the matcher conjunction into one predicate per + // conjunct on the Scan. assert_eq!(predicates.len(), 2); + assert!(predicates + .iter() + .all(|p| matches!(&p.0, L3Expr::Compare { .. }))); } #[test] @@ -67,148 +44,117 @@ fn regex_matcher_lowers_to_regex_compareop() { panic!("expected Scan, got {qe:?}"); }; let L3Expr::Compare { left, op, right } = &predicates[0].0 else { - panic!("expected Compare predicate, got {:?}", predicates[0].0); + panic!("expected Compare, got {:?}", predicates[0].0); }; assert_eq!(*op, CompareOp::Regex); - assert!(matches!(left.as_ref(), L3Expr::Column(c) if c.0 == "path")); + assert!(matches!(left.as_ref(), L3Expr::Column(ColumnRef::Named(n)) if n == "path")); assert!(matches!(right.as_ref(), L3Expr::Literal(L3Scalar::Utf8(v)) if v == "/api/.*")); } -// ── *_over_time → Window → Aggregate ───────────────────────────────────────────── +// ── *_over_time → Window over Aggregate ───────────────────────────────────────── #[test] fn quantile_over_time_is_window_over_aggregate() { let qe = lower(r#"quantile_over_time(0.99, http_request_duration{env="prod"}[5m])"#); - let QueryExpr::TimeWindow { + let QueryExpr::Window { kind, size, child, .. } = &qe else { - panic!("expected TimeWindow, got {qe:?}"); + panic!("expected Window, got {qe:?}"); }; - assert_eq!(*kind, TimeWindowKind::Tumbling); + assert_eq!(*kind, WindowKind::Tumbling); assert_eq!(*size, Duration::from_secs(300)); - match &child.expr { - QueryExpr::Aggregate { - by, aggs, child, .. - } => { - assert!(by.is_empty()); - assert!(matches!( - aggs.as_slice(), - [AggIntent::Quantile { q, .. }] if (*q - 0.99).abs() < 1e-9 - )); - // The label matcher rides on the Scan, not a separate Filter node. - assert!( - matches!(&child.expr, QueryExpr::Scan { predicates, .. } if predicates.len() == 1) - ); - } - other => panic!("expected Aggregate under TimeWindow, got {other:?}"), - } + let QueryExpr::Aggregate { + by, aggs, child, .. + } = child.as_ref() + else { + panic!("expected Aggregate under Window, got {child:?}"); + }; + assert!(by.is_empty()); + assert!(matches!(aggs.as_slice(), [AggIntent::Quantile { q, .. }] if (*q - 0.99).abs() < 1e-9)); + // The label matcher folded onto the Scan. + assert!(matches!(child.as_ref(), QueryExpr::Scan { predicates, .. } if predicates.len() == 1)); } #[test] -fn outer_sum_by_pushes_group_keys_onto_inner_aggregate() { +fn outer_sum_by_wraps_in_partition() { + // `sum by (host) (...)` → grouping rides on a `Partition` (backend model). let qe = lower(r#"sum by (host) (quantile_over_time(0.99, latency{service="web"}[5m]))"#); - let QueryExpr::TimeWindow { child, .. } = &qe else { - panic!("expected TimeWindow, got {qe:?}"); + let QueryExpr::Partition { keys, child } = &qe else { + panic!("expected Partition, got {qe:?}"); }; - let QueryExpr::Aggregate { by, aggs, .. } = &child.expr else { - panic!("expected Aggregate, got {:?}", child.expr); + assert_eq!(keys, &PartitionKeys::By(vec!["host".into()])); + // Inner: Window over Aggregate{Quantile} (the inner func wins the intent). + let QueryExpr::Window { child, .. } = child.as_ref() else { + panic!("expected Window under Partition"); }; - assert_eq!(by.len(), 1); - assert_eq!(by[0].0, "host"); - // The inner quantile_over_time supplies the intent; the outer `sum` only groups. - assert!(matches!(aggs.as_slice(), [AggIntent::Quantile { .. }])); + assert!(matches!( + child.as_ref(), + QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Quantile { .. }]) + )); } #[test] fn avg_over_time_maps_to_avg_intent() { let qe = lower("avg_over_time(cpu_seconds_total[10m])"); - let QueryExpr::TimeWindow { size, child, .. } = &qe else { - panic!("expected TimeWindow, got {qe:?}"); + let QueryExpr::Window { size, child, .. } = &qe else { + panic!("expected Window, got {qe:?}"); }; assert_eq!(*size, Duration::from_secs(600)); assert!(matches!( - &child.expr, + child.as_ref(), QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Avg]) )); } -#[test] -fn min_max_sum_over_time_map_directly() { - for (q, want) in [ - ("min_over_time(m[5m])", "min"), - ("max_over_time(m[5m])", "max"), - ("sum_over_time(m[5m])", "sum"), - ] { - let qe = lower(q); - let QueryExpr::TimeWindow { child, .. } = &qe else { - panic!("expected TimeWindow for {q}"); - }; - let QueryExpr::Aggregate { aggs, .. } = &child.expr else { - panic!("expected Aggregate for {q}"); - }; - let ok = matches!( - (want, &aggs[0]), - ("min", AggIntent::Min) | ("max", AggIntent::Max) | ("sum", AggIntent::Sum) - ); - assert!(ok, "{q} produced {:?}", aggs[0]); - } -} - #[test] fn stddev_and_stdvar_over_time() { let qe = lower("stddev_over_time(m[5m])"); - let QueryExpr::TimeWindow { child, .. } = &qe else { - panic!("expected TimeWindow"); + let QueryExpr::Window { child, .. } = &qe else { + panic!("expected Window"); }; assert!(matches!( - &child.expr, + child.as_ref(), QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::StdDev { population: false }]) )); let qe = lower("stdvar_over_time(m[5m])"); - let QueryExpr::TimeWindow { child, .. } = &qe else { - panic!("expected TimeWindow"); + let QueryExpr::Window { child, .. } = &qe else { + panic!("expected Window"); }; assert!(matches!( - &child.expr, + child.as_ref(), QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Variance { population: false }]) )); } -// ── histogram_quantile ─────────────────────────────────────────────────────────── - #[test] fn histogram_quantile_substitutes_to_quantile() { let qe = lower(r#"histogram_quantile(0.95, rate(http_duration_seconds_bucket{le="0.5"}[5m]))"#); - let QueryExpr::TimeWindow { size, child, .. } = &qe else { - panic!("expected TimeWindow, got {qe:?}"); + let QueryExpr::Window { size, child, .. } = &qe else { + panic!("expected Window, got {qe:?}"); }; assert_eq!(*size, Duration::from_secs(300)); - let QueryExpr::Aggregate { aggs, child, .. } = &child.expr else { + let QueryExpr::Aggregate { aggs, child, .. } = child.as_ref() else { panic!("expected Aggregate"); }; - assert!(matches!( - aggs.as_slice(), - [AggIntent::Quantile { q, .. }] if (*q - 0.95).abs() < 1e-9 - )); - // The `le` matcher is preserved on the bucket scan. - assert!(matches!(&child.expr, QueryExpr::Scan { predicates, .. } if predicates.len() == 1)); + assert!(matches!(aggs.as_slice(), [AggIntent::Quantile { q, .. }] if (*q - 0.95).abs() < 1e-9)); + assert!(matches!(child.as_ref(), QueryExpr::Scan { predicates, .. } if predicates.len() == 1)); } -// ── rate / increase carry their own window ─────────────────────────────────────── +// ── rate / increase carry their own window (no Window node) ───────────────────── #[test] -fn rate_has_no_timewindow_node() { +fn rate_has_no_window_node() { let qe = lower("rate(http_requests_total[5m])"); let QueryExpr::Aggregate { aggs, child, .. } = &qe else { - panic!("expected Aggregate (no TimeWindow) for rate, got {qe:?}"); + panic!("expected Aggregate (no Window) for rate, got {qe:?}"); }; assert!(matches!( aggs.as_slice(), [AggIntent::Rate { window }] if *window == Duration::from_secs(300) )); - assert!(matches!(&child.expr, QueryExpr::Scan { .. })); + assert!(matches!(child.as_ref(), QueryExpr::Scan { .. })); } #[test] @@ -223,16 +169,16 @@ fn increase_maps_to_increase_intent() { )); } -// ── count / cardinality ─────────────────────────────────────────────────────────── +// ── count / cardinality ─────────────────────────────────────────────────────── #[test] fn count_over_time_is_count_intent() { let qe = lower("count_over_time(m[5m])"); - let QueryExpr::TimeWindow { child, .. } = &qe else { - panic!("expected TimeWindow"); + let QueryExpr::Window { child, .. } = &qe else { + panic!("expected Window"); }; assert!(matches!( - &child.expr, + child.as_ref(), QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Count { .. }]) )); } @@ -240,69 +186,59 @@ fn count_over_time_is_count_intent() { #[test] fn outer_count_is_cardinality() { let qe = lower("count by (symbol) (count_over_time(financial_last_trade_price[5m]))"); - let QueryExpr::TimeWindow { child, .. } = &qe else { - panic!("expected TimeWindow, got {qe:?}"); + let QueryExpr::Partition { keys, child } = &qe else { + panic!("expected Partition, got {qe:?}"); }; - let QueryExpr::Aggregate { by, aggs, .. } = &child.expr else { - panic!("expected Aggregate"); + assert_eq!(keys, &PartitionKeys::By(vec!["symbol".into()])); + let QueryExpr::Window { child, .. } = child.as_ref() else { + panic!("expected Window"); }; - assert_eq!(by.len(), 1); - assert_eq!(by[0].0, "symbol"); - assert!(matches!(aggs.as_slice(), [AggIntent::Cardinality { .. }])); + assert!(matches!( + child.as_ref(), + QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Cardinality { .. }]) + )); } -// ── topk / bottomk ──────────────────────────────────────────────────────────────── +// ── topk / bottomk ──────────────────────────────────────────────────────────── #[test] fn topk_over_count_is_heavy_hitter_topk() { let qe = lower(r#"topk by (service) (10, count_over_time(requests{env="prod"}[1m]))"#); - // Window over Aggregate: the heavy-hitter top-k is computed per 1m window. - let QueryExpr::TimeWindow { size, child, .. } = &qe else { - panic!("expected TimeWindow, got {qe:?}"); - }; - assert_eq!(*size, Duration::from_secs(60)); + // Heavy-hitter: Aggregate{TopK} with grouping resolved to positional ids. let QueryExpr::Aggregate { by, aggs, child, .. - } = &child.expr + } = &qe else { - panic!("expected Aggregate with TopK, got {:?}", child.expr); + panic!("expected Aggregate with TopK, got {qe:?}"); }; - assert!(by.is_empty(), "heavy-hitter keys live in the TopK intent"); - match aggs.as_slice() { - [AggIntent::TopK { k, by, .. }] => { - assert_eq!(*k, 10); - assert_eq!(by.len(), 1); - assert_eq!(by[0].0, "service"); - } - other => panic!("expected TopK intent, got {other:?}"), - } - // The heavy-hitter sketch counts directly off the scan in one pass. - assert!(matches!(&child.expr, QueryExpr::Scan { .. })); + // `service` is the only group key → resolved to a positional ColumnId. + assert_eq!(by.len(), 1); + assert!(matches!(aggs.as_slice(), [AggIntent::TopK { k: 10, .. }])); + // The heavy-hitter sketch counts directly off the windowed scan. + let QueryExpr::Window { size, child, .. } = child.as_ref() else { + panic!("expected Window under TopK Aggregate, got {child:?}"); + }; + assert_eq!(*size, Duration::from_secs(60)); + assert!(matches!(child.as_ref(), QueryExpr::Scan { .. })); } #[test] fn topk_over_avg_is_generic_sort_limit() { - // Ranking by avg value has no heavy-hitter sketch → generic Sort + Limit. let qe = lower("topk by (host) (5, avg_over_time(cpu[5m]))"); let QueryExpr::Limit { n, offset, child } = &qe else { panic!("expected Limit, got {qe:?}"); }; - assert_eq!(*n, Some(5)); + assert_eq!(*n, 5); assert_eq!(*offset, 0); - let QueryExpr::Sort { keys, child } = &child.expr else { - panic!("expected Sort under Limit, got {:?}", child.expr); + let QueryExpr::Sort { keys, child } = child.as_ref() else { + panic!("expected Sort under Limit, got {child:?}"); }; assert_eq!(keys.len(), 1); assert!(!keys[0].ascending, "topk ranks descending"); - // Underneath: the windowed avg aggregate grouped by host. - let QueryExpr::TimeWindow { child, .. } = &child.expr else { - panic!("expected TimeWindow under Sort"); - }; - assert!(matches!( - &child.expr, - QueryExpr::Aggregate { by, aggs, .. } - if by.len() == 1 && by[0].0 == "host" && matches!(aggs.as_slice(), [AggIntent::Avg]) - )); + // Underneath: the windowed avg aggregate, grouped via Partition. + assert!( + matches!(child.as_ref(), QueryExpr::Partition { keys, .. } if *keys == PartitionKeys::By(vec!["host".into()])) + ); } #[test] @@ -311,14 +247,14 @@ fn bottomk_is_always_generic_sort_ascending() { let QueryExpr::Limit { n, child, .. } = &qe else { panic!("expected Limit, got {qe:?}"); }; - assert_eq!(*n, Some(3)); - let QueryExpr::Sort { keys, .. } = &child.expr else { + assert_eq!(*n, 3); + let QueryExpr::Sort { keys, .. } = child.as_ref() else { panic!("expected Sort"); }; assert!(keys[0].ascending, "bottomk ranks ascending"); } -// ── binary ops ──────────────────────────────────────────────────────────────────── +// ── binary ops ──────────────────────────────────────────────────────────────── #[test] fn binary_op_division() { @@ -328,10 +264,10 @@ fn binary_op_division() { }; assert_eq!(*op, BinaryOpKind::Div); assert!( - matches!(&lhs.expr, QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Rate { .. }])) + matches!(lhs.as_ref(), QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Rate { .. }])) ); assert!( - matches!(&rhs.expr, QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Rate { .. }])) + matches!(rhs.as_ref(), QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Rate { .. }])) ); } @@ -342,49 +278,36 @@ fn binary_op_with_on_grouping() { panic!("expected BinaryOp, got {qe:?}"); }; let vm = vector_match.as_ref().expect("vector_match present"); - assert!(vm.on); + use asap_control_core::intent_algebra::VectorMatchKind; + assert_eq!(vm.kind, VectorMatchKind::On); assert_eq!(vm.labels, vec!["host".to_string()]); } -// ── without resolution ────────────────────────────────────────────────────────── - -#[test] -fn without_resolves_kept_labels_from_catalog() { - let catalog = catalog_with("m", &["host", "region", "instance"]); - let qe = lower_promql( - "sum without (instance) (rate(m[5m]))", - &catalog, - AccuracyTarget::Exact, - ) - .expect("lower with catalog"); - let QueryExpr::Aggregate { by, .. } = &qe else { - panic!("expected Aggregate, got {qe:?}"); - }; - let mut got: Vec<&str> = by.iter().map(|g| g.0.as_str()).collect(); - got.sort(); - assert_eq!(got, vec!["host", "region"]); -} +// ── without is unsupported (no label registry) ────────────────────────────────── #[test] -fn without_without_catalog_is_unsupported() { +fn without_grouping_is_unsupported() { let err = lower_promql( "sum without (instance) (rate(m[5m]))", - &empty_catalog(), AccuracyTarget::Exact, ) .unwrap_err(); assert!(format!("{err}").contains("without"), "got {err}"); } -// ── accuracy propagation ────────────────────────────────────────────────────────── +// ── accuracy propagation ────────────────────────────────────────────────────── #[test] fn accuracy_target_flows_into_quantile_intent() { - let qe = lower_eps("quantile_over_time(0.9, m[5m])"); - let QueryExpr::TimeWindow { child, .. } = &qe else { - panic!("expected TimeWindow"); + let qe = lower_promql( + "quantile_over_time(0.9, m[5m])", + AccuracyTarget::Epsilon(0.01), + ) + .unwrap(); + let QueryExpr::Window { child, .. } = &qe else { + panic!("expected Window"); }; - let QueryExpr::Aggregate { aggs, .. } = &child.expr else { + let QueryExpr::Aggregate { aggs, .. } = child.as_ref() else { panic!("expected Aggregate"); }; assert!(matches!( @@ -393,61 +316,50 @@ fn accuracy_target_flows_into_quantile_intent() { )); } -// ── schema population ─────────────────────────────────────────────────────────── +// ── schema flow (positional, carried on Scan; derived on demand) ───────────────── #[test] -fn schema_population_for_quantile_over_time() { - let catalog = catalog_with("http_request_duration", &["env", "host"]); - let qe = lower_promql( - r#"quantile_over_time(0.99, http_request_duration{env="prod"}[5m])"#, - &catalog, - AccuracyTarget::Exact, - ) - .unwrap(); - let typed = populate_schemas(qe, &catalog); - // Root TimeWindow passes the aggregate's schema through: a single Float64 - // `value` column (group-by is empty). - assert_eq!(typed.schema.fields.len(), 1); - assert_eq!(typed.schema.fields[0].name, "value"); - assert_eq!(typed.schema.fields[0].dtype, L3DataType::Float64); - - // Walk to the Scan leaf: timestamp + value + the two catalog labels. - fn scan_schema(n: &asap_control_core::intent_algebra::L3Node) -> Vec { - match &n.expr { - QueryExpr::Scan { .. } => n.schema.fields.iter().map(|f| f.name.clone()).collect(), - QueryExpr::TimeWindow { child, .. } +fn aggregate_output_schema_is_single_quantile_column() { + let qe = lower(r#"quantile_over_time(0.99, http_request_duration{env="prod"}[5m])"#); + // Window requires its child to carry a time axis; the Aggregate beneath it + // strips it, so derive the schema at the Aggregate node. + let QueryExpr::Window { child, .. } = &qe else { + panic!("expected Window"); + }; + let schema = child.output_schema().expect("aggregate schema"); + let names: Vec<&str> = schema.columns.iter().map(|c| c.name.as_str()).collect(); + assert_eq!(names, vec!["quantile_0_99"]); + assert!( + schema.time_index.is_none(), + "aggregate strips the time axis" + ); +} + +#[test] +fn scan_schema_carries_ts_value_and_group_keys() { + // `service` is a group key → the Binder lands it in the self-contained + // Scan schema (positional). `env` is only a filter, so it is not a column. + let qe = lower("count by (service) (count_over_time(requests[1m]))"); + fn find_scan(n: &QueryExpr) -> &QueryExpr { + match n { + QueryExpr::Scan { .. } => n, + QueryExpr::Partition { child, .. } + | QueryExpr::Window { child, .. } | QueryExpr::Aggregate { child, .. } - | QueryExpr::Filter { child, .. } => scan_schema(child), + | QueryExpr::Filter { child, .. } => find_scan(child), other => panic!("unexpected node {other:?}"), } } - let mut leaf = scan_schema(&typed); - leaf.sort(); - assert_eq!(leaf, vec!["env", "host", "timestamp", "value"]); -} - -#[test] -fn schema_population_for_heavy_hitter_topk() { - let catalog = catalog_with("requests", &["service", "env"]); - let qe = lower_promql( - "topk by (service) (10, count_over_time(requests[1m]))", - &catalog, - AccuracyTarget::Exact, - ) - .unwrap(); - let typed = populate_schemas(qe, &catalog); - // TopK output: the by-column(s) + a synthetic `count` column. - let names: Vec<&str> = typed - .schema - .fields - .iter() - .map(|f| f.name.as_str()) - .collect(); - assert_eq!(names, vec!["service", "count"]); - assert_eq!(typed.schema.fields[1].dtype, L3DataType::Int64); + let QueryExpr::Scan { schema, .. } = find_scan(&qe) else { + unreachable!() + }; + let mut names: Vec<&str> = schema.columns.iter().map(|c| c.name.as_str()).collect(); + names.sort(); + assert_eq!(names, vec!["service", "ts", "value"]); + assert_eq!(schema.time_index, Some(0)); // ts } -// ── batch entry point ───────────────────────────────────────────────────────────── +// ── batch entry point ───────────────────────────────────────────────────────── #[test] fn batch_lowers_each_entry_and_reads_per_query_accuracy() { @@ -469,7 +381,7 @@ fn batch_lowers_each_entry_and_reads_per_query_accuracy() { repeating_queries: None, data_characteristics: None, }; - let results = lower_promql_batch(&workload, &empty_catalog()); + let results = lower_promql_batch(&workload); assert_eq!(results.len(), 2); assert!(results[0].is_ok()); assert!(results[1].is_ok()); @@ -487,10 +399,7 @@ fn batch_rejects_non_promql_language() { repeating_queries: None, data_characteristics: None, }; - let results = lower_promql_batch(&workload, &empty_catalog()); + let results = lower_promql_batch(&workload); assert_eq!(results.len(), 1); - assert!(matches!( - results[0], - Err(asap_control_lower::LoweringError::WrongLanguage(_)) - )); + assert!(matches!(results[0], Err(LoweringError::WrongLanguage(_)))); } From 62c21916d9e17cb61e9d0e07e815244d773b2057 Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 22 May 2026 08:21:04 -0600 Subject: [PATCH 03/40] docs(promql): explain Binder, positional ColumnId, and CSE with a traced example MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add docs/promql-lowering.md documenting the two-IR + Binder architecture: why column identity is positional (ColumnId), why name resolution is an explicit Binder pass, and why unique_keys gates workload-level CSE. Includes a full trace of `topk by (service) (10, count_over_time(requests{env="prod"}[1m]))` through all four stages (parse → L2 names → Binder → canonical positions), ending with the CSE legality gate refusing to share under the default usage-derived catalog. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/promql-lowering.md | 261 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 261 insertions(+) create mode 100644 docs/promql-lowering.md diff --git a/docs/promql-lowering.md b/docs/promql-lowering.md new file mode 100644 index 00000000..837773de --- /dev/null +++ b/docs/promql-lowering.md @@ -0,0 +1,261 @@ +# PromQL lowering — positional `ColumnId`, the Binder, and CSE + +How `asap-control-lower` turns a PromQL string into the canonical Layer-3 +intent algebra, and *why* the IR uses positional column identity, an explicit +name-resolution pass, and unique-key metadata. + +This is the PromQL companion to [`design.md` §6](design.md) ("Core crate +details — Layer 3"). It mirrors the `asapquery-backend` control-plane IR: +**two IRs joined by a Binder.** + +## The pipeline at a glance + +``` +PromQL string + │ L1: parse promql-parser 0.8 → Expr AST + ▼ +Expr AST + │ L2: front-end lowering crates/lower/src/promql.rs + ▼ +relational::QueryExpr ← columns are NAMES (ColumnRef::Named, Aggregate.keys: Vec) + │ Binder: build Schema + resolve crates/core/src/intent_algebra/binder.rs + │ + column_resolution.rs + ▼ +query_expr::QueryExpr ← columns are POSITIONS (Aggregate.by: Vec) + + a self-contained Schema rides on each Scan +``` + +`convert_root` (`intent_algebra/lower.rs`) runs the Binder first, then converts +the L2 tree structurally: + +```rust +pub fn convert_root(legacy: &LQueryExpr, accuracy: &AccuracyTarget) + -> Result +{ + let schema = Binder::new().bind(legacy); // ← name resolution, once + convert(legacy, &schema, accuracy) // ← purely structural after this +} +``` + +The four stages below — **parse → L2 (names) → Binder (schema) → canonical +(positions)** — are what we trace at the end. + +--- + +## Why positional `ColumnId` + +`ColumnId = usize` (`schema.rs`), an index into `Schema::columns`. Everywhere +the *canonical* IR names a column — `Aggregate.by`, `Schema::unique_keys`, +`Schema::time_index` — it is a position, not a string. + +The IR is deliberately split in two: + +| | Layer-2 `relational::QueryExpr` | Canonical `query_expr::QueryExpr` | +|---|---|---| +| Column identity | `ColumnRef::Named(String)`, `Aggregate.keys: Vec` | `ColumnId = usize`, `Aggregate.by: Vec` | +| Source of names | whatever the PromQL parser emits | resolved against a `Schema` | + +Why convert names → positions at all: + +1. **Identity is settled once.** A string `"service"` means nothing until you + know which schema it lives in and at what offset. If every downstream pass + (schema flow, push-down, CSE, cost model, L5 emitters) carried names, each + would re-resolve and each would own a "column not found" failure path. A + `ColumnId` is an array index that **cannot dangle** — resolution already + happened. +2. **The canonical tree is self-describing.** The `Scan` node carries the + `Schema`, so any sub-tree's output schema is computable without surrounding + context (`QueryExpr::output_schema_in`). Positions index straight into it. +3. **It matches the backend wire format.** `ColumnId` is aliased to `usize` + specifically to line up with `design.md`'s `unique_keys: Vec>`. + The point of the restructure was convergence with the backend IR, not a + parallel L3. + +The named alias is kept (rather than a bare `usize`) so code can still pattern +on intent — *"this is a column position, not just any number."* + +--- + +## Why the Binder is its own pass + +`Binder::bind` (`binder.rs`) walks the L2 tree and returns **one** +self-contained `Schema { columns, time_index, unique_keys }` that every +`ColumnId` in the converted tree indexes into. It: + +1. Seeds columns from the catalog (`SchemaCatalog::columns_for`), or the + `(ts, value)` floor if the catalog knows nothing. +2. Guarantees that `(ts, value)` floor is present. +3. Appends one `Utf8` column per referenced-but-unknown name — collected from + `Aggregate.keys`, `TopK.by`, `Partition.keys` (`collect_referenced_columns`). + +Why isolate this instead of resolving inline during lowering: + +- **The converter becomes purely structural and total.** Once the schema + exists, positional resolution downstream can't fail to *find* a column. Every + failure mode (`ResolveError::NotFound`, `NoSampleValue`, `WildcardNotPositional`) + is concentrated in this one pass. +- **Schema/catalog policy is swappable without touching lowering.** The default + `UsageDerivedCatalog` knows nothing — honest for observability, where metric + label sets are open-ended. A registry-backed `SchemaCatalog` is future work, + and the Binder pass does not change when it lands — only the catalog impl + swaps. +- **It is the natural home for resolution-policy errors.** `without(...)` is + rejected here: a usage-derived schema can't enumerate "all labels *except* + these," so the error belongs in binding, not smeared across lowering. + +--- + +## Why `unique_keys` / CSE + +`unique_keys: Vec>` (`schema.rs`). Each inner vec is a set of +column positions that *together* uniquely identify a row; the outer vec allows +several such sets. It is populated by the per-node output-schema rule — +`Aggregate { by, .. }` emits `unique_keys = [by-positions]` when `by` is +non-empty (`query_expr.rs`), most other nodes pass through. + +**What it is for: workload-level CSE.** When several queries are planned +together, `cse::dedupe_subtrees` hoists a shared sub-DAG into a `LetBinding` so +the cost model credits the producer once, with each root referencing it via +`Ref`. But sharing is only sound if the producer emits *the same rows* for every +consumer — and a unique key is exactly what proves that. + +`cse_reuse_is_legal` is the gatekeeper (`schema.rs`): + +```rust +pub fn cse_reuse_is_legal(producer_schema: &Schema, consumer_count: usize) + -> Result<(), CseError> +{ + if consumer_count < 2 { return Err(CseError::InsufficientConsumers(consumer_count)); } + if !producer_schema.has_unique_key() { return Err(CseError::NoUniqueKeys); } + Ok(()) +} +``` + +Why `unique_keys` rather than just deduping structurally-identical subtrees: +structural identity (`format!("{child:?}")`) tells you two consumers *want* the +same producer — it does **not** tell you the producer's output is *stable across +reads*. Without a provable unique key, two `Ref`s could observe different row +sets, and crediting the sharing would be unsound. Structural identity is the +candidate-finder; `unique_keys` is the correctness predicate. Expressing keys as +`ColumnId` sets is what lets the deduper assert this cheaply — another reason +positions exist. + +**Status:** this PR lands the scaffolding (`dedupe_subtrees`, +`cse_reuse_is_legal`, `Schema::unique_keys`, `LetBinding`/`Ref`). The cost-model +integration that makes it influence planning is tracked in **#6**. Single-query +plans never read `unique_keys`. + +--- + +## Worked example — one query through all four stages + +```promql +topk by (service) (10, count_over_time(requests{env="prod"}[1m])) +``` + +This is the heavy-hitter case (`topk` over `count` → one-pass sketch), exercised +by `topk_over_count_is_heavy_hitter_topk` in `crates/lower/tests/promql_lowering.rs`. + +### Stage 1 — L1 parse (`promql-parser`) + +``` +Expr::Aggregate { + op: topk, + param: NumberLiteral(10), + modifier: by (service), + expr: Expr::Call { + func: count_over_time, + args: [ Expr::MatrixSelector { vs: requests{env="prod"}, range: 1m } ], + }, +} +``` + +### Stage 2 — L2 relational IR (names) · `promql.rs` + +`walk_aggregate` resolves the group modifier to `keys = ["service"]`, lowers the +inner `count_over_time(...[1m])` to `Inner { metric: "requests", +matchers: [env=="prod"], window: 1m, func: Count }`, and — because the op is +`topk` *and* the inner func is `Count` — picks the heavy-hitter branch +(`Outer::TopK { k: 10, descending: true }` → `heavy_hitter == true`): + +``` +TopK { k: 10, by: ["service"], ← columns are still NAMES + input: Window { duration: 1m, slide: None, + input: Filter { pred: Compare { left: Column("env"), op: Eq, right: "prod" }, + input: Source(SourceSpec { name: "requests" }) } } } +``` + +No `Aggregate` wraps the scan — the heavy-hitter sketch counts directly off the +windowed scan (`window_scan`). Grouping rides as a *name list* on `TopK.by`, +awaiting resolution. + +### Stage 3 — Binder builds the Schema, resolves names → `ColumnId` + +`Binder::bind` walks the L2 tree: + +- `source_name() == "requests"`; `UsageDerivedCatalog` returns `None` → start + from the `(ts, value)` floor. +- `collect_referenced_columns` finds `TopK.by = ["service"]` → append `service` + as a `Utf8` column. + +Result — the single self-contained schema: + +``` +Schema { + columns: [ ts:Timestamp(0), value:Float64(1), service:Utf8(2) ], + time_index: Some(0), + unique_keys: [], ← UsageDerivedCatalog proves no unique key +} +``` + +`resolve_named_keys(["service"], schema)` → `"service"` is at position 2 → +`by = [2]`. + +### Stage 4 — canonical IR (positions) · `lower.rs convert` + +The `TopK` arm rewrites to the canonical `Aggregate{TopK}`, threading the +accuracy target and the resolved positional `by`; the `Filter`-over-`Source` +folds into `Scan.predicates`; the bound schema rides on the `Scan`: + +``` +Aggregate { + by: [2], ← service, POSITIONAL now + aggs: [ TopK { k: 10, accuracy: } ], + having: None, + child: Window { kind: Tumbling, size: 1m, slide: None, + child: Scan { + source: TimeSeries { metric: "requests" }, + predicates: [ Compare { left: Column("env"), op: Eq, right: "prod" } ], + schema: Schema { [ts, value, service], time_index: Some(0), unique_keys: [] }, + } } } +``` + +### Schema flow & the CSE gate on this tree + +`output_schema_in` for the top `Aggregate`: `by = [2]` is non-empty, so its +output schema is + +``` +Schema { + columns: [ service:Utf8, topk_10:Utf8 ], ← group key + TopK output + time_index: None, + unique_keys: [[0]], ← the group key is now a unique key +} +``` + +Now suppose a second query shared the same `Window → Scan` producer. The deduper +would propose hoisting it and call +`cse_reuse_is_legal(window.output_schema(), 2)`. The window passes the Scan's +schema through unchanged — and that schema's `unique_keys` is **empty** (the +`UsageDerivedCatalog` couldn't prove one). So the gate returns +`Err(NoUniqueKeys)` and the producer is **not** shared — each consumer +recomputes it. + +That refusal is the design working as intended: under the default catalog we +cannot assert that a raw windowed scan yields identical rows across reads, so we +decline to share rather than risk an unsound plan. A registry-backed +`SchemaCatalog` that declared, say, `(ts, service)` unique on `requests` would +populate `Scan.schema.unique_keys`, flip the gate green, and let the windowed +scan be hoisted into a `LetBinding` — without any change to the Binder or +converter (cf. the `dedupe_subtrees_basic` test, which constructs exactly such a +Scan). From 133bf61703993a0eefad6ef84b16f49902409426 Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 22 May 2026 08:41:35 -0600 Subject: [PATCH 04/40] docs(promql): add unique_keys propagation example MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend docs/promql-lowering.md with a "unique_keys propagation" section: a per-operator propagation table (Scan verbatim, Window/Filter/etc. pass-through, Aggregate replaces with re-based group keys, Distinct adds) plus a bottom-to-top trace of the worked-example tree under a registry catalog that declares (ts, service) unique. Shows the leaf key flowing through Window unchanged and Aggregate re-basing it from ColumnId 2 to 0 — which is why the CSE gate flips green under a registry catalog but stays empty under UsageDerivedCatalog. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/promql-lowering.md | 72 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/docs/promql-lowering.md b/docs/promql-lowering.md index 837773de..71a76510 100644 --- a/docs/promql-lowering.md +++ b/docs/promql-lowering.md @@ -258,4 +258,74 @@ decline to share rather than risk an unsound plan. A registry-backed populate `Scan.schema.unique_keys`, flip the gate green, and let the windowed scan be hoisted into a `LetBinding` — without any change to the Binder or converter (cf. the `dedupe_subtrees_basic` test, which constructs exactly such a -Scan). +Scan). The next section traces exactly that. + +--- + +## `unique_keys` propagation + +`unique_keys` is **not** something the Binder computes — `Binder::bind` always +emits `unique_keys: Vec::new()`. It enters at the leaf (from the catalog) and is +then derived edge-by-edge by each operator's output-schema rule +(`QueryExpr::output_schema_in`). The rules: + +| Operator | `unique_keys` of its output | +|---|---| +| `Scan` | **verbatim** from the schema the Binder/catalog built | +| `Window`, `Filter`, `Partition`, `Sort`, `Limit`, `Subquery`, `Project` | **pass through** the child's unchanged | +| `Aggregate { by }` | **replaced** with `[[0..by.len()]]` — the group keys, *re-based to output positions*; empty when `by` is empty | +| `Distinct { cols }` | child's keys **plus** `cols` added as a new key (`add_unique_key`) | +| `Merge` | first child's | +| `SetOp`, `Join`, `BinaryOp` | left / `lhs` child's | + +Two rules carry the weight: leaf-bearing operators **pass keys through** untouched, +while `Aggregate` **manufactures** a key — grouping by a column makes that column +unique in the result, so it becomes the new key (and the input's keys are +dropped, because the grouped output no longer has those rows). + +### Same tree, under a registry catalog + +Take the worked example's canonical tree, but bind it with a `SchemaCatalog` that +declares `(ts, service)` unique on `requests`. Now the leaf schema arrives with a +key, and we can watch it flow up (bottom → top): + +``` + ── unique_keys on this edge ── +Scan { requests, [[0, 2]] ← from catalog + schema: [ts(0), value(1), service(2)], (ts, service) + unique_keys = [[0, 2]] } + ▲ +Window { 1m } [[0, 2]] ← pass-through + ▲ (time_index present) +Aggregate { by:[2]=service, aggs:[TopK{10}] } [[0]] ← REPLACED + output cols: [service(0), topk_10(1)] by re-based to + output position 0 +``` + +Three things to read off this: + +1. **Scan** hands up the catalog's `[[0, 2]]` verbatim. +2. **Window** (and any `Filter`/`Sort`/`Limit` between) passes `[[0, 2]]` straight + through — these operators don't change which rows are distinct. +3. **Aggregate** does *not* forward `[[0, 2]]`. After `GROUP BY service`, the old + per-sample identity is gone; what's unique now is `service` itself — and in the + output schema `service` sits at **position 0**, so the derived key is `[[0]]`, + not `[[2]]`. This re-basing is why keys are positional `ColumnId`s, not names: + the same column is id `2` below the aggregate and id `0` above it. + +### Why this makes CSE legal here + +The shared producer the deduper would hoist is the `Window → Scan` sub-tree. Its +output edge now carries `unique_keys = [[0, 2]]`, so: + +``` +cse_reuse_is_legal( window.output_schema() // unique_keys = [[0, 2]] + , 2 /* consumers */ ) ==> Ok(()) +``` + +The gate fires green, the `Window → Scan` is hoisted into a `LetBinding`, and both +queries `Ref` it — scan + window computed once. Under the default +`UsageDerivedCatalog` the very same tree carries `unique_keys = []` at every edge +(nothing manufactures a key below the top `Aggregate`), so the gate returns +`Err(NoUniqueKeys)` and each consumer recomputes. **The only thing that changed +was the leaf key the catalog supplied; propagation and the gate did the rest.** From 96d2ecd64a04eaef2c28df87be15573cb3d21e5f Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 22 May 2026 08:46:40 -0600 Subject: [PATCH 05/40] docs(promql): add input-vs-output ColumnId (2 vs 0) table Clarify why the same `service` column is ColumnId 2 below an Aggregate and 0 above it: input and output edges are different schemas, so a ColumnId is a position within one schema and is re-based per edge. Adds two sample-row tables (input: service@2, unique_keys [[0,2]]; output: service@0, unique_keys [[0]]) and a ColumnId-vs-unique_keys distinction ("which column" pointer vs "which columns are jointly unique" fact written using ColumnIds). Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/promql-lowering.md | 42 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/docs/promql-lowering.md b/docs/promql-lowering.md index 71a76510..79ed9301 100644 --- a/docs/promql-lowering.md +++ b/docs/promql-lowering.md @@ -313,6 +313,48 @@ Three things to read off this: not `[[2]]`. This re-basing is why keys are positional `ColumnId`s, not names: the same column is id `2` below the aggregate and id `0` above it. +### `ColumnId` is relative to a schema — the `2` vs `0` + +A `ColumnId` is a position *within one schema*. The input and output edges of +`Aggregate` are **different schemas**, so the *same* logical `service` column +gets a different id on each. Watch it with sample rows. + +**Input edge** (below the aggregate) — `service` is column **2**: + +| `ts` · id 0 | `value` · id 1 | `service` · id 2 | +|---|---|---| +| 100 | 0.5 | api | +| 100 | 0.3 | web | +| 200 | 0.7 | api | +| 200 | 0.4 | web | + +→ "group by `service`" is written `by = [2]`. And every `(ts, service)` combo +occurs once, so the edge carries `unique_keys = [[0, 2]]`. + +**Output edge** (above the aggregate) — a *brand-new* table; `service` is now +column **0**: + +| `service` · id 0 | `topk_10` · id 1 | +|---|---| +| api | … | +| web | … | + +→ after grouping, each `service` appears exactly once, so `service` *alone* is +the key — at its **new** position: `unique_keys = [[0]]`. + +So `2` and `0` both name `service`; they differ only because input and output are +different schemas. This is also the cleanest way to see how the two concepts +divide up: + +- **`ColumnId`** answers *"which column"* — a pointer (`by = [2]`: group by the + column at position 2). Used everywhere a column must be named. +- **`unique_keys`** answers *"which set(s) of columns are jointly non-duplicating"* + — a fact about the data, *written using* `ColumnId`s (`[[0, 2]]`: columns 0 and + 2 together identify a row). Read only by the CSE gate. + +The outer `Vec` allows several such sets: `[[0, 2]]` = one key (the pair); +`[[0], [1, 2]]` = two independent keys. + ### Why this makes CSE legal here The shared producer the deduper would hoist is the `Window → Scan` sub-tree. Its From 5203733a0f144a9a392fd14519eb7a085135b3a0 Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 22 May 2026 08:50:17 -0600 Subject: [PATCH 06/40] docs(promql): label L3 explicitly in the pipeline diagram MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The diagram mixed layer names (L1/L2) with a pass name (Binder) and a property (canonical), so L3 looked absent. Label the boxes L1/L2/L3 directly, state that the Binder is the L2→L3 pass (not a layer), and align the worked-example stage headers (Stage 4 = L3 canonical, Stage 3 = Binder pass on the L2→L3 edge). Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/promql-lowering.md | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/docs/promql-lowering.md b/docs/promql-lowering.md index 79ed9301..7d5536b8 100644 --- a/docs/promql-lowering.md +++ b/docs/promql-lowering.md @@ -12,33 +12,36 @@ details — Layer 3"). It mirrors the `asapquery-backend` control-plane IR: ``` PromQL string - │ L1: parse promql-parser 0.8 → Expr AST + │ parse promql-parser 0.8 ▼ -Expr AST - │ L2: front-end lowering crates/lower/src/promql.rs +Expr AST ← L1 + │ front-end lowering crates/lower/src/promql.rs ▼ -relational::QueryExpr ← columns are NAMES (ColumnRef::Named, Aggregate.keys: Vec) - │ Binder: build Schema + resolve crates/core/src/intent_algebra/binder.rs - │ + column_resolution.rs +relational::QueryExpr ← L2 — columns are NAMES (ColumnRef::Named, Aggregate.keys: Vec) + │ Binder pass: build Schema + crates/core/src/intent_algebra/binder.rs + │ resolve names → ColumnId + column_resolution.rs ▼ -query_expr::QueryExpr ← columns are POSITIONS (Aggregate.by: Vec) +query_expr::QueryExpr ← L3 — columns are POSITIONS (Aggregate.by: Vec) + a self-contained Schema rides on each Scan ``` -`convert_root` (`intent_algebra/lower.rs`) runs the Binder first, then converts -the L2 tree structurally: +The three layers are **L1 (`Expr AST`) → L2 (`relational::QueryExpr`, names) → +L3 (`query_expr::QueryExpr`, positions)**. The **Binder is not a layer** — it is +the pass that sits on the L2→L3 edge, turning names into positional `ColumnId`s. +`convert_root` (`intent_algebra/lower.rs`) runs it first, then converts the L2 +tree structurally: ```rust pub fn convert_root(legacy: &LQueryExpr, accuracy: &AccuracyTarget) -> Result { - let schema = Binder::new().bind(legacy); // ← name resolution, once + let schema = Binder::new().bind(legacy); // ← L2→L3 name resolution, once convert(legacy, &schema, accuracy) // ← purely structural after this } ``` -The four stages below — **parse → L2 (names) → Binder (schema) → canonical -(positions)** — are what we trace at the end. +The worked example at the end traces a query through all three layers (with the +Binder pass shown explicitly between L2 and L3). --- @@ -147,7 +150,10 @@ plans never read `unique_keys`. --- -## Worked example — one query through all four stages +## Worked example — one query through L1 → L2 → L3 + +Four steps: the three layers, plus the Binder pass shown explicitly on the +L2→L3 edge. ```promql topk by (service) (10, count_over_time(requests{env="prod"}[1m])) @@ -189,7 +195,7 @@ No `Aggregate` wraps the scan — the heavy-hitter sketch counts directly off th windowed scan (`window_scan`). Grouping rides as a *name list* on `TopK.by`, awaiting resolution. -### Stage 3 — Binder builds the Schema, resolves names → `ColumnId` +### Stage 3 — Binder pass (L2→L3 edge): build the Schema, resolve names → `ColumnId` `Binder::bind` walks the L2 tree: @@ -211,7 +217,7 @@ Schema { `resolve_named_keys(["service"], schema)` → `"service"` is at position 2 → `by = [2]`. -### Stage 4 — canonical IR (positions) · `lower.rs convert` +### Stage 4 — L3 canonical IR (positions) · `lower.rs convert` The `TopK` arm rewrites to the canonical `Aggregate{TopK}`, threading the accuracy target and the resolved positional `by`; the `Filter`-over-`Source` From e6a3b182082febe6f185c05ab18dda4987b710c8 Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 22 May 2026 10:01:45 -0600 Subject: [PATCH 07/40] =?UTF-8?q?fix(promql):=20address=20PR=20#5=20review?= =?UTF-8?q?=20=E2=80=94=20outer-agg=20drop,=20histogram,=20CSE=20key,=20pe?= =?UTF-8?q?r-branch=20binding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four review findings, all with regression tests: 1. Outer aggregation over an inner range-vector func dropped the outer op. `sum(rate(m[w]))` lowered to just `Aggregate{Rate}`, silently losing the sum. `build()` now emits a two-level aggregate (outer op over inner func) for both `Outer::Plain` and the `Outer::Count` sibling. This also fixes the same latent bug for `sum by (..)(quantile_over_time(..))` and `count by (..)(count_over_time(..))` — the two existing tests that asserted the collapsed shape are updated to the correct two-level structure. 2. `histogram_quantile(φ, sum by (le)(rate(..)))` errored — `extract_matrix` couldn't see through the `sum by (le)` aggregate. `histogram_quantile` is now special-cased in `walk`: it lowers its argument in full (preserving the `sum by (le)` / `rate` structure) and wraps it in `Aggregate{[Quantile]}`. 3. CSE used `format!("{child:?}")` as a structural key. Replaced with grouping by `PartialEq` over collected candidates — no reliance on Debug being an injective, stable identity. 4. `convert` threaded a single root schema (derived from the left leaf) to both sides of `BinaryOp`/`Join`/`SetOp`, so the right branch could resolve columns against the wrong metric's schema. Each branch is now bound independently via `convert_root`. Tests: 22 core + 27 lower pass under --locked; clippy -D warnings and fmt clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/core/src/intent_algebra/cse.rs | 24 ++-- crates/core/src/intent_algebra/lower.rs | 18 ++- crates/lower/src/promql.rs | 75 ++++++++---- crates/lower/tests/promql_lowering.rs | 145 ++++++++++++++++++++++-- 4 files changed, 215 insertions(+), 47 deletions(-) diff --git a/crates/core/src/intent_algebra/cse.rs b/crates/core/src/intent_algebra/cse.rs index b88ffe26..23d6f67e 100644 --- a/crates/core/src/intent_algebra/cse.rs +++ b/crates/core/src/intent_algebra/cse.rs @@ -10,8 +10,6 @@ //! case. The fully-general algorithm (alpha-equivalence, schema-merge, //! nested CSE) is a downstream optimisation, not part of the IR contract. -use std::collections::HashMap; - use crate::intent_algebra::names::{BindingName, QueryId}; use crate::intent_algebra::query_expr::QueryExpr; use crate::intent_algebra::schema::cse_reuse_is_legal; @@ -38,24 +36,30 @@ pub fn dedupe_subtrees(roots: Vec<(QueryId, QueryExpr)>) -> CseWorkloadPlan { }; } - // Count `Aggregate`-child sub-trees that appear in ≥2 roots. - let mut candidate_counts: HashMap = HashMap::new(); + // Count `Aggregate`-child sub-trees that appear in ≥2 roots. Group by + // structural equality (`QueryExpr: PartialEq`) rather than `Debug` + // output: `{:?}` is not a guaranteed-injective, stable identity contract. + // The candidate set is one entry per distinct root child, so this linear + // scan is bounded by the number of distinct queries. + let mut candidate_counts: Vec<(QueryExpr, usize)> = Vec::new(); for (_, root) in &roots { if let QueryExpr::Aggregate { child, .. } = root { if matches!(**child, QueryExpr::Ref { .. }) { continue; } - let key = format!("{child:?}"); - let entry = candidate_counts - .entry(key) - .or_insert_with(|| ((**child).clone(), 0)); - entry.1 += 1; + match candidate_counts + .iter_mut() + .find(|(e, _)| e == child.as_ref()) + { + Some(entry) => entry.1 += 1, + None => candidate_counts.push(((**child).clone(), 1)), + } } } // Pick the most-shared legal candidate (biggest reuse first). let mut chosen: Option<(QueryExpr, usize)> = None; - for (_key, (expr, count)) in candidate_counts.into_iter() { + for (expr, count) in candidate_counts.into_iter() { if count < 2 { continue; } diff --git a/crates/core/src/intent_algebra/lower.rs b/crates/core/src/intent_algebra/lower.rs index 3716fb50..51ba3062 100644 --- a/crates/core/src/intent_algebra/lower.rs +++ b/crates/core/src/intent_algebra/lower.rs @@ -187,8 +187,9 @@ pub fn convert( crate::intent_algebra::expr_ir::L3Scalar::Boolean(true), ), )), - left: Box::new(convert(left, schema, acc)?), - right: Box::new(convert(right, schema, acc)?), + // Each branch is bound independently — see the `BinaryOp` arm. + left: Box::new(convert_root(left, acc)?), + right: Box::new(convert_root(right, acc)?), }, LQueryExpr::SetOp { @@ -199,8 +200,8 @@ pub fn convert( } => CQueryExpr::SetOp { kind: kind.clone(), all: *all, - left: Box::new(convert(left, schema, acc)?), - right: Box::new(convert(right, schema, acc)?), + left: Box::new(convert_root(left, acc)?), + right: Box::new(convert_root(right, acc)?), }, LQueryExpr::Sort { keys, input } => CQueryExpr::Sort { @@ -237,8 +238,13 @@ pub fn convert( vector_match, } => CQueryExpr::BinaryOp { op: op.clone(), - lhs: Box::new(convert(lhs, schema, acc)?), - rhs: Box::new(convert(rhs, schema, acc)?), + // A binary op's two sides may scan different metrics with different + // label sets, so each branch must resolve against its OWN bound + // schema. `convert_root` re-runs the Binder per sub-tree; threading + // the parent `schema` (derived from the left leaf only) would bind + // the right side's columns to the wrong positions. + lhs: Box::new(convert_root(lhs, acc)?), + rhs: Box::new(convert_root(rhs, acc)?), vector_match: vector_match.clone(), }, }) diff --git a/crates/lower/src/promql.rs b/crates/lower/src/promql.rs index c7ed56ba..a22c741d 100644 --- a/crates/lower/src/promql.rs +++ b/crates/lower/src/promql.rs @@ -13,7 +13,8 @@ //! | PromQL | L2 shape (→ canonical via `convert_root`) | //! |---|---| //! | `quantile_over_time(φ, m{f}[w])` | `Aggregate{[Quantile(φ)], Window{w, Filter(Source)}}` | -//! | `histogram_quantile(φ, rate(m[w]))` | `Aggregate{[Quantile(φ)], Window{w, …}}` | +//! | `histogram_quantile(φ, )` | `Aggregate{[Quantile(φ)]}` over the fully-lowered `` (preserves any `sum by (le)`/`rate`) | +//! | `OUTER_op(inner_func(m[w]))` (e.g. `sum(rate(m[w]))`) | `Aggregate{[OUTER_op]}` over `Aggregate{[inner_func]}` — two levels | //! | `avg/min/max/sum_over_time(m[w])` | `Aggregate{[Avg/Min/Max/Sum], Window{w}}` | //! | `stddev/stdvar_over_time(m[w])` | `Aggregate{[StdDev/Variance], Window{w}}` | //! | `count_over_time(m[w])` | `Aggregate{[Count], Window{w}}` | @@ -99,6 +100,7 @@ impl PromqlLowerer { fn walk(expr: &Expr) -> Result { match expr { Expr::Aggregate(agg) => walk_aggregate(agg), + Expr::Call(call) if call.func.name == "histogram_quantile" => walk_histogram_quantile(call), Expr::Call(call) => build(lower_inner_call(call)?, vec![], Outer::None), Expr::Binary(bin) => walk_binary(bin), Expr::Paren(p) => walk(&p.expr), @@ -169,6 +171,20 @@ fn walk_aggregate(agg: &AggregateExpr) -> Result { build(inner, keys, outer) } +/// `histogram_quantile(φ, )` lowers `` in full — preserving any +/// `sum by (le)` / `rate` structure inside it — and wraps the result in an +/// `Aggregate{[Quantile(φ)]}`. The φ-quantile reduces across the `le` buckets, +/// so the wrapper carries no grouping keys: the usage-derived schema can't +/// enumerate the non-`le` labels to group by (the same limitation that rejects +/// `without`). This handles the canonical +/// `histogram_quantile(φ, sum by (le) (rate(m_bucket[w])))` pattern, which the +/// old "extract the matrix and substitute a bare Quantile" path could not. +fn walk_histogram_quantile(call: &Call) -> Result { + let phi = num_arg(call, 0)?; + let inner = walk(arg(call, 1)?)?; + Ok(outer_aggregate(vec![], AggFunc::Quantile(phi), inner)) +} + fn walk_binary(bin: &BinaryExpr) -> Result { let lhs = walk(&bin.lhs)?; let rhs = walk(&bin.rhs)?; @@ -273,17 +289,6 @@ fn lower_inner_call(call: &Call) -> Result { func: Some(InnerFunc::Quantile(phi)), }) } - // Substitute histogram_quantile(φ, buckets) with a plain Quantile(φ). - "histogram_quantile" => { - let phi = num_arg(call, 0)?; - let (metric, matchers, window) = extract_matrix(arg(call, 1)?)?; - Ok(Inner { - metric, - matchers, - window: Some(window), - func: Some(InnerFunc::Quantile(phi)), - }) - } "avg_over_time" => at0(InnerFunc::Avg), "min_over_time" => at0(InnerFunc::Min), "max_over_time" => at0(InnerFunc::Max), @@ -306,14 +311,27 @@ fn build(inner: Inner, keys: Vec, outer: Outer) -> Result { Ok(windowed_aggregate(inner, keys, func)) } }, - Outer::Plain(outer_intent) => { - let func = match &inner.func { - Some(f) => inner_func(f), - None => outer_func(&outer_intent), - }; - Ok(windowed_aggregate(inner, keys, func)) - } - Outer::Count => Ok(windowed_aggregate(inner, keys, AggFunc::CountDistinct)), + // An OUTER aggregation operator (`sum`/`avg`/…/`count`) over an inner + // range-vector function (`rate`/`increase`/`*_over_time`) is a + // two-level reduction: the inner func runs per series, the outer op + // then aggregates across series. Collapsing them into one aggregate + // silently drops a level — e.g. `sum(rate(m[w]))` must keep the `sum`. + Outer::Plain(outer_intent) => Ok(match &inner.func { + None => windowed_aggregate(inner, keys, outer_func(&outer_intent)), + Some(f) => { + let inner_f = inner_func(f); + let inner_agg = windowed_aggregate(inner, vec![], inner_f); + outer_aggregate(keys, outer_func(&outer_intent), inner_agg) + } + }), + Outer::Count => Ok(match &inner.func { + None => windowed_aggregate(inner, keys, AggFunc::CountDistinct), + Some(f) => { + let inner_f = inner_func(f); + let inner_agg = windowed_aggregate(inner, vec![], inner_f); + outer_aggregate(keys, AggFunc::CountDistinct, inner_agg) + } + }), Outer::TopK { k, descending } => { // Heavy-hitter only when ranking by frequency (`count`): a dedicated // sketch serves it in one pass → first-class `TopK`. Any other @@ -378,6 +396,23 @@ fn windowed_aggregate(inner: Inner, keys: Vec, func: AggFunc) -> L2 { } } +/// `Aggregate{keys, [func]}` directly over an existing L2 subtree — the OUTER +/// level of a two-level aggregation such as `sum(rate(…))` or the +/// `Aggregate{[Quantile]}` that wraps a `histogram_quantile` argument. +fn outer_aggregate(keys: Vec, func: AggFunc, input: L2) -> L2 { + L2::Aggregate { + keys, + aggs: vec![AggItem { + alias: "value".into(), + func, + col: ColumnRef::SampleValue, + distinct: false, + }], + having: None, + input: Box::new(input), + } +} + /// `[Window{w}] → Filter(Source)` with no aggregate (the heavy-hitter TopK /// child — the sketch counts directly off the scan). fn window_scan(inner: Inner) -> L2 { diff --git a/crates/lower/tests/promql_lowering.rs b/crates/lower/tests/promql_lowering.rs index 72bf0a47..fa697225 100644 --- a/crates/lower/tests/promql_lowering.rs +++ b/crates/lower/tests/promql_lowering.rs @@ -78,15 +78,22 @@ fn quantile_over_time_is_window_over_aggregate() { #[test] fn outer_sum_by_wraps_in_partition() { - // `sum by (host) (...)` → grouping rides on a `Partition` (backend model). + // `sum by (host) (quantile_over_time(...))` is a two-level reduction: an + // inner per-series quantile-over-time, then an outer cross-series sum. + // Grouping rides on a `Partition` wrapping the outer Sum (backend model). let qe = lower(r#"sum by (host) (quantile_over_time(0.99, latency{service="web"}[5m]))"#); let QueryExpr::Partition { keys, child } = &qe else { panic!("expected Partition, got {qe:?}"); }; assert_eq!(keys, &PartitionKeys::By(vec!["host".into()])); - // Inner: Window over Aggregate{Quantile} (the inner func wins the intent). + // Outer cross-series Sum. + let QueryExpr::Aggregate { aggs, child, .. } = child.as_ref() else { + panic!("expected outer Aggregate{{Sum}} under Partition, got {child:?}"); + }; + assert!(matches!(aggs.as_slice(), [AggIntent::Sum])); + // Inner: Window over Aggregate{Quantile}. let QueryExpr::Window { child, .. } = child.as_ref() else { - panic!("expected Window under Partition"); + panic!("expected Window under the outer Sum, got {child:?}"); }; assert!(matches!( child.as_ref(), @@ -129,19 +136,39 @@ fn stddev_and_stdvar_over_time() { } #[test] -fn histogram_quantile_substitutes_to_quantile() { +fn histogram_quantile_wraps_inner_in_quantile() { + // The argument's structure (here `rate`) is preserved *under* the Quantile, + // not squashed away — `Aggregate{Quantile}` over `Aggregate{Rate}` over Scan. let qe = lower(r#"histogram_quantile(0.95, rate(http_duration_seconds_bucket{le="0.5"}[5m]))"#); - let QueryExpr::Window { size, child, .. } = &qe else { - panic!("expected Window, got {qe:?}"); + let QueryExpr::Aggregate { aggs, child, .. } = &qe else { + panic!("expected outer Aggregate{{Quantile}}, got {qe:?}"); }; - assert_eq!(*size, Duration::from_secs(300)); + assert!(matches!(aggs.as_slice(), [AggIntent::Quantile { q, .. }] if (*q - 0.95).abs() < 1e-9)); let QueryExpr::Aggregate { aggs, child, .. } = child.as_ref() else { - panic!("expected Aggregate"); + panic!("expected inner Aggregate{{Rate}}, got {child:?}"); }; - assert!(matches!(aggs.as_slice(), [AggIntent::Quantile { q, .. }] if (*q - 0.95).abs() < 1e-9)); + assert!( + matches!(aggs.as_slice(), [AggIntent::Rate { window }] if *window == Duration::from_secs(300)) + ); assert!(matches!(child.as_ref(), QueryExpr::Scan { predicates, .. } if predicates.len() == 1)); } +#[test] +fn histogram_quantile_over_sum_by_le_preserves_grouping() { + // The canonical Prometheus histogram pattern. Previously returned + // UnsupportedFeature because `extract_matrix` couldn't see through the + // `sum by (le)` aggregate; now the `le` grouping survives into L3. + let qe = lower(r#"histogram_quantile(0.99, sum by (le) (rate(http_requests_bucket[5m])))"#); + let QueryExpr::Aggregate { aggs, child, .. } = &qe else { + panic!("expected outer Aggregate{{Quantile}}, got {qe:?}"); + }; + assert!(matches!(aggs.as_slice(), [AggIntent::Quantile { q, .. }] if (*q - 0.99).abs() < 1e-9)); + assert!( + matches!(child.as_ref(), QueryExpr::Partition { keys, .. } if *keys == PartitionKeys::By(vec!["le".into()])), + "expected `sum by (le)` to survive as a Partition under the quantile, got {child:?}" + ); +} + // ── rate / increase carry their own window (no Window node) ───────────────────── #[test] @@ -169,6 +196,59 @@ fn increase_maps_to_increase_intent() { )); } +// ── outer aggregation over an inner range-vector func is two levels ───────────── + +#[test] +fn sum_over_rate_keeps_both_levels() { + // Regression: `sum(rate(m[w]))` — the most common PromQL shape — must keep + // the cross-series Sum, not collapse to a bare per-series Rate. + let qe = lower("sum(rate(http_requests_total[5m]))"); + let QueryExpr::Aggregate { aggs, child, .. } = &qe else { + panic!("expected outer Aggregate{{Sum}}, got {qe:?}"); + }; + assert!(matches!(aggs.as_slice(), [AggIntent::Sum])); + let QueryExpr::Aggregate { aggs, child, .. } = child.as_ref() else { + panic!("expected inner Aggregate{{Rate}}, got {child:?}"); + }; + assert!( + matches!(aggs.as_slice(), [AggIntent::Rate { window }] if *window == Duration::from_secs(300)) + ); + assert!(matches!(child.as_ref(), QueryExpr::Scan { .. })); +} + +#[test] +fn sum_by_over_rate_groups_the_outer_sum() { + // `sum by (job) (rate(...))`: the grouping belongs to the OUTER sum, landing + // on a Partition that wraps the two-level aggregate. + let qe = lower("sum by (job) (rate(http_requests_total[5m]))"); + let QueryExpr::Partition { keys, child } = &qe else { + panic!("expected Partition by job, got {qe:?}"); + }; + assert_eq!(*keys, PartitionKeys::By(vec!["job".into()])); + let QueryExpr::Aggregate { aggs, child, .. } = child.as_ref() else { + panic!("expected Aggregate{{Sum}} under Partition, got {child:?}"); + }; + assert!(matches!(aggs.as_slice(), [AggIntent::Sum])); + assert!(matches!( + child.as_ref(), + QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Rate { .. }]) + )); +} + +#[test] +fn count_over_rate_keeps_both_levels() { + // The `Outer::Count` sibling of the `sum(rate(...))` bug. + let qe = lower("count(rate(http_requests_total[5m]))"); + let QueryExpr::Aggregate { aggs, child, .. } = &qe else { + panic!("expected outer Aggregate{{Cardinality}}, got {qe:?}"); + }; + assert!(matches!(aggs.as_slice(), [AggIntent::Cardinality { .. }])); + assert!(matches!( + child.as_ref(), + QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Rate { .. }]) + )); +} + // ── count / cardinality ─────────────────────────────────────────────────────── #[test] @@ -185,17 +265,25 @@ fn count_over_time_is_count_intent() { #[test] fn outer_count_is_cardinality() { + // `count by (symbol) (count_over_time(...))`: inner per-series sample count + // over the window, outer cross-series cardinality grouped by symbol. let qe = lower("count by (symbol) (count_over_time(financial_last_trade_price[5m]))"); let QueryExpr::Partition { keys, child } = &qe else { panic!("expected Partition, got {qe:?}"); }; assert_eq!(keys, &PartitionKeys::By(vec!["symbol".into()])); + // Outer cardinality (count of series). + let QueryExpr::Aggregate { aggs, child, .. } = child.as_ref() else { + panic!("expected outer Aggregate{{Cardinality}} under Partition, got {child:?}"); + }; + assert!(matches!(aggs.as_slice(), [AggIntent::Cardinality { .. }])); + // Inner: Window over Aggregate{Count} (count_over_time). let QueryExpr::Window { child, .. } = child.as_ref() else { - panic!("expected Window"); + panic!("expected Window under the outer cardinality, got {child:?}"); }; assert!(matches!( child.as_ref(), - QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Cardinality { .. }]) + QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Count { .. }]) )); } @@ -283,6 +371,41 @@ fn binary_op_with_on_grouping() { assert_eq!(vm.labels, vec!["host".to_string()]); } +#[test] +fn binary_op_binds_each_branch_against_its_own_schema() { + // Each side scans a different metric and groups by a different label. With a + // single root schema threaded to both branches, the left scan would leak the + // right's group key (and vice-versa). Per-branch binding keeps them separate. + let qe = lower("count by (job) (a) / count by (region) (b)"); + let QueryExpr::BinaryOp { lhs, rhs, .. } = &qe else { + panic!("expected BinaryOp, got {qe:?}"); + }; + let lcols = scan_columns(lhs); + let rcols = scan_columns(rhs); + assert!( + lcols.iter().any(|c| c == "job") && !lcols.iter().any(|c| c == "region"), + "lhs scan schema leaked the rhs key: {lcols:?}" + ); + assert!( + rcols.iter().any(|c| c == "region") && !rcols.iter().any(|c| c == "job"), + "rhs scan schema leaked the lhs key: {rcols:?}" + ); +} + +/// Column names on the first `Scan` reachable by descending single-child nodes. +fn scan_columns(e: &QueryExpr) -> Vec { + match e { + QueryExpr::Scan { schema, .. } => schema.columns.iter().map(|c| c.name.clone()).collect(), + QueryExpr::Partition { child, .. } + | QueryExpr::Aggregate { child, .. } + | QueryExpr::Window { child, .. } + | QueryExpr::Filter { child, .. } + | QueryExpr::Sort { child, .. } + | QueryExpr::Limit { child, .. } => scan_columns(child), + _ => vec![], + } +} + // ── without is unsupported (no label registry) ────────────────────────────────── #[test] From 0e5d7c441de40e50cc21faad26aeb8ed1d081231 Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 22 May 2026 10:13:40 -0600 Subject: [PATCH 08/40] test(promql): add semantic conformance suite vs PromQL spec/cheat-sheet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add crates/lower/tests/promql_conformance.rs: 32 tests mapping canonical PromQL queries (from the Prometheus querying-basics docs, the PromLabs cheat sheet, and the Prometheus promqltest corpus) to their documented semantics, asserting the L3 lowering encodes the same intent. Because we lower (not execute), each test asserts the L3 *structure* matches the semantic rather than numeric results. Tests are grouped by category (selectors, counters, cross-series aggregation, two-level sum(rate), over-time, histograms, binary/set ops, topk/sort, subqueries, time-shift modifiers, unsupported functions) and cite the source + the matching Prometheus .test file. The suite also pins, rather than hides, where we diverge from PromQL — each flagged with a `__GAP` test name: - `group(v)` lowered as Sum (PromQL: constant 1 per group) - scalar/number-literal operands rejected (`v > 10*1024*1024`) - `topk(k, sum by(..)(rate(..)))` rejected (no nested-aggregate arg) - `max_over_time(rate(..)[1h:])` rejected (no subquery range-vector arg) - `offset` / `@` modifiers silently dropped by vs_parts - unsupported funcs (time, timestamp, absent, deriv, delta, predict_linear, label_replace, clamp_max) cleanly rejected All 32 pass; clippy -D warnings and fmt clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/lower/tests/promql_conformance.rs | 534 +++++++++++++++++++++++ 1 file changed, 534 insertions(+) create mode 100644 crates/lower/tests/promql_conformance.rs diff --git a/crates/lower/tests/promql_conformance.rs b/crates/lower/tests/promql_conformance.rs new file mode 100644 index 00000000..06b438ef --- /dev/null +++ b/crates/lower/tests/promql_conformance.rs @@ -0,0 +1,534 @@ +//! PromQL **semantic conformance** for the L1→L3 lowering. +//! +//! We *lower* PromQL to the intent algebra; we do not *execute* it. So "same +//! semantic job as Prometheus" here means: for each canonical query, does the +//! L3 tree encode the **documented PromQL meaning** — and where we knowingly +//! diverge (reject, approximate, or drop a modifier), is that pinned by a test +//! so it stays visible? +//! +//! Sources for the queries + their semantics: +//! - PromQL basics (data types, selectors, offset/@/subquery): +//! +//! - PromLabs PromQL cheat sheet (common real-world queries by category): +//! +//! - Prometheus' own engine test corpus (these are *execution* tests — +//! load → eval → expect values — so they define semantics we mirror as +//! *structure*): +//! Relevant files, mapped to the sections below: selectors.test, +//! aggregators.test, functions.test, histograms.test, operators.test, +//! subquery.test, at_modifier.test, literals.test, limit.test +//! +//! Legend used in test names: +//! - (no suffix) — we lower it and the L3 intent matches PromQL. +//! - `__rejected` — we cleanly return an error (capability not implemented). +//! - `__GAP` — we lower it but the result **diverges** from PromQL +//! semantics (silent loss or over-aggregation). Pinned so a future fix +//! flips the assertion deliberately. + +// `__GAP`-suffixed test names intentionally SHOUT the documented divergences. +#![allow(non_snake_case)] + +use std::time::Duration; + +use asap_control_core::intent_algebra::{ + AggIntent, BinaryOpKind, PartitionKeys, QueryExpr, Source, +}; +use asap_control_core::types::AccuracyTarget; +use asap_control_lower::{lower_promql, LoweringError}; + +// ── harness helpers ───────────────────────────────────────────────────────────── + +/// Lower, expecting success. +fn ok(q: &str) -> QueryExpr { + lower_promql(q, AccuracyTarget::Exact) + .unwrap_or_else(|e| panic!("expected {q:?} to lower, got error: {e}")) +} + +/// Lower, expecting a clean `LoweringError` (an unsupported capability). +fn rejected(q: &str) -> LoweringError { + match lower_promql(q, AccuracyTarget::Exact) { + Err(e) => e, + Ok(tree) => panic!("expected {q:?} to be rejected, but it lowered to: {tree:?}"), + } +} + +/// Every `AggIntent` anywhere in the tree, root-to-leaf. +fn intents(e: &QueryExpr) -> Vec { + let mut out = Vec::new(); + collect(e, &mut out); + out +} + +fn collect(e: &QueryExpr, out: &mut Vec) { + match e { + QueryExpr::Aggregate { aggs, child, .. } => { + out.extend(aggs.iter().cloned()); + collect(child, out); + } + QueryExpr::Window { child, .. } + | QueryExpr::Partition { child, .. } + | QueryExpr::Filter { child, .. } + | QueryExpr::Sort { child, .. } + | QueryExpr::Limit { child, .. } + | QueryExpr::Subquery { child, .. } + | QueryExpr::Distinct { child, .. } + | QueryExpr::Project { child, .. } => collect(child, out), + QueryExpr::BinaryOp { lhs, rhs, .. } => { + collect(lhs, out); + collect(rhs, out); + } + QueryExpr::Join { left, right, .. } | QueryExpr::SetOp { left, right, .. } => { + collect(left, out); + collect(right, out); + } + QueryExpr::Merge { children } => children.iter().for_each(|c| collect(c, out)), + QueryExpr::LetBinding { expr, child, .. } => { + collect(expr, out); + collect(child, out); + } + QueryExpr::Scan { .. } | QueryExpr::Ref { .. } => {} + } +} + +/// The first `Scan` reached by descending single-child nodes, with its metric +/// name and predicate count. +fn first_scan(e: &QueryExpr) -> (String, usize) { + match e { + QueryExpr::Scan { + source, predicates, .. + } => { + let name = match source { + Source::TimeSeries { metric } => metric.clone(), + Source::Table { table_ref } => table_ref.clone(), + }; + (name, predicates.len()) + } + QueryExpr::Window { child, .. } + | QueryExpr::Aggregate { child, .. } + | QueryExpr::Partition { child, .. } + | QueryExpr::Filter { child, .. } + | QueryExpr::Sort { child, .. } + | QueryExpr::Limit { child, .. } + | QueryExpr::Subquery { child, .. } => first_scan(child), + other => panic!("no Scan reachable from {other:?}"), + } +} + +fn has bool>(e: &QueryExpr, pred: F) -> bool { + intents(e).iter().any(pred) +} + +// ───────────────────────────────────────────────────────────────────────────── +// A. Selectors & label matchers (basics §"Instant/Range Vector +// Selectors"; selectors.test) +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn instant_vector_selector() { + // SEMANTICS: bare metric → instant vector (latest sample per series). + let (metric, preds) = first_scan(&ok("node_cpu_seconds_total")); + assert_eq!(metric, "node_cpu_seconds_total"); + assert_eq!(preds, 0, "no label matchers → no predicates"); +} + +#[test] +fn label_matchers_become_scan_predicates() { + // SEMANTICS: `=`, `!=`, `=~`, `!~` filter series; one conjunct per matcher. + let (_, preds) = first_scan(&ok( + r#"http_requests_total{job!="x",path=~"/api/.*",env!~"dev"}"#, + )); + assert_eq!(preds, 3, "three matchers → three Scan predicates"); +} + +#[test] +fn name_label_selects_the_metric() { + // SEMANTICS: the metric name is the internal `__name__` label. + let (metric, preds) = first_scan(&ok(r#"{__name__="up"}"#)); + assert_eq!(metric, "up"); + assert_eq!(preds, 0, "__name__ is the metric, not a residual predicate"); +} + +#[test] +fn range_vector_selector_is_a_window() { + // SEMANTICS: `[5m]` turns an instant vector into a range vector. + let qe = ok("node_cpu_seconds_total[5m]"); + let QueryExpr::Window { size, .. } = &qe else { + panic!("expected Window for a range-vector selector, got {qe:?}"); + }; + assert_eq!(*size, Duration::from_secs(300)); +} + +// ───────────────────────────────────────────────────────────────────────────── +// B. Counters: rate / irate / increase (cheat sheet "Rates of Increase"; +// functions.test) +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn rate_carries_its_window_in_the_intent() { + // SEMANTICS: per-second average rate over the range; the window IS the rate + // parameter, so no separate Window node. + let qe = ok("rate(http_requests_total[5m])"); + assert!(matches!(&qe, QueryExpr::Aggregate { .. })); + assert!(has( + &qe, + |i| matches!(i, AggIntent::Rate { window } if *window == Duration::from_secs(300)) + )); +} + +#[test] +fn irate_maps_to_rate_intent() { + // SEMANTICS: instant rate from the last two samples; same intent vocabulary. + assert!(has(&ok("irate(http_requests_total[1m])"), |i| matches!( + i, + AggIntent::Rate { .. } + ))); +} + +#[test] +fn increase_maps_to_increase_intent() { + assert!(has(&ok("increase(http_requests_total[1h])"), |i| matches!( + i, + AggIntent::Increase { window } if *window == Duration::from_secs(3600) + ))); +} + +// ───────────────────────────────────────────────────────────────────────────── +// C. Aggregation across series (cheat sheet "Aggregating Over +// Multiple Series"; aggregators.test) +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn sum_collapses_all_series() { + // SEMANTICS: `sum(v)` → one output series. No grouping → no Partition. + let qe = ok("sum(node_filesystem_size_bytes)"); + assert!(matches!(&qe, QueryExpr::Aggregate { .. })); + assert!(has(&qe, |i| matches!(i, AggIntent::Sum))); +} + +#[test] +fn sum_by_preserves_dimensions_as_partition() { + // SEMANTICS: `by(job,instance)` keeps those labels; grouping rides on Partition. + let qe = ok("sum by(job, instance) (node_filesystem_size_bytes)"); + let QueryExpr::Partition { keys, .. } = &qe else { + panic!("expected Partition for `by(...)`, got {qe:?}"); + }; + assert_eq!( + *keys, + PartitionKeys::By(vec!["job".into(), "instance".into()]) + ); + assert!(has(&qe, |i| matches!(i, AggIntent::Sum))); +} + +#[test] +fn count_is_cardinality() { + assert!(has(&ok("count(up)"), |i| matches!( + i, + AggIntent::Cardinality { .. } + ))); +} + +#[test] +fn avg_min_max_stddev_stdvar_quantile_aggregators() { + assert!(has(&ok("avg(up)"), |i| matches!(i, AggIntent::Avg))); + assert!(has(&ok("min(up)"), |i| matches!(i, AggIntent::Min))); + assert!(has(&ok("max(up)"), |i| matches!(i, AggIntent::Max))); + assert!(has(&ok("stddev(up)"), |i| matches!( + i, + AggIntent::StdDev { .. } + ))); + assert!(has(&ok("stdvar(up)"), |i| matches!( + i, + AggIntent::Variance { .. } + ))); + assert!(has(&ok("quantile(0.5, up)"), |i| matches!( + i, + AggIntent::Quantile { .. } + ))); +} + +#[test] +fn sum_without_is_rejected() { + // SEMANTICS: `without(instance)` = group by all labels EXCEPT instance. + // We can't enumerate a metric's full label set (usage-derived schema), so + // the complement is rejected rather than silently mis-grouped. + let e = rejected("sum without(instance) (node_filesystem_size_bytes)"); + assert!(format!("{e}").contains("without"), "got {e}"); +} + +#[test] +fn group_aggregator_is_lowered_as_sum__GAP() { + // SEMANTICS (PromQL): `group(v)` returns a constant 1 per group (presence), + // NOT a sum. We currently fold it onto `Sum`. Pinned as a known divergence. + assert!(has(&ok("group by (job) (up)"), |i| matches!( + i, + AggIntent::Sum + ))); +} + +// ───────────────────────────────────────────────────────────────────────────── +// D. Two-level: outer aggregation OVER an inner counter (the canonical +// `sum(rate(...))` shape; aggregators.test + functions.test) +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn sum_of_rate_is_two_levels() { + // SEMANTICS: per-series rate, THEN cross-series sum. Both must survive. + let qe = ok("sum(rate(http_requests_total[5m]))"); + let QueryExpr::Aggregate { aggs, child, .. } = &qe else { + panic!("expected outer Aggregate{{Sum}}, got {qe:?}"); + }; + assert!(matches!(aggs.as_slice(), [AggIntent::Sum])); + assert!(matches!( + child.as_ref(), + QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Rate { .. }]) + )); +} + +#[test] +fn sum_by_of_rate_groups_outer_level() { + let qe = ok("sum by(instance) (rate(node_network_receive_bytes_total[5m]))"); + let QueryExpr::Partition { keys, child } = &qe else { + panic!("expected Partition, got {qe:?}"); + }; + assert_eq!(*keys, PartitionKeys::By(vec!["instance".into()])); + // Partition → Sum → Rate. + assert!(matches!( + child.as_ref(), + QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Sum]) + )); + assert!(has(&qe, |i| matches!(i, AggIntent::Rate { .. }))); +} + +// ───────────────────────────────────────────────────────────────────────────── +// E. Aggregation over time (per-series) (cheat sheet "Aggregating Over +// Time"; functions.test) +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn over_time_functions_window_then_reduce() { + // SEMANTICS: reduce the samples WITHIN each series over the range → Window + // over the matching reduce intent. + for (q, want) in [ + ("avg_over_time(go_goroutines[5m])", "avg"), + ("max_over_time(process_resident_memory_bytes[1d])", "max"), + ("min_over_time(go_goroutines[5m])", "min"), + ("sum_over_time(go_goroutines[5m])", "sum"), + ("count_over_time(go_goroutines[5m])", "count"), + ] { + let qe = ok(q); + assert!( + matches!(&qe, QueryExpr::Window { .. }), + "{q}: expected Window" + ); + let matched = intents(&qe).iter().any(|i| match want { + "avg" => matches!(i, AggIntent::Avg), + "max" => matches!(i, AggIntent::Max), + "min" => matches!(i, AggIntent::Min), + "sum" => matches!(i, AggIntent::Sum), + "count" => matches!(i, AggIntent::Count { .. }), + _ => unreachable!(), + }); + assert!(matched, "{q}: missing {want} intent"); + } +} + +#[test] +fn quantile_over_time_is_window_over_quantile() { + let qe = ok("quantile_over_time(0.9, request_latency_seconds[5m])"); + assert!(matches!(&qe, QueryExpr::Window { .. })); + assert!(has( + &qe, + |i| matches!(i, AggIntent::Quantile { q, .. } if (*q - 0.9).abs() < 1e-9) + )); +} + +// ───────────────────────────────────────────────────────────────────────────── +// F. Histograms (cheat sheet "Quantiles from +// Histograms"; histograms.test) +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn histogram_quantile_over_rate() { + // SEMANTICS: φ-quantile estimated from bucket rates. + let qe = ok("histogram_quantile(0.9, rate(demo_api_request_duration_seconds_bucket[5m]))"); + let QueryExpr::Aggregate { aggs, .. } = &qe else { + panic!("expected Aggregate{{Quantile}}, got {qe:?}"); + }; + assert!(matches!(aggs.as_slice(), [AggIntent::Quantile { q, .. }] if (*q - 0.9).abs() < 1e-9)); + assert!(has(&qe, |i| matches!(i, AggIntent::Rate { .. }))); +} + +#[test] +fn histogram_quantile_over_sum_by_le_preserves_le_grouping() { + // SEMANTICS: the standard pattern — bucket rates summed by `le`, then the + // quantile. The `sum by (le)` aggregation must survive into L3. + let qe = ok( + "histogram_quantile(0.99, sum by(le) (rate(demo_api_request_duration_seconds_bucket[5m])))", + ); + let QueryExpr::Aggregate { aggs, child, .. } = &qe else { + panic!("expected outer Aggregate{{Quantile}}, got {qe:?}"); + }; + assert!(matches!(aggs.as_slice(), [AggIntent::Quantile { .. }])); + assert!(matches!( + child.as_ref(), + QueryExpr::Partition { keys, .. } if *keys == PartitionKeys::By(vec!["le".into()]) + )); +} + +// ───────────────────────────────────────────────────────────────────────────── +// G. Binary ops: math, matching, comparison (cheat sheet "Math Between +// Series" / "Filtering Series by Value"; operators.test) +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn vector_arithmetic() { + let qe = ok("node_memory_MemFree_bytes + node_memory_Cached_bytes"); + let QueryExpr::BinaryOp { op, .. } = &qe else { + panic!("expected BinaryOp, got {qe:?}"); + }; + assert_eq!(*op, BinaryOpKind::Add); +} + +#[test] +fn on_matching_with_group_left() { + // SEMANTICS: many-to-one matching on a label subset. + let qe = + ok("rate(demo_cpu_usage_seconds_total[1m]) / on(instance, job) group_left demo_num_cpus"); + let QueryExpr::BinaryOp { + op, vector_match, .. + } = &qe + else { + panic!("expected BinaryOp, got {qe:?}"); + }; + assert_eq!(*op, BinaryOpKind::Div); + let vm = vector_match.as_ref().expect("on(...) group_left present"); + assert_eq!(vm.labels, vec!["instance".to_string(), "job".to_string()]); + assert!( + vm.grouping.is_some(), + "group_left should set the grouping side" + ); +} + +#[test] +fn vector_comparison_filters() { + // SEMANTICS: `>` between two vectors keeps the LHS series where it holds. + let qe = ok("go_goroutines > go_threads"); + assert!(matches!(&qe, QueryExpr::BinaryOp { op, .. } if *op == BinaryOpKind::Gt)); +} + +#[test] +fn scalar_literal_operand_is_rejected__GAP() { + // SEMANTICS (PromQL): `v > 10*1024*1024` filters by a scalar threshold. + // We have no scalar/number-literal expression in L2, so a literal operand + // is rejected. Common real-world thresholds therefore don't lower yet. + let _ = rejected("node_filesystem_avail_bytes > 10*1024*1024"); +} + +// ───────────────────────────────────────────────────────────────────────────── +// H. Set operations (cheat sheet "Set Operations"; +// operators.test) +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn set_ops_lower_to_binaryop() { + // SEMANTICS: or = union of label sets; and = intersection; unless = difference. + assert!(matches!(&ok("up{job=\"a\"} or up{job=\"b\"}"), + QueryExpr::BinaryOp { op, .. } if *op == BinaryOpKind::Or)); + assert!(matches!(&ok("node_network_mtu_bytes and node_up"), + QueryExpr::BinaryOp { op, .. } if *op == BinaryOpKind::And)); + assert!(matches!(&ok("node_network_mtu_bytes unless node_down"), + QueryExpr::BinaryOp { op, .. } if *op == BinaryOpKind::Unless)); +} + +// ───────────────────────────────────────────────────────────────────────────── +// I. Sorting / top-k (cheat sheet "Sorting"/topk; +// functions.test, limit.test) +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn topk_over_count_is_heavy_hitter() { + // SEMANTICS: top-k by frequency → single-pass heavy-hitter sketch. + let qe = ok("topk(10, count_over_time(http_requests_total[1m]))"); + assert!(has( + &qe, + |i| matches!(i, AggIntent::TopK { k, .. } if *k == 10) + )); +} + +#[test] +fn bottomk_is_generic_sort_limit() { + // SEMANTICS: bottom-k → generic ascending order + limit (no sketch). + let qe = ok("bottomk(3, count_over_time(http_requests_total[5m]))"); + assert!(matches!(&qe, QueryExpr::Limit { .. })); +} + +#[test] +fn topk_over_aggregate_arg_is_rejected__GAP() { + // SEMANTICS (PromQL): `topk(3, sum by(x)(rate(...)))` is extremely common. + // Our aggregate-argument lowering only accepts selectors/calls, not a + // nested aggregate, so this is rejected today. + let _ = rejected("topk(3, sum by(instance) (rate(node_cpu_seconds_total[5m])))"); +} + +// ───────────────────────────────────────────────────────────────────────────── +// J. Subqueries (basics §Subqueries; subquery.test) +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn subquery_wraps_inner_query() { + // SEMANTICS: `[range:res]` evaluates the inner query across a range. + let qe = ok("rate(demo_api_request_duration_seconds_count[5m])[1h:]"); + assert!(matches!(&qe, QueryExpr::Subquery { .. })); + assert!(has(&qe, |i| matches!(i, AggIntent::Rate { .. }))); +} + +#[test] +fn over_time_of_subquery_is_rejected__GAP() { + // SEMANTICS (PromQL): `max_over_time(rate(...)[1h:])` chains a subquery into + // a range-vector function. `extract_matrix` doesn't accept a subquery arg, + // so this canonical pattern is rejected today. + let _ = rejected("max_over_time(rate(demo_api_request_duration_seconds_count[5m])[1h:])"); +} + +// ───────────────────────────────────────────────────────────────────────────── +// K. Time-shift modifiers (basics §Offset/@; at_modifier.test) +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn offset_modifier_is_silently_dropped__GAP() { + // SEMANTICS (PromQL): `offset 5m` shifts the lookback 5m into the past. + // `vs_parts` reads only name + matchers, so the offset is lost: the query + // lowers as if there were no offset. Pinned so the silent loss is visible. + let (metric, _) = first_scan(&ok("http_requests_total offset 5m")); + assert_eq!(metric, "http_requests_total"); +} + +#[test] +fn at_modifier_is_silently_dropped__GAP() { + // SEMANTICS (PromQL): `@ ` pins the evaluation time. Also dropped today. + let (metric, _) = first_scan(&ok("http_requests_total @ 1609746000")); + assert_eq!(metric, "http_requests_total"); +} + +// ───────────────────────────────────────────────────────────────────────────── +// L. Unsupported functions (functions.test) — clean rejection +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn unsupported_functions_are_rejected() { + // These parse fine but have no intent-algebra lowering yet. Each must return + // a clean LoweringError rather than mislower. + for q in [ + "time()", + "timestamp(up)", + "absent(up)", + "absent_over_time(up[5m])", + "deriv(demo_disk_usage_bytes[1h])", + "delta(demo_disk_usage_bytes[1h])", + "predict_linear(demo_disk_usage_bytes[4h], 3600)", + r#"label_replace(up, "host", "$1", "instance", "(.+):.*")"#, + "clamp_max(go_goroutines, 5)", + ] { + let _ = rejected(q); + } +} From 127e98baa69faa6c95a0c0d19977459e0bbaf3c2 Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 22 May 2026 10:30:00 -0600 Subject: [PATCH 09/40] test(promql): semantic-equivalence suite + fix the divergences it exposed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add crates/lower/tests/promql_equivalence.rs (10 tests) proving the lowering is a sound normalizer against the PromQL spec / cheat sheet / Prometheus engine tests: equivalence classes collapse to one canonical L3, distinct meanings stay distinct, and nothing distinct is silently merged. Writing it surfaced five real divergences; each is now fixed in promql.rs: - Label-matcher order: `{a,b}` and `{b,a}` select the same series but lowered to different predicate orders. vs_parts now canonicalises matchers by (name,value). - Group-key order: `by(a,b)` ≡ `by(b,a)`; resolve_group now sorts+dedups keys. - `changes` / `resets`: were aliased to `count_over_time` → Count (wrong count). Now rejected (distinct semantics, no intent yet). - `group`: was folded onto `Sum` (sum of values, not constant-1 presence). Now rejected. - `offset` / `@`: were silently dropped by vs_parts, changing the query's meaning. Now rejected (no intent-algebra representation). `rate` ≡ `irate` is kept as an intentional intent-level equivalence (the avg-vs-last-two-samples difference is an L4 estimation method, not an L3 intent) and documented as such. Conformance suite updated to match: the formerly-silent `group`/`offset`/`@` GAP tests now assert clean rejection; `changes`/`resets` added to the rejected list; the sum-by key-order expectation is normalised. Tests: 22 core + 27 lowering + 32 conformance + 10 equivalence pass under --locked; clippy -D warnings and fmt clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/lower/src/promql.rs | 57 ++++++-- crates/lower/tests/promql_conformance.rs | 47 ++++--- crates/lower/tests/promql_equivalence.rs | 172 +++++++++++++++++++++++ 3 files changed, 241 insertions(+), 35 deletions(-) create mode 100644 crates/lower/tests/promql_equivalence.rs diff --git a/crates/lower/src/promql.rs b/crates/lower/src/promql.rs index a22c741d..eeeae41a 100644 --- a/crates/lower/src/promql.rs +++ b/crates/lower/src/promql.rs @@ -18,7 +18,9 @@ //! | `avg/min/max/sum_over_time(m[w])` | `Aggregate{[Avg/Min/Max/Sum], Window{w}}` | //! | `stddev/stdvar_over_time(m[w])` | `Aggregate{[StdDev/Variance], Window{w}}` | //! | `count_over_time(m[w])` | `Aggregate{[Count], Window{w}}` | -//! | `rate/irate/increase(m[w])` | `Aggregate{[Rate{w}/Increase{w}]}` (no Window) | +//! | `rate/irate(m[w])` | `Aggregate{[Rate{w}]}` (no Window) — `irate` shares the `rate` *intent*; the avg-vs-last-two-samples difference is an L4 estimation method | +//! | `increase(m[w])` | `Aggregate{[Increase{w}]}` (no Window) | +//! | `changes` / `resets` / `group` / `offset` / `@` | **rejected** — distinct semantics with no intent-algebra representation yet | //! | `OUTER by (dims) (…)` | `Aggregate.keys = dims` (→ `Partition` in L3) | //! | `count by (d) (…)` | `Aggregate{[CountDistinct], …}` (→ `Cardinality`) | //! | `topk(k, count_over_time(…))` | `TopK{k, by}` (heavy-hitter, one pass) | @@ -111,11 +113,11 @@ fn walk(expr: &Expr) -> Result { input: Box::new(walk(&sq.expr)?), }), Expr::VectorSelector(vs) => { - let (metric, matchers) = vs_parts(vs); + let (metric, matchers) = vs_parts(vs)?; Ok(filtered_source(metric, matchers)) } Expr::MatrixSelector(ms) => { - let (metric, matchers) = vs_parts(&ms.vs); + let (metric, matchers) = vs_parts(&ms.vs)?; Ok(L2::Window { duration: ms.range, slide: None, @@ -148,8 +150,15 @@ fn walk_aggregate(agg: &AggregateExpr) -> Result { } } else if op == token::T_COUNT { Outer::Count - } else if op == token::T_SUM || op == token::T_GROUP { + } else if op == token::T_SUM { Outer::Plain(OuterIntent::Sum) + } else if op == token::T_GROUP { + // `group(v)` yields a constant 1 per group (presence), not a sum of + // values. Folding it onto `Sum` changed the result; reject until a + // distinct group-presence intent exists. + return Err(LoweringError::UnsupportedAggregateOp( + "`group` (constant-1 presence) is not `sum`; no distinct intent yet".into(), + )); } else if op == token::T_AVG { Outer::Plain(OuterIntent::Avg) } else if op == token::T_MIN { @@ -223,7 +232,7 @@ fn walk_binary(bin: &BinaryExpr) -> Result { fn lower_inner(expr: &Expr) -> Result { match expr { Expr::VectorSelector(vs) => { - let (metric, matchers) = vs_parts(vs); + let (metric, matchers) = vs_parts(vs)?; Ok(Inner { metric, matchers, @@ -232,7 +241,7 @@ fn lower_inner(expr: &Expr) -> Result { }) } Expr::MatrixSelector(ms) => { - let (metric, matchers) = vs_parts(&ms.vs); + let (metric, matchers) = vs_parts(&ms.vs)?; Ok(Inner { metric, matchers, @@ -295,7 +304,10 @@ fn lower_inner_call(call: &Call) -> Result { "sum_over_time" => at0(InnerFunc::Sum), "stddev_over_time" => at0(InnerFunc::StdDev), "stdvar_over_time" => at0(InnerFunc::Variance), - "count_over_time" | "changes" | "resets" => at0(InnerFunc::Count), + "count_over_time" => at0(InnerFunc::Count), + // `changes` (value changes) and `resets` (counter resets) are NOT + // sample counts — aliasing them to `count_over_time` silently produced + // the wrong number. Reject until they have distinct intents. other => Err(LoweringError::UnsupportedFunction(other.to_string())), } } @@ -477,7 +489,14 @@ fn outer_func(o: &OuterIntent) -> AggFunc { fn resolve_group(agg: &AggregateExpr) -> Result> { match &agg.modifier { None => Ok(vec![]), - Some(LabelModifier::Include(ls)) => Ok(ls.labels.clone()), + Some(LabelModifier::Include(ls)) => { + // Grouping labels are a set: `by (a, b)` ≡ `by (b, a)`. Canonicalise + // so equivalent groupings lower to identical keys. + let mut keys = ls.labels.clone(); + keys.sort(); + keys.dedup(); + Ok(keys) + } Some(LabelModifier::Exclude(_)) => Err(LoweringError::UnsupportedFeature( "`without(...)` grouping requires a registry-backed catalog of the \ metric's label set (the usage-derived schema can't enumerate the \ @@ -489,7 +508,15 @@ fn resolve_group(agg: &AggregateExpr) -> Result> { // ── Free helpers ────────────────────────────────────────────────────────────── -fn vs_parts(vs: &VectorSelector) -> (String, Vec) { +fn vs_parts(vs: &VectorSelector) -> Result<(String, Vec)> { + // `offset` / `@` shift the evaluation/lookback time. The intent algebra has + // no representation for either, so silently lowering them (as if absent) + // would change the query's meaning. Reject rather than mislower. + if vs.offset.is_some() || vs.at.is_some() { + return Err(LoweringError::UnsupportedFeature( + "`offset` / `@` time-shift modifiers have no intent-algebra representation".into(), + )); + } let metric = vs.name.clone().unwrap_or_else(|| { vs.matchers .matchers @@ -498,14 +525,18 @@ fn vs_parts(vs: &VectorSelector) -> (String, Vec) { .map(|m| m.value.clone()) .unwrap_or_default() }); - let matchers = vs + // Label matchers are an unordered set: `{a="1",b="2"}` and `{b="2",a="1"}` + // select the same series. Canonicalise by (name, value) so equivalent + // selectors lower to identical predicates. + let mut ms: Vec<&Matcher> = vs .matchers .matchers .iter() .filter(|m| m.name != "__name__") - .map(matcher_to_l3expr) .collect(); - (metric, matchers) + ms.sort_by(|a, b| a.name.cmp(&b.name).then_with(|| a.value.cmp(&b.value))); + let matchers = ms.into_iter().map(matcher_to_l3expr).collect(); + Ok((metric, matchers)) } fn matcher_to_l3expr(m: &Matcher) -> L3Expr { @@ -525,7 +556,7 @@ fn matcher_to_l3expr(m: &Matcher) -> L3Expr { fn extract_matrix(expr: &Expr) -> Result<(String, Vec, Duration)> { match expr { Expr::MatrixSelector(ms) => { - let (metric, matchers) = vs_parts(&ms.vs); + let (metric, matchers) = vs_parts(&ms.vs)?; Ok((metric, matchers, ms.range)) } Expr::Paren(p) => extract_matrix(&p.expr), diff --git a/crates/lower/tests/promql_conformance.rs b/crates/lower/tests/promql_conformance.rs index 06b438ef..49dc8d37 100644 --- a/crates/lower/tests/promql_conformance.rs +++ b/crates/lower/tests/promql_conformance.rs @@ -20,10 +20,13 @@ //! //! Legend used in test names: //! - (no suffix) — we lower it and the L3 intent matches PromQL. -//! - `__rejected` — we cleanly return an error (capability not implemented). -//! - `__GAP` — we lower it but the result **diverges** from PromQL -//! semantics (silent loss or over-aggregation). Pinned so a future fix -//! flips the assertion deliberately. +//! - `__GAP` — a PromQL capability we don't *yet* support. It is **cleanly +//! rejected** (never silently mislowered), and pinned here so adding support +//! later flips the assertion deliberately. +//! +//! NOTE: the formerly-silent divergences (`group`→sum, dropped `offset`/`@`, +//! `changes`/`resets`→count) are now rejected rather than mislowered — see the +//! equivalence suite (`promql_equivalence.rs`) and section L below. // `__GAP`-suffixed test names intentionally SHOUT the documented divergences. #![allow(non_snake_case)] @@ -207,14 +210,15 @@ fn sum_collapses_all_series() { #[test] fn sum_by_preserves_dimensions_as_partition() { - // SEMANTICS: `by(job,instance)` keeps those labels; grouping rides on Partition. + // SEMANTICS: `by(job,instance)` keeps those labels; grouping rides on + // Partition. Keys are canonicalised (sorted), so order is normalised. let qe = ok("sum by(job, instance) (node_filesystem_size_bytes)"); let QueryExpr::Partition { keys, .. } = &qe else { panic!("expected Partition for `by(...)`, got {qe:?}"); }; assert_eq!( *keys, - PartitionKeys::By(vec!["job".into(), "instance".into()]) + PartitionKeys::By(vec!["instance".into(), "job".into()]) ); assert!(has(&qe, |i| matches!(i, AggIntent::Sum))); } @@ -256,13 +260,10 @@ fn sum_without_is_rejected() { } #[test] -fn group_aggregator_is_lowered_as_sum__GAP() { +fn group_aggregator_is_rejected() { // SEMANTICS (PromQL): `group(v)` returns a constant 1 per group (presence), - // NOT a sum. We currently fold it onto `Sum`. Pinned as a known divergence. - assert!(has(&ok("group by (job) (up)"), |i| matches!( - i, - AggIntent::Sum - ))); + // NOT a sum. Rather than fold it onto `Sum` (wrong value) we reject it. + let _ = rejected("group by (job) (up)"); } // ───────────────────────────────────────────────────────────────────────────── @@ -495,19 +496,18 @@ fn over_time_of_subquery_is_rejected__GAP() { // ───────────────────────────────────────────────────────────────────────────── #[test] -fn offset_modifier_is_silently_dropped__GAP() { - // SEMANTICS (PromQL): `offset 5m` shifts the lookback 5m into the past. - // `vs_parts` reads only name + matchers, so the offset is lost: the query - // lowers as if there were no offset. Pinned so the silent loss is visible. - let (metric, _) = first_scan(&ok("http_requests_total offset 5m")); - assert_eq!(metric, "http_requests_total"); +fn offset_modifier_is_rejected() { + // SEMANTICS (PromQL): `offset 5m` shifts the lookback 5m into the past. The + // intent algebra can't represent it, so we reject rather than silently drop + // it (which would change the query's meaning). + let _ = rejected("http_requests_total offset 5m"); } #[test] -fn at_modifier_is_silently_dropped__GAP() { - // SEMANTICS (PromQL): `@ ` pins the evaluation time. Also dropped today. - let (metric, _) = first_scan(&ok("http_requests_total @ 1609746000")); - assert_eq!(metric, "http_requests_total"); +fn at_modifier_is_rejected() { + // SEMANTICS (PromQL): `@ ` pins the evaluation time. Rejected for the + // same reason as `offset`. + let _ = rejected("http_requests_total @ 1609746000"); } // ───────────────────────────────────────────────────────────────────────────── @@ -528,6 +528,9 @@ fn unsupported_functions_are_rejected() { "predict_linear(demo_disk_usage_bytes[4h], 3600)", r#"label_replace(up, "host", "$1", "instance", "(.+):.*")"#, "clamp_max(go_goroutines, 5)", + // changes / resets are NOT sample counts (formerly aliased to Count). + "changes(demo_disk_usage_bytes[1h])", + "resets(http_requests_total[1h])", ] { let _ = rejected(q); } diff --git a/crates/lower/tests/promql_equivalence.rs b/crates/lower/tests/promql_equivalence.rs new file mode 100644 index 00000000..b8423a9e --- /dev/null +++ b/crates/lower/tests/promql_equivalence.rs @@ -0,0 +1,172 @@ +//! PromQL **semantic-equivalence proving** for the L1→L3 lowering. +//! +//! The lowering is a *normalizer*: it should map a whole class of +//! semantically-equivalent PromQL strings to **one** canonical L3 tree, and +//! must keep semantically-*distinct* queries distinct. This suite proves: +//! +//! 1. Equivalence classes collapse to identical L3 (`assert_equiv`). +//! 2. Distinct meanings stay distinct (`assert_distinct`). +//! 3. The lowering never *wrongly* equates distinct semantics — the cases it +//! cannot faithfully distinguish are **rejected**, not silently merged. +//! +//! Equivalences are grammar/spec-level facts, drawn from: +//! - PromQL basics: +//! - PromLabs cheat sheet: +//! - Prometheus engine tests (operators.test, aggregators.test, selectors.test): +//! + +#![allow(non_snake_case)] + +use asap_control_core::intent_algebra::QueryExpr; +use asap_control_core::types::AccuracyTarget; +use asap_control_lower::lower_promql; + +fn lo(q: &str) -> QueryExpr { + lower_promql(q, AccuracyTarget::Exact).unwrap_or_else(|e| panic!("{q:?} should lower: {e}")) +} + +/// Every member of an equivalence class must lower to the *same* L3 tree. +fn assert_equiv(class: &[&str]) { + let first = lo(class[0]); + for q in &class[1..] { + assert_eq!( + lo(q), + first, + "expected {q:?} ≡ {:?}, but they lowered to different L3", + class[0] + ); + } +} + +/// Two semantically-distinct queries must lower to *different* L3 trees. +fn assert_distinct(a: &str, b: &str) { + assert_ne!( + lo(a), + lo(b), + "{a:?} and {b:?} must not collapse to the same L3" + ); +} + +/// A query whose semantics we can't faithfully represent must be rejected +/// (never silently mislowered into a different meaning). +fn assert_rejected(q: &str) { + assert!( + lower_promql(q, AccuracyTarget::Exact).is_err(), + "{q:?} should be rejected, not silently lowered" + ); +} + +// ───────────────────────────────────────────────────────────────────────────── +// 1. Equivalence classes the lowering canonicalises to one L3. +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn aggregation_modifier_placement_is_equivalent() { + // ` by (..) (expr)` and `(expr) by (..)` are the same query. + assert_equiv(&["sum by (job) (up)", "sum(up) by (job)"]); + assert_equiv(&["count by (instance) (up)", "count(up) by (instance)"]); +} + +#[test] +fn parentheses_are_transparent() { + assert_equiv(&[ + "sum(rate(http_requests_total[5m]))", + "(sum(rate(http_requests_total[5m])))", + "sum((rate(http_requests_total[5m])))", + ]); +} + +#[test] +fn whitespace_is_irrelevant() { + assert_equiv(&[ + "rate(http_requests_total[5m])", + "rate( http_requests_total [5m] )", + "rate(http_requests_total[5m] )", + ]); +} + +#[test] +fn label_matcher_order_is_equivalent() { + // A matcher set is unordered: same series, so same L3 (FIX: predicates are + // now canonicalised by (name, value) at lowering time). + assert_equiv(&[r#"up{job="a",env="prod"}"#, r#"up{env="prod",job="a"}"#]); +} + +#[test] +fn group_key_order_is_equivalent() { + // Grouping labels are a set: `by (a, b)` ≡ `by (b, a)` (FIX: keys sorted). + assert_equiv(&["sum by (instance, job) (up)", "sum by (job, instance) (up)"]); +} + +// ───────────────────────────────────────────────────────────────────────────── +// 2. Distinct semantics must NOT collapse. +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn distinct_semantics_stay_distinct() { + // Outer aggregation matters (the sum(rate) two-level fix). + assert_distinct("sum(rate(m[5m]))", "rate(m[5m])"); + // Window size matters. + assert_distinct("rate(m[5m])", "rate(m[10m])"); + // Operand order matters for non-commutative binary ops. + assert_distinct("a / b", "b / a"); + // Operator identity matters. + assert_distinct("a and b", "a or b"); + // Quantile parameter matters. + assert_distinct( + "quantile_over_time(0.9, m[5m])", + "quantile_over_time(0.5, m[5m])", + ); + // Grouping dimension matters. + assert_distinct("sum by (job) (up)", "sum by (instance) (up)"); + // Aggregator identity matters. + assert_distinct("sum(up)", "avg(up)"); + // Heavy-hitter topk vs generic bottomk are different plans. + assert_distinct( + "topk(5, count_over_time(m[5m]))", + "bottomk(5, count_over_time(m[5m]))", + ); +} + +// ───────────────────────────────────────────────────────────────────────────── +// 3. Intentional intent-level equivalence (documented, not a bug). +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn rate_and_irate_share_the_same_intent() { + // L3 captures *intent* ("per-second rate of a counter"), not the estimation + // method. `rate` (windowed average) and `irate` (last two samples) differ + // only in HOW the rate is estimated — an L4/execution concern — so they + // share one L3 intent by design. + assert_equiv(&["rate(m[5m])", "irate(m[5m])"]); +} + +// ───────────────────────────────────────────────────────────────────────────── +// 4. Distinct semantics we cannot faithfully represent are REJECTED, not +// silently merged into a wrong intent. (Each previously mislowered.) +// ───────────────────────────────────────────────────────────────────────────── + +#[test] +fn changes_and_resets_are_not_count_over_time() { + // PromQL: count_over_time = #samples, changes = #value-changes, + // resets = #counter-resets. They previously all collapsed to `Count`. + assert_rejected("changes(m[5m])"); + assert_rejected("resets(m[5m])"); +} + +#[test] +fn group_is_not_sum() { + // PromQL `group` returns a constant 1 per group; it previously collapsed + // onto `sum` (sum of values). + assert_rejected("group(up)"); + assert_rejected("group by (job) (up)"); +} + +#[test] +fn offset_and_at_are_not_dropped() { + // Time-shift modifiers change the query's meaning; they previously lowered + // identically to the un-shifted query (silent loss). + assert_rejected("http_requests_total offset 5m"); + assert_rejected("http_requests_total @ 1609746000"); + assert_rejected("rate(http_requests_total[5m] offset 1h)"); +} From 87530942d11bc8252fd133cd7fba41d548447737 Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 22 May 2026 10:39:27 -0600 Subject: [PATCH 10/40] test(promql): exhaustive query corpus from the three PromQL sources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the full set of PromQL query strings from all three sources as lowering inputs, plus a data-driven totality test: - tests/data/promql_corpus_docs.txt — verbatim example queries from the Prometheus querying-basics docs and the PromLabs cheat sheet (49 queries). - tests/data/promql_corpus_testdata.txt — every `eval` expression (deduped, source-tagged) from the Prometheus engine test suite, 1823 queries (Apache-2.0, attributed in the file header). - tests/promql_corpus.rs — runs all ~1870 strings through `lower_promql`. The property proven is TOTALITY: for every real-world PromQL string the lowerer returns Ok or a clean Err and never panics (a panic in the loop fails the test). Current breakdown — docs: 29 lowered / 20 rejected; testdata: 530 lowered / 899 cleanly rejected / 394 unparseable (native-histogram syntax etc.). A coverage floor guards against a change silently tanking how much we can lower. No panics found across the entire corpus. Tests: 22 core + 27 lowering + 32 conformance + 10 equivalence + 1 corpus pass under --locked; clippy -D warnings and fmt clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../lower/tests/data/promql_corpus_docs.txt | 86 + .../tests/data/promql_corpus_testdata.txt | 1849 +++++++++++++++++ crates/lower/tests/promql_corpus.rs | 85 + 3 files changed, 2020 insertions(+) create mode 100644 crates/lower/tests/data/promql_corpus_docs.txt create mode 100644 crates/lower/tests/data/promql_corpus_testdata.txt create mode 100644 crates/lower/tests/promql_corpus.rs diff --git a/crates/lower/tests/data/promql_corpus_docs.txt b/crates/lower/tests/data/promql_corpus_docs.txt new file mode 100644 index 00000000..0f9daa7b --- /dev/null +++ b/crates/lower/tests/data/promql_corpus_docs.txt @@ -0,0 +1,86 @@ +# Verbatim example PromQL queries from the official docs and the PromLabs cheat sheet. +# Sources: +# https://prometheus.io/docs/prometheus/latest/querying/basics/ +# https://promlabs.com/promql-cheat-sheet/ +# Used only as lowering INPUT (we do not execute) — see promql_corpus.rs. + +# --- basics: selectors & matchers --- +http_requests_total +http_requests_total{job="prometheus",group="canary"} +http_requests_total{environment=~"staging|testing|development",method!="GET"} +http_requests_total{replica!="rep-a",replica=~"rep.*"} +{__name__=~"job:.*"} +http_requests_total{job="prometheus"}[5m] + +# --- basics: time-shift modifiers & subqueries --- +http_requests_total offset 5m +sum(http_requests_total{method="GET"} offset 5m) +http_requests_total @ 1609746000 +http_requests_total @ 1609746000 offset 5m +rate(http_requests_total[5m])[30m:1m] +rate(http_requests_total[5m])[30m:] + +# --- cheat sheet: selecting series --- +node_cpu_seconds_total +node_cpu_seconds_total[5m] +node_cpu_seconds_total{cpu="0",mode="idle"} + +# --- cheat sheet: rates of increase for counters --- +rate(demo_api_request_duration_seconds_count[5m]) +irate(demo_api_request_duration_seconds_count[1m]) +increase(demo_api_request_duration_seconds_count[1h]) + +# --- cheat sheet: aggregating over multiple series --- +sum(node_filesystem_size_bytes) +sum by(job, instance) (node_filesystem_size_bytes) +sum without(instance, job) (node_filesystem_size_bytes) + +# --- cheat sheet: math between series --- +node_memory_MemFree_bytes + node_memory_Cached_bytes +node_memory_MemFree_bytes + on(instance, job) node_memory_Cached_bytes +rate(demo_cpu_usage_seconds_total[1m]) / on(instance, job) group_left demo_num_cpus + +# --- cheat sheet: filtering series by value --- +node_filesystem_avail_bytes > 10*1024*1024 +go_goroutines > go_threads +go_goroutines > bool on(job, instance) go_threads + +# --- cheat sheet: set operations --- +up{job="prometheus"} or up{job="node"} +node_network_mtu_bytes and (node_network_address_assign_type == 0) +node_network_mtu_bytes unless (node_network_address_assign_type == 1) + +# --- cheat sheet: quantiles from histograms --- +histogram_quantile(0.9, rate(demo_api_request_duration_seconds_bucket[5m])) +histogram_quantile(0.9, sum by(le, path, method) (rate(demo_api_request_duration_seconds_bucket[5m]))) + +# --- cheat sheet: changes in gauges --- +deriv(demo_disk_usage_bytes[1h]) +delta(demo_disk_usage_bytes[1h]) +predict_linear(demo_disk_usage_bytes[4h], 3600) + +# --- cheat sheet: aggregating over time --- +avg_over_time(go_goroutines[5m]) +max_over_time(process_resident_memory_bytes[1d]) +count_over_time(process_resident_memory_bytes[5m]) + +# --- cheat sheet: time --- +time() +time() - demo_batch_last_success_timestamp_seconds +time() - demo_batch_last_success_timestamp_seconds > 3600 + +# --- cheat sheet: dealing with missing data --- +absent(up{job="some-job"}) +absent_over_time(up{job="some-job"}[5m]) + +# --- cheat sheet: manipulating labels --- +label_replace(up, "hostname", "$1", "instance", "(.+):(\\d+)") + +# --- cheat sheet: subqueries --- +rate(demo_api_request_duration_seconds_count[5m])[1h:] +rate(demo_api_request_duration_seconds_count[5m])[1h:15s] +max_over_time(rate(demo_api_request_duration_seconds_count[5m])[1h:]) + +# --- cheat sheet: sorting / top-k --- +topk(3, sum by(method, path) (rate(demo_api_request_duration_seconds_count[5m]))) +sort_desc(sum by(method, path) (rate(demo_api_request_duration_seconds_count[5m]))) diff --git a/crates/lower/tests/data/promql_corpus_testdata.txt b/crates/lower/tests/data/promql_corpus_testdata.txt new file mode 100644 index 00000000..b07c42d9 --- /dev/null +++ b/crates/lower/tests/data/promql_corpus_testdata.txt @@ -0,0 +1,1849 @@ +# PromQL expressions extracted from the Prometheus engine test corpus. +# Source: https://github.com/prometheus/prometheus/tree/main/promql/promqltest/testdata (Apache-2.0). +# Each line is the from an `eval ...` directive; deduped, grouped by source file. +# Used only as lowering INPUT (we do not execute) — see promql_corpus.rs. +# +# --- prometheus testdata: aggregators.test --- +SUM BY (group) (http_requests{job="api-server"}) +SUM BY (group) (((http_requests{job="api-server"}))) +sum by (group) (http_requests{job="api-server"}) +avg by (group) (http_requests{job="api-server"}) +count by (group) (http_requests{job="api-server"}) +sum without (instance) (http_requests{job="api-server"}) +sum by () (http_requests{job="api-server"}) +sum(http_requests{job="api-server"}) +sum without () (http_requests{job="api-server",group="production"}) +sum without (instance) (http_requests{job="api-server"} or foo) +sum(http_requests) by (job) + min(http_requests) by (job) + max(http_requests) by (job) + avg(http_requests) by (job) +sum(sum by (group) (http_requests{job="api-server"})) by (job) +SUM(http_requests) +SUM(http_requests{instance="0"}) BY(job) +SUM(http_requests) BY (job) +SUM(http_requests) BY (job, nonexistent) +COUNT(http_requests) BY (job) +SUM(http_requests) BY (job, group) +AVG(http_requests) BY (job) +MIN(http_requests) BY (job) +MAX(http_requests) BY (job) +abs(-1 * http_requests{group="production",job="api-server"}) +floor(0.004 * http_requests{group="production",job="api-server"}) +ceil(0.004 * http_requests{group="production",job="api-server"}) +round(0.004 * http_requests{group="production",job="api-server"}) +round(-1 * (0.004 * http_requests{group="production",job="api-server"})) +round(0.005 * http_requests{group="production",job="api-server"}) +round(-1 * (0.005 * http_requests{group="production",job="api-server"})) +round(1 + 0.005 * http_requests{group="production",job="api-server"}) +round(-1 * (1 + 0.005 * http_requests{group="production",job="api-server"})) +round(0.0005 * http_requests{group="production",job="api-server"}, 0.1) +round(2.1 + 0.0005 * http_requests{group="production",job="api-server"}, 0.1) +round(5.2 + 0.0005 * http_requests{group="production",job="api-server"}, 0.1) +round(-1 * (5.2 + 0.0005 * http_requests{group="production",job="api-server"}), 0.1) +round(0.025 * http_requests{group="production",job="api-server"}, 5) +round(0.045 * http_requests{group="production",job="api-server"}, 5) +stddev(http_requests) +stddev by (instance)(http_requests) +stdvar(http_requests) +stdvar by (instance)(http_requests) +sum(label_grouping_test) by (a, b) +max(http_requests) +max({job="api-server"}) +max(http_requests_histogram) +min(http_requests) +min({job="api-server"}) +min(http_requests_histogram) +max by (group) (http_requests) +min by (group) (http_requests) +topk(3, http_requests) +topk((3), (http_requests)) +topk(5, http_requests{group="canary",job="app-server"}) +bottomk(3, http_requests) +bottomk(5, http_requests{group="canary",job="app-server"}) +topk by (group) (1, http_requests) +bottomk by (group) (2, http_requests) +bottomk by (group) (2, http_requests{group="production"}) +topk(3, http_requests{job="api-server",group="production"}) +bottomk(3, http_requests{job="api-server",group="production"}) +bottomk(9999999999, http_requests{job="app-server",group="canary"}) +topk(9999999999, http_requests{job="api-server",group="production"}) +topk(scalar(foo), http_requests) +count(topk(scalar(foo), http_requests)) +count(bottomk(scalar(foo), http_requests)) +topk(100, http_requests_histogram) +topk(1, {__name__=~"http_requests(_histogram)?"}) +count(topk(1000, {__name__=~"http_requests(_histogram)?"})) +topk by (instance) (1, {__name__=~"http_requests(_histogram)?"}) +bottomk(100, http_requests_histogram) +bottomk(1, {__name__=~"http_requests(_histogram)?"}) +count(bottomk(1000, {__name__=~"http_requests(_histogram)?"})) +bottomk by (instance) (1, {__name__=~"http_requests(_histogram)?"}) +topk(NaN, non_existent) +limitk(NaN, non_existent) +limit_ratio(NaN, non_existent) +count_values("version", version) +count_values(((("version"))), version) +count_values without (instance)("version", version) +count_values without (instance)("job", version) +count_values by (job, group)("job", version) +count_values("a\xc5z", version) +quantile without(point)(0.8, data) +quantile without(point)(0.2, data) +quantile without(point)(0.8, {__name__=~"data(_histogram)?"}) +quantile(0.8, data_histogram) +quantile without(point)(scalar(foo), data) +quantile without(point)((scalar(foo)), data) +quantile without(point)(NaN, data) +quantile without(point) (scalar(foo), data) +group without(point)(data) +group(foo) +avg(data{test="ten"}) +avg(data{test="inf"}) +avg(data{test="inf2"}) +avg(data{test="inf3"}) +avg(data{test="-inf"}) +avg(data{test="-inf2"}) +avg(data{test="-inf3"}) +avg(data{test="nan"}) +avg(data{test="big"}) +avg(data{test="-big"}) +avg(data{test="bigzero"}) +avg(data) +sum(data{test="ten"}) +sum by (group) (data{test="pos_inf"}) +avg by (group) (data{test="pos_inf"}) +sum by (group) (data{test="neg_inf"}) +avg by (group) (data{test="neg_inf"}) +sum(data{test="inf_inf"}) +avg(data{test="inf_inf"}) +sum by (group) (data{test="nan"}) +avg by (group) (data{test="nan"}) +avg(foo) - 52 +avg(topk(11, foo)) - 52 +avg(topk(10, foo)) - 52 +avg(topk(9, foo)) - 52 +avg(topk(8, foo)) - 52 +avg(foo) == 52 +avg(topk(11, foo)) == 52 +avg(topk(10, foo)) == 52 +avg(topk(9, foo)) == 52 +avg(topk(8, foo)) == 52 +stddev(series) +stdvar(series) +stddev({label="c"}) +stdvar({label="c"}) +stddev by (label) (series) +stdvar by (label) (series) +stddev (series) +stdvar (series) +# --- prometheus testdata: at_modifier.test --- +metric @ 100 +metric @ 100s +metric @ 1m40s +metric @ 100 offset 50s +metric @ 100 offset 50 +metric offset 50s @ 100 +metric offset 50 @ 100 +metric @ 0 offset -50s +metric @ 0 offset -50 +metric offset -50s @ 0 +metric offset -50 @ 0 +-metric @ 100 +---metric @ 100 +metric_ms @ 1.234 +sum_over_time(metric{job="1"}[100s] @ 100) +sum_over_time(metric{job="1"}[100s] @ 100 offset 50s) +sum_over_time(metric{job="1"}[100s] offset 50s @ 100) +sum_over_time(metric{job="1"}[100] @ 100 offset 50) +sum_over_time(metric{job="1"}[100] offset 50s @ 100) +metric{job="1"} @ 50 + metric{job="1"} @ 100 +rate(metric{job="1"}[100s] @ 100) + label_replace(rate(metric{job="2"}[123s] @ 200), "job", "1", "", "") +sum_over_time(metric{job="1"}[100s] @ 100) + label_replace(sum_over_time(metric{job="2"}[100s] @ 100), "job", "1", "", "") +sum_over_time(metric{job="1"}[100] @ 100) + label_replace(sum_over_time(metric{job="2"}[100] @ 100), "job", "1", "", "") +sum_over_time(metric{job="1"}[100s:1s] @ 100) +sum_over_time(metric{job="1"}[100s:1s] @ 100 offset 20s) +sum_over_time(metric{job="1"}[100s:1s] offset 20s @ 100) +sum_over_time(metric{job="1"}[100:1] offset 20 @ 100) +sum_over_time(sum_over_time(metric{job="1"}[100s] @ 100)[100s:25s] @ 50) +sum_over_time(sum_over_time(sum_over_time(metric{job="1"}[100s] @ 100)[100s:25s] @ 50)[3s:1s] @ 3000) +sum_over_time(sum_over_time(sum_over_time(metric{job="1"}[10s])[100s:25s] @ 50)[3s:1s] @ 200) +sum_over_time(sum_over_time(sum_over_time(metric{job="1"}[10s])[100s:25s] @ 200)[3s:1s] @ 50) +sum_over_time(sum_over_time(sum_over_time(metric{job="1"}[20s])[20s:10s] offset 10s)[100s:25s] @ 1000) +minute(metric @ 1500) +timestamp(metric{job="1"} @ 10) +timestamp(timestamp(metric{job="1"} @ 10)) +sum_over_time(minute(metric @ 1500)[100s:10s]) +sum_over_time(minute()[50m:1m] @ 6000) +sum_over_time(minute()[50m:1m] @ 6000 offset 5m) +sum_over_time(vector(time())[100s:1s] @ 3000) +sum_over_time(vector(time())[100s:1s] @ 3000 offset 600s) +sum_over_time(timestamp(metric{job="1"} @ 10)[100s:10s] @ 3000) +sum_over_time(timestamp(timestamp(metric{job="1"} @ 999))[10s:1s] @ 10) +quantile_over_time(scalar(up) + 1, {__name__="up"}[1h:1m] @ 1111111) +predict_linear({__name__="up"}[1h:1m] @ 1111111, 0.1) +deriv({__name__="up"}[1h:1m] @ 1111111) +changes({__name__="up"}[1h:1m] @ 1111111) +resets({__name__="up"}[1h:1m] @ 1111111) +first_over_time({__name__="up"}[1h:1m] @ 1111111) +last_over_time({__name__="up"}[1h:1m] @ 1111111) +sum_over_time({__name__="up"}[1h:1m] @ 1111111) +avg_over_time({__name__="up"}[1h:1m] @ 1111111) +min_over_time({__name__="up"}[1h:1m] @ 1111111) +max_over_time({__name__="up"}[1h:1m] @ 1111111) +count_over_time({__name__="up"}[1h:1m] @ 1111111) +stddev_over_time({__name__="up"}[1h:1m] @ 1111111) +stdvar_over_time({__name__="up"}[1h:1m] @ 1111111) +mad_over_time({__name__="up"}[1h:1m] @ 1111111) +metric @ 11 +abs(metric @ 11) +timestamp(metric) +timestamp(metric @ 11) +timestamp(metric @ 19) +timestamp(metric @ 20) +timestamp(metric_missing @ 0) +timestamp(metric_missing @ 10) +timestamp(metric_missing @ 20) +timestamp(abs(metric @ 11)) +timestamp(abs(metric_missing @ 11)) +# --- prometheus testdata: collision.test --- +count by(namespace, pod, cpu) (node_cpu_seconds_total{cpu=~".*",job="node-exporter",mode="idle",namespace="observability",pod="node-exporter-l454v"}) * on(namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{namespace="observability",pod="node-exporter-l454v"} +ceil({__name__=~'testmetric1|testmetric2'}) +# --- prometheus testdata: duration_expression.test --- +changes(http_requests[30m]) +changes(http_requests[26m+4m]) +changes(http_requests[30m+0s]) +changes(http_requests[1800]) +changes(http_requests[60*30]) +changes(http_requests[2m*15]) +changes(http_requests[2m*(10+5)]) +changes(http_requests[29m+60s]) +changes(http_requests[24m+((1.5*2m)+2m)]) +changes(http_requests[-5m+35m]) +changes(http_requests[1h/2]) +changes(http_requests[1h30m % 1h]) +changes(http_requests[30m1s-30m1s % 1m]) +changes(http_requests[(9m30s+30s)*3]) +sum_over_time(metric1_total[29s+1s:5s+5s]) +sum_over_time(metric1_total[29s+1s:((((8 - 2) / 3) * 7s) % 4) + 8000ms]) +sum_over_time(metric1_total[29s+1s:20*500ms] offset (20*(((((8 - 2) / 3) * 7s) % 4) + 8000ms))) +sum_over_time(metric1_total[29s+1s:20*500ms] offset -(20*(((((8 - 2) / 3) * 7s) % 4) + 8000ms))) +metric1_total offset (100 + 2) +metric1_total offset 100 + 2 +(metric1_total offset 2) ^ 2 +metric1_total offset 2 ^ 2 +metric1_total offset -2 ^ 2 +metric1_total offset (2 ^ 2) +metric1_total offset (2 * 2) +metric1_total offset -2 * 2 +metric1_total offset (-2 * 2) +metric1_total offset -4 +metric1_total offset (-2 ^ 2) +count_over_time(metric1_total[step()]) +count_over_time(metric1_total[step()+1ms]) +count_over_time(metric1_total[(step())+1]) +count_over_time(metric1_total[1+(STep()-5)*2]) +count_over_time(metric1_total[step()+1]) +count_over_time(metric1_total[min_of(step()+1,1h)]) +count_over_time(metric1_total[max_of(min_of(step()+1,1h),1ms)]) +count_over_time(metric1_total[((max_of(min_of((step()+1),((1h))),1ms)))]) +metric1_total offset STEP() +metric1_total offset step() +metric1_total offset step()*0 +metric1_total offset (-step()*2) +metric1_total offset -step()*2 +metric1_total offset step()^0 +metric1_total offset (STEP()/10) +metric1_total offset (step()) +metric1_total offset min_of(step(), 1s) +metric1_total offset min_of(step(), 1s)+8000 +metric1_total offset -min_of(step(), 1s)+8000 +metric1_total offset -(min_of(step(), 1s))+8000 +metric1_total offset -min_of(step(), 1s)^0 +metric1_total offset +min_of(step(), 1s)^0 +metric1_total offset min_of(step(), 1s)^0 +metric1_total offset max_of(3s,min_of(step(), 1s))+8000 +metric1_total offset -(min_of(step(), 2s)-5)+8000 +count_over_time(metric1_total[range()]) +metric1_total offset range() +metric1_total offset min_of(range(), 8s) +# --- prometheus testdata: extended_vectors.test --- +increase(metric[1m]) +increase(metric[1m] anchored) +increase(metric[1m] smoothed) +delta(metric[1m]) +delta(metric[1m] anchored) +increase(metric[5m]) +increase(metric[5m] smoothed) +increase(metric[5m] anchored) +delta(metric[5m] smoothed) +changes(metric[5m]) +changes(metric[5m] anchored) +resets(metric[5m]) +resets(metric[5m] anchored) +changes(metric[1m]) +changes(metric[1m] anchored) +changes(metric[1m1ms] anchored) +resets(metric[1m]) +resets(metric[1m] anchored) +resets(metric[1m1ms] anchored) +increase(metric[2m] smoothed) +rate(metric[10s] smoothed) +deriv(foo[3m] smoothed) +resets(foo[3m] smoothed) +changes(foo[3m] smoothed) +max_over_time(foo[3m] smoothed) +predict_linear(foo[3m] smoothed, 4) +deriv(foo[3m] anchored) +resets(foo[3m] anchored) +changes(foo[3m] anchored) +max_over_time(foo[3m] anchored) +predict_linear(foo[3m] anchored, 4) +metric smoothed +withreset smoothed +notregular smoothed +rate(metric[5s] smoothed) +increase(metric[5s] smoothed) +increase(metric[10s] smoothed) +metric @ 100 smoothed +metric @ 100 smoothed + 0 +metric offset -100 +metric offset -100 smoothed +metric offset -100 smoothed + 0 +histogram_count(rate(hist_counter[1m])) +histogram_count(increase(hist_counter[1m])) +histogram_count(increase(hist_counter[1m] anchored)) +histogram_sum(increase(hist_counter[1m] anchored)) +histogram_count(increase(hist_counter[1m] smoothed)) +histogram_sum(increase(hist_counter[1m] smoothed)) +histogram_count(rate(hist_counter[1m] smoothed)) +histogram_count(delta(hist_counter[1m] smoothed)) +histogram_count(rate(hist_counter[1m] anchored)) +histogram_count(delta(hist_counter[1m] anchored)) +histogram_count(hist_counter smoothed) +rate(mixed_hist[1m] anchored) +rate(mixed_hist[1m] smoothed) +histogram_count(increase(reset_custom_hist[90s])) +histogram_sum(increase(reset_custom_hist[90s])) +histogram_count(increase(reset_custom_hist[90s] anchored)) +histogram_sum(increase(reset_custom_hist[90s] anchored)) +histogram_count(rate(mid_gauge_hist[90s] anchored)) +histogram_count(increase(mid_gauge_hist[90s] smoothed)) +histogram_count(delta(mid_gauge_hist[90s] anchored)) +histogram_count(increase(custom_only[1m] smoothed)) +histogram_sum(increase(custom_only[1m] smoothed)) +histogram_count(reset_middle smoothed) +histogram_count(rate(reset_boundary[1s] smoothed)) +histogram_count(increase(anchored_reset_at_end[70s] anchored)) +histogram_count(increase(anchored_two_sample_reset[30s] anchored)) +histogram_count(increase(smoothed_double_reset[14s] smoothed)) +histogram_count(increase(smoothed_two_sample_both_interp[10s] smoothed)) +histogram_count(smoothed_mix smoothed) +sort(mixed_types smoothed) +histogram_count(rate(right_boundary_reset[10s] smoothed)) +# --- prometheus testdata: fill-modifier.test --- +left_vector + fill(0) right_vector +left_vector + fill_left(0) right_vector +left_vector + fill_right(0) right_vector +left_vector + fill_left(5) fill_right(7) right_vector +left_vector + fill(NaN) right_vector +left_vector + fill(Inf) right_vector +left_vector + fill(-Inf) right_vector +left_vector == fill(30) right_vector +left_vector != fill(30) right_vector +left_vector > fill(25) right_vector +left_vector == bool fill(30) right_vector +left_vector != bool fill(30) right_vector +left_vector > bool fill(25) right_vector +left_vector + on(job, instance) fill(0) right_vector +left_vector + on(job, instance) fill_right(0) right_vector +left_vector + on(job, instance) fill_left(0) right_vector +left_vector + ignoring(job) group_left fill(0) right_vector +requests / on(status) group_left fill_right(1) limits +requests + on(status) group_left fill_left(0) limits +requests + on(status) group_left fill(0) limits +node_meta * on(instance) group_right fill_left(1) cpu_info +node_meta * on(instance) group_right fill_right(0) cpu_info +node_meta * on(instance) group_right fill(1) cpu_info +requests + on(status) group_left(owner) fill_right(0) limits +only_left + fill(0) only_right +only_left + fill_left(0) only_right +only_left + fill_right(0) only_right +complete_left + fill(99) complete_right +range_left + fill(0) range_right +range_left + fill_right(0) range_right +range_left + fill_left(0) range_right +intermittent_left + fill(0) intermittent_right +intermittent_left + fill_right(0) intermittent_right +intermittent_left + fill_left(0) intermittent_right +non_empty + fill_right(0) nonexistent +non_empty + fill_left(0) nonexistent +nonexistent + fill_left(0) non_empty +nonexistent + fill_right(0) non_empty +non_empty + fill(0) nonexistent +nonexistent + fill(0) non_empty +fill + fill(0) other +other + fill +other + fill(0) fill +other + fill_left(0) fill_left +other + fill_right(0) fill_right +# --- prometheus testdata: functions.test --- +resets(http_requests[5m]) +resets(http_requests[10m]) +resets(http_requests[600]) +resets(http_requests[20m]) +resets(http_requests[30m]) +resets(http_requests[32m]) +resets(http_requests[50m]) +resets(nonexistent_metric[50m]) +resets(http_requests_histogram[6m]) +resets(http_requests_histogram[60m]) +changes(http_requests[5m]) +changes(http_requests[6m]) +changes(http_requests[20m]) +changes(http_requests[50m]) +changes((http_requests[50m])) +changes(nonexistent_metric[50m]) +changes(http_requests_histogram[5m]) +changes(http_requests_histogram[6m]) +changes(http_requests_histogram[60m]) +changes(x[20m]) +increase(http_requests_total[50m]) +increase(http_requests_total[100m]) +increase(http_requests_total[30m]) +rate(testcounter_reset_middle_total[50m]) +rate(testcounter_reset_end_total[5m]) +rate(testcounter_reset_end_total[6m]) +rate(calculate_rate_window_total[50m]) +rate(calculate_rate_offset_total[10m] offset 5m) +rate(testcounter_zero_cutoff_total[20m]) +irate(http_requests_total[50m]) +irate(http_requests_nan[15m1s]) +irate(http_requests_histogram{path="/a"}[20m]) +irate(http_requests_histogram{path="/b"}[20m]) +irate(http_requests_histogram{path="/b"}[6m]) +irate(http_requests_histogram{path="/c"}[20m]) +irate(http_requests_histogram{path="/d"}[20m]) +irate(http_requests_histogram{path="/e"}[20m]) +irate(http_requests_histogram{path="/f"}[20m]) +irate(http_requests_histogram{path="/g"}[20m]) +delta(http_requests[20m]) +delta(http_requests_gauge[20m]) +delta(http_requests_counter[20m]) +delta(http_requests_mix[20m]) +idelta(http_requests[20m]) +idelta(http_requests_nan[15m1s]) +idelta(http_requests_histogram{path="/a"}[20m]) +idelta(http_requests_histogram{path="/b"}[20m]) +idelta(http_requests_histogram{path="/b"}[6m]) +idelta(http_requests_histogram{path="/c"}[20m]) +idelta(http_requests_histogram{path="/d"}[20m]) +idelta(http_requests_histogram{path="/e"}[20m]) +idelta(http_requests_histogram{path="/f"}[20m]) +idelta(http_requests_histogram{path="/g"}[20m]) +rate(http_requests_total{group="canary", instance="1", job="app-server"}[50m]) +deriv(http_requests_total{group="canary", instance="1", job="app-server"}[50m]) +deriv(testcounter_reset_middle_total[100m]) +deriv(http_requests_mix{group="canary", instance="1", job="app-server"}[110m]) +deriv(testcounter_reset_middle_mix[110m]) +deriv(http_requests_histogram[60m]) +deriv(http_requests_inf[100m]) +predict_linear(testcounter_reset_middle_total[50m], 3600) +predict_linear(testcounter_reset_middle_total[50m], 1h) +predict_linear(testcounter_reset_middle_total[55m] @ 3000, 3600) +predict_linear(testcounter_reset_middle_total[55m] @ 3000, 1h) +predict_linear(testcounter_reset_middle_mix[60m], 3000) +predict_linear(testcounter_reset_middle_mix[60m], 50m) +predict_linear(http_requests_histogram[60m], 50m) +predict_linear(http_requests_inf[100m], 6000) +predict_linear(http_requests_total[50m], 3600) - (http_requests_total + deriv(http_requests_total[50m]) * 3600) +label_replace(testmetric, "dst", "destination-value-$1", "src", "source-value-(.*)") +label_replace(testmetric, "dst", "destination-value-$1", "src", "value-(.*)") +label_replace(testmetric, "dst", "$1-value-$2", "src", "(.*)-value-(.*)") +label_replace(testmetric, "dst", "value-$1", "nonexistent-src", "source-value-(.*)") +label_replace(testmetric, "dst", "value-$1", "nonexistent-src", "(.*)") +label_replace(testmetric, "dst", "value-$1", "src", "non-matching-regex") +label_replace((((testmetric))), (("dst")), (("value-$1")), (("src")), (("non-matching-regex"))) +label_replace(testmetric, "dst", "", "dst", ".*") +label_replace(testmetric, "dst", "value-$1", "src", "(.*") +label_replace(testmetric, "\xff", "", "src", "(.*)") +label_replace(testmetric, "src", "", "", "") +timestamp(((metric))) +label_join(testmetric, "dst", "-", "src", "src1", "src2") +label_join(testmetric, "dst", "-", "src", "src3", "src1") +label_join(testmetric, "dst", "", "emptysrc", "emptysrc1", "emptysrc2") +label_join(testmetric, "dst", ", ") +label_join(testmetric1, "dst", ", ", "src", "src1", "src2") +label_join(dup, "label", "", "this") +vector(1) +vector(time()) +clamp_max(test_clamp, 75) +clamp_min(test_clamp, -25) +clamp(test_clamp, -25, 75) +clamp_max(clamp_min(test_clamp, -20), 70) +clamp_max((clamp_min(test_clamp, (-20))), (70)) +clamp(test_clamp, 0, NaN) +clamp(test_clamp, NaN, 0) +clamp(test_clamp, 5, -5) +clamp(mixed_metric, 2, 5) +clamp_min(mixed_metric, 2) +clamp_max(mixed_metric, 2) +sgn(test_sgn) +sort(http_requests) +sort_desc(http_requests) +sort_by_label(http_requests, "instance") +sort_by_label(http_requests, "instance", "group") +sort_by_label(http_requests, "group", "instance", "job") +sort_by_label(http_requests, "job", "instance", "group") +sort_by_label_desc(http_requests, "instance") +sort_by_label_desc(http_requests, "instance", "group") +sort_by_label_desc(http_requests, "instance", "group", "job") +sort_by_label(cpu_time_total, "cpu") +sort_by_label(node_uname_info, "instance") +sort_by_label(node_uname_info, "release") +double_exponential_smoothing(http_requests[1m], 0.01, 0.1) +double_exponential_smoothing(http_requests_mix[1m], 0.01, 0.1) +double_exponential_smoothing(http_requests_histogram[1m], 0.01, 0.1) +avg_over_time(metric[10s]) +avg_over_time(metric[20s]) +avg_over_time(metric[1m]) +sum_over_time(metric[1m])/count_over_time(metric[1m]) +avg_over_time(metric2[1m]) +sum_over_time(metric2[1m])/count_over_time(metric2[1m]) +avg_over_time(metric3[1m]) +sum_over_time(metric3[1m])/count_over_time(metric3[1m]) +avg_over_time(metric4[1m]) +sum_over_time(metric4[1m])/count_over_time(metric4[1m]) +avg_over_time(metric5[1m]) +sum_over_time(metric5[1m])/count_over_time(metric5[1m]) +avg_over_time(metric5b[1m]) +sum_over_time(metric5b[1m])/count_over_time(metric5b[1m]) +avg_over_time(metric5c[1m]) +sum_over_time(metric5c[1m])/count_over_time(metric5c[1m]) +avg_over_time(metric6[1m]) +sum_over_time(metric6[1m])/count_over_time(metric6[1m]) +avg_over_time(metric6b[1m]) +sum_over_time(metric6b[1m])/count_over_time(metric6b[1m]) +avg_over_time(metric6c[1m]) +sum_over_time(metric6c[1m])/count_over_time(metric6c[1m]) +avg_over_time(metric7[1m]) +sum_over_time(metric7[1m])/count_over_time(metric7[1m]) +avg_over_time(metric8[1m]) +sum_over_time(metric8[1m])/count_over_time(metric8[1m]) +avg_over_time(metric9[1m]) +sum_over_time(metric9[1m])/count_over_time(metric9[1m]) +avg_over_time(metric10[1m]) +sum_over_time(metric10[1m])/count_over_time(metric10[1m]) +avg_over_time(metric11[1m]) +sum_over_time(metric11[1m])/count_over_time(metric11[1m]) +sum_over_time(metric12[1m]) +avg_over_time(metric12[1m]) +sum_over_time(metric13[1m]) +avg_over_time(metric13[1m]) +sum_over_time(metric13[1m])/count_over_time(metric13[1m]) +sum_over_time(metric[2m]) +avg_over_time(metric[2m]) +avg_over_time(metric1[1m]) +avg_over_time(foo[100s]) - 52 +avg_over_time(foo[110s]) - 52 +avg_over_time(foo[120s]) - 52 +avg_over_time(foo[130s]) - 52 +avg_over_time(foo[100s]) == 52 +avg_over_time(foo[110s]) == 52 +avg_over_time(foo[120s]) == 52 +avg_over_time(foo[130s]) == 52 +sum_over_time(metric[1000ms]) +sum_over_time(metric[1001ms]) +sum_over_time(metric[1002ms]) +sum_over_time(metric[1003ms]) +sum_over_time(metric[2000ms]) +sum_over_time(metric[2001ms]) +sum_over_time(metric[2002ms]) +sum_over_time(metric[2003ms]) +sum_over_time(metric[3000ms]) +sum_over_time(metric[3001ms]) +sum_over_time(metric[3002ms]) +sum_over_time(metric[3003ms]) +stdvar_over_time(metric[2m]) +stddev_over_time(metric[2m]) +stddev_over_time((metric[2m])) +stddev_over_time(metric_histogram{type="only_histogram"}[2m]) +stddev_over_time(metric_histogram{type="mix"}[2m]) +stdvar_over_time(metric_histogram{type="only_histogram"}[2m]) +stdvar_over_time(metric_histogram{type="mix"}[2m]) +stdvar_over_time(metric[1m]) +stddev_over_time(metric[1m]) +mad_over_time(metric[70s]) +mad_over_time(metric_histogram{type="only_histogram"}[70s]) +mad_over_time(metric_histogram{type="mix"}[70s]) +ts_of_min_over_time(metric[90s]) +ts_of_max_over_time(metric[90s]) +ts_of_last_over_time(metric[90s]) +ts_of_last_over_time(metric_histogram{type="only_histogram"}[90s]) +ts_of_last_over_time(metric_histogram{type="mix"}[90s]) +ts_of_first_over_time(metric[90s]) +ts_of_first_over_time(metric_histogram{type="only_histogram"}[90s]) +ts_of_first_over_time(metric_histogram{type="mix"}[90s]) +quantile_over_time(0, data[2m]) +quantile_over_time(0.5, data[2m]) +quantile_over_time(0.75, data[2m]) +quantile_over_time(0.8, data[2m]) +quantile_over_time(1, data[2m]) +quantile_over_time(-1, data[2m]) +quantile_over_time(2, data[2m]) +(quantile_over_time(2, (data[2m]))) +quantile_over_time(0.5, data_histogram{test="only histogram samples"}[2m]) +quantile_over_time(0.5, data_histogram{test="mix samples"}[2m]) +year() +time() +year(vector(1136239445)) +month() +month(vector(1136239445)) +day_of_month() +day_of_month(vector(1136239445)) +day_of_year() +day_of_year(vector(1136239445)) +day_of_week() +day_of_week(vector(1136239445)) +hour() +hour(vector(1136239445)) +minute() +minute(vector(1136239445)) +year(vector(1230767999)) +year(vector(1230768000)) +month(vector(1456790399)) + day_of_month(vector(1456790399)) / 100 +month(vector(1456790400)) + day_of_month(vector(1456790400)) / 100 +day_of_year(vector(1483191420)) +day_of_year(vector(1672493820)) +days_in_month(vector(1454284800)) +days_in_month(vector(1485907200)) +day_of_month(histogram_sample) +day_of_week(histogram_sample) +day_of_year(histogram_sample) +days_in_month(histogram_sample) +hour(histogram_sample) +minute(histogram_sample) +month(histogram_sample) +year(histogram_sample) +changes({__name__=~'testmetric1|testmetric2'}[5m]) +min_over_time(data[2m]) +min_over_time(data_histogram{type="only_histogram"}[2m]) +min_over_time(data_histogram{type=~"mix_samples.*"}[2m]) +min_over_time(data_sparse[2m]) +max_over_time(data[2m]) +max_over_time(data_histogram{type="only_histogram"}[2m]) +max_over_time(data_histogram{type=~"mix_samples.*"}[2m]) +max_over_time(data_sparse[2m]) +last_over_time({__name__=~"data(_histogram|_sparse|_empty)?"}[2m]) +first_over_time({__name__=~"data(_histogram|_sparse|_empty)?"}[2m]) +count_over_time({__name__=~"data(_histogram|_sparse|_empty)?"}[2m]) +abs(data) +ceil(data) +floor(data) +round(data) +absent(nonexistent) +absent(nonexistent{job="testjob", instance="testinstance", method=~".x"}) +absent(nonexistent{job="testjob",job="testjob2",foo="bar"}) +absent(nonexistent{job="testjob",job="testjob2",job="three",foo="bar"}) +absent(nonexistent{job="testjob",job=~"testjob2",foo="bar"}) +absent(http_requests) +absent(sum(http_requests)) +absent(http_requests_histogram) +absent(sum(http_requests_histogram)) +absent(sum(nonexistent{job="testjob", instance="testinstance"})) +absent(max(nonexistent)) +absent(nonexistent > 1) +absent(a + b) +absent(a and b) +absent(rate(nonexistent[5m])) +absent_over_time(http_requests_total[5m]) +absent_over_time(http_requests_total{handler="/foo"}[5m]) +absent_over_time(http_requests_total{handler!="/foo"}[5m]) +absent_over_time(http_requests_total{handler="/foo", handler="/bar", handler="/foobar"}[5m]) +absent_over_time(rate(nonexistent[5m])[5m:]) +absent_over_time(http_requests_total{handler="/foo", handler="/bar", instance="127.0.0.1"}[5m]) +absent_over_time(rate(http_requests_total[5m])[5m:1m]) +absent_over_time(httpd_log_lines_total[30s]) +absent_over_time(http_requests_total[10m]) +absent_over_time(http_requests_total[6m]) +absent_over_time(http_requests_total[16m]) +absent_over_time(httpd_handshake_failures_total[1m]) +absent_over_time(httpd_handshake_failures_total[2m]) +absent_over_time({instance="127.0.0.1"}[5m]) +absent_over_time({instance="127.0.0.1"}[20m]) +absent_over_time({job="grok"}[20m]) +absent_over_time({instance="127.0.0.1"}[5m:5s]) +absent_over_time({job="ingress"}[4m]) +absent_over_time(http_requests_histogram[5m]) +absent_over_time(rate(http_requests_histogram[5m])[5m:1m]) +present_over_time(http_requests_total[5m]) +present_over_time(http_requests_total{handler="/foo"}[5m]) +present_over_time(http_requests_total{handler!="/foo"}[5m]) +present_over_time(http_requests_total{handler="/foo", handler="/bar", handler="/foobar"}[5m]) +present_over_time(rate(nonexistent[5m])[5m:]) +present_over_time(http_requests_total{handler="/foo", handler="/bar", instance="127.0.0.1"}[5m]) +present_over_time(rate(http_requests_total[5m])[5m:1m]) +present_over_time(httpd_log_lines_total[30s]) +present_over_time(http_requests_total[10m]) +present_over_time(http_requests_total[6m]) +present_over_time(http_requests_total[16m]) +present_over_time(httpd_handshake_failures_total[1m]) +present_over_time({instance="127.0.0.1"}[5m]) +present_over_time({job="grok"}[20m]) +present_over_time({instance="127.0.0.1"}[5m:5s]) +present_over_time({job="ingress"}[4m]) +exp(exp_root_log) +exp({__name__=~"exp_root_log(_h)?"}) +exp(exp_root_log - 10) +exp(exp_root_log - 20) +ln(exp_root_log) +ln({__name__=~"exp_root_log(_h)?"}) +ln(exp_root_log - 10) +ln(exp_root_log - 20) +exp(ln(exp_root_log)) +exp(ln({__name__=~"exp_root_log(_h)?"})) +sqrt(exp_root_log) +sqrt({__name__=~"exp_root_log(_h)?"}) +log2(exp_root_log) +log2({__name__=~"exp_root_log(_h)?"}) +log2(exp_root_log - 10) +log2(exp_root_log - 20) +log10(exp_root_log) +log10({__name__=~"exp_root_log(_h)?"}) +log10(exp_root_log - 10) +log10(exp_root_log - 20) +round(mixed_metric) +scalar(metric) +scalar({type="histogram"}) +scalar({l="x"}) +label_replace(series, "idx", "replaced", "idx", ".*") +label_join(series, "idx", ",", "label", "label") +label_replace(overlap, "idx", "same", "idx", ".*") +label_join(overlap, "idx", ",", "label", "label") +step() +range() +vector(step()) +vector(range()) +metric * step() +metric + range() +start() +end() +vector(start()) +vector(end()) +end() - start() +(end() + start()) / 2 +start() + range() +end() - range() +metric_for_at @ start() +metric_for_at @ end() +min_of(3, 5) +min_of(5, 3) +max_of(3, 5) +max_of(5, 3) +min_of(4, 4) +max_of(4, 4) +min_of(-2, -5) +max_of(-2, -5) +min_of(0, 1) +max_of(0, 1) +min_of(NaN, 3) +min_of(3, NaN) +max_of(NaN, 3) +max_of(3, NaN) +# --- prometheus testdata: histograms.test --- +histogram_count(testhistogram3) +testhistogram3_count +histogram_sum(testhistogram3) +testhistogram3_sum +histogram_avg(testhistogram3) +histogram_stddev(testhistogram3) +histogram_stdvar(testhistogram3) +histogram_fraction(0, 4, testhistogram2) +histogram_fraction(0, 4, testhistogram2_bucket) +histogram_fraction(0, 6, testhistogram2) +histogram_fraction(0, 6, testhistogram2_bucket) +histogram_fraction(0, 3.5, testhistogram2) +histogram_fraction(0, 3.5, testhistogram2_bucket) +histogram_fraction(0, 0.2, testhistogram3) +histogram_fraction(0, 0.2, testhistogram3_bucket) +histogram_fraction(0, 0.2, rate(testhistogram3[10m])) +histogram_fraction(0, 0.2, rate(testhistogram3_bucket[10m])) +histogram_fraction(0, 1.5, positive_buckets_lower_falls_in_the_first_bucket_bucket) +histogram_fraction(0, 1.5, positive_buckets_lower_falls_in_the_first_bucket) +histogram_fraction(-4, -2, negative_buckets_lower_falls_in_the_first_bucket_bucket) +histogram_fraction(-4, -2, negative_buckets_lower_falls_in_the_first_bucket) +histogram_fraction(-Inf, -1.5, lower_is_negative_Inf_bucket) +histogram_fraction(-Inf, -1.5, lower_is_negative_Inf) +histogram_fraction(-Inf, +Inf, lower_is_negative_Inf_and_upper_is_positive_Inf__positive_buckets__bucket) +histogram_fraction(-Inf, +Inf, lower_is_negative_Inf_and_upper_is_positive_Inf__positive_buckets_) +histogram_fraction(-Inf, +Inf, lower_is_negative_Inf_and_upper_is_positive_Inf__negative_buckets__bucket) +histogram_fraction(-Inf, +Inf, lower_is_negative_Inf_and_upper_is_positive_Inf__negative_buckets_) +histogram_fraction(4, 5, lower_and_upper_fall_in_last_bucket__positive_buckets__bucket) +histogram_fraction(4, 5, lower_and_upper_fall_in_last_bucket__positive_buckets_) +histogram_fraction(0, 1, lower_and_upper_fall_in_last_bucket__negative_buckets__bucket) +histogram_fraction(0, 1, lower_and_upper_fall_in_last_bucket__negative_buckets_) +histogram_fraction(2, 5, upper_falls_in_last_bucket_bucket) +histogram_fraction(2, 5, upper_falls_in_last_bucket) +histogram_fraction(400, +Inf, upper_is_positive_Inf_bucket) +histogram_fraction(400, +Inf, upper_is_positive_Inf) +histogram_fraction(2, 2, lower_equals_upper_bucket) +histogram_fraction(2, 2, lower_equals_upper) +histogram_fraction(3, 2, lower_greater_than_upper_bucket) +histogram_fraction(3, 2, lower_greater_than_upper) +histogram_fraction(0, 1, single_bucket_bucket) +histogram_fraction(0, 1, single_bucket) +histogram_fraction(0, 5, all_zero_counts_bucket) +histogram_fraction(0, 5, all_zero_counts) +histogram_fraction(2, 3.5, lower_exactly_on_bucket_boundary_bucket) +histogram_fraction(2, 3.5, lower_exactly_on_bucket_boundary) +histogram_fraction(0.5, 2, upper_exactly_on_bucket_boundary_bucket) +histogram_fraction(0.5, 2, upper_exactly_on_bucket_boundary) +histogram_fraction(1, 3, both_bounds_exactly_on_bucket_boundaries_bucket) +histogram_fraction(1, 3, both_bounds_exactly_on_bucket_boundaries) +histogram_fraction(0.1, 0.75, fractional_bucket_bounds_bucket) +histogram_fraction(0.1, 0.75, fractional_bucket_bounds) +histogram_fraction(-1, 1, range_crosses_zero_bucket) +histogram_fraction(-1, 1, range_crosses_zero) +histogram_fraction(NaN, 1, lower_is_NaN_bucket) +histogram_fraction(NaN, 1, lower_is_NaN) +histogram_fraction(0, NaN, upper_is_NaN_bucket) +histogram_fraction(0, NaN, upper_is_NaN) +histogram_fraction(-10, -5, range_entirely_below_all_buckets_bucket) +histogram_fraction(-10, -5, range_entirely_below_all_buckets) +histogram_fraction(5, 10, range_entirely_above_all_buckets_bucket) +histogram_fraction(5, 10, range_entirely_above_all_buckets) +testhistogram3_bucket{le=".2"} / ignoring(le) testhistogram3_count +rate(testhistogram3_bucket{le=".2"}[10m]) / ignoring(le) rate(testhistogram3_count[10m]) +histogram_quantile(0, testhistogram3) +histogram_quantile(0, testhistogram3_bucket) +histogram_quantile(0.25, testhistogram3) +histogram_quantile(0.25, testhistogram3_bucket) +histogram_quantile(0.5, testhistogram3) +histogram_quantile(0.5, testhistogram3_bucket) +histogram_quantile(0.75, testhistogram3) +histogram_quantile(0.75, testhistogram3_bucket) +histogram_quantile(1, testhistogram3) +histogram_quantile(1, testhistogram3_bucket) +histogram_quantiles(testhistogram3, "q", 0, 0.25, 0.5, 0.75, 1) +histogram_quantiles(testhistogram3_bucket, "q", 0, 0.25, 0.5, 0.75, 1) +histogram_quantiles(testhistogram3, "start", 0, 0.25, 0.5, 0.75, 1) +histogram_quantiles(testhistogram3_bucket, "start", 0, 0.25, 0.5, 0.75, 1) +histogram_quantile(-0.1, testhistogram) +histogram_quantile(-0.1, testhistogram_bucket) +histogram_quantiles(testhistogram, "q", -0.1) +histogram_quantiles(testhistogram_bucket, "q", -0.1) +histogram_quantile(1.01, testhistogram) +histogram_quantile(1.01, testhistogram_bucket) +histogram_quantiles(testhistogram, "q", 1.01) +histogram_quantiles(testhistogram_bucket, "q", 1.01) +histogram_quantile(NaN, testhistogram) +histogram_quantile(NaN, testhistogram_bucket) +histogram_quantiles(testhistogram, "q", NaN) +histogram_quantiles(testhistogram_bucket, "q", NaN) +histogram_quantile(NaN, non_existent) +histogram_quantiles(non_existent, "q", NaN) +histogram_quantile(0, testhistogram) +histogram_quantile(0, testhistogram_bucket) +histogram_quantile(1, testhistogram) +histogram_quantile(1, testhistogram_bucket) +histogram_quantile(0.2, testhistogram) +histogram_quantile(0.2, testhistogram_bucket) +histogram_quantile(0.5, testhistogram) +histogram_quantile(0.5, testhistogram_bucket) +histogram_quantile(0.8, testhistogram) +histogram_quantile(0.8, testhistogram_bucket) +histogram_quantile(0.2, rate(testhistogram[10m])) +histogram_quantile(0.2, rate(testhistogram_bucket[10m])) +histogram_quantile(0.5, rate(testhistogram[10m])) +histogram_quantile(0.5, rate(testhistogram_bucket[10m])) +histogram_quantile(0.8, rate(testhistogram[10m])) +histogram_quantile(0.8, rate(testhistogram_bucket[10m])) +histogram_quantile(1./6., testhistogram2) +histogram_quantile(1./6., testhistogram2_bucket) +histogram_quantile(0.5, testhistogram2) +histogram_quantile(0.5, testhistogram2_bucket) +histogram_quantile(5./6., testhistogram2) +histogram_quantile(5./6., testhistogram2_bucket) +histogram_quantile(1./6., rate(testhistogram2[15m])) +histogram_quantile(1./6., rate(testhistogram2_bucket[15m])) +histogram_quantile(0.5, rate(testhistogram2[15m])) +histogram_quantile(0.5, rate(testhistogram2_bucket[15m])) +histogram_quantile(5./6., rate(testhistogram2[15m])) +histogram_quantile(5./6., rate(testhistogram2_bucket[15m])) +histogram_quantile(0.3, sum(rate(request_duration_seconds[10m]))) +histogram_quantile(0.3, sum(rate(request_duration_seconds_bucket[10m])) by (le)) +histogram_quantile(0.5, sum(rate(request_duration_seconds[10m]))) +histogram_quantile(0.5, sum(rate(request_duration_seconds_bucket[10m])) by (le)) +histogram_quantile(0.3, avg(rate(request_duration_seconds[10m]))) +histogram_quantile(0.3, avg(rate(request_duration_seconds_bucket[10m])) by (le)) +histogram_quantile(0.5, avg(rate(request_duration_seconds[10m]))) +histogram_quantile(0.5, avg(rate(request_duration_seconds_bucket[10m])) by (le)) +histogram_quantile(0.3, sum(rate(request_duration_seconds[10m])) by (instance)) +histogram_quantile(0.3, sum(rate(request_duration_seconds_bucket[10m])) by (le, instance)) +histogram_quantile(0.5, sum(rate(request_duration_seconds[10m])) by (instance)) +histogram_quantile(0.5, sum(rate(request_duration_seconds_bucket[10m])) by (le, instance)) +histogram_quantile(0.3, sum(rate(request_duration_seconds[10m])) by (job)) +histogram_quantile(0.3, sum(rate(request_duration_seconds_bucket[10m])) by (le, job)) +histogram_quantile(0.5, sum(rate(request_duration_seconds[10m])) by (job)) +histogram_quantile(0.5, sum(rate(request_duration_seconds_bucket[10m])) by (le, job)) +histogram_quantile(0.3, sum(rate(request_duration_seconds[10m])) by (job, instance)) +histogram_quantile(0.3, sum(rate(request_duration_seconds_bucket[10m])) by (le, job, instance)) +histogram_quantile(0.5, sum(rate(request_duration_seconds[10m])) by (job, instance)) +histogram_quantile(0.5, sum(rate(request_duration_seconds_bucket[10m])) by (le, job, instance)) +histogram_quantile(0.3, rate(request_duration_seconds[10m])) +histogram_quantile(0.3, rate(request_duration_seconds_bucket[10m])) +histogram_quantile(0.5, rate(request_duration_seconds[10m])) +histogram_quantile(0.5, rate(request_duration_seconds_bucket[10m])) +sum(request_duration_seconds) +sum(request_duration_seconds{job="job1",instance="ins1"} + ignoring(job,instance) request_duration_seconds{job="job1",instance="ins2"} + ignoring(job,instance) request_duration_seconds{job="job2",instance="ins1"} + ignoring(job,instance) request_duration_seconds{job="job2",instance="ins2"}) +avg(request_duration_seconds) +avg (request_duration_seconds_bucket{le="0.1"}) +avg (request_duration_seconds_bucket{le="0.2"}) - avg (request_duration_seconds_bucket{le="0.1"}) +avg (request_duration_seconds_bucket{le="+Inf"}) - avg (request_duration_seconds_bucket{le="0.2"}) +count(request_duration_seconds) +histogram_quantile(0.01, nonmonotonic_bucket) +histogram_quantile(0.5, nonmonotonic_bucket) +histogram_quantile(0.99, nonmonotonic_bucket) +histogram_quantiles(nonmonotonic_bucket, "q", 0.01, 0.5, 0.99) +histogram_quantile(0.5, rate(mixed_bucket[10m])) +histogram_quantile(0.5, rate(mixed[10m])) +histogram_quantile(0.75, rate(mixed_bucket[10m])) +histogram_quantile(1, rate(mixed_bucket[10m])) +histogram_quantile(0.2, rate(empty_bucket[10m])) +histogram_quantile(0.99, {__name__=~"request_duration_seconds\\d*_bucket"}) +histogram_quantiles({__name__=~"request_duration_seconds\\d*_bucket"}, "q", 0.99) +histogram_quantile(0.99, {__name__=~"request_duration_seconds\\d*"}) +histogram_quantiles({__name__=~"request_duration_seconds\\d*"}, "q", 0.99) +rate(const_histogram_bucket[5m]) +rate(const_histogram[5m]) +histogram_quantile(1.0, sum by (le) (rate(const_histogram_bucket[5m]))) +histogram_quantile(1.0, sum(rate(const_histogram[5m]))) +sum_over_time(histogram_over_time[4m:1m]) +avg_over_time(histogram_over_time[4m:1m]) +increase(histogram_with_reset[15m]) +resets(histogram_with_reset[15m]) +histogram_count(increase(histogram_with_reset[15m])) +histogram_sum(increase(histogram_with_reset[15m])) +histogram_quantile(0.8, series) +histogram_quantiles(series, "q", 0.1, 0.2) +histogram_fraction(-Inf, 1, series) +# --- prometheus testdata: info.test --- +info(metric, {data=~".+"}) +info(metric) +info(metric_not_matching_target_info) +info(metric_not_matching_target_info, {data=~".*"}) +info(metric_not_matching_target_info, {data=~".+"}) +info(metric, {non_existent=~".+"}) +info(metric, {non_existent=~".*"}) +info(metric_with_overlapping_label) +info(metric_with_overlapping_label, {data="info"}) +info(metric_with_overlapping_label, {data=~".+"}) +info(metric_with_overlapping_label, {instance="a"}) +info(metric, {__name__="target_info"}) +info(metric, {__name__="non_existent"}) +info(metric, {__name__="non_existent", data=~".+"}) +info(metric, {__name__="build_info"}) +info(metric, {__name__=~".+_info"}) +info(build_info, {__name__=~".+_info", another_data=~".+"}) +info(build_info, {__name__=~".+_info"}) +info(metric, {__name__!~".+_info", data=~".+"}) +info(metric, {__name__!~".+_info", data=~".*"}) +info(metric, {__name__!="target_info"}) +info(build_info, {__name__=~"target_.+", __name__=~".+_info"}) +info({__name__=~"websvc_.+"}, {__name__=~".+_info", __name__!~"websvc_.+"}) +info(metric, {__name__=~"target_.+", __name__=~".+_info"}) +info(metric, {__name__=~".+_info", __name__!~".*build.*"}) +info(metric, {data=~".*"}) +info(metric, {__name__="histogram"}) +info(metric @ 60) +info(metric offset 1m) +info(data_metric, {__name__="info_metric"}) +info({job="work"}, {__name__="info_metric"}) +info(metric, {__name__="custom_info"}) +# --- prometheus testdata: limit.test --- +count(limitk by (group) (0, http_requests)) +count(limitk by (group) (-1, http_requests)) +count(limitk by (group) (1, http_requests) and http_requests) +count(limitk by (group) (2, http_requests) and http_requests) +count(limitk(100, http_requests) and http_requests) +limitk(1, http_requests{instance="histogram_1"}) +limitk(8, http_requests{instance=~"(histogram_2|0)"}) +count(limitk(2, http_requests{instance=~"histogram_[0-9]"})) +count(limitk(1000, http_requests{instance=~"histogram_[0-9]"})) +count(limitk(scalar(foo), http_requests)) +count(limit_ratio(0.0, http_requests)) +count(limitk(2, http_requests) and http_requests) +count(limit_ratio(0.5, http_requests) and http_requests) <= bool (4+1) +count(limit_ratio(0.5, http_requests) and http_requests) >= bool (4-1) +count(limit_ratio(1.0, http_requests) and http_requests) +count(limit_ratio(-1.0, http_requests) and http_requests) +count(limit_ratio(1.1, http_requests) and http_requests) +count(limit_ratio(-1.1, http_requests) and http_requests) +count(limit_ratio(0.2, http_requests) or limit_ratio(-0.8, http_requests)) +count(limit_ratio(0.2, http_requests) and limit_ratio(-0.8, http_requests)) +count(limit_ratio(0.5, http_requests) or limit_ratio(-0.5, http_requests)) +count(limit_ratio(0.5, http_requests) and limit_ratio(-0.5, http_requests)) +count(limit_ratio(0.8, http_requests) or limit_ratio(-0.2, http_requests)) +count(limit_ratio(0.8, http_requests) and limit_ratio(-0.2, http_requests)) +count(limit_ratio(time() % 17/17, http_requests) or limit_ratio( - (1.0 - (time() % 17/17)), http_requests)) +count(limit_ratio(time() % 17/17, http_requests) and limit_ratio( - (1.0 - (time() % 17/17)), http_requests)) +abs(avg(limit_ratio(0.5, http_requests{instance!~"histogram_[0-9]"})) - avg(limit_ratio(-0.5, http_requests{instance!~"histogram_[0-9]"}))) <= bool stddev(http_requests{instance!~"histogram_[0-9]"}) +limit_ratio(1, http_requests{instance="histogram_1"}) +count(limit_ratio(scalar(bar), http_requests)) +# --- prometheus testdata: literals.test --- +12.34e6 +12.34e+6 +12.34e-6 +1+1 +1-1 +1 - -1 +.2 ++0.2 +-0.2e-6 ++Inf +inF +-inf +NaN +nan +2. +1 / 0 +((1) / (0)) +-1 / 0 +0 / 0 +1 % 0 +("Foo") +"Foo" +" Foo " +("") +"" +# --- prometheus testdata: name_label_dropping.test --- +metric_total{env="1"} +-metric_total +metric_total + another_metric_total +metric_total <= another_metric_total +metric_total <= bool another_metric_total +metric_total * 2 +clamp(metric_total, 0, 100) +round(metric_total) +rate(metric_total{env="1"}[10m]) +last_over_time(metric_total{env="1"}[10m]) +first_over_time(metric_total{env="1"}[10m]) +last_over_time(abs(metric_total{env="1"})[10m:]) +max_over_time(metric_total{env="1"}[10m]) +label_replace(rate({env="1"}[10m]), "my_name", "rate_$1", "__name__", "(.+)") +label_replace(rate({env="1"}[10m]), "__name__", "rate_$1", "__name__", "(.+)") +label_join(rate({env="1"}[10m]), "my_name", "_", "__name__") +label_join(rate({env="1"}[10m]), "__name__", "_", "__name__", "env") +sum by (__name__, env) (metric_total{env="1"}) +sum by (__name__) (rate({env="1"}[10m])) +sum(rate({env="1"}[10m])) by (env) +topk(10, sum by (__name__, env) (metric_total{env="1"})) +topk(10, sum by (__name__, env) (rate(metric_total{env="1"}[10m]))) +sum by (__name__) (metric_total{env="1"}) +sum by (__name__) (rate(metric_total{env="2"}[5m])) +label_replace(sum by (__name__) (rate(metric_total{env="2"}[5m])), "__name__", "$1", "__name__", "(.+)") +sum by (__name__) (metric_total{env="1"} or rate(metric_total{env="2"}[5m])) +sum by (__name__) (rate(metric_total{env="2"}[5m]) or metric_total{env="1"}) +sum by (__name__) (metric_total{env="3"} or rate(metric_total{env="2"}[5m])) +sum by (__name__) (rate(metric_total{env="3"}[5m]) or metric_total{env="1"}) +-metric_a or -metric_b +# --- prometheus testdata: native_histograms.test --- +empty_histogram +histogram_count(empty_histogram) +histogram_sum(empty_histogram) +histogram_avg(empty_histogram) +histogram_fraction(-Inf, +Inf, empty_histogram) +histogram_fraction(0, 8, empty_histogram) +histogram_count(single_histogram) +histogram_sum(single_histogram) +histogram_avg(single_histogram) +histogram_fraction(1, 2, single_histogram) +histogram_fraction(0, 8, single_histogram) +histogram_quantile(0.5, single_histogram) +histogram_quantiles(single_histogram, "q", 0.5) +histogram_count(multi_histogram) +histogram_sum(multi_histogram) +histogram_avg(multi_histogram) +histogram_fraction(1, 2, multi_histogram) +histogram_quantile(0.5, multi_histogram) +histogram_count(incr_histogram) +histogram_sum(incr_histogram) +histogram_avg(incr_histogram) +histogram_fraction(1, 2, incr_histogram) +histogram_quantile(0.5, incr_histogram) +incr_histogram +rate(incr_histogram[10m]) +histogram_quantile(0.5, rate(incr_histogram[10m])) +low_res_histogram +histogram_count(low_res_histogram) +histogram_sum(low_res_histogram) +histogram_avg(low_res_histogram) +histogram_fraction(1, 4, low_res_histogram) +histogram_count(single_zero_histogram) +histogram_sum(single_zero_histogram) +histogram_avg(single_zero_histogram) +histogram_fraction(-0.5, 0.5, single_zero_histogram) +histogram_quantile(0.5, single_zero_histogram) +histogram_count(negative_histogram) +histogram_sum(negative_histogram) +histogram_avg(negative_histogram) +histogram_fraction(-2, -1, negative_histogram) +histogram_quantile(0.5, negative_histogram) +histogram_count(two_samples_histogram) +histogram_sum(two_samples_histogram) +histogram_avg(two_samples_histogram) +histogram_fraction(-2, -1, two_samples_histogram) +histogram_quantile(0.5, two_samples_histogram) +histogram_count(balanced_histogram) +histogram_sum(balanced_histogram) +histogram_avg(balanced_histogram) +histogram_fraction(0, 4, balanced_histogram) +histogram_quantile(0.5, balanced_histogram) +histogram_sum(sum(incr_sum_histogram)) +histogram_sum(sum(last_over_time(incr_sum_histogram[5m]))) +rate(histogram_rate[45s]) +histogram_count(histogram_count_sum_2) +histogram_sum(histogram_count_sum_2) +histogram_stddev(histogram_stddev_stdvar_1) +histogram_stdvar(histogram_stddev_stdvar_1) +histogram_stddev(histogram_stddev_stdvar_2) +histogram_stdvar(histogram_stddev_stdvar_2) +histogram_stddev(histogram_stddev_stdvar_3) +histogram_stdvar(histogram_stddev_stdvar_3) +histogram_stddev(histogram_stddev_stdvar_4) +histogram_stdvar(histogram_stddev_stdvar_4) +histogram_stddev(histogram_stddev_stdvar_5) +histogram_stdvar(histogram_stddev_stdvar_5) +histogram_stddev(histogram_stddev_stdvar_6) +histogram_stdvar(histogram_stddev_stdvar_6) +histogram_stddev(histogram_stddev_stdvar_7) +histogram_stdvar(histogram_stddev_stdvar_7) +histogram_quantile(1.001, histogram_quantile_1) +histogram_quantile(1, histogram_quantile_1) +histogram_quantile(0.99, histogram_quantile_1) +histogram_quantile(0.9, histogram_quantile_1) +histogram_quantile(0.6, histogram_quantile_1) +histogram_quantile(0.5, histogram_quantile_1) +histogram_quantile(0.1, histogram_quantile_1) +histogram_quantile(0, histogram_quantile_1) +histogram_quantile(-1, histogram_quantile_1) +histogram_quantile(1.001, histogram_quantile_2) +histogram_quantile(1, histogram_quantile_2) +histogram_quantile(0.99, histogram_quantile_2) +histogram_quantile(0.9, histogram_quantile_2) +histogram_quantile(0.5, histogram_quantile_2) +histogram_quantile(0.1, histogram_quantile_2) +histogram_quantile(0, histogram_quantile_2) +histogram_quantile(-1, histogram_quantile_2) +histogram_quantile(1.001, histogram_quantile_3) +histogram_quantile(1, histogram_quantile_3) +histogram_quantile(0.99, histogram_quantile_3) +histogram_quantile(0.9, histogram_quantile_3) +histogram_quantile(0.7, histogram_quantile_3) +histogram_quantile(0.55, histogram_quantile_3) +histogram_quantile(0.5, histogram_quantile_3) +histogram_quantile(0.45, histogram_quantile_3) +histogram_quantile(0.3, histogram_quantile_3) +histogram_quantile(0.1, histogram_quantile_3) +histogram_quantile(0.01, histogram_quantile_3) +histogram_quantile(0, histogram_quantile_3) +histogram_quantile(-1, histogram_quantile_3) +histogram_quantile(0.5, var_res_histogram) +histogram_fraction(0, 2, var_res_histogram{schema="-1"}) +histogram_fraction(0, 1.4142135623730951, var_res_histogram{schema="0"}) +histogram_fraction(0, 1.189207, var_res_histogram{schema="+1"}) +histogram_fraction(0, 8, var_res_histogram{schema="-1"}) +histogram_fraction(0, 2.82842712474619, var_res_histogram{schema="0"}) +histogram_fraction(0, 1.6817928305074292, var_res_histogram{schema="+1"}) +histogram_fraction(-2, 0, var_res_histogram{schema="-1"}) +histogram_fraction(-1.4142135623730951, 0, var_res_histogram{schema="0"}) +histogram_fraction(-1.189207, 0, var_res_histogram{schema="+1"}) +histogram_fraction(-8, 0, var_res_histogram{schema="-1"}) +histogram_fraction(-2.82842712474619, 0, var_res_histogram{schema="0"}) +histogram_fraction(-1.6817928305074292, 0, var_res_histogram{schema="+1"}) +histogram_fraction(3.1415, 42, histogram_fraction_1) +histogram_fraction(0, +Inf, histogram_fraction_2) +histogram_fraction(-Inf, 0, histogram_fraction_2) +histogram_fraction(-0.001, 0, histogram_fraction_2) +histogram_fraction(0, 0.001, histogram_fraction_2) +histogram_fraction(0.001, inf, histogram_fraction_2) +histogram_fraction(0, 0.0005, histogram_fraction_2) +histogram_quantile(0.08333333333333333, histogram_fraction_2) +histogram_fraction(-inf, -0.001, histogram_fraction_2) +histogram_fraction(1, 2, histogram_fraction_2) +histogram_fraction(0, 1.5, histogram_fraction_2) +histogram_fraction(1.5, 2, histogram_fraction_2) +histogram_fraction(1, 8, histogram_fraction_2) +histogram_fraction(0, 6, histogram_fraction_2) +histogram_quantile(0.6320802083934297, histogram_fraction_2) +histogram_fraction(1, 6, histogram_fraction_2) +histogram_fraction(1.5, 6, histogram_fraction_2) +histogram_fraction(-2, -1, histogram_fraction_2) +histogram_fraction(-2, -1.5, histogram_fraction_2) +histogram_fraction(-8, -1, histogram_fraction_2) +histogram_fraction(-6, -1, histogram_fraction_2) +histogram_fraction(-6, -1.5, histogram_fraction_2) +histogram_fraction(42, 3.1415, histogram_fraction_2) +histogram_fraction(0, 0, histogram_fraction_2) +histogram_fraction(0.000001, 0.000001, histogram_fraction_2) +histogram_fraction(42, 42, histogram_fraction_2) +histogram_fraction(-3.1, -3.1, histogram_fraction_2) +histogram_fraction(3.1415, NaN, histogram_fraction_2) +histogram_fraction(NaN, 42, histogram_fraction_2) +histogram_fraction(NaN, NaN, histogram_fraction_2) +histogram_fraction(-Inf, +Inf, histogram_fraction_2) +histogram_fraction(0, +Inf, histogram_fraction_3) +histogram_fraction(-Inf, 0, histogram_fraction_3) +histogram_fraction(-0.001, 0, histogram_fraction_3) +histogram_fraction(0, 0.001, histogram_fraction_3) +histogram_fraction(-0.0005, 0, histogram_fraction_3) +histogram_fraction(-inf, -0.0005, histogram_fraction_3) +histogram_quantile(0.9166666666666666, histogram_fraction_3) +histogram_fraction(0.001, inf, histogram_fraction_3) +histogram_fraction(-inf, -0.001, histogram_fraction_3) +histogram_fraction(1, 2, histogram_fraction_3) +histogram_fraction(1.5, 2, histogram_fraction_3) +histogram_fraction(1, 8, histogram_fraction_3) +histogram_fraction(1, 6, histogram_fraction_3) +histogram_fraction(1.5, 6, histogram_fraction_3) +histogram_fraction(-2, -1, histogram_fraction_3) +histogram_fraction(-2, -1.5, histogram_fraction_3) +histogram_fraction(-8, -1, histogram_fraction_3) +histogram_fraction(-inf, -6, histogram_fraction_3) +histogram_quantile(0.36791979160657035, histogram_fraction_3) +histogram_fraction(-6, -1, histogram_fraction_3) +histogram_fraction(-6, -1.5, histogram_fraction_3) +histogram_fraction(42, 3.1415, histogram_fraction_3) +histogram_fraction(0, 0, histogram_fraction_3) +histogram_fraction(0.000001, 0.000001, histogram_fraction_3) +histogram_fraction(42, 42, histogram_fraction_3) +histogram_fraction(-3.1, -3.1, histogram_fraction_3) +histogram_fraction(3.1415, NaN, histogram_fraction_3) +histogram_fraction(NaN, 42, histogram_fraction_3) +histogram_fraction(NaN, NaN, histogram_fraction_3) +histogram_fraction(-Inf, +Inf, histogram_fraction_3) +histogram_fraction(0, +Inf, histogram_fraction_4) +histogram_fraction(-Inf, 0, histogram_fraction_4) +histogram_fraction(-0.001, 0, histogram_fraction_4) +histogram_fraction(0, 0.001, histogram_fraction_4) +histogram_fraction(-0.0005, 0.0005, histogram_fraction_4) +histogram_fraction(-inf, 0.0005, histogram_fraction_4) +histogram_quantile(0.5416666666666666, histogram_fraction_4) +histogram_fraction(-inf, -0.0005, histogram_fraction_4) +histogram_quantile(0.4583333333333333, histogram_fraction_4) +histogram_fraction(0.001, inf, histogram_fraction_4) +histogram_fraction(-inf, -0.001, histogram_fraction_4) +histogram_fraction(1, 2, histogram_fraction_4) +histogram_fraction(1.5, 2, histogram_fraction_4) +histogram_fraction(1, 8, histogram_fraction_4) +histogram_fraction(1, 6, histogram_fraction_4) +histogram_fraction(1.5, 6, histogram_fraction_4) +histogram_fraction(-2, -1, histogram_fraction_4) +histogram_fraction(-2, -1.5, histogram_fraction_4) +histogram_fraction(-8, -1, histogram_fraction_4) +histogram_fraction(-6, -1, histogram_fraction_4) +histogram_fraction(-6, -1.5, histogram_fraction_4) +histogram_fraction(42, 3.1415, histogram_fraction_4) +histogram_fraction(0, 0, histogram_fraction_4) +histogram_fraction(0.000001, 0.000001, histogram_fraction_4) +histogram_fraction(42, 42, histogram_fraction_4) +histogram_fraction(-3.1, -3.1, histogram_fraction_4) +histogram_fraction(3.1415, NaN, histogram_fraction_4) +histogram_fraction(NaN, 42, histogram_fraction_4) +histogram_fraction(NaN, NaN, histogram_fraction_4) +histogram_fraction(-Inf, +Inf, histogram_fraction_4) +histogram_sum(scalar(histogram_fraction(-Inf, +Inf, sum(histogram_fraction_4))) * histogram_fraction_4) +histogram_mul_div*3 +histogram_mul_div*-1 +-histogram_mul_div +histogram_mul_div*-3 +3*histogram_mul_div +histogram_mul_div*float_series_3 +float_series_3*histogram_mul_div +histogram_mul_div/3 +histogram_mul_div/-3 +histogram_mul_div/float_series_3 +histogram_mul_div*0 +0*histogram_mul_div +histogram_mul_div*float_series_0 +float_series_0*histogram_mul_div +histogram_mul_div/0 +histogram_mul_div/float_series_0 +histogram_mul_div*0/0 +histogram_mul_div*histogram_mul_div +histogram_mul_div/histogram_mul_div +float_series_3/histogram_mul_div +0/histogram_mul_div +float_sample+histogram_sample +histogram_sample+float_sample +float_sample-histogram_sample +histogram_sample-float_sample +increase(reset_in_bucket[15m]) +histogram_count(increase(reset_in_bucket[15m])) +histogram_sum(increase(reset_in_bucket[15m])) +histogram_fraction(5, 10, custom_buckets_histogram) +histogram_quantile(0.5, custom_buckets_histogram) +sum(custom_buckets_histogram) +rate(some_metric[1m]) +rate(some_metric[1m30s]) +histogram_count(rate(some_metric[1m30s])) +histogram_avg(rate(const_histogram[5m])) +histogram_count(rate(const_histogram[5m])) +histogram_sum(rate(const_histogram[5m])) +histogram_fraction(0.0, 1.0, rate(const_histogram[5m])) +histogram_count(rate(const_histogram[5m])) == 0.0 or histogram_fraction(0.0, 1.0, rate(const_histogram[5m])) * histogram_count(rate(const_histogram[5m])) +histogram_quantile(1.0, rate(const_histogram[5m])) +histogram_stddev(rate(const_histogram[5m])) +histogram_stdvar(rate(const_histogram[5m])) +sum(metric) +avg(metric) +count(metric) +group(metric) +count(limitk(1, metric)) +limitk(3, metric) +limit_ratio(1, metric) +metric{series="1"} and ignoring(series) metric{series="2"} +metric{series="1"} or ignoring(series) metric{series="2"} +metric{series="2"} + ignoring (series) metric{series="3"} +metric{series="2"} - ignoring (series) metric{series="3"} +metric1 == metric2 +metric1 != metric2 +metric2 > metric2 +sum_over_time(nhcb_metric[13m]) +avg_over_time(nhcb_metric[13m]) +last_over_time(nhcb_metric[13m]) +count_over_time(nhcb_metric[13m]) +present_over_time(nhcb_metric[13m]) +changes(nhcb_metric[13m]) +delta(nhcb_metric[13m]) +increase(nhcb_metric[13m]) +rate(nhcb_metric[13m]) +resets(nhcb_metric[13m]) +sum by (group) (metric) +sum(histogram_sum) +sum({idx="0"}) +sum(histogram_sum{idx="0"} + ignoring(idx) histogram_sum{idx="3"}) +count(histogram_sum) +avg(histogram_sum) +avg(histogram_avg_incremental) +sum_over_time(histogram_sum_over_time[4m:1m]) +avg_over_time(histogram_sum_over_time[4m:1m]) +sum_over_time(histogram_sum_over_time_2[8m:1m]) +avg_over_time(histogram_sum_over_time_2[8m:1m]) +sum_over_time(histogram_sum_over_time_3[4m:1m]) +avg_over_time(histogram_sum_over_time_3[4m:1m]) +sum_over_time(histogram_sum_over_time_4[7m:1m]) +avg_over_time(histogram_sum_over_time_4[7m:1m]) +sum_over_time(histogram_sum_over_time_incremental[8m:1m]) +avg_over_time(histogram_sum_over_time_incremental[8m:1m]) +sum_over_time(histogram_sum_over_time_incremental_2[7m:1m]) +avg_over_time(histogram_sum_over_time_incremental_2[7m:1m]) +sum_over_time(histogram_sum_over_time_incremental_3[7m:1m]) +avg_over_time(histogram_sum_over_time_incremental_3[7m:1m]) +sum_over_time(histogram_sum_over_time_incremental_4[7m:1m]) +avg_over_time(histogram_sum_over_time_incremental_4[7m:1m]) +sum_over_time(histogram_sum_over_time_incremental_6[4m:1m]) +avg_over_time(histogram_sum_over_time_incremental_6[4m:1m]) +avg_over_time(single_histogram_sample[1m]) +avg_over_time(single_nhcb_sample[1m]) +histogram_sub_1{idx="0"} - ignoring(idx) histogram_sub_1{idx="1"} +histogram_sub_2{idx="0"} - ignoring(idx) histogram_sub_2{idx="1"} +histogram_sub_3{idx="0"} - ignoring(idx) histogram_sub_3{idx="1"} +last_over_time({__name__="http_request_duration_seconds"} @ start()[1h:1m] offset 1m16s) +histogram_quantile(1, histogram_nan) +histogram_quantile(0.81, histogram_nan) +histogram_quantiles(histogram_nan, "q", 0.81) +histogram_quantile(0.8, histogram_nan{case="100% NaNs"}) +histogram_quantile(0.8, histogram_nan{case="20% NaNs"}) +histogram_quantile(0.4, histogram_nan{case="100% NaNs"}) +histogram_quantile(0.4, histogram_nan{case="20% NaNs"}) +histogram_fraction(-Inf, 0.7071067811865475, histogram_nan) +histogram_fraction(-Inf, +Inf, histogram_nan) +increase(metric[90m]) +increase(metric[55m15s]) +increase(metric[54m45s]) +histogram_count(increase(metric[90m])) +histogram_count(increase(metric[55m15s])) +histogram_count(increase(metric[54m45s])) +-metric +metric - 0.5 * metric +metric - 2 * metric +sum_over_time(mixed[3m]) +avg_over_time(mixed[3m]) +sum_over_time(mixed[10m]) +avg_over_time(mixed[10m]) +sum(metric{type=~"counter.*"}) +avg(metric{type=~"counter.*"}) +histogram_count(sum(metric)) +histogram_count(avg(metric)) +histogram_count(sum_over_time(mixed[10m])) +histogram_count(avg_over_time(mixed[10m])) +histogram_count(sum_over_time(mixed[2m])) +histogram_count(avg_over_time(mixed[2m])) +histogram_quantile(0.5, myHistogram1) +histogram_quantile(0.5, myHistogram2) +histogram_quantile(0.5, mixedHistogram) +histogram_quantiles(mixedHistogram, "q", 0.5) +histogram_count(increase(h[40m:9m])) +increase(h[40m:9m]) +histogram_count(sum_over_time(reset{timing="late"}[5m])) +histogram_count(sum(reset)) +histogram_count(avg(reset)) +histogram_count(rate(reset{timing="late"}[5m])) +histogram_count(histogram unless histogram_quantile(0.5, histogram) < 3) +histogram_quantile(0.5, histogram unless histogram_count(histogram) == 0) +histogram_quantiles(histogram unless histogram_count(histogram) == 0, "q", 0.5) +mixed_metric1 +mixed_metric2 +irate(nhcb_add_buckets[2m]) * 60 +irate(nhcb_remove_buckets[2m]) * 60 +irate(nhcb_add_bucket[2m]) * 60 +h_test >/ -Inf +h_test / +Inf +h_test / 0 +h_test / 1.4142135624 +h_test_2 / 1.13 +h_test_2 >/ -1.3 +h_test_2 / 2 +h_test >/ -1 +h_test / 0.5 +h_positive_buckets >/ 0.1 +h_positive_buckets >/ 0 +h_positive_buckets / -0.5 +h_negative_buckets >/ -0.1 +h_negative_buckets >/ 0 +zero_bucket_only >/ 0.1 +zero_bucket_only / 0.05 +zero_bucket_only / 0 +zero_bucket_only / -0.05 +zero_bucket_only / -0.1 +cbh / 15 +cbh / 13 +cbh / +Inf +cbh / -Inf +cbh >/ 0 +cbh / 0 +cbh_one_bucket / 10.0 +cbh_one_bucket / +Inf +cbh_one_bucket >/ -Inf +cbh_one_bucket / -10.0 +cbh_two_buckets_split_at_zero >/ 0.0 +cbh_two_buckets_split_at_zero >/ 10.0 +cbh_two_buckets_split_at_positive / -10.0 +cbh_two_buckets_split_at_positive >/ 0.0 +cbh_two_buckets_split_at_positive >/ 2.0 +cbh_two_buckets_split_at_positive >/ 10.0 +cbh_two_buckets_split_at_negative / -10.0 +cbh_two_buckets_split_at_negative >/ -2.0 +cbh_two_buckets_split_at_negative >/ 0.0 +cbh_two_buckets_split_at_negative >/ 10.0 +histogram_sum(cbh_two_buckets_split_at_negative >/ 10.0) +histogram_count(cbh_two_buckets_split_at_negative >/ 10.0) +cbh_for_join >/ on (label) float_for_join +empty / -Inf +empty >/ -5 +empty >/ 0 +empty >/ 5 +empty >/ +Inf +histogram_count(h_test / 2) +histogram_fraction(2, +Inf, h_test) * histogram_count(h_test) +histogram_count(h_test / -1) +histogram_fraction(-1, +Inf, h_test) * histogram_count(h_test) +histogram_count(h_test / 0) +histogram_fraction(0, +Inf, h_test) * histogram_count(h_test) +histogram_count(h_test / 1.4142135624) +histogram_fraction(1.4142135624, +Inf, h_test) * histogram_count(h_test) +histogram_count(h_test_2 / 1.13) +histogram_fraction(1.13, +Inf, h_test_2) * histogram_count(h_test_2) +histogram_count(cbh / 15) +histogram_fraction(15, +Inf, cbh) * histogram_count(cbh) +histogram_count(cbh / 13) +histogram_fraction(13, +Inf, cbh) * histogram_count(cbh) +# --- prometheus testdata: operators.test --- +SUM(http_requests_total) BY (job) - COUNT(http_requests_total) BY (job) +2 - SUM(http_requests_total) BY (job) +-http_requests_total{job="api-server",instance="0",group="production"} ++http_requests_total{job="api-server",instance="0",group="production"} +- - - SUM(http_requests_total) BY (job) +- - - 1 +-2^---1*3 +2/-2^---1*3+2 +-10^3 * - SUM(http_requests_total) BY (job) ^ -1 +1000 / SUM(http_requests_total) BY (job) +SUM(http_requests_total) BY (job) - 2 +SUM(http_requests_total) BY (job) % 3 +SUM(http_requests_total) BY (job) % 0.3 +SUM(http_requests_total) BY (job) ^ 2 +SUM(http_requests_total) BY (job) % 3 ^ 2 +SUM(http_requests_total) BY (job) % 2 ^ (3 ^ 2) +SUM(http_requests_total) BY (job) % 2 ^ 3 ^ 2 +SUM(http_requests_total) BY (job) % 2 ^ 3 ^ 2 ^ 2 +COUNT(http_requests_total) BY (job) ^ COUNT(http_requests_total) BY (job) +SUM(http_requests_total) BY (job) / 0 +http_requests_total{group="canary", instance="0", job="api-server"} / 0 +-1 * http_requests_total{group="canary", instance="0", job="api-server"} / 0 +0 * http_requests_total{group="canary", instance="0", job="api-server"} / 0 +0 * http_requests_total{group="canary", instance="0", job="api-server"} % 0 +SUM(http_requests_total) BY (job) + SUM(http_requests_total) BY (job) +(SUM((http_requests_total)) BY (job)) + SUM(http_requests_total) BY (job) +http_requests_total{job="api-server", group="canary"} +http_requests_total{job="api-server", group="canary"} + rate(http_requests_total{job="api-server"}[10m]) * 5 * 60 +rate(http_requests_total[25m]) * 25 * 60 +(rate((http_requests_total[25m])) * 25) * 60 +http_requests_total{group="canary"} and http_requests_total{instance="0"} +(http_requests_total{group="canary"} + 1) and http_requests_total{instance="0"} +(http_requests_total{group="canary"} + 1) and on(instance, job) http_requests_total{instance="0", group="production"} +(http_requests_total{group="canary"} + 1) and on(instance) http_requests_total{instance="0", group="production"} +(http_requests_total{group="canary"} + 1) and ignoring(group) http_requests_total{instance="0", group="production"} +(http_requests_total{group="canary"} + 1) and ignoring(group, job) http_requests_total{instance="0", group="production"} +http_requests_total{group="canary"} or http_requests_total{group="production"} +(http_requests_total{group="canary"} + 1) or http_requests_total{instance="1"} +(http_requests_total{group="canary"} + 1) or on(instance) (http_requests_total or cpu_count or vector_matching_a) +(http_requests_total{group="canary"} + 1) or ignoring(l, group, job) (http_requests_total or cpu_count or vector_matching_a) +http_requests_total{group="canary"} unless http_requests_total{instance="0"} +http_requests_total{group="canary"} unless on(job) http_requests_total{instance="0"} +http_requests_total{group="canary"} unless on(job, instance) http_requests_total{instance="0"} +http_requests_total{group="canary"} / on(instance,job) http_requests_total{group="production"} +http_requests_total{group="canary"} unless ignoring(group, instance) http_requests_total{instance="0"} +http_requests_total{group="canary"} unless ignoring(group) http_requests_total{instance="0"} +http_requests_total{group="canary"} / ignoring(group) http_requests_total{group="production"} +http_requests_total AND ON (dummy) vector(1) +http_requests_total AND IGNORING (group, instance, job) vector(1) +SUM(http_requests_total) BY (job) > 1000 +1000 < SUM(http_requests_total) BY (job) +SUM(http_requests_total) BY (job) <= 1000 +SUM(http_requests_total) BY (job) != 1000 +SUM(http_requests_total) BY (job) == 1000 +SUM(http_requests_total) BY (job) == bool 1000 +SUM(http_requests_total) BY (job) == bool SUM(http_requests_total) BY (job) +SUM(http_requests_total) BY (job) != bool SUM(http_requests_total) BY (job) +0 == bool 1 +1 == bool 1 +http_requests_total{job="api-server", instance="0", group="production"} == bool 100 +{job="app-server"} == 80 +http_requests_histogram != 80 +http_requests_histogram > 80 +http_requests_histogram < 80 +http_requests_histogram >= 80 +http_requests_histogram <= 80 +http_requests_histogram == http_requests_histogram +http_requests_histogram != http_requests_histogram +many_side > on(job) one_side +many_side >= on(job) one_side +many_side <= on(job) one_side +node_role * on (instance) group_right (role) node_var +node_var * on (instance) group_left (role) node_role +node_var * ignoring (role) group_left (role) node_role +node_role * ignoring (role) group_right (role) node_var +node_cpu * ignoring (role, mode) group_left (role) node_role +node_cpu * on (instance) group_left (role) node_role +node_cpu / on (instance) group_left sum by (instance,job)(node_cpu) +sum by (mode, job)(node_cpu) / on (job) group_left sum by (job)(node_cpu) +sum(sum by (mode, job)(node_cpu) / on (job) group_left sum by (job)(node_cpu)) +node_cpu / ignoring (mode) group_left sum without (mode)(node_cpu) +node_cpu / ignoring (mode) group_left(dummy) sum without (mode)(node_cpu) +sum without (instance)(node_cpu) / ignoring (mode) group_left sum without (instance, mode)(node_cpu) +sum(sum without (instance)(node_cpu) / ignoring (mode) group_left sum without (instance, mode)(node_cpu)) +node_cpu + on(dummy) group_left(foo) random*0 +node_cpu > on(job, instance) group_left(target) threshold +node_cpu > on(job, instance) group_left(target) (threshold or on (job, instance) (sum by (job, instance)(node_cpu) * 0 + 1)) +node_cpu + 2 +node_cpu - 2 +node_cpu / 2 +node_cpu * 2 +node_cpu ^ 2 +node_cpu % 2 +random + on() metricA +metricA + ignoring() metricB +metricA + metricB +-{__name__=~'testmetric1|testmetric2'} +test_total > bool test_smaller +test_total > test_smaller +test_total < bool test_smaller +test_total < test_smaller +trigy atan2 trigx +trigy atan2 trigNaN +10 atan2 20 +10 atan2 NaN +left_floats == right_floats +left_floats == bool right_floats +left_floats == does_not_match +left_histograms == right_histograms +left_histograms == bool right_histograms +left_histograms == right_floats_for_histograms +left_histograms == bool right_floats_for_histograms +left_floats != right_floats +left_floats != bool right_floats +left_histograms != right_histograms +left_histograms != bool right_histograms +left_histograms != right_floats_for_histograms +left_histograms != bool right_floats_for_histograms +left_floats > right_floats +left_floats > bool right_floats +left_histograms > right_histograms +left_histograms > bool right_histograms +left_histograms > right_floats_for_histograms +left_histograms > bool right_floats_for_histograms +left_floats >= right_floats +left_floats >= bool right_floats +left_histograms >= right_histograms +left_histograms >= bool right_histograms +left_histograms >= right_floats_for_histograms +left_histograms >= bool right_floats_for_histograms +left_floats < right_floats +left_floats < bool right_floats +left_histograms < right_histograms +left_histograms < bool right_histograms +left_histograms < right_floats_for_histograms +left_histograms < bool right_floats_for_histograms +left_floats <= right_floats +left_floats <= bool right_floats +left_histograms <= right_histograms +left_histograms <= bool right_histograms +left_histograms <= right_floats_for_histograms +left_histograms <= bool right_floats_for_histograms +left_floats == 3 +left_floats != 3 +left_floats > 3 +left_floats >= 3 +left_floats < 3 +left_floats <= 3 +left_floats == bool 3 +left_floats == Inf +left_floats == bool Inf +left_floats == NaN +left_floats == bool NaN +left_histograms == 3 +left_histograms == 0 +left_histograms != 3 +left_histograms != 0 +left_histograms > 3 +left_histograms > 0 +left_histograms >= 3 +left_histograms >= 0 +left_histograms < 3 +left_histograms < 0 +left_histograms <= 3 +left_histograms <= 0 +left_histograms == bool 3 +left_histograms == bool 0 +left_histograms != bool 3 +left_histograms != bool 0 +left_histograms > bool 3 +left_histograms > bool 0 +left_histograms >= bool 3 +left_histograms >= bool 0 +left_histograms < bool 3 +left_histograms < bool 0 +left_histograms <= bool 3 +left_histograms <= bool 0 +3 == left_floats +3 != left_floats +3 < left_floats +3 <= left_floats +3 > left_floats +3 >= left_floats +3 == bool left_floats +Inf == left_floats +Inf == bool left_floats +NaN == left_floats +NaN == bool left_floats +3 == left_histograms +0 == left_histograms +3 != left_histograms +0 != left_histograms +3 < left_histograms +0 < left_histograms +3 > left_histograms +0 > left_histograms +3 >= left_histograms +0 >= left_histograms +(testhistogram_bucket) and on() (vector(1) == 1) +(testhistogram_bucket) and on() (vector(-1) == 1) +(testhistogram) and on() (vector(1) == 1) +(testhistogram) and on() (vector(-1) == 1) +-{job="api"} +# --- prometheus testdata: range_queries.test --- +sum_over_time(bar[30s]) +metric +foo > 2 or bar +requests * 2 +some_metric[1m] +some_metric[2m] +some_metric_with_stale_marker[3m] +some_nonexistent_metric[1m] +sort(series) +sort_desc(series) +sort_by_label(series) +sort_by_label_desc(series) +sum(sort(series)) +# --- prometheus testdata: selectors.test --- +rate(http_requests_total[1m]) +rate(http_requests_total{group=~"pro.*"}[1m]) +rate(http_requests_total{group=~".*ry", instance="1"}[1m]) +rate(http_requests_total{instance!="3"}[1m] offset 10000s) +rate(http_requests_total{instance!="3"}[1m] offset -4000s) +rate(http_requests_total[40s]) - rate(http_requests_total[1m] offset 10000s) +http_requests_total{foo!="bar"} +http_requests_total{foo!="bar", job="api-server"} +http_requests_total{foo!~"bar", job="api-server"} +http_requests_total{foo!~"bar", job="api-server", instance="1", x!="y", z="", group!=""} +rate(http_requests_total{group=~"(?i:PRO).*"}[1m]) +rate(http_requests_total{group=~".*?(?i:PRO).*"}[1m]) +rate(http_requests_total{group=~".*(?i:DUC).*"}[1m]) +rate(http_requests_total{group=~".*(?i:TION)"}[1m]) +rate(http_requests_total{group=~".*(?i:TION).*?"}[1m]) +rate(http_requests_total{group=~"((?i)PRO).*"}[1m]) +rate(http_requests_total{group=~".*((?i)DUC).*"}[1m]) +rate(http_requests_total{group=~".*((?i)TION)"}[1m]) +rate(http_requests_total{group=~"(?i:PRODUCTION)"}[1m]) +rate(http_requests_total{group=~".*(?i:C).*"}[1m]) +metric1 offset 15m or metric2 offset 45m +x{y="testvalue"} +{__name__=~".+"} +{job=~".+-server", job!~"api-.+"} +http_requests_total{group!="canary"} +http_requests_total{job=~".+-server",group!="canary"} +http_requests_total{job!~"api-.+",group!="canary"} +http_requests_total{group="production",job=~"api-.+"} +http_requests_total{group="production",job="api-server"} offset 5m +testmetric +# --- prometheus testdata: staleness.test --- +count_over_time(metric[1m]) +count_over_time(metric[1s]) +count_over_time(metric[10s]) +count_over_time(metric[20s]) +count_over_time(metric[10]) +count_over_time(metric[20]) +# --- prometheus testdata: start_timestamps.test --- +increase(cumulative[5m]) +rate(cumulative[5m]) +irate(cumulative[5m]) +round(increase(cumulative[5m1ms])) +increase(cumulative[5m:1m]) +resets(cumulative[5m]) +increase(delta[5m]) +rate(delta[5m]) +irate(delta[5m]) +increase(delta[5m:1m]) +round(increase(series[1m1ms])) +# --- prometheus testdata: subquery.test --- +sum_over_time(metric_total[50s:10s]) +sum_over_time(metric_total[50s:5s]) +sum_over_time(metric_total[60s:10s]) +rate(metric_total[20s:10s]) +rate(metric_total[20s:5s]) +rate(http_requests_total{group=~"pro.*"}[1m:10s]) +avg_over_time(rate(http_requests_total[1m])[1m:1s]) +sum_over_time(metric1_total[30s:10s]) +sum_over_time(metric1_total[30s:5s]) +sum_over_time(metric1_total[30s:10s] offset 10s) +sum_over_time(metric1_total[30s:10s] offset 9s) +sum_over_time(metric1_total[30s:10s] offset 7s) +sum_over_time(metric1_total[30s:10s] offset 5s) +sum_over_time(metric1_total[30s:10s] offset 3s) +sum_over_time((metric1_total)[30s:10s] offset 3s) +sum_over_time(metric1_total[30:10] offset 3) +sum_over_time((metric1_total)[30:10s] offset 3s) +sum_over_time((metric1_total)[30:10] offset 3s) +sum_over_time((metric1_total)[30:10] offset 3) +rate(sum_over_time(metric1_total[30s:10s])[50s:10s]) +rate(sum_over_time(metric2_total[30s:10s])[50s:10s]) +rate(sum_over_time(metric3_total[30s:10s])[50s:10s]) +rate(sum_over_time((metric1_total+metric2_total+metric3_total)[30s:10s])[30s:10s]) +rate(metric_total[1m]) +rate(metric_total[1m500ms:10s]) +rate(metric_total[1m1s:10s]) +min_over_time(metric_total[10s]) +min_over_time(metric_total[15s:10s]) +min_over_time(rate(metric_total[5m])[20m:1m]) +increase(native_histogram[10m:3m]) +increase(native_histogram[10m:15s]) +min_over_time((topk(1, foo))[1m:5m]) +# --- prometheus testdata: trig_functions.test --- +sin(trig) +cos(trig) +tan(trig) +asin(trig - 10.1) +acos(trig - 10.1) +atan(trig) +sinh(trig) +cosh(trig) +tanh(trig) +asinh(trig) +acosh(trig) +atanh(trig - 10.1) +rad(trig) +rad(trig - 10) +rad(trig - 20) +deg(trig) +deg(trig - 10) +deg(trig - 20) +pi() +# --- prometheus testdata: type_and_unit.test --- +SUM(http_requests_total) BY (job) +SUM(http_requests_total{__type__="counter", __unit__="request"}) BY (job) +SUM({__type__="counter"}) BY (job) +SUM({__unit__="request"}) BY (job) +SUM({__type__="counter", __unit__="request"}) BY (job) +http_requests_total{__type__="counter", job="api-server", group="canary"} diff --git a/crates/lower/tests/promql_corpus.rs b/crates/lower/tests/promql_corpus.rs new file mode 100644 index 00000000..c55ff422 --- /dev/null +++ b/crates/lower/tests/promql_corpus.rs @@ -0,0 +1,85 @@ +//! Exhaustive PromQL **corpus** — every query string from the three sources, +//! run through the lowerer. +//! +//! Two corpora (in `tests/data/`): +//! - `promql_corpus_docs.txt` — verbatim examples from the PromQL basics +//! docs and the PromLabs cheat sheet. +//! - `promql_corpus_testdata.txt` — every `eval` expression (deduped) from the +//! Prometheus engine test suite. +//! +//! We *lower* (not execute), so the property under test is **totality**: for +//! every real-world PromQL string, `lower_promql` returns `Ok` or a clean +//! `Err` and **never panics**. A panic anywhere in the loop fails the test — +//! that is the guarantee. A coverage floor guards against a change silently +//! tanking how much of the corpus we can lower. + +use asap_control_core::types::AccuracyTarget; +use asap_control_lower::{lower_promql, LoweringError}; + +const DOCS: &str = include_str!("data/promql_corpus_docs.txt"); +const TESTDATA: &str = include_str!("data/promql_corpus_testdata.txt"); + +/// Non-comment, non-blank query lines. +fn queries(corpus: &str) -> impl Iterator { + corpus + .lines() + .map(str::trim) + .filter(|l| !l.is_empty() && !l.starts_with('#')) +} + +#[derive(Default, Debug)] +struct Tally { + lowered: usize, + rejected: usize, + unparseable: usize, +} + +impl Tally { + fn total(&self) -> usize { + self.lowered + self.rejected + self.unparseable + } +} + +/// Lower every query; a panic here fails the test (the totality guarantee). +fn tally(corpus: &str) -> Tally { + let mut t = Tally::default(); + for q in queries(corpus) { + match lower_promql(q, AccuracyTarget::Exact) { + Ok(_) => t.lowered += 1, + Err(LoweringError::Parse(_)) => t.unparseable += 1, + Err(_) => t.rejected += 1, + } + } + t +} + +#[test] +fn lowering_is_total_over_the_entire_corpus() { + let docs = tally(DOCS); + let td = tally(TESTDATA); + eprintln!("docs corpus: {docs:?}"); + eprintln!("testdata corpus: {td:?}"); + + // Totality: reaching here means no query panicked. Sanity-check that every + // query was classified into exactly one bucket. + assert!( + docs.total() >= 45, + "docs corpus unexpectedly small: {docs:?}" + ); + assert!( + td.total() > 1500, + "testdata corpus unexpectedly small: {td:?}" + ); + + // Coverage tripwire: a code change that breaks lowering for a large slice of + // real PromQL trips this. Set well below the current numbers (docs≈30, + // testdata≈530 lowered); it guards regressions, it is not an exact count. + assert!( + docs.lowered >= 20, + "docs lowering coverage regressed: {docs:?}" + ); + assert!( + td.lowered >= 450, + "testdata lowering coverage regressed: {td:?}" + ); +} From 6b00acce6e3027bd04853ec0f2d71959c7728142 Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 22 May 2026 11:27:53 -0600 Subject: [PATCH 11/40] chore(deps): upgrade promql-parser 0.8 -> 0.9 (more of the corpus parses) The PromQL corpus is extracted from Prometheus `main`, which uses grammar newer than our pinned parser. Bumping promql-parser 0.8 -> 0.9 is API-compatible (no source changes needed) and recovers 78 previously-unparseable corpus queries: testdata corpus: 530 -> 574 lowered, 394 -> 316 unparseable. 0.9 adds `limitk`/`limit_ratio` (limit.test: 29 parse-failures -> 0) and the `fill`/`fill_left` modifiers (fill-modifier.test: 44 -> 0), among others. Still unparseable on 0.9 (no Rust-parser release supports them yet): native histograms (114), `anchored`/`smoothed` range modifiers (61), duration expressions like `[26m+4m]` (49), and experimental functions incl. `histogram_quantiles`, `mad_over_time`, `info`, `sort_by_label`. These remain a parser-version ceiling, not a lowering issue. Corpus coverage floor bumped 450 -> 520 to match the new baseline. All tests (22 core + 27 lowering + 32 conformance + 10 equivalence + 1 corpus) pass under --locked; clippy -D warnings and fmt clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- Cargo.lock | 4 ++-- crates/lower/Cargo.toml | 2 +- crates/lower/tests/promql_corpus.rs | 7 ++++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e61301ae..f90c3cc6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -388,9 +388,9 @@ dependencies = [ [[package]] name = "promql-parser" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df2791a28f8ea7e48f2838999c06d089184d44adb860feab682d45dd190ef718" +checksum = "c28f36f755c7046e83aeb11802c9a0fb8c4507088654dac0fdd12a1035080d1c" dependencies = [ "cfgrammar", "chrono", diff --git a/crates/lower/Cargo.toml b/crates/lower/Cargo.toml index 9e266588..a23768e7 100644 --- a/crates/lower/Cargo.toml +++ b/crates/lower/Cargo.toml @@ -5,4 +5,4 @@ edition = "2021" [dependencies] asap-control-core = { path = "../core" } -promql-parser = "0.8" +promql-parser = "0.9" diff --git a/crates/lower/tests/promql_corpus.rs b/crates/lower/tests/promql_corpus.rs index c55ff422..db3e9127 100644 --- a/crates/lower/tests/promql_corpus.rs +++ b/crates/lower/tests/promql_corpus.rs @@ -72,14 +72,15 @@ fn lowering_is_total_over_the_entire_corpus() { ); // Coverage tripwire: a code change that breaks lowering for a large slice of - // real PromQL trips this. Set well below the current numbers (docs≈30, - // testdata≈530 lowered); it guards regressions, it is not an exact count. + // real PromQL trips this. Set well below the current numbers (docs≈29, + // testdata≈574 lowered / 316 unparseable on promql-parser 0.9); it guards + // regressions, it is not an exact count. assert!( docs.lowered >= 20, "docs lowering coverage regressed: {docs:?}" ); assert!( - td.lowered >= 450, + td.lowered >= 520, "testdata lowering coverage regressed: {td:?}" ); } From 95da3d62b83f818284e32aa328a6ab606bd4770b Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 22 May 2026 11:46:31 -0600 Subject: [PATCH 12/40] chore(deps): vendor promql-parser via private mirror (ProjectASAP/promql-parser) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switch the promql-parser dependency from crates.io "0.9" to a git dependency on a private mirror of GreptimeTeam/promql-parser (Apache-2.0) under the ProjectASAP org, so we can carry local PromQL grammar/function additions ahead of upstream releases. Cargo.lock pins the exact rev. The mirror's main is currently API-identical to 0.9.0 — no source changes needed; all tests (22 core + 27 lowering + 32 conformance + 10 equivalence + 1 corpus) pass, clippy -D warnings and fmt clean. NOTE: builds now require read access to the private repo. Local dev works with `gh auth setup-git`; CI needs CARGO_NET_GIT_FETCH_WITH_CLI=true plus a token / deploy key with access to ProjectASAP/promql-parser. Co-Authored-By: Claude Opus 4.7 (1M context) --- Cargo.lock | 3 +-- crates/lower/Cargo.toml | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f90c3cc6..087cbf88 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -389,8 +389,7 @@ dependencies = [ [[package]] name = "promql-parser" version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28f36f755c7046e83aeb11802c9a0fb8c4507088654dac0fdd12a1035080d1c" +source = "git+https://github.com/ProjectASAP/promql-parser?branch=main#2e4ebde7cef1351459229b2fbca4822c43d0cdfc" dependencies = [ "cfgrammar", "chrono", diff --git a/crates/lower/Cargo.toml b/crates/lower/Cargo.toml index a23768e7..7a9c4527 100644 --- a/crates/lower/Cargo.toml +++ b/crates/lower/Cargo.toml @@ -5,4 +5,6 @@ edition = "2021" [dependencies] asap-control-core = { path = "../core" } -promql-parser = "0.9" +# Private mirror of GreptimeTeam/promql-parser (Apache-2.0), vendored so we can +# carry local PromQL grammar/function additions ahead of upstream releases. +promql-parser = { git = "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/ProjectASAP/promql-parser", branch = "main" } From c5a5f8bab9acaac8284c6c9ce8237974c95efa8b Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 22 May 2026 11:59:40 -0600 Subject: [PATCH 13/40] docs: add THIRD_PARTY.md (dependency licenses + promql-parser attribution) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Record the direct third-party crates and their licenses (all permissive, MIT/Apache-2.0), and document the vendored Apache-2.0 `promql-parser` private mirror: what Apache-2.0 permits, the §4 redistribution obligations (only triggered on external distribution), and how to sync the mirror with upstream. Co-Authored-By: Claude Opus 4.7 (1M context) --- THIRD_PARTY.md | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 THIRD_PARTY.md diff --git a/THIRD_PARTY.md b/THIRD_PARTY.md new file mode 100644 index 00000000..ed87c0c1 --- /dev/null +++ b/THIRD_PARTY.md @@ -0,0 +1,62 @@ +# Third-party software + +ASAPController bundles third-party Rust crates. This file records the notable +direct dependencies, their licenses, and any attribution obligations. It is a +convenience summary, **not legal advice** — for an exhaustive, machine-generated +list (including transitive dependencies) run e.g. `cargo tree` or +[`cargo about`](https://github.com/EmbarkStudios/cargo-about). + +All licenses below are OSI-approved and **permissive** (MIT / Apache-2.0); none +are copyleft. ASAPController itself is therefore not obligated to be open-sourced +on account of these dependencies. + +## Direct dependencies + +| Crate | License | Source | Notes | +|---|---|---|---| +| `promql-parser` | Apache-2.0 | **Private mirror** `ProjectASAP/promql-parser` of [`GreptimeTeam/promql-parser`](https://github.com/GreptimeTeam/promql-parser) | L1 PromQL parsing. See below. | +| `serde` (+ derive) | MIT OR Apache-2.0 | crates.io | Serialization of the intent-algebra IR. | +| `serde_json` | MIT OR Apache-2.0 | crates.io | JSON (de)serialization in tests/IR. | +| `thiserror` | MIT OR Apache-2.0 | crates.io | Error types. | + +Transitive dependencies pulled in by the above (notably the `lrpar` / `lrlex` / +`cfgrammar` parser-toolkit stack behind `promql-parser`, and `regex`) carry their +own licenses — overwhelmingly MIT and/or Apache-2.0. Regenerate the full set with +`cargo about generate` if a complete NOTICE bundle is needed for a release. + +## `promql-parser` — vendored Apache-2.0 mirror + +`promql-parser` is consumed as a **git dependency on a private mirror** +(`ProjectASAP/promql-parser`) of the upstream Apache-2.0 project +`GreptimeTeam/promql-parser`, so we can carry local PromQL grammar/function +additions ahead of upstream releases (see `docs/` and the `crates/lower` manifest). + +Apache-2.0 explicitly permits copying, modifying, **keeping modifications +private**, and commercial use, and it is **not copyleft**. The conditions in +§4 ("Redistribution") apply only when the software is **distributed outside the +organization**. If/when ASAPController (with this parser compiled in) is +distributed externally, retain the following: + +- a copy of the **Apache-2.0 license** text (kept in the mirror as `LICENSE`); +- the upstream **`NOTICE`** file's attribution content, if present; +- original copyright / patent / attribution notices in the source; and +- a prominent note in **each file we modify** stating that it was changed + (Apache-2.0 §4(b)). + +Purely internal use (private mirror, internal builds, no external distribution) +carries essentially none of these obligations beyond keeping `LICENSE`/`NOTICE` +in the mirror, which the mirror already does. + +### Keeping the mirror in sync with upstream + +The mirror was created as a one-way copy (not a GitHub fork). To pull future +upstream changes: + +```sh +git clone --bare https://github.com/GreptimeTeam/promql-parser.git +cd promql-parser.git +git push --mirror https://github.com/ProjectASAP/promql-parser.git # if main is unmodified +``` + +If local modifications live on `main`, keep upstream on a separate branch and +merge/rebase instead of mirror-pushing (which overwrites). From 867e1438dca12ed8348902b321d16d1dd1535485 Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 22 May 2026 12:14:03 -0600 Subject: [PATCH 14/40] chore(promql): track private parser `asap` branch (added functions) + CI auth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - crates/lower: pin promql-parser to the private mirror's `asap` branch, which adds 12 experimental functions missing from upstream (mad_over_time, first_over_time, ts_of_{first,last,max,min}_over_time, histogram_quantiles, info, max_of, min_of, step, range). They now parse instead of erroring; corpus unparseable drops 316 -> 235. They still have no L3 intent, so they parse-then- reject (lowered stays 574) — by design. `start()`/`end()` deferred (reserved lexer keywords for the @ modifier; need grammar work). - CI: rust.yml authenticates to the private dep via secret CARGO_PRIVATE_GIT_TOKEN + CARGO_NET_GIT_FETCH_WITH_CLI in both jobs. - THIRD_PARTY.md: document main(pristine)/asap(edits) branch model, upstream-sync flow, the §4(b) modification, and the required CI secret. All tests pass under --locked; clippy -D warnings and fmt clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/rust.yml | 18 +++++++++++++++++ Cargo.lock | 2 +- THIRD_PARTY.md | 31 +++++++++++++++++++++++------ crates/lower/Cargo.toml | 7 ++++--- crates/lower/tests/promql_corpus.rs | 4 ++-- 5 files changed, 50 insertions(+), 12 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index bc8950db..80e2a43d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -24,6 +24,10 @@ concurrency: env: CARGO_TERM_COLOR: always + # The promql-parser dependency is a private git repo (ProjectASAP/promql-parser). + # Cargo must fetch it via the git CLI so it picks up the credential helper + # configured by the "Authenticate private git deps" step below. + CARGO_NET_GIT_FETCH_WITH_CLI: "true" jobs: format-and-lint: @@ -32,6 +36,13 @@ jobs: steps: - uses: actions/checkout@v4 + # Grant read access to the private promql-parser mirror. Requires an org/repo + # secret CARGO_PRIVATE_GIT_TOKEN (fine-grained PAT or GitHub App token with + # read access to ProjectASAP/promql-parser). + - name: Authenticate private git deps + run: | + git config --global url."https://x-access-token:${{ secrets.CARGO_PRIVATE_GIT_TOKEN }}@github.com/".insteadOf "/" + - name: Install Rust uses: dtolnay/rust-toolchain@stable with: @@ -64,6 +75,13 @@ jobs: steps: - uses: actions/checkout@v4 + # Grant read access to the private promql-parser mirror. Requires an org/repo + # secret CARGO_PRIVATE_GIT_TOKEN (fine-grained PAT or GitHub App token with + # read access to ProjectASAP/promql-parser). + - name: Authenticate private git deps + run: | + git config --global url."https://x-access-token:${{ secrets.CARGO_PRIVATE_GIT_TOKEN }}@github.com/".insteadOf "/" + - name: Install Rust uses: dtolnay/rust-toolchain@stable with: diff --git a/Cargo.lock b/Cargo.lock index 087cbf88..e0c8dad7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -389,7 +389,7 @@ dependencies = [ [[package]] name = "promql-parser" version = "0.9.0" -source = "git+https://github.com/ProjectASAP/promql-parser?branch=main#2e4ebde7cef1351459229b2fbca4822c43d0cdfc" +source = "git+https://github.com/ProjectASAP/promql-parser?branch=asap#c51beafb361af4cc95ed62ae377862c660ceb757" dependencies = [ "cfgrammar", "chrono", diff --git a/THIRD_PARTY.md b/THIRD_PARTY.md index ed87c0c1..b39fe008 100644 --- a/THIRD_PARTY.md +++ b/THIRD_PARTY.md @@ -29,7 +29,15 @@ own licenses — overwhelmingly MIT and/or Apache-2.0. Regenerate the full set w `promql-parser` is consumed as a **git dependency on a private mirror** (`ProjectASAP/promql-parser`) of the upstream Apache-2.0 project `GreptimeTeam/promql-parser`, so we can carry local PromQL grammar/function -additions ahead of upstream releases (see `docs/` and the `crates/lower` manifest). +additions ahead of upstream releases. Branch layout: + +- **`main`** — an untouched mirror of upstream `GreptimeTeam/promql-parser`. +- **`asap`** — our working branch; ASAPController's `crates/lower` manifest pins + this branch. It currently adds the experimental functions present in + Prometheus `promql/parser/functions.go` but missing upstream (`mad_over_time`, + `first_over_time`, `ts_of_{first,last,max,min}_over_time`, `histogram_quantiles`, + `info`, `max_of`, `min_of`, `step`, `range`). Modified files are marked per + Apache-2.0 §4(b). Apache-2.0 explicitly permits copying, modifying, **keeping modifications private**, and commercial use, and it is **not copyleft**. The conditions in @@ -49,14 +57,25 @@ in the mirror, which the mirror already does. ### Keeping the mirror in sync with upstream -The mirror was created as a one-way copy (not a GitHub fork). To pull future -upstream changes: +The mirror was created as a one-way copy (not a GitHub fork). `main` stays a +pristine upstream mirror; local edits live on `asap`. To pull future upstream +changes, refresh `main` then rebase `asap`: ```sh +# refresh the pristine mirror branch git clone --bare https://github.com/GreptimeTeam/promql-parser.git cd promql-parser.git -git push --mirror https://github.com/ProjectASAP/promql-parser.git # if main is unmodified +git push https://github.com/ProjectASAP/promql-parser.git +refs/heads/main:refs/heads/main +# then, in a normal clone: git checkout asap && git rebase main && git push --force-with-lease ``` -If local modifications live on `main`, keep upstream on a separate branch and -merge/rebase instead of mirror-pushing (which overwrites). +### CI / build access + +Because `promql-parser` is a private git dependency, any build needs read access +to `ProjectASAP/promql-parser`: + +- **Local dev:** `gh auth setup-git` (uses your GitHub credentials). +- **CI:** the `rust.yml` workflow sets `CARGO_NET_GIT_FETCH_WITH_CLI=true` and + configures a git credential helper from the secret **`CARGO_PRIVATE_GIT_TOKEN`** + — add that org/repo secret (a fine-grained PAT or GitHub App token with read + access to the mirror) or CI will fail to fetch the dependency. diff --git a/crates/lower/Cargo.toml b/crates/lower/Cargo.toml index 7a9c4527..e026b92e 100644 --- a/crates/lower/Cargo.toml +++ b/crates/lower/Cargo.toml @@ -5,6 +5,7 @@ edition = "2021" [dependencies] asap-control-core = { path = "../core" } -# Private mirror of GreptimeTeam/promql-parser (Apache-2.0), vendored so we can -# carry local PromQL grammar/function additions ahead of upstream releases. -promql-parser = { git = "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/ProjectASAP/promql-parser", branch = "main" } +# Private mirror of GreptimeTeam/promql-parser (Apache-2.0). `main` tracks +# upstream untouched; the `asap` branch carries our local grammar/function +# additions (see THIRD_PARTY.md). +promql-parser = { git = "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/ProjectASAP/promql-parser", branch = "asap" } diff --git a/crates/lower/tests/promql_corpus.rs b/crates/lower/tests/promql_corpus.rs index db3e9127..458aa84d 100644 --- a/crates/lower/tests/promql_corpus.rs +++ b/crates/lower/tests/promql_corpus.rs @@ -73,8 +73,8 @@ fn lowering_is_total_over_the_entire_corpus() { // Coverage tripwire: a code change that breaks lowering for a large slice of // real PromQL trips this. Set well below the current numbers (docs≈29, - // testdata≈574 lowered / 316 unparseable on promql-parser 0.9); it guards - // regressions, it is not an exact count. + // testdata≈574 lowered / 1014 rejected / 235 unparseable on the private + // promql-parser `asap` branch); it guards regressions, not an exact count. assert!( docs.lowered >= 20, "docs lowering coverage regressed: {docs:?}" From 2c50467a77df15a53763a5b2abf4e669bc7f5e9d Mon Sep 17 00:00:00 2001 From: zz_y Date: Fri, 22 May 2026 12:33:31 -0600 Subject: [PATCH 15/40] ci: disable Rust CI auto-triggers until private-dep credential is set CI can't fetch the private promql-parser dependency until the CARGO_PRIVATE_GIT_TOKEN secret exists, so the push/pull_request triggers are commented out (workflow_dispatch only) to avoid a perpetually-red check. The auth step + CARGO_NET_GIT_FETCH_WITH_CLI scaffolding stays in place; re-enable the triggers once the credential is configured. Documented in THIRD_PARTY.md. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/rust.yml | 34 +++++++++++++++++++--------------- THIRD_PARTY.md | 4 +++- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 80e2a43d..20fe2945 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -1,22 +1,26 @@ name: Rust CI +# TEMPORARILY DISABLED: auto-triggers are off because CI can't fetch the private +# promql-parser dependency until the CARGO_PRIVATE_GIT_TOKEN secret is added (see +# THIRD_PARTY.md). The auth scaffolding below is ready — re-enable the push / +# pull_request triggers once the credential is configured. Manual runs still work. on: - push: - branches: [ main ] - paths: - - 'crates/**' - - 'Cargo.toml' - - 'Cargo.lock' - - '.github/workflows/rust.yml' - pull_request: - types: [opened, synchronize, reopened, ready_for_review] - branches: [ main ] - paths: - - 'crates/**' - - 'Cargo.toml' - - 'Cargo.lock' - - '.github/workflows/rust.yml' workflow_dispatch: + # push: + # branches: [ main ] + # paths: + # - 'crates/**' + # - 'Cargo.toml' + # - 'Cargo.lock' + # - '.github/workflows/rust.yml' + # pull_request: + # types: [opened, synchronize, reopened, ready_for_review] + # branches: [ main ] + # paths: + # - 'crates/**' + # - 'Cargo.toml' + # - 'Cargo.lock' + # - '.github/workflows/rust.yml' concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/THIRD_PARTY.md b/THIRD_PARTY.md index b39fe008..dce20c37 100644 --- a/THIRD_PARTY.md +++ b/THIRD_PARTY.md @@ -78,4 +78,6 @@ to `ProjectASAP/promql-parser`: - **CI:** the `rust.yml` workflow sets `CARGO_NET_GIT_FETCH_WITH_CLI=true` and configures a git credential helper from the secret **`CARGO_PRIVATE_GIT_TOKEN`** — add that org/repo secret (a fine-grained PAT or GitHub App token with read - access to the mirror) or CI will fail to fetch the dependency. + access to the mirror) or CI will fail to fetch the dependency. Until the secret + exists, `rust.yml`'s auto-triggers are **disabled** (manual `workflow_dispatch` + only); re-enable the commented `push` / `pull_request` triggers afterward. From 17a287644a6e9901ab9333afdd50bd61caf30283 Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 24 May 2026 12:25:22 -0600 Subject: [PATCH 16/40] =?UTF-8?q?docs:=20add=20intent=5Falgebra=20reconcil?= =?UTF-8?q?iation=20plan=20(ASAPController=20=E2=87=84=20control=5Fplane)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit File-by-file plan to unify the two diverged intent_algebra copies into one shared L3 (base = control_plane's richer IR + ASAPController's fixes), as the first step of the L4/L5 consolidation. Includes the drift evidence, three sign-off decisions (StdDev/Variance representation, scalar-IR location, shared types home), per-file tasks with effort/risk, and a recommended order. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/intent-algebra-reconciliation.md | 100 ++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 docs/intent-algebra-reconciliation.md diff --git a/docs/intent-algebra-reconciliation.md b/docs/intent-algebra-reconciliation.md new file mode 100644 index 00000000..f0084459 --- /dev/null +++ b/docs/intent-algebra-reconciliation.md @@ -0,0 +1,100 @@ +# `intent_algebra` reconciliation plan (ASAPController ⇄ ASAPQuery-backend) + +ASAPController's `crates/core/src/intent_algebra` is a slimmed, refactored fork +of the canonical L3 IR in `ASAPQuery-backend/control_plane/src/intent_algebra`. +This is the first concrete step of the L4/L5 consolidation: produce **one** +canonical `intent_algebra` so that (a) both repos stop drifting, and (b) +control_plane's L4 (optimizer + sketch_algebra) and L5 (physical + emit) can be +ported onto the shared IR the PromQL/SQL front-ends already lower into. + +**Base = control_plane's L3** (the richer original); ASAPController's +correctness fixes and multi-language front-end are layered on top. + +## Drift summary (evidence) + +Line counts, `intent_algebra/*.rs`, control_plane (CP) vs ASAPController (ASAP): + +| file | CP | ASAP | finding | +|---|---:|---:|---| +| `schema.rs` | 332 | 332 | **byte-identical** (already shared) | +| `query_expr.rs` | 1193 | 445 | ASAP relational nodes ⊆ CP; CP's "extra" variants are the **inlined scalar IR** ASAP split into `expr_ir.rs` | +| `agg_intent.rs` | 673 | 245 | **additive both ways**: CP has 11 ASAP lacks; ASAP has `StdDev`/`Variance` CP lacks | +| `relational.rs` | 920 | 212 | CP superset, **incl. `Project`** (ASAP lacks; SQL needs it) | +| `lower.rs` | 845 | 294 | CP richer; ASAP carries the per-branch-binding fix + accuracy threading | +| `binder.rs` | 300 | 189 | CP richer; same `SchemaCatalog`/`UsageDerivedCatalog` API | +| `cse.rs` | 324 | 228 | ASAP carries the structural-`PartialEq` key fix | +| `column_resolution.rs` | 465 | 177 | CP richer | +| `mod.rs` | 125 | 41 | **same export shape**, CP exports more | +| `expr_ir.rs`, `names.rs` | inlined | separate | file-organization difference | + +Verdict: a **contained merge**, not a rewrite. `schema.rs` is already shared, the +node set is a clean superset, `AggIntent` is a union, and the public module API +matches. Cost concentrates in `lower.rs` and `expr_ir.rs`. + +Behavioral specifics found in CP: +- `convert_root(legacy)` takes **no accuracy** — it hardcodes `Exact` / + `Epsilon(0.05)` in the converter. ASAP threads a per-query `AccuracyTarget`. +- CP's converter threads **one root schema to all branches** — i.e. it has the + same per-branch-binding bug ASAP already fixed. +- CP `AggIntent` carries `accuracy` and the 11 extra intents + (`Changes/Resets/Deriv/Delta/PredictLinear/Absent/Present/Idelta/Irate/HoltWinters/Frequency`), + but **lacks `StdDev`/`Variance`** — it fans those out into a `Merge` of + quantile aggregates in `lower.rs`. +- CP sources `AccuracyTarget` / `BindingName` from a `types_v2` module. + +Two payoffs from basing on CP's L3: +- Several functions ASAPController currently **rejects** (`changes`, `resets`, + `deriv`, `delta`, `predict_linear`, `absent`, `present`) gain intents → become + lowerable. +- `irate` becomes distinguishable from `rate` (CP has a separate `Irate` intent); + our `rate≡irate` equivalence was a consequence of the slimmer vocabulary. + +## Decisions to settle (sign-off needed) + +| # | Fork | Options | Recommendation | +|---|---|---|---| +| **D1** | `StdDev` / `Variance` | CP's `Merge`-of-quantiles fan-out **vs** ASAP's first-class `AggIntent` | **CP fan-out** (less L4 work); add first-class intents only if L4 can bind them | +| **D2** | scalar IR location | CP inlines in `query_expr`/`relational` **vs** ASAP's separate `expr_ir.rs` (`L3Expr`) | **ASAP's `expr_ir.rs`**, extended to CP's scalar superset | +| **D3** | shared scalar types home | CP `types_v2` **vs** ASAP `names.rs` + `types.rs` (`AccuracyTarget`, `BindingName`, `QueryId`) | **one `core::types` module**; both already expose the same names | + +## File-by-file tasks + +Base = control_plane's file unless noted; "port" = bring ASAP's delta onto the CP base. + +| file | base | tasks | effort | risk | +|---|---|---|---|---| +| `schema.rs` | identical | **no-op** — adopt as-is | none | none | +| `query_expr.rs` | CP | adopt CP node set (ASAP ⊆ CP); per **D2** reference `expr_ir::L3Expr` instead of inline scalars; confirm CP covers ASAP methods (`output_schema_in`, …) | low | low | +| `agg_intent.rs` | CP | adopt CP's full vocabulary; resolve **D1**; reconcile `output_column` / `agg_accuracy` | low–med | low | +| `relational.rs` | CP | adopt CP (incl. `Project`); per **D2** point scalar refs at `expr_ir` | low | low | +| `lower.rs` | CP | **main work**: (1) port per-branch binding to `BinaryOp`/`Join`/`SetOp` arms; (2) add `acc: &AccuracyTarget` to `convert`/`convert_root`, replace hardcoded defaults; (3) verify nested-aggregate convert supports the two-level `sum(rate)` shape; (4) reflect **D1** | **med** | med | +| `binder.rs` | CP | adopt CP; verify `SchemaCatalog`/`UsageDerivedCatalog` parity | low | low | +| `cse.rs` | CP | **port** the structural-`PartialEq` key fix if CP keys by `Debug` | low | low | +| `column_resolution.rs` | CP | adopt CP | low | low | +| `expr_ir.rs` | ASAP (keep, **D2**) | **med work**: extend `L3Expr` to CP's scalar superset (`FunctionCall`, `InList`, `Between`, `IsNull`, `ScalarSubquery`, `Cast`) + our `Regex`/`NotRegex` | med | low | +| `names.rs` | merge → `types` | fold `BindingName`/`QueryId` into the shared types module (**D3**); update CP's `types_v2` imports | low | low | +| `mod.rs` | CP + ASAP | union the exports; keep `expr_ir` + shared `types` | low | low | + +## Cross-cutting (part of the merge, outside `intent_algebra`) + +- **`types_v2` → `core::types`** (D3): one home for `AccuracyTarget`, `BindingName`, `QueryId`. +- **Front-ends**: keep ASAPController's **PromQL + SQL** lowering (CP has PromQL only) and the correctness fixes (`sum(rate)`, `histogram_quantile`, matcher canonicalization) — retarget onto the unified `QueryExpr`. Lives in `crates/lower`. +- **Tests**: bring the `conformance` / `equivalence` / `corpus` suites onto the unified crate and merge with CP's `lower.rs` / `cse.rs` unit tests. Use the full suite as the acceptance gate. + +## Recommended order + +1. `schema.rs` (free) + `types`/`names.rs` (D3) — shared primitives. +2. `expr_ir.rs` to CP's scalar superset (D2) — unblocks the type layer. +3. `query_expr.rs` + `relational.rs` + `agg_intent.rs` (D1) — the type layer. +4. `binder.rs` + `column_resolution.rs` + `cse.rs` (+ port the CSE key fix). +5. `lower.rs` — per-branch binding + accuracy threading (the one med-risk file). +6. Retarget front-ends + bring tests; run the full ASAPController suite against the unified crate. + +## Effort summary + +- **adopt-CP (low):** schema, query_expr, relational, binder, column_resolution, mod. +- **port-our-fix (low):** cse, names/types. +- **real work (med):** `lower.rs` (per-branch binding + accuracy), `expr_ir.rs` (scalar superset). +- **decision with semantic weight:** D1 only. + +Open prerequisite (tracked separately): host the unified crate as **(A)** a cross-repo shared crate or **(B)** absorb control_plane into the ASAPController monorepo (design.md §5/§8). Recommendation: **B** for the core IR — a cross-repo git dependency on the central IR forces lock-step two-repo changes (cf. the `promql-parser` private-mirror friction). From d36c23f2653a70e8b86b8848c63508a720a10d3f Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 26 May 2026 09:47:56 -0600 Subject: [PATCH 17/40] =?UTF-8?q?feat(core):=20make=20L3=20IR=20SQL-ready?= =?UTF-8?q?=20=E2=80=94=20scalar=20superset,=20col-carrying=20AggIntent,?= =?UTF-8?q?=20SQL-node=20schema=20flow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase A of the #4⇄#5 intent_algebra reconciliation (positional IR as base). Extends the positional L3 IR so both language front ends can target it, keeping the PromQL path green (99 tests). - expr_ir: L3Expr/CompareOp extended to the SQL∪PromQL superset (Arith/Cast/InList/FunctionCall/Case/IsNull/IsNotNull + Like/ILike, keeping Regex/NotRegex). [D2] - agg_intent: Sum/Min/Max/Avg/StdDev/Variance carry col: Option (None = PromQL sample value); first-class StdDev/Variance kept. [D1/D4] - query_expr: Aggregate schema derivation binds each reducer to its own input column; Project recomputes schema from its items (was passthrough); Join concatenates left+right with per-JoinKind nullability (was left-only); SetOp drops unique_keys. + unit tests for each. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/core/src/intent_algebra/agg_intent.rs | 102 +++++- .../src/intent_algebra/column_resolution.rs | 8 +- crates/core/src/intent_algebra/cse.rs | 2 +- crates/core/src/intent_algebra/expr_ir.rs | 100 +++++- crates/core/src/intent_algebra/lower.rs | 13 +- crates/core/src/intent_algebra/mod.rs | 2 +- crates/core/src/intent_algebra/query_expr.rs | 332 +++++++++++++++++- crates/lower/tests/promql_conformance.rs | 22 +- crates/lower/tests/promql_lowering.rs | 12 +- 9 files changed, 540 insertions(+), 53 deletions(-) diff --git a/crates/core/src/intent_algebra/agg_intent.rs b/crates/core/src/intent_algebra/agg_intent.rs index 6c8d90b5..b6ca2d8f 100644 --- a/crates/core/src/intent_algebra/agg_intent.rs +++ b/crates/core/src/intent_algebra/agg_intent.rs @@ -14,7 +14,7 @@ use std::time::Duration; use serde::{Deserialize, Serialize}; use crate::intent_algebra::query_expr::DataModel; -use crate::intent_algebra::schema::{Column, DataType}; +use crate::intent_algebra::schema::{Column, ColumnId, DataType}; use crate::types::AccuracyTarget; /// "What to compute" at L3 — the vocabulary the planner pivots on. @@ -22,6 +22,12 @@ use crate::types::AccuracyTarget; /// Grouping for `TopK` rides on the enclosing `QueryExpr::Aggregate.by` /// (positional `ColumnId`s), like every other aggregate; the intent itself /// carries only `k` + the accuracy target. +/// +/// The single-column reducers (`Sum` / `Min` / `Max` / `Avg` / `StdDev` / +/// `Variance`) carry `col: Option` — the positional input column +/// they reduce. `None` is the PromQL convention "the time-series sample +/// value"; SQL `SUM(bytes), AVG(latency)` sets distinct `Some(id)`s so a +/// multi-aggregate node binds each reducer to the right column. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case")] pub enum AggIntent { @@ -29,17 +35,33 @@ pub enum AggIntent { Count { accuracy: AccuracyTarget, }, - Sum, - Min, - Max, - Avg, + Sum { + #[serde(default)] + col: Option, + }, + Min { + #[serde(default)] + col: Option, + }, + Max { + #[serde(default)] + col: Option, + }, + Avg { + #[serde(default)] + col: Option, + }, /// Sample standard deviation when `population == false`; population stddev - /// otherwise. PromQL `stddev` / `stddev_over_time`. + /// otherwise. PromQL `stddev` / `stddev_over_time`; SQL `STDDEV(col)`. StdDev { + #[serde(default)] + col: Option, population: bool, }, - /// Variance — PromQL `stdvar` / `stdvar_over_time`. + /// Variance — PromQL `stdvar` / `stdvar_over_time`; SQL `VARIANCE(col)`. Variance { + #[serde(default)] + col: Option, population: bool, }, Quantile { @@ -77,6 +99,22 @@ impl AggIntent { } } + /// The positional input column this intent reduces, if it carries one. + /// `None` = the synthetic time-series sample value (PromQL) or an + /// argument-less aggregate (`Count` / `Cardinality` / `TopK`). Used by + /// schema derivation to resolve each reducer's input column. + pub fn input_col(&self) -> Option { + match self { + AggIntent::Sum { col } + | AggIntent::Min { col } + | AggIntent::Max { col } + | AggIntent::Avg { col } + | AggIntent::StdDev { col, .. } + | AggIntent::Variance { col, .. } => *col, + _ => None, + } + } + /// Output column name + type produced by this intent over `input`. /// Used by `QueryExpr::Aggregate`'s schema-derivation rule. The PromQL /// convention names the column after the intent kind so consumers can @@ -84,10 +122,10 @@ impl AggIntent { pub fn output_column(&self, input: &Column) -> Column { match self { AggIntent::Count { .. } => col("count", DataType::Int64, false), - AggIntent::Sum => col("sum", input.dtype.clone(), false), - AggIntent::Min => col("min", input.dtype.clone(), input.nullable), - AggIntent::Max => col("max", input.dtype.clone(), input.nullable), - AggIntent::Avg => col("avg", DataType::Float64, false), + AggIntent::Sum { .. } => col("sum", input.dtype.clone(), false), + AggIntent::Min { .. } => col("min", input.dtype.clone(), input.nullable), + AggIntent::Max { .. } => col("max", input.dtype.clone(), input.nullable), + AggIntent::Avg { .. } => col("avg", DataType::Float64, false), AggIntent::StdDev { .. } => col("stddev", DataType::Float64, false), AggIntent::Variance { .. } => col("variance", DataType::Float64, false), AggIntent::Quantile { q, .. } => col( @@ -131,7 +169,7 @@ fn quantile_suffix(q: f64) -> String { pub fn agg_is_mergeable(op: &AggIntent) -> bool { !matches!( op, - AggIntent::Avg | AggIntent::StdDev { .. } | AggIntent::Variance { .. } + AggIntent::Avg { .. } | AggIntent::StdDev { .. } | AggIntent::Variance { .. } ) } @@ -140,7 +178,11 @@ pub fn agg_is_mergeable(op: &AggIntent) -> bool { pub fn agg_is_exact(op: &AggIntent) -> bool { matches!( op, - AggIntent::Sum | AggIntent::Count { .. } | AggIntent::Avg | AggIntent::Min | AggIntent::Max + AggIntent::Sum { .. } + | AggIntent::Count { .. } + | AggIntent::Avg { .. } + | AggIntent::Min { .. } + | AggIntent::Max { .. } ) } @@ -203,7 +245,7 @@ mod tests { .name, "count" ); - assert_eq!(AggIntent::Sum.output_column(&v).name, "sum"); + assert_eq!(AggIntent::Sum { col: None }.output_column(&v).name, "sum"); assert_eq!( AggIntent::Quantile { q: 0.99, @@ -218,20 +260,42 @@ mod tests { #[test] fn sum_preserves_input_dtype() { assert!(matches!( - AggIntent::Sum.output_column(&c("c", DataType::Int64)).dtype, + AggIntent::Sum { col: None } + .output_column(&c("c", DataType::Int64)) + .dtype, DataType::Int64 )); } #[test] fn mergeability_and_exactness() { - assert!(agg_is_mergeable(&AggIntent::Sum)); - assert!(!agg_is_mergeable(&AggIntent::Avg)); - assert!(!agg_is_mergeable(&AggIntent::StdDev { population: false })); - assert!(agg_is_exact(&AggIntent::Min)); + assert!(agg_is_mergeable(&AggIntent::Sum { col: None })); + assert!(!agg_is_mergeable(&AggIntent::Avg { col: None })); + assert!(!agg_is_mergeable(&AggIntent::StdDev { + col: None, + population: false + })); + assert!(agg_is_exact(&AggIntent::Min { col: None })); assert!(!agg_is_exact(&default_cardinality())); } + #[test] + fn input_col_tracks_only_reducers() { + assert_eq!(AggIntent::Sum { col: Some(3) }.input_col(), Some(3)); + assert_eq!( + AggIntent::Avg { col: None }.input_col(), + None, + "None = PromQL sample value" + ); + assert_eq!( + AggIntent::Count { + accuracy: AccuracyTarget::Exact + } + .input_col(), + None + ); + } + #[test] fn agg_intent_serde_roundtrip() { let v = AggIntent::Quantile { diff --git a/crates/core/src/intent_algebra/column_resolution.rs b/crates/core/src/intent_algebra/column_resolution.rs index 580b994b..9d25272e 100644 --- a/crates/core/src/intent_algebra/column_resolution.rs +++ b/crates/core/src/intent_algebra/column_resolution.rs @@ -120,7 +120,11 @@ pub fn output_schema_for_aggregate(input: &Schema, by: &[ColumnId], aggs: &[AggI nullable: false, }); for intent in aggs { - out_cols.push(intent.output_column(&probe)); + let in_col = intent + .input_col() + .and_then(|id| input.columns.get(id)) + .unwrap_or(&probe); + out_cols.push(intent.output_column(in_col)); } let unique_keys = if by.is_empty() { Vec::new() @@ -167,7 +171,7 @@ mod tests { dtype: DataType::Utf8, nullable: false, }); - let out = output_schema_for_aggregate(&input, &[2usize], &[AggIntent::Sum]); + let out = output_schema_for_aggregate(&input, &[2usize], &[AggIntent::Sum { col: None }]); assert_eq!(out.columns.len(), 2); // host, sum assert_eq!(out.columns[0].name, "host"); assert_eq!(out.columns[1].name, "sum"); diff --git a/crates/core/src/intent_algebra/cse.rs b/crates/core/src/intent_algebra/cse.rs index 23d6f67e..54e17e55 100644 --- a/crates/core/src/intent_algebra/cse.rs +++ b/crates/core/src/intent_algebra/cse.rs @@ -218,7 +218,7 @@ mod tests { }; let mk = || QueryExpr::Aggregate { by: vec![], - aggs: vec![AggIntent::Sum], + aggs: vec![AggIntent::Sum { col: None }], having: None, child: Box::new(scan_no_uk.clone()), }; diff --git a/crates/core/src/intent_algebra/expr_ir.rs b/crates/core/src/intent_algebra/expr_ir.rs index 464ddcc4..f0180930 100644 --- a/crates/core/src/intent_algebra/expr_ir.rs +++ b/crates/core/src/intent_algebra/expr_ir.rs @@ -4,10 +4,16 @@ //! and projection / sort-key expressions. Carried by both the Layer-2 //! `relational` IR and the canonical L3 `query_expr` IR so the predicate //! representation is identical across the lowering boundary. +//! +//! The variant set is the **union** of what the two front ends need: PromQL +//! contributes `Regex` / `NotRegex` (`=~` / `!~`); SQL contributes arithmetic, +//! `CASE`, `IN`, `CAST`, `IS [NOT] NULL`, scalar function calls, and the +//! `LIKE` / `ILIKE` comparison family. use serde::{Deserialize, Serialize}; use super::query_expr::ColumnRef; +use super::schema::DataType; /// A typed scalar constant. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -23,6 +29,7 @@ pub enum L3Scalar { /// /// `Regex` / `NotRegex` carry PromQL/RE2 regex-match semantics (`=~` / `!~`): /// the right-hand side is a regular-expression pattern, not a literal value. +/// `Like` / `ILike` (+ negations) are the SQL pattern-match analogues. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum CompareOp { Eq, @@ -31,12 +38,30 @@ pub enum CompareOp { Le, Gt, Ge, + /// SQL `LIKE` — RHS is a `%`/`_` glob pattern. + Like, + /// SQL `NOT LIKE`. + NotLike, + /// SQL `ILIKE` — case-insensitive `LIKE`. + ILike, + /// SQL `NOT ILIKE`. + NotILike, /// RHS is a regular-expression pattern; matches PromQL `=~`. Regex, /// RHS is a regular-expression pattern; matches PromQL `!~`. NotRegex, } +/// Binary arithmetic operators for `L3Expr::Arith`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum ArithOp { + Add, + Sub, + Mul, + Div, + Mod, +} + /// A scalar expression. Flat conjunctions (`BoolAnd`) / disjunctions /// (`BoolOr`) make per-conjunct selectivity estimation and label-matcher /// lowering straightforward without recursive descent. @@ -58,6 +83,39 @@ pub enum L3Expr { BoolOr(Vec), /// Logical NOT. Not(Box), + /// `expr IS NULL`. + IsNull(Box), + /// `expr IS NOT NULL`. + IsNotNull(Box), + /// `CAST(expr AS to)`. `try_cast` is `true` for SQL `TRY_CAST`, which + /// returns `NULL` on conversion failure instead of raising an error. + Cast { + expr: Box, + to: DataType, + try_cast: bool, + }, + /// `expr [NOT] IN (v1, v2, …)`. + InList { + expr: Box, + list: Vec, + negated: bool, + }, + /// Scalar function call, e.g. `LOWER(col)`, `ABS(x)`. + FunctionCall { name: String, args: Vec }, + /// Binary arithmetic: `left op right`. + Arith { + op: ArithOp, + left: Box, + right: Box, + }, + /// SQL `CASE` expression (both searched and simple forms). + /// `operand` is present for simple CASE (`CASE expr WHEN ...`), + /// absent for searched CASE (`CASE WHEN condition THEN ...`). + Case { + operand: Option>, + branches: Vec<(L3Expr, L3Expr)>, + else_expr: Option>, + }, } impl L3Expr { @@ -70,13 +128,22 @@ impl L3Expr { } } + /// If this expression is a `BoolOr`, return its elements; otherwise a + /// single-element slice containing `self`. + pub fn disjuncts(&self) -> &[L3Expr] { + match self { + L3Expr::BoolOr(v) => v.as_slice(), + _ => std::slice::from_ref(self), + } + } + /// Recursively collect every `ColumnRef` referenced anywhere in this - /// expression. + /// expression. Used by L4 for column-lineage and selectivity estimation. pub fn columns_referenced(&self) -> Vec<&ColumnRef> { match self { L3Expr::Column(c) => vec![c], L3Expr::Literal(_) => vec![], - L3Expr::Compare { left, right, .. } => { + L3Expr::Compare { left, right, .. } | L3Expr::Arith { left, right, .. } => { let mut v = left.columns_referenced(); v.extend(right.columns_referenced()); v @@ -84,7 +151,34 @@ impl L3Expr { L3Expr::BoolAnd(parts) | L3Expr::BoolOr(parts) => { parts.iter().flat_map(|e| e.columns_referenced()).collect() } - L3Expr::Not(e) => e.columns_referenced(), + L3Expr::Not(e) | L3Expr::IsNull(e) | L3Expr::IsNotNull(e) => e.columns_referenced(), + L3Expr::Cast { expr, .. } => expr.columns_referenced(), + L3Expr::InList { expr, list, .. } => { + let mut v = expr.columns_referenced(); + v.extend(list.iter().flat_map(|e| e.columns_referenced())); + v + } + L3Expr::FunctionCall { args, .. } => { + args.iter().flat_map(|e| e.columns_referenced()).collect() + } + L3Expr::Case { + operand, + branches, + else_expr, + } => { + let mut v = vec![]; + if let Some(op) = operand { + v.extend(op.columns_referenced()); + } + for (when, then) in branches { + v.extend(when.columns_referenced()); + v.extend(then.columns_referenced()); + } + if let Some(e) = else_expr { + v.extend(e.columns_referenced()); + } + v + } } } } diff --git a/crates/core/src/intent_algebra/lower.rs b/crates/core/src/intent_algebra/lower.rs index 51ba3062..aa5449ff 100644 --- a/crates/core/src/intent_algebra/lower.rs +++ b/crates/core/src/intent_algebra/lower.rs @@ -267,14 +267,19 @@ fn agg_func_to_intent(func: &AggFunc, acc: &AccuracyTarget) -> AggIntent { AggFunc::Count => AggIntent::Count { accuracy: acc.clone(), }, - AggFunc::Sum => AggIntent::Sum, - AggFunc::Avg => AggIntent::Avg, - AggFunc::Min => AggIntent::Min, - AggFunc::Max => AggIntent::Max, + // PromQL reduces the synthetic sample value, so `col = None`. SQL's + // per-column binding (`SUM(bytes)`) is threaded in `agg_func_to_intent` + // callers that carry an `AggItem.col` resolved to a `ColumnId`. + AggFunc::Sum => AggIntent::Sum { col: None }, + AggFunc::Avg => AggIntent::Avg { col: None }, + AggFunc::Min => AggIntent::Min { col: None }, + AggFunc::Max => AggIntent::Max { col: None }, AggFunc::StdDev { population } => AggIntent::StdDev { + col: None, population: *population, }, AggFunc::Variance { population } => AggIntent::Variance { + col: None, population: *population, }, AggFunc::Quantile(q) => AggIntent::Quantile { diff --git a/crates/core/src/intent_algebra/mod.rs b/crates/core/src/intent_algebra/mod.rs index 0b688544..169cd0c3 100644 --- a/crates/core/src/intent_algebra/mod.rs +++ b/crates/core/src/intent_algebra/mod.rs @@ -30,7 +30,7 @@ pub use column_resolution::{ resolve_column_refs, resolve_named_keys, ResolveError, }; pub use cse::{dedupe_subtrees, CseWorkloadPlan}; -pub use expr_ir::{CompareOp, L3Expr, L3Scalar}; +pub use expr_ir::{ArithOp, CompareOp, L3Expr, L3Scalar}; pub use lower::{convert, convert_root, ConvertError}; pub use names::{BindingName, QueryId}; pub use query_expr::{ diff --git a/crates/core/src/intent_algebra/query_expr.rs b/crates/core/src/intent_algebra/query_expr.rs index c94dced3..c4ad78be 100644 --- a/crates/core/src/intent_algebra/query_expr.rs +++ b/crates/core/src/intent_algebra/query_expr.rs @@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize}; use thiserror::Error; use super::agg_intent::AggIntent; -use super::expr_ir::L3Expr; +use super::expr_ir::{L3Expr, L3Scalar}; use super::names::BindingName; use super::schema::{Column, ColumnId, DataType, Schema}; @@ -363,8 +363,15 @@ impl QueryExpr { dtype: DataType::Float64, nullable: false, }); + // Each reducer types off its own input column (`SUM(bytes)` vs + // `AVG(latency)` in one node); `None` falls back to the sample- + // value probe (PromQL's single-column convention). for intent in aggs { - out_cols.push(intent.output_column(&probe)); + let in_col = intent + .input_col() + .and_then(|id| in_schema.columns.get(id)) + .unwrap_or(&probe); + out_cols.push(intent.output_column(in_col)); } let unique_keys = if by.is_empty() { Vec::new() @@ -392,8 +399,37 @@ impl QueryExpr { | QueryExpr::Partition { child, .. } | QueryExpr::Sort { child, .. } | QueryExpr::Limit { child, .. } - | QueryExpr::Subquery { child, .. } - | QueryExpr::Project { child, .. } => child.output_schema_in(scope), + | QueryExpr::Subquery { child, .. } => child.output_schema_in(scope), + + // π — one output column per projection item. Each item's type is + // inferred from its expression against the child schema; the name + // is the explicit alias or a derived default. Projection may drop + // the grouping/time columns, so unique_keys reset and time_index + // is re-found by name. + QueryExpr::Project { cols, child } => { + let in_schema = child.output_schema_in(scope)?; + let columns: Vec = cols + .iter() + .enumerate() + .map(|(i, item)| { + let (dtype, nullable) = infer_expr_type(&item.expr, &in_schema); + Column { + name: item + .alias + .clone() + .unwrap_or_else(|| default_proj_name(&item.expr, i)), + dtype, + nullable, + } + }) + .collect(); + let time_index = columns.iter().position(|c| c.name == "ts"); + Ok(Schema { + columns, + time_index, + unique_keys: Vec::new(), + }) + } QueryExpr::Distinct { cols, child } => { let in_schema = child.output_schema_in(scope)?; @@ -416,14 +452,120 @@ impl QueryExpr { .first() .ok_or(QueryExprError::EmptyMerge) .and_then(|c| c.output_schema_in(scope)), - QueryExpr::SetOp { left, .. } | QueryExpr::Join { left, .. } => { - left.output_schema_in(scope) + // Set operations are union-compatible: both sides share the left's + // column shape, so the output schema is the left's. (Row identity + // is not preserved across a UNION, so unique_keys are dropped.) + QueryExpr::SetOp { left, .. } => { + let mut s = left.output_schema_in(scope)?; + s.unique_keys.clear(); + Ok(s) + } + // ⋈ — output is the concatenation of both inputs' columns. Outer + // joins make the non-preserved side nullable. Post-join row + // identity isn't provable in general, so unique_keys reset. + QueryExpr::Join { + kind, left, right, .. + } => { + let l = left.output_schema_in(scope)?; + let r = right.output_schema_in(scope)?; + let (left_null, right_null) = match kind { + JoinKind::Left => (false, true), + JoinKind::Right => (true, false), + JoinKind::Full => (true, true), + JoinKind::Inner | JoinKind::Cross => (false, false), + }; + let l_len = l.columns.len(); + let mut columns = Vec::with_capacity(l_len + r.columns.len()); + columns.extend(l.columns.iter().cloned().map(|mut c| { + c.nullable |= left_null; + c + })); + columns.extend(r.columns.iter().cloned().map(|mut c| { + c.nullable |= right_null; + c + })); + let time_index = l.time_index.or(r.time_index.map(|i| i + l_len)); + Ok(Schema { + columns, + time_index, + unique_keys: Vec::new(), + }) } QueryExpr::BinaryOp { lhs, .. } => lhs.output_schema_in(scope), } } } +/// Infer the `(DataType, nullable)` a scalar [`L3Expr`] produces against an +/// input [`Schema`]. Used by `Project` schema derivation. Approximate at L3: +/// unknown columns and bare `FunctionCall`s fall back to a permissive default +/// (the L4/emit layer refines with a real function/type registry). +fn infer_expr_type(expr: &L3Expr, schema: &Schema) -> (DataType, bool) { + match expr { + L3Expr::Column(ColumnRef::Named(name)) => schema + .column_id(name) + .and_then(|id| schema.columns.get(id)) + .map(|c| (c.dtype.clone(), c.nullable)) + .unwrap_or((DataType::Utf8, true)), + L3Expr::Column(ColumnRef::SampleValue) => schema + .column_id("value") + .and_then(|id| schema.columns.get(id)) + .map(|c| (c.dtype.clone(), c.nullable)) + .unwrap_or((DataType::Float64, false)), + L3Expr::Column(ColumnRef::Wildcard) => (DataType::Float64, false), + L3Expr::Literal(s) => match s { + L3Scalar::Int64(_) => (DataType::Int64, false), + L3Scalar::Float64(_) => (DataType::Float64, false), + L3Scalar::Utf8(_) => (DataType::Utf8, false), + L3Scalar::Boolean(_) => (DataType::Bool, false), + L3Scalar::Null => (DataType::Float64, true), + }, + // Boolean-valued expressions (SQL three-valued logic → nullable). + L3Expr::Compare { .. } + | L3Expr::BoolAnd(_) + | L3Expr::BoolOr(_) + | L3Expr::Not(_) + | L3Expr::IsNull(_) + | L3Expr::IsNotNull(_) + | L3Expr::InList { .. } => (DataType::Bool, true), + L3Expr::Arith { left, right, .. } => { + let (lt, ln) = infer_expr_type(left, schema); + let (rt, rn) = infer_expr_type(right, schema); + let dtype = if matches!(lt, DataType::Int64) && matches!(rt, DataType::Int64) { + DataType::Int64 + } else { + DataType::Float64 + }; + (dtype, ln || rn) + } + L3Expr::Cast { to, try_cast, expr } => { + let (_, nullable) = infer_expr_type(expr, schema); + (to.clone(), *try_cast || nullable) + } + // No function/type registry at L3 — default permissive. + L3Expr::FunctionCall { .. } => (DataType::Float64, true), + L3Expr::Case { + branches, + else_expr, + .. + } => branches + .first() + .map(|(_, then)| (infer_expr_type(then, schema).0, true)) + .or_else(|| else_expr.as_ref().map(|e| infer_expr_type(e, schema))) + .unwrap_or((DataType::Float64, true)), + } +} + +/// Default output-column name for a projection item with no explicit alias: +/// a bare column keeps its name; anything else gets a positional `col_{i}`. +fn default_proj_name(expr: &L3Expr, idx: usize) -> String { + match expr { + L3Expr::Column(ColumnRef::Named(n)) => n.clone(), + L3Expr::Column(ColumnRef::SampleValue) => "value".to_string(), + _ => format!("col_{idx}"), + } +} + /// Lexical scope for `LetBinding` / `Ref` resolution. #[derive(Debug, Default, Clone)] pub struct BindingScope { @@ -443,3 +585,181 @@ impl BindingScope { self.bindings.get(name.as_str()) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::intent_algebra::expr_ir::{ArithOp, CompareOp}; + + fn col(name: &str, dtype: DataType, nullable: bool) -> Column { + Column { + name: name.into(), + dtype, + nullable, + } + } + + fn scan( + columns: Vec, + time_index: Option, + uk: Vec>, + ) -> QueryExpr { + QueryExpr::Scan { + source: Source::Table { + table_ref: "t".into(), + }, + predicates: vec![], + schema: Schema { + columns, + time_index, + unique_keys: uk, + }, + } + } + + #[test] + fn project_retypes_and_renames_per_item() { + let child = scan( + vec![ + col("ts", DataType::Timestamp, false), + col("host", DataType::Utf8, false), + col("value", DataType::Float64, false), + ], + Some(0), + vec![vec![0, 1]], + ); + let q = QueryExpr::Project { + cols: vec![ + // bare column passthrough keeps its name + type + ProjectItem { + alias: None, + expr: L3Expr::Column(ColumnRef::Named("host".into())), + }, + // arithmetic over the sample value → Float64 + ProjectItem { + alias: Some("dbl".into()), + expr: L3Expr::Arith { + op: ArithOp::Add, + left: Box::new(L3Expr::Column(ColumnRef::SampleValue)), + right: Box::new(L3Expr::Column(ColumnRef::SampleValue)), + }, + }, + // comparison → Bool (nullable under 3-valued logic) + ProjectItem { + alias: Some("flag".into()), + expr: L3Expr::Compare { + left: Box::new(L3Expr::Column(ColumnRef::SampleValue)), + op: CompareOp::Gt, + right: Box::new(L3Expr::Literal(L3Scalar::Float64(0.0))), + }, + }, + ], + child: Box::new(child), + }; + let s = q.output_schema().unwrap(); + assert_eq!(s.columns.len(), 3); + assert_eq!(s.columns[0], col("host", DataType::Utf8, false)); + assert_eq!(s.columns[1], col("dbl", DataType::Float64, false)); + assert_eq!(s.columns[2], col("flag", DataType::Bool, true)); + // projection drops the time axis + unique keys (ts not retained) + assert!(s.time_index.is_none()); + assert!(s.unique_keys.is_empty()); + } + + #[test] + fn project_keeps_time_index_when_ts_passed_through() { + let child = scan( + vec![ + col("ts", DataType::Timestamp, false), + col("value", DataType::Float64, false), + ], + Some(0), + vec![], + ); + let q = QueryExpr::Project { + cols: vec![ + ProjectItem { + alias: None, + expr: L3Expr::Column(ColumnRef::SampleValue), + }, + ProjectItem { + alias: None, + expr: L3Expr::Column(ColumnRef::Named("ts".into())), + }, + ], + child: Box::new(child), + }; + let s = q.output_schema().unwrap(); + assert_eq!(s.columns[0].name, "value"); + assert_eq!(s.columns[1].name, "ts"); + assert_eq!(s.time_index, Some(1)); + } + + fn join(kind: JoinKind) -> QueryExpr { + let left = scan(vec![col("a", DataType::Int64, false)], None, vec![vec![0]]); + let right = scan(vec![col("b", DataType::Utf8, false)], None, vec![]); + QueryExpr::Join { + kind, + pred: Predicate(L3Expr::Literal(L3Scalar::Boolean(true))), + left: Box::new(left), + right: Box::new(right), + } + } + + #[test] + fn inner_join_concatenates_both_sides() { + let s = join(JoinKind::Inner).output_schema().unwrap(); + assert_eq!(s.columns.len(), 2); + assert_eq!(s.columns[0], col("a", DataType::Int64, false)); + assert_eq!(s.columns[1], col("b", DataType::Utf8, false)); + // post-join row identity not provable → no unique keys + assert!(s.unique_keys.is_empty()); + } + + #[test] + fn left_join_makes_right_side_nullable() { + let s = join(JoinKind::Left).output_schema().unwrap(); + assert!(!s.columns[0].nullable, "preserved left side stays non-null"); + assert!(s.columns[1].nullable, "right side nullable under LEFT JOIN"); + } + + #[test] + fn full_join_makes_both_sides_nullable() { + let s = join(JoinKind::Full).output_schema().unwrap(); + assert!(s.columns[0].nullable); + assert!(s.columns[1].nullable); + } + + #[test] + fn setop_takes_left_shape_and_drops_unique_keys() { + let left = scan( + vec![ + col("k", DataType::Utf8, false), + col("v", DataType::Int64, false), + ], + None, + vec![vec![0]], + ); + let right = scan( + vec![ + col("k", DataType::Utf8, false), + col("v", DataType::Int64, false), + ], + None, + vec![vec![0]], + ); + let q = QueryExpr::SetOp { + kind: SetOpKind::Union, + all: false, + left: Box::new(left), + right: Box::new(right), + }; + let s = q.output_schema().unwrap(); + assert_eq!(s.columns.len(), 2); + assert_eq!(s.columns[0].name, "k"); + assert!( + s.unique_keys.is_empty(), + "UNION does not preserve row identity" + ); + } +} diff --git a/crates/lower/tests/promql_conformance.rs b/crates/lower/tests/promql_conformance.rs index 49dc8d37..83f1ebf7 100644 --- a/crates/lower/tests/promql_conformance.rs +++ b/crates/lower/tests/promql_conformance.rs @@ -205,7 +205,7 @@ fn sum_collapses_all_series() { // SEMANTICS: `sum(v)` → one output series. No grouping → no Partition. let qe = ok("sum(node_filesystem_size_bytes)"); assert!(matches!(&qe, QueryExpr::Aggregate { .. })); - assert!(has(&qe, |i| matches!(i, AggIntent::Sum))); + assert!(has(&qe, |i| matches!(i, AggIntent::Sum { .. }))); } #[test] @@ -220,7 +220,7 @@ fn sum_by_preserves_dimensions_as_partition() { *keys, PartitionKeys::By(vec!["instance".into(), "job".into()]) ); - assert!(has(&qe, |i| matches!(i, AggIntent::Sum))); + assert!(has(&qe, |i| matches!(i, AggIntent::Sum { .. }))); } #[test] @@ -233,9 +233,9 @@ fn count_is_cardinality() { #[test] fn avg_min_max_stddev_stdvar_quantile_aggregators() { - assert!(has(&ok("avg(up)"), |i| matches!(i, AggIntent::Avg))); - assert!(has(&ok("min(up)"), |i| matches!(i, AggIntent::Min))); - assert!(has(&ok("max(up)"), |i| matches!(i, AggIntent::Max))); + assert!(has(&ok("avg(up)"), |i| matches!(i, AggIntent::Avg { .. }))); + assert!(has(&ok("min(up)"), |i| matches!(i, AggIntent::Min { .. }))); + assert!(has(&ok("max(up)"), |i| matches!(i, AggIntent::Max { .. }))); assert!(has(&ok("stddev(up)"), |i| matches!( i, AggIntent::StdDev { .. } @@ -278,7 +278,7 @@ fn sum_of_rate_is_two_levels() { let QueryExpr::Aggregate { aggs, child, .. } = &qe else { panic!("expected outer Aggregate{{Sum}}, got {qe:?}"); }; - assert!(matches!(aggs.as_slice(), [AggIntent::Sum])); + assert!(matches!(aggs.as_slice(), [AggIntent::Sum { .. }])); assert!(matches!( child.as_ref(), QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Rate { .. }]) @@ -295,7 +295,7 @@ fn sum_by_of_rate_groups_outer_level() { // Partition → Sum → Rate. assert!(matches!( child.as_ref(), - QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Sum]) + QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Sum { .. }]) )); assert!(has(&qe, |i| matches!(i, AggIntent::Rate { .. }))); } @@ -322,10 +322,10 @@ fn over_time_functions_window_then_reduce() { "{q}: expected Window" ); let matched = intents(&qe).iter().any(|i| match want { - "avg" => matches!(i, AggIntent::Avg), - "max" => matches!(i, AggIntent::Max), - "min" => matches!(i, AggIntent::Min), - "sum" => matches!(i, AggIntent::Sum), + "avg" => matches!(i, AggIntent::Avg { .. }), + "max" => matches!(i, AggIntent::Max { .. }), + "min" => matches!(i, AggIntent::Min { .. }), + "sum" => matches!(i, AggIntent::Sum { .. }), "count" => matches!(i, AggIntent::Count { .. }), _ => unreachable!(), }); diff --git a/crates/lower/tests/promql_lowering.rs b/crates/lower/tests/promql_lowering.rs index fa697225..3ac06cb1 100644 --- a/crates/lower/tests/promql_lowering.rs +++ b/crates/lower/tests/promql_lowering.rs @@ -90,7 +90,7 @@ fn outer_sum_by_wraps_in_partition() { let QueryExpr::Aggregate { aggs, child, .. } = child.as_ref() else { panic!("expected outer Aggregate{{Sum}} under Partition, got {child:?}"); }; - assert!(matches!(aggs.as_slice(), [AggIntent::Sum])); + assert!(matches!(aggs.as_slice(), [AggIntent::Sum { .. }])); // Inner: Window over Aggregate{Quantile}. let QueryExpr::Window { child, .. } = child.as_ref() else { panic!("expected Window under the outer Sum, got {child:?}"); @@ -110,7 +110,7 @@ fn avg_over_time_maps_to_avg_intent() { assert_eq!(*size, Duration::from_secs(600)); assert!(matches!( child.as_ref(), - QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Avg]) + QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Avg { .. }]) )); } @@ -122,7 +122,7 @@ fn stddev_and_stdvar_over_time() { }; assert!(matches!( child.as_ref(), - QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::StdDev { population: false }]) + QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::StdDev { population: false, .. }]) )); let qe = lower("stdvar_over_time(m[5m])"); @@ -131,7 +131,7 @@ fn stddev_and_stdvar_over_time() { }; assert!(matches!( child.as_ref(), - QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Variance { population: false }]) + QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Variance { population: false, .. }]) )); } @@ -206,7 +206,7 @@ fn sum_over_rate_keeps_both_levels() { let QueryExpr::Aggregate { aggs, child, .. } = &qe else { panic!("expected outer Aggregate{{Sum}}, got {qe:?}"); }; - assert!(matches!(aggs.as_slice(), [AggIntent::Sum])); + assert!(matches!(aggs.as_slice(), [AggIntent::Sum { .. }])); let QueryExpr::Aggregate { aggs, child, .. } = child.as_ref() else { panic!("expected inner Aggregate{{Rate}}, got {child:?}"); }; @@ -228,7 +228,7 @@ fn sum_by_over_rate_groups_the_outer_sum() { let QueryExpr::Aggregate { aggs, child, .. } = child.as_ref() else { panic!("expected Aggregate{{Sum}} under Partition, got {child:?}"); }; - assert!(matches!(aggs.as_slice(), [AggIntent::Sum])); + assert!(matches!(aggs.as_slice(), [AggIntent::Sum { .. }])); assert!(matches!( child.as_ref(), QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Rate { .. }]) From 3f7f3602536c96eb9cf373690462edd0059cbba2 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 26 May 2026 09:51:45 -0600 Subject: [PATCH 18/40] =?UTF-8?q?feat(core):=20thread=20per-aggregate=20in?= =?UTF-8?q?put=20column=20through=20L2=E2=86=92L3=20convert=20(D4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `convert` now resolves each relational `AggItem.col` (a name/SampleValue) to a positional `ColumnId` and threads it onto the L3 reducer, so a SQL-shaped `SUM(bytes), AVG(latency)` lowers to `[Sum{col:1}, Avg{col:2}]` and the derived output schema types each result off its own input column. PromQL's SampleValue stays `col: None`. Unit-tested both paths. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/core/src/intent_algebra/lower.rs | 146 +++++++++++++++++++++--- 1 file changed, 131 insertions(+), 15 deletions(-) diff --git a/crates/core/src/intent_algebra/lower.rs b/crates/core/src/intent_algebra/lower.rs index aa5449ff..09068a98 100644 --- a/crates/core/src/intent_algebra/lower.rs +++ b/crates/core/src/intent_algebra/lower.rs @@ -17,10 +17,11 @@ use crate::intent_algebra::binder::Binder; use crate::intent_algebra::column_resolution::{resolve_named_keys, ResolveError}; use crate::intent_algebra::names::BindingName; use crate::intent_algebra::query_expr::{ - PartitionKeys as CPartitionKeys, Predicate, QueryExpr as CQueryExpr, Source, WindowKind, + ColumnRef, PartitionKeys as CPartitionKeys, Predicate, QueryExpr as CQueryExpr, Source, + WindowKind, }; use crate::intent_algebra::relational::{AggFunc, QueryExpr as LQueryExpr}; -use crate::intent_algebra::schema::Schema; +use crate::intent_algebra::schema::{ColumnId, Schema}; use crate::types::AccuracyTarget; /// Errors produced while converting a Layer-2 tree to canonical. @@ -78,7 +79,8 @@ pub fn convert( // becomes `Window { Aggregate { by: [] } }`; GROUP BY keys wrap the // result in a `Partition`. if aggs.len() == 1 && having.is_none() { - let intent = agg_func_to_intent(&aggs[0].func, acc); + let intent = + agg_func_to_intent(&aggs[0].func, acc, resolve_agg_col(&aggs[0].col, schema)); let sketch = match input.as_ref() { LQueryExpr::Window { duration, @@ -120,7 +122,7 @@ pub fn convert( let by = resolve_named_keys(keys, schema)?; let intents = aggs .iter() - .map(|item| agg_func_to_intent(&item.func, acc)) + .map(|item| agg_func_to_intent(&item.func, acc, resolve_agg_col(&item.col, schema))) .collect(); CQueryExpr::Aggregate { by, @@ -260,26 +262,36 @@ fn scan(metric: String, schema: &Schema, predicates: Vec) -> CQueryEx } } +/// Resolve a Layer-2 aggregate-input [`ColumnRef`] to a positional input +/// column. `SampleValue` / `Wildcard` carry no specific column → `None` (the +/// PromQL sample-value convention); a named column (`SUM(bytes)`) resolves to +/// its position so the L3 reducer types off the right input. +fn resolve_agg_col(col: &ColumnRef, schema: &Schema) -> Option { + match col { + ColumnRef::Named(name) => schema.column_id(name), + ColumnRef::SampleValue | ColumnRef::Wildcard => None, + } +} + /// Map a Layer-2 [`AggFunc`] to its canonical [`AggIntent`], threading the -/// workload's accuracy target onto the approximate intents. -fn agg_func_to_intent(func: &AggFunc, acc: &AccuracyTarget) -> AggIntent { +/// workload's accuracy target onto the approximate intents and the resolved +/// input column (`col`) onto the single-column reducers. `col = None` is the +/// PromQL sample-value convention. +fn agg_func_to_intent(func: &AggFunc, acc: &AccuracyTarget, col: Option) -> AggIntent { match func { AggFunc::Count => AggIntent::Count { accuracy: acc.clone(), }, - // PromQL reduces the synthetic sample value, so `col = None`. SQL's - // per-column binding (`SUM(bytes)`) is threaded in `agg_func_to_intent` - // callers that carry an `AggItem.col` resolved to a `ColumnId`. - AggFunc::Sum => AggIntent::Sum { col: None }, - AggFunc::Avg => AggIntent::Avg { col: None }, - AggFunc::Min => AggIntent::Min { col: None }, - AggFunc::Max => AggIntent::Max { col: None }, + AggFunc::Sum => AggIntent::Sum { col }, + AggFunc::Avg => AggIntent::Avg { col }, + AggFunc::Min => AggIntent::Min { col }, + AggFunc::Max => AggIntent::Max { col }, AggFunc::StdDev { population } => AggIntent::StdDev { - col: None, + col, population: *population, }, AggFunc::Variance { population } => AggIntent::Variance { - col: None, + col, population: *population, }, AggFunc::Quantile(q) => AggIntent::Quantile { @@ -297,3 +309,107 @@ fn agg_func_to_intent(func: &AggFunc, acc: &AccuracyTarget) -> AggIntent { AggFunc::Increase { window } => AggIntent::Increase { window: *window }, } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::intent_algebra::agg_intent::AggIntent; + use crate::intent_algebra::query_expr::QueryExpr as CQueryExpr; + use crate::intent_algebra::relational::{ + AggFunc, AggItem, QueryExpr as LQueryExpr, SourceSpec, + }; + use crate::intent_algebra::schema::{Column, DataType, Schema}; + + fn col(name: &str, dtype: DataType) -> Column { + Column { + name: name.into(), + dtype, + nullable: false, + } + } + + /// A SQL-shaped `SELECT SUM(bytes), AVG(latency) FROM t` lowers each + /// reducer onto its own input column (positional), and the derived output + /// schema types each result off that column (`SUM(bytes:Int64)→Int64`). + #[test] + fn multi_column_aggregate_threads_per_agg_col() { + let schema = Schema::with_time_index( + vec![ + col("ts", DataType::Timestamp), + col("bytes", DataType::Int64), + col("latency", DataType::Float64), + col("value", DataType::Float64), + ], + 0, + vec![], + ); + let tree = LQueryExpr::Aggregate { + keys: vec![], + aggs: vec![ + AggItem { + alias: "total_bytes".into(), + func: AggFunc::Sum, + col: ColumnRef::Named("bytes".into()), + distinct: false, + }, + AggItem { + alias: "avg_latency".into(), + func: AggFunc::Avg, + col: ColumnRef::Named("latency".into()), + distinct: false, + }, + ], + having: None, + input: Box::new(LQueryExpr::Source(SourceSpec { name: "t".into() })), + }; + + let l3 = convert(&tree, &schema, &AccuracyTarget::Exact).unwrap(); + let CQueryExpr::Aggregate { by, aggs, .. } = &l3 else { + panic!("expected Aggregate, got {l3:?}"); + }; + assert!(by.is_empty()); + // bytes is column 1, latency is column 2 in the input schema. + assert_eq!( + aggs, + &vec![ + AggIntent::Sum { col: Some(1) }, + AggIntent::Avg { col: Some(2) }, + ] + ); + + // Output schema types each reducer off its own input column. + let out = l3.output_schema().unwrap(); + assert_eq!(out.columns[0], col("sum", DataType::Int64)); // SUM(bytes:Int64) + assert_eq!(out.columns[1], col("avg", DataType::Float64)); // AVG(latency)→Float64 + } + + /// PromQL's single sample-value reducer stays `col: None`. + #[test] + fn promql_sample_value_agg_stays_col_none() { + let schema = Schema::with_time_index( + vec![ + col("ts", DataType::Timestamp), + col("value", DataType::Float64), + ], + 0, + vec![], + ); + let tree = LQueryExpr::Aggregate { + keys: vec![], + aggs: vec![AggItem { + alias: "value".into(), + func: AggFunc::Sum, + col: ColumnRef::SampleValue, + distinct: false, + }], + having: None, + input: Box::new(LQueryExpr::Source(SourceSpec { name: "m".into() })), + }; + let l3 = convert(&tree, &schema, &AccuracyTarget::Exact).unwrap(); + // single-agg fused path → bare Aggregate (no keys → no Partition) + let CQueryExpr::Aggregate { aggs, .. } = &l3 else { + panic!("expected Aggregate, got {l3:?}"); + }; + assert_eq!(aggs, &vec![AggIntent::Sum { col: None }]); + } +} From 90a9193d00420b1cf33951c4006279ec4bbe58af Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 26 May 2026 10:04:16 -0600 Subject: [PATCH 19/40] =?UTF-8?q?feat(core):=20make=20L2=E2=86=92L3=20conv?= =?UTF-8?q?ert=20bottom-up=20schema-aware=20for=20SQL=20trees?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The converter previously threaded one Binder-built root schema everywhere — correct for single-leaf PromQL but wrong for SQL, where a JOIN's schema is the concatenation of both sides and table leaves carry real typed columns. - `relational::SourceSpec` gains `schema: Option` — SQL leaves carry their DataFusion-resolved schema (→ `Source::Table`); PromQL leaves stay `None` (→ `Source::TimeSeries`, Binder-synthesized). - `convert` resolves `Aggregate` keys + per-reducer input columns and `TopK` keys against the **converted child's** `output_schema`, not a single root schema — so names bind to the right positions across joins/projects. - `Window.output_schema` now passes the child schema through (was erroring on the canonical Window-over-Aggregate fused shape, which has no time_index). Unit test: `SELECT region, SUM(bytes), COUNT(*) FROM logs JOIN meta GROUP BY region` resolves region→col 3 and bytes→col 1 of the concatenated schema. 102 tests green. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/core/src/intent_algebra/binder.rs | 2 +- crates/core/src/intent_algebra/lower.rs | 206 ++++++++++++++----- crates/core/src/intent_algebra/query_expr.rs | 14 +- crates/core/src/intent_algebra/relational.rs | 25 +++ crates/lower/src/promql.rs | 2 +- 5 files changed, 186 insertions(+), 63 deletions(-) diff --git a/crates/core/src/intent_algebra/binder.rs b/crates/core/src/intent_algebra/binder.rs index 4d0d2bc4..9f594db2 100644 --- a/crates/core/src/intent_algebra/binder.rs +++ b/crates/core/src/intent_algebra/binder.rs @@ -132,7 +132,7 @@ mod tests { use crate::intent_algebra::relational::{QueryExpr as LQueryExpr, SourceSpec}; fn src(name: &str) -> LQueryExpr { - LQueryExpr::Source(SourceSpec { name: name.into() }) + LQueryExpr::Source(SourceSpec::new(name)) } #[test] diff --git a/crates/core/src/intent_algebra/lower.rs b/crates/core/src/intent_algebra/lower.rs index 09068a98..2a59d58f 100644 --- a/crates/core/src/intent_algebra/lower.rs +++ b/crates/core/src/intent_algebra/lower.rs @@ -20,17 +20,21 @@ use crate::intent_algebra::query_expr::{ ColumnRef, PartitionKeys as CPartitionKeys, Predicate, QueryExpr as CQueryExpr, Source, WindowKind, }; -use crate::intent_algebra::relational::{AggFunc, QueryExpr as LQueryExpr}; +use crate::intent_algebra::relational::{AggFunc, QueryExpr as LQueryExpr, SourceSpec}; use crate::intent_algebra::schema::{ColumnId, Schema}; use crate::types::AccuracyTarget; /// Errors produced while converting a Layer-2 tree to canonical. #[derive(Debug, Error)] pub enum ConvertError { - /// A column reference (`Aggregate` key, `Partition` / `TopK` key) did not - /// resolve against the inherited schema. + /// A column reference (`Aggregate` key, `TopK` key, aggregate input column) + /// did not resolve against the child's derived schema. #[error("column resolution failed: {0}")] Resolve(#[from] ResolveError), + /// Deriving the schema of an already-converted child failed (needed to + /// resolve positional column references against it). + #[error("schema derivation failed: {0}")] + Schema(#[from] crate::intent_algebra::query_expr::QueryExprError), } /// Lower a Layer-2 tree to canonical L3, threading `accuracy` onto every @@ -39,18 +43,26 @@ pub fn convert_root( legacy: &LQueryExpr, accuracy: &AccuracyTarget, ) -> Result { - let schema = Binder::new().bind(legacy); - convert(legacy, &schema, accuracy) + let fallback = Binder::new().bind(legacy); + convert(legacy, &fallback, accuracy) } -/// Convert a Layer-2 tree to canonical against an explicit inherited schema. +/// Convert a Layer-2 tree to canonical L3. +/// +/// `fallback` is the leaf schema used for schema-less (PromQL) `Source`s — +/// the Binder's usage-derived `(ts, value)` floor + referenced labels. SQL +/// leaves carry their own resolved schema on [`SourceSpec::schema`], so the +/// fallback is unused for them. Positional column references (`Aggregate` +/// keys + input columns, `TopK` keys) resolve against the **converted child's +/// derived output schema**, so a `JOIN`'s concatenated schema and a table's +/// real columns bind to the right positions. pub fn convert( legacy: &LQueryExpr, - schema: &Schema, + fallback: &Schema, acc: &AccuracyTarget, ) -> Result { Ok(match legacy { - LQueryExpr::Source(spec) => scan(spec.name.clone(), schema, Vec::new()), + LQueryExpr::Source(spec) => scan(spec, fallback, Vec::new()), LQueryExpr::Ref(name) => CQueryExpr::Ref { name: BindingName::new(name.clone()), @@ -61,11 +73,11 @@ pub fn convert( LQueryExpr::Filter { pred, input } => match input.as_ref() { LQueryExpr::Source(spec) => { let predicates = pred.conjuncts().iter().cloned().map(Predicate).collect(); - scan(spec.name.clone(), schema, predicates) + scan(spec, fallback, predicates) } other => CQueryExpr::Filter { pred: Predicate(pred.clone()), - child: Box::new(convert(other, schema, acc)?), + child: Box::new(convert(other, fallback, acc)?), }, }, @@ -77,36 +89,42 @@ pub fn convert( } => { // Single-statistic aggregate (no HAVING) fuses: a `Window` input // becomes `Window { Aggregate { by: [] } }`; GROUP BY keys wrap the - // result in a `Partition`. + // result in a `Partition`. The reducer's input column resolves + // against the aggregate's *direct* input (the scan under any window). if aggs.len() == 1 && having.is_none() { - let intent = - agg_func_to_intent(&aggs[0].func, acc, resolve_agg_col(&aggs[0].col, schema)); - let sketch = match input.as_ref() { + let (agg_input_l2, window): (&LQueryExpr, Option<(_, _)>) = match input.as_ref() { LQueryExpr::Window { duration, slide, input: win_input, - } => CQueryExpr::Window { + } => (win_input, Some((*duration, *slide))), + other => (other, None), + }; + let agg_child = convert(agg_input_l2, fallback, acc)?; + let agg_in_schema = agg_child.output_schema()?; + let intent = agg_func_to_intent( + &aggs[0].func, + acc, + resolve_agg_col(&aggs[0].col, &agg_in_schema), + ); + let aggregate = CQueryExpr::Aggregate { + by: Vec::new(), + aggs: vec![intent], + having: None, + child: Box::new(agg_child), + }; + let sketch = match window { + Some((duration, slide)) => CQueryExpr::Window { kind: if slide.is_some() { WindowKind::Sliding } else { WindowKind::Tumbling }, - size: *duration, - slide: *slide, - child: Box::new(CQueryExpr::Aggregate { - by: Vec::new(), - aggs: vec![intent], - having: None, - child: Box::new(convert(win_input, schema, acc)?), - }), - }, - other => CQueryExpr::Aggregate { - by: Vec::new(), - aggs: vec![intent], - having: None, - child: Box::new(convert(other, schema, acc)?), + size: duration, + slide, + child: Box::new(aggregate), }, + None => aggregate, }; return Ok(if keys.is_empty() { sketch @@ -118,17 +136,22 @@ pub fn convert( }); } - // Plain canonical `Aggregate`: multi-agg or HAVING-bearing. - let by = resolve_named_keys(keys, schema)?; + // Plain canonical `Aggregate`: multi-agg or HAVING-bearing. Keys + + // per-reducer input columns resolve against the child's schema. + let child = convert(input, fallback, acc)?; + let child_schema = child.output_schema()?; + let by = resolve_named_keys(keys, &child_schema)?; let intents = aggs .iter() - .map(|item| agg_func_to_intent(&item.func, acc, resolve_agg_col(&item.col, schema))) + .map(|item| { + agg_func_to_intent(&item.func, acc, resolve_agg_col(&item.col, &child_schema)) + }) .collect(); CQueryExpr::Aggregate { by, aggs: intents, having: having.clone().map(Predicate), - child: Box::new(convert(input, schema, acc)?), + child: Box::new(child), } } @@ -144,21 +167,23 @@ pub fn convert( }, size: *duration, slide: *slide, - child: Box::new(convert(input, schema, acc)?), + child: Box::new(convert(input, fallback, acc)?), }, LQueryExpr::Partition { keys, input } => CQueryExpr::Partition { keys: keys.clone(), - child: Box::new(convert(input, schema, acc)?), + child: Box::new(convert(input, fallback, acc)?), }, LQueryExpr::Distinct { cols, input } => CQueryExpr::Distinct { cols: cols.clone(), - child: Box::new(convert(input, schema, acc)?), + child: Box::new(convert(input, fallback, acc)?), }, LQueryExpr::TopK { k, by, input } => { - let by = resolve_named_keys(by, schema)?; + let child = convert(input, fallback, acc)?; + let child_schema = child.output_schema()?; + let by = resolve_named_keys(by, &child_schema)?; CQueryExpr::Aggregate { by, aggs: vec![AggIntent::TopK { @@ -166,14 +191,14 @@ pub fn convert( accuracy: acc.clone(), }], having: None, - child: Box::new(convert(input, schema, acc)?), + child: Box::new(child), } } LQueryExpr::Merge { inputs } => CQueryExpr::Merge { children: inputs .iter() - .map(|i| convert(i, schema, acc)) + .map(|i| convert(i, fallback, acc)) .collect::, _>>()?, }, @@ -208,19 +233,19 @@ pub fn convert( LQueryExpr::Sort { keys, input } => CQueryExpr::Sort { keys: keys.clone(), - child: Box::new(convert(input, schema, acc)?), + child: Box::new(convert(input, fallback, acc)?), }, LQueryExpr::Limit { n, offset, input } => CQueryExpr::Limit { n: *n as usize, offset: *offset as usize, - child: Box::new(convert(input, schema, acc)?), + child: Box::new(convert(input, fallback, acc)?), }, LQueryExpr::LetBinding { name, expr, body } => CQueryExpr::LetBinding { name: BindingName::new(name.clone()), - expr: Box::new(convert(expr, schema, acc)?), - child: Box::new(convert(body, schema, acc)?), + expr: Box::new(convert(expr, fallback, acc)?), + child: Box::new(convert(body, fallback, acc)?), }, LQueryExpr::PromQLSubquery { @@ -230,7 +255,7 @@ pub fn convert( } => CQueryExpr::Subquery { range: *range, resolution: *resolution, - child: Box::new(convert(input, schema, acc)?), + child: Box::new(convert(input, fallback, acc)?), }, LQueryExpr::BinaryOp { @@ -252,13 +277,28 @@ pub fn convert( }) } -/// Build a canonical `Scan` over a time-series source carrying the Binder's -/// self-contained schema. -fn scan(metric: String, schema: &Schema, predicates: Vec) -> CQueryExpr { +/// Build a canonical `Scan`. A schema-bearing [`SourceSpec`] (SQL table) emits +/// a `Source::Table` carrying that resolved schema; a schema-less one (PromQL) +/// emits a `Source::TimeSeries` carrying the Binder's usage-derived `fallback`. +fn scan(spec: &SourceSpec, fallback: &Schema, predicates: Vec) -> CQueryExpr { + let (source, schema) = match &spec.schema { + Some(s) => ( + Source::Table { + table_ref: spec.name.clone(), + }, + s.clone(), + ), + None => ( + Source::TimeSeries { + metric: spec.name.clone(), + }, + fallback.clone(), + ), + }; CQueryExpr::Scan { - source: Source::TimeSeries { metric }, + source, predicates, - schema: schema.clone(), + schema, } } @@ -314,7 +354,7 @@ fn agg_func_to_intent(func: &AggFunc, acc: &AccuracyTarget, col: Option Ok(schema.clone()), - QueryExpr::Window { child, .. } => { - let in_schema = child.output_schema_in(scope)?; - if in_schema.time_index.is_none() { - return Err(QueryExprError::WindowMissingTimeIndex); - } - Ok(in_schema) - } + // ψ — a window reshapes the time axis but not the column set. Over a + // time-indexed Scan it preserves the time_index; over an Aggregate + // (the canonical Window-over-Aggregate fused shape) the child has + // already consumed the time axis, so the child schema passes through. + QueryExpr::Window { child, .. } => child.output_schema_in(scope), QueryExpr::Aggregate { by, aggs, child, .. diff --git a/crates/core/src/intent_algebra/relational.rs b/crates/core/src/intent_algebra/relational.rs index 592ef590..c04d530f 100644 --- a/crates/core/src/intent_algebra/relational.rs +++ b/crates/core/src/intent_algebra/relational.rs @@ -11,12 +11,37 @@ use std::time::Duration; use super::expr_ir::L3Expr; pub use super::query_expr::{BinaryOpKind, ColumnRef, PartitionKeys, SortKey, VectorMatch}; +use super::schema::Schema; /// Base relation / metric stream source. #[derive(Debug, Clone, PartialEq)] pub struct SourceSpec { /// Metric name (PromQL) or table name (SQL). pub name: String, + /// Front-end-resolved leaf schema. `Some` for SQL tables (DataFusion knows + /// the columns); `None` for PromQL, where the [`Binder`](super::binder) + /// synthesises a usage-derived schema (the `(ts, value)` floor + referenced + /// labels). The presence of a schema also selects the L3 `Source` variant: + /// `Some` → `Source::Table`, `None` → `Source::TimeSeries`. + pub schema: Option, +} + +impl SourceSpec { + /// A PromQL-style leaf whose schema the Binder synthesises. + pub fn new(name: impl Into) -> Self { + Self { + name: name.into(), + schema: None, + } + } + + /// A SQL-style leaf carrying its front-end-resolved schema. + pub fn with_schema(name: impl Into, schema: Schema) -> Self { + Self { + name: name.into(), + schema: Some(schema), + } + } } /// One aggregate function in a GROUP BY / AGGREGATE node. diff --git a/crates/lower/src/promql.rs b/crates/lower/src/promql.rs index eeeae41a..d3c2a5ac 100644 --- a/crates/lower/src/promql.rs +++ b/crates/lower/src/promql.rs @@ -440,7 +440,7 @@ fn window_scan(inner: Inner) -> L2 { } fn filtered_source(metric: String, matchers: Vec) -> L2 { - let source = L2::Source(SourceSpec { name: metric }); + let source = L2::Source(SourceSpec::new(metric)); if matchers.is_empty() { source } else { From 4594ac66504caaed4a65ea5fa00090bdbd63699c Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 26 May 2026 10:10:52 -0600 Subject: [PATCH 20/40] feat(core): add Project to relational L2 + convert arm (SQL projection) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit relational L2 lacked a Project node (PromQL never needs one); SQL SELECT lists do. Adds `relational::Project { cols, input }` (re-using the shared ProjectItem), its walk/source_name traversal, and the convert arm → L3 `Project`. Column refs resolve by name against the child schema in L3 schema derivation. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/core/src/intent_algebra/lower.rs | 8 ++++++++ crates/core/src/intent_algebra/relational.rs | 13 ++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/crates/core/src/intent_algebra/lower.rs b/crates/core/src/intent_algebra/lower.rs index 2a59d58f..70e15901 100644 --- a/crates/core/src/intent_algebra/lower.rs +++ b/crates/core/src/intent_algebra/lower.rs @@ -170,6 +170,14 @@ pub fn convert( child: Box::new(convert(input, fallback, acc)?), }, + // π — column refs in the project items resolve by name against the + // child schema during L3 schema derivation, so the conversion is a + // structural pass-through of the (shared) `ProjectItem` list. + LQueryExpr::Project { cols, input } => CQueryExpr::Project { + cols: cols.clone(), + child: Box::new(convert(input, fallback, acc)?), + }, + LQueryExpr::Partition { keys, input } => CQueryExpr::Partition { keys: keys.clone(), child: Box::new(convert(input, fallback, acc)?), diff --git a/crates/core/src/intent_algebra/relational.rs b/crates/core/src/intent_algebra/relational.rs index c04d530f..01d4d4c9 100644 --- a/crates/core/src/intent_algebra/relational.rs +++ b/crates/core/src/intent_algebra/relational.rs @@ -10,7 +10,9 @@ use std::time::Duration; use super::expr_ir::L3Expr; -pub use super::query_expr::{BinaryOpKind, ColumnRef, PartitionKeys, SortKey, VectorMatch}; +pub use super::query_expr::{ + BinaryOpKind, ColumnRef, PartitionKeys, ProjectItem, SortKey, VectorMatch, +}; use super::schema::Schema; /// Base relation / metric stream source. @@ -97,6 +99,13 @@ pub enum QueryExpr { /// σ — row-level filter (WHERE / PromQL label matchers). Filter { pred: L3Expr, input: Box }, + /// π — projection / SELECT list (SQL). Column refs in `cols` resolve by + /// name against the child schema during conversion. + Project { + cols: Vec, + input: Box, + }, + /// γ + α — GROUP BY (`keys`) followed by aggregate functions. Aggregate { keys: Vec, @@ -183,6 +192,7 @@ impl QueryExpr { match self { QueryExpr::Source(_) | QueryExpr::Ref(_) => {} QueryExpr::Filter { input, .. } + | QueryExpr::Project { input, .. } | QueryExpr::Aggregate { input, .. } | QueryExpr::Window { input, .. } | QueryExpr::Partition { input, .. } @@ -218,6 +228,7 @@ impl QueryExpr { match self { QueryExpr::Source(s) => Some(&s.name), QueryExpr::Filter { input, .. } + | QueryExpr::Project { input, .. } | QueryExpr::Aggregate { input, .. } | QueryExpr::Window { input, .. } | QueryExpr::Partition { input, .. } From 99c2747a8b2e0e6fe7afe3292321631069b611d7 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 26 May 2026 10:38:46 -0600 Subject: [PATCH 21/40] feat(sql): re-target DataFusion front end to emit relational L2 (positional IR) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-points #4's SQL lowerer from name-based L3 onto the unified positional IR: both PromQL and SQL now lower through the same convert_root. - sql/mod.rs: walks DataFusion's LogicalPlan and emits relational::QueryExpr (L2). TableScan → Source carrying the catalog's resolved Schema; aggregates → AggItem{AggFunc, col} (the converter binds col→ColumnId + applies accuracy); projection → the new L2 Project; filter/sort/limit/distinct/union/topk as L2. JOIN / subquery / window funcs are rejected (no L3 analytic node yet). - sql/types.rs: SqlCatalog (table → L3 Schema) + Arrow⇄L3 DataType bridges. - sql/expr.rs: reused verbatim except ColumnRef → ColumnRef::Named. - Dropped schema_pass.rs (schema flows via Binder/SourceSpec now) and sql/time.rs (time-range pushdown isn't on #5's Source::Table). - error.rs: union PromQL + SQL variants; lib.rs: lower_sql / lower_sql_batch. - Cargo: add datafusion 43 + tokio dev-dep. Fresh sql_lowering tests (positional): WHERE folds onto Scan; multi-agg GROUP BY binds SUM(bytes)→col 3 / AVG(latency)→col 2 / service→col 1; COUNT(*)→Count; COUNT(DISTINCT)→Cardinality; JOIN rejected. Workspace green: 107 tests, clippy -D warnings clean, fmt clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- Cargo.lock | 3294 ++++++++++++++++++++++++---- crates/lower/Cargo.toml | 5 + crates/lower/src/error.rs | 26 +- crates/lower/src/lib.rs | 79 +- crates/lower/src/schema_pass.rs | 166 -- crates/lower/src/sql/expr.rs | 18 +- crates/lower/src/sql/mod.rs | 692 ++---- crates/lower/src/sql/time.rs | 288 --- crates/lower/src/sql/types.rs | 96 +- crates/lower/tests/sql_lowering.rs | 114 + 10 files changed, 3357 insertions(+), 1421 deletions(-) delete mode 100644 crates/lower/src/schema_pass.rs delete mode 100644 crates/lower/src/sql/time.rs create mode 100644 crates/lower/tests/sql_lowering.rs diff --git a/Cargo.lock b/Cargo.lock index e0c8dad7..e922417e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,26 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -11,6 +31,33 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -27,666 +74,3153 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] -name = "asap-control-core" -version = "0.1.0" +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "arrow" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3a3ec4fe573f9d1f59d99c085197ef669b00b088ba1d7bb75224732d9357a74" dependencies = [ - "serde", - "serde_json", - "thiserror", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", ] [[package]] -name = "asap-control-lower" -version = "0.1.0" +name = "arrow-arith" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dcf19f07792d8c7f91086c67b574a79301e367029b17fcf63fb854332246a10" dependencies = [ - "asap-control-core", - "promql-parser", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "num", ] [[package]] -name = "autocfg" -version = "1.5.0" +name = "arrow-array" +version = "53.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "7845c32b41f7053e37a075b3c2f29c6f5ea1b3ca6e5df7a2d325ee6e1b4a63cf" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "chrono-tz", + "half", + "hashbrown 0.15.5", + "num", +] [[package]] -name = "bincode" -version = "1.3.3" +name = "arrow-buffer" +version = "53.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +checksum = "5b5c681a99606f3316f2a99d9c8b6fa3aad0b1d34d8f6d7a1b471893940219d8" dependencies = [ - "serde", + "bytes", + "half", + "num", ] [[package]] -name = "bumpalo" -version = "3.20.2" +name = "arrow-cast" +version = "53.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +checksum = "6365f8527d4f87b133eeb862f9b8093c009d41a210b8f101f91aa2392f61daac" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "atoi", + "base64", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num", + "ryu", +] [[package]] -name = "cactus" -version = "1.0.7" +name = "arrow-csv" +version = "53.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbc26382d871df4b7442e3df10a9402bf3cf5e55cbd66f12be38861425f0564" +checksum = "30dac4d23ac769300349197b845e0fd18c7f9f15d260d4659ae6b5a9ca06f586" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "chrono", + "csv", + "csv-core", + "lazy_static", + "lexical-core", + "regex", +] [[package]] -name = "cc" -version = "1.2.62" +name = "arrow-data" +version = "53.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" +checksum = "cd962fc3bf7f60705b25bcaa8eb3318b2545aa1d528656525ebdd6a17a6cd6fb" dependencies = [ - "find-msvc-tools", - "shlex", + "arrow-buffer", + "arrow-schema", + "half", + "num", ] [[package]] -name = "cfg-if" -version = "1.0.4" +name = "arrow-ipc" +version = "53.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +checksum = "c3527365b24372f9c948f16e53738eb098720eea2093ae73c7af04ac5e30a39b" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "flatbuffers", + "lz4_flex", +] [[package]] -name = "cfgrammar" -version = "0.13.10" +name = "arrow-json" +version = "53.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe45e18904af7af10e4312df7c97251e98af98c70f42f1f2587aecfcbee56bf" +checksum = "acdec0024749fc0d95e025c0b0266d78613727b3b3a5d4cf8ea47eb6d38afdd1" dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "chrono", + "half", "indexmap", - "lazy_static", - "num-traits", - "regex", + "lexical-core", + "num", "serde", - "vob", + "serde_json", ] [[package]] -name = "chrono" -version = "0.4.44" +name = "arrow-ord" +version = "53.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "79af2db0e62a508d34ddf4f76bfd6109b6ecc845257c9cba6f939653668f89ac" dependencies = [ - "iana-time-zone", - "js-sys", - "num-traits", - "wasm-bindgen", - "windows-link", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "half", + "num", ] [[package]] -name = "core-foundation-sys" -version = "0.8.7" +name = "arrow-row" +version = "53.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +checksum = "da30e9d10e9c52f09ea0cf15086d6d785c11ae8dcc3ea5f16d402221b6ac7735" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] [[package]] -name = "deranged" -version = "0.5.8" +name = "arrow-schema" +version = "53.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +checksum = "35b0f9c0c3582dd55db0f136d3b44bfa0189df07adcf7dc7f2f2e74db0f52eb8" + +[[package]] +name = "arrow-select" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92fc337f01635218493c23da81a364daf38c694b05fc20569c3193c11c561984" dependencies = [ - "powerfmt", + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num", ] [[package]] -name = "equivalent" -version = "1.0.2" +name = "arrow-string" +version = "53.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +checksum = "d596a9fc25dae556672d5069b090331aca8acb93cae426d8b7dcdf1c558fa0ce" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num", + "regex", + "regex-syntax", +] [[package]] -name = "filetime" -version = "0.2.29" +name = "asap-control-core" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "asap-control-lower" +version = "0.1.0" +dependencies = [ + "asap-control-core", + "datafusion", + "promql-parser", + "tokio", +] + +[[package]] +name = "async-compression" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c" dependencies = [ - "cfg-if", - "libc", + "bzip2 0.5.2", + "flate2", + "futures-core", + "futures-io", + "memchr", + "pin-project-lite", + "tokio", + "xz2", + "zstd", + "zstd-safe", ] [[package]] -name = "find-msvc-tools" -version = "0.1.9" +name = "async-trait" +version = "0.1.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] -name = "fnv" -version = "1.0.7" +name = "atoi" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] [[package]] -name = "futures-core" -version = "0.3.32" +name = "autocfg" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] -name = "futures-task" -version = "0.3.32" +name = "base64" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] -name = "futures-util" -version = "0.3.32" +name = "bincode" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" dependencies = [ - "futures-core", - "futures-task", - "pin-project-lite", - "slab", + "serde", ] [[package]] -name = "getopts" -version = "0.2.24" +name = "bitflags" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" -dependencies = [ - "unicode-width", -] +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] -name = "hashbrown" -version = "0.17.1" +name = "bitflags" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" [[package]] -name = "iana-time-zone" -version = "0.1.65" +name = "blake2" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "log", - "wasm-bindgen", - "windows-core", + "digest", ] [[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" +name = "blake3" +version = "1.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" dependencies = [ + "arrayref", + "arrayvec", "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.0", ] [[package]] -name = "indexmap" -version = "2.14.0" +name = "block-buffer" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "equivalent", - "hashbrown", + "generic-array", ] [[package]] -name = "itoa" -version = "1.0.18" +name = "brotli" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] [[package]] -name = "js-sys" -version = "0.3.98" +name = "brotli-decompressor" +version = "4.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" +checksum = "a334ef7c9e23abf0ce748e8cd309037da93e606ad52eb372e4ce327a0dcfbdfd" dependencies = [ - "cfg-if", - "futures-util", - "once_cell", - "wasm-bindgen", + "alloc-no-stdlib", + "alloc-stdlib", ] [[package]] -name = "lazy_static" -version = "1.5.0" +name = "bumpalo" +version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" [[package]] -name = "libc" -version = "0.2.186" +name = "byteorder" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] -name = "log" -version = "0.4.29" +name = "bytes" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] -name = "lrlex" -version = "0.13.10" +name = "bzip2" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c71364e868116ee891b0f93559eb9eca5675bec28b22d33c58481e66c3951d7e" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" dependencies = [ - "cfgrammar", - "getopts", - "lazy_static", - "lrpar", - "num-traits", - "quote", - "regex", - "regex-syntax", - "serde", - "vergen", + "bzip2-sys", + "libc", ] [[package]] -name = "lrpar" -version = "0.13.10" +name = "bzip2" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51b265a81193d94c92d1c9c715498d6fa505bce3f789ceecb24ab5d6fa2dbc71" +checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" dependencies = [ - "bincode", - "cactus", - "cfgrammar", - "filetime", - "indexmap", - "lazy_static", - "lrtable", - "num-traits", - "packedvec", - "regex", - "serde", - "static_assertions", - "vergen", - "vob", + "bzip2-sys", ] [[package]] -name = "lrtable" -version = "0.13.10" +name = "bzip2-sys" +version = "0.1.13+1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc36d15214ca997a5097845be1f932b7ee6125c36f5c5e55f6c49e027ddeb6de" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" dependencies = [ - "cfgrammar", - "fnv", + "cc", + "pkg-config", +] + +[[package]] +name = "cactus" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbc26382d871df4b7442e3df10a9402bf3cf5e55cbd66f12be38861425f0564" + +[[package]] +name = "cc" +version = "1.2.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfgrammar" +version = "0.13.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe45e18904af7af10e4312df7c97251e98af98c70f42f1f2587aecfcbee56bf" +dependencies = [ + "indexmap", + "lazy_static", "num-traits", + "regex", "serde", - "sparsevec", "vob", ] [[package]] -name = "memchr" -version = "2.8.0" +name = "chrono" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-targets", +] [[package]] -name = "num-conv" -version = "0.2.2" +name = "chrono-tz" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf", +] [[package]] -name = "num-traits" -version = "0.2.19" +name = "comfy-table" +version = "7.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" dependencies = [ - "autocfg", + "unicode-segmentation", + "unicode-width", ] [[package]] -name = "num_threads" +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "datafusion" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbba0799cf6913b456ed07a94f0f3b6e12c62a5d88b10809e2284a0f2b915c05" +dependencies = [ + "ahash", + "arrow", + "arrow-array", + "arrow-ipc", + "arrow-schema", + "async-compression", + "async-trait", + "bytes", + "bzip2 0.4.4", + "chrono", + "dashmap", + "datafusion-catalog", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-nested", + "datafusion-functions-window", + "datafusion-optimizer", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-optimizer", + "datafusion-physical-plan", + "datafusion-sql", + "flate2", + "futures", + "glob", + "half", + "hashbrown 0.14.5", + "indexmap", + "itertools", + "log", + "num_cpus", + "object_store", + "parking_lot", + "parquet", + "paste", + "pin-project-lite", + "rand", + "sqlparser", + "tempfile", + "tokio", + "tokio-util", + "url", + "uuid", + "xz2", + "zstd", +] + +[[package]] +name = "datafusion-catalog" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7493c5c2d40eec435b13d92e5703554f4efc7059451fcb8d3a79580ff0e45560" +dependencies = [ + "arrow-schema", + "async-trait", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-plan", + "parking_lot", +] + +[[package]] +name = "datafusion-common" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24953049ebbd6f8964f91f60aa3514e121b5e81e068e33b60e77815ab369b25c" dependencies = [ + "ahash", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.14.5", + "indexmap", + "instant", "libc", + "num_cpus", + "object_store", + "parquet", + "paste", + "sqlparser", + "tokio", ] [[package]] -name = "once_cell" -version = "1.21.4" +name = "datafusion-common-runtime" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +checksum = "f06df4ef76872e11c924d3c814fd2a8dd09905ed2e2195f71c857d78abd19685" +dependencies = [ + "log", + "tokio", +] [[package]] -name = "packedvec" -version = "1.2.5" +name = "datafusion-execution" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69e0a534dd2e6aefce319af62a0aa0066a76bdfcec0201dfe02df226bc9ec70" +checksum = "6bbdcb628d690f3ce5fea7de81642b514486d58ff9779a51f180a69a4eadb361" dependencies = [ - "num-traits", - "serde", + "arrow", + "chrono", + "dashmap", + "datafusion-common", + "datafusion-expr", + "futures", + "hashbrown 0.14.5", + "log", + "object_store", + "parking_lot", + "rand", + "tempfile", + "url", ] [[package]] -name = "pin-project-lite" -version = "0.2.17" +name = "datafusion-expr" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +checksum = "8036495980e3131f706b7d33ab00b4492d73dc714e3cb74d11b50f9602a73246" +dependencies = [ + "ahash", + "arrow", + "arrow-array", + "arrow-buffer", + "chrono", + "datafusion-common", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr-common", + "indexmap", + "paste", + "serde_json", + "sqlparser", + "strum", + "strum_macros", +] [[package]] -name = "powerfmt" -version = "0.2.0" +name = "datafusion-expr-common" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4da0f3cb4669f9523b403d6b5a0ec85023e0ab3bf0183afd1517475b3e64fdd2" +dependencies = [ + "arrow", + "datafusion-common", + "itertools", + "paste", +] + +[[package]] +name = "datafusion-functions" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52c4012648b34853e40a2c6bcaa8772f837831019b68aca384fb38436dba162" +dependencies = [ + "arrow", + "arrow-buffer", + "base64", + "blake2", + "blake3", + "chrono", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "hashbrown 0.14.5", + "hex", + "itertools", + "log", + "md-5", + "rand", + "regex", + "sha2", + "unicode-segmentation", + "uuid", +] + +[[package]] +name = "datafusion-functions-aggregate" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5b8bb624597ba28ed7446df4a9bd7c7a7bde7c578b6b527da3f47371d5f6741" +dependencies = [ + "ahash", + "arrow", + "arrow-schema", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "half", + "indexmap", + "log", + "paste", +] + +[[package]] +name = "datafusion-functions-aggregate-common" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fb06208fc470bc8cf1ce2d9a1159d42db591f2c7264a8c1776b53ad8f675143" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "datafusion-expr-common", + "datafusion-physical-expr-common", + "rand", +] + +[[package]] +name = "datafusion-functions-nested" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fca25bbb87323716d05e54114666e942172ccca23c5a507e9c7851db6e965317" +dependencies = [ + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-physical-expr-common", + "itertools", + "log", + "paste", + "rand", +] + +[[package]] +name = "datafusion-functions-window" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ae23356c634e54c59f7c51acb7a5b9f6240ffb2cf997049a1a24a8a88598dbe" +dependencies = [ + "datafusion-common", + "datafusion-expr", + "datafusion-functions-window-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "log", + "paste", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b3d6ff7794acea026de36007077a06b18b89e4f9c3fea7f2215f9f7dd9059b" +dependencies = [ + "datafusion-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-optimizer" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec6241eb80c595fa0e1a8a6b69686b5cf3bd5fdacb8319582a0943b0bd788aa" +dependencies = [ + "arrow", + "async-trait", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr", + "hashbrown 0.14.5", + "indexmap", + "itertools", + "log", + "paste", + "regex-syntax", +] + +[[package]] +name = "datafusion-physical-expr" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3370357b8fc75ec38577700644e5d1b0bc78f38babab99c0b8bd26bafb3e4335" +dependencies = [ + "ahash", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", + "arrow-string", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr-common", + "half", + "hashbrown 0.14.5", + "indexmap", + "itertools", + "log", + "paste", + "petgraph", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8b7734d94bf2fa6f6e570935b0ddddd8421179ce200065be97874e13d46a47b" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "datafusion-expr-common", + "hashbrown 0.14.5", + "rand", +] + +[[package]] +name = "datafusion-physical-optimizer" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eee8c479522df21d7b395640dff88c5ed05361852dce6544d7c98e9dbcebffe" +dependencies = [ + "arrow", + "arrow-schema", + "datafusion-common", + "datafusion-execution", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-plan", + "itertools", +] + +[[package]] +name = "datafusion-physical-plan" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17e1fc2e2c239d14e8556f2622b19a726bf6bc6962cc00c71fc52626274bee24" +dependencies = [ + "ahash", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", + "async-trait", + "chrono", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "futures", + "half", + "hashbrown 0.14.5", + "indexmap", + "itertools", + "log", + "once_cell", + "parking_lot", + "pin-project-lite", + "rand", + "tokio", +] + +[[package]] +name = "datafusion-sql" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e3a4ed41dbee20a5d947a59ca035c225d67dc9cbe869c10f66dcdf25e7ce51" +dependencies = [ + "arrow", + "arrow-array", + "arrow-schema", + "datafusion-common", + "datafusion-expr", + "indexmap", + "log", + "regex", + "sqlparser", + "strum", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "flatbuffers" +version = "24.12.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f1baf0dbf96932ec9a3038d57900329c015b0bfb7b63d904f3bc27e2b02a096" +dependencies = [ + "bitflags 1.3.2", + "rustc_version", +] + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getopts" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "humantime" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" +dependencies = [ + "cfg-if", + "futures-util", + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "lrlex" +version = "0.13.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c71364e868116ee891b0f93559eb9eca5675bec28b22d33c58481e66c3951d7e" +dependencies = [ + "cfgrammar", + "getopts", + "lazy_static", + "lrpar", + "num-traits", + "quote", + "regex", + "regex-syntax", + "serde", + "vergen", +] + +[[package]] +name = "lrpar" +version = "0.13.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b265a81193d94c92d1c9c715498d6fa505bce3f789ceecb24ab5d6fa2dbc71" +dependencies = [ + "bincode", + "cactus", + "cfgrammar", + "filetime", + "indexmap", + "lazy_static", + "lrtable", + "num-traits", + "packedvec", + "regex", + "serde", + "static_assertions", + "vergen", + "vob", +] + +[[package]] +name = "lrtable" +version = "0.13.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc36d15214ca997a5097845be1f932b7ee6125c36f5c5e55f6c49e027ddeb6de" +dependencies = [ + "cfgrammar", + "fnv", + "num-traits", + "serde", + "sparsevec", + "vob", +] + +[[package]] +name = "lz4_flex" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" +dependencies = [ + "twox-hash 2.1.2", +] + +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + +[[package]] +name = "object_store" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cfccb68961a56facde1163f9319e0d15743352344e7808a11795fb99698dcaf" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "futures", + "humantime", + "itertools", + "parking_lot", + "percent-encoding", + "snafu", + "tokio", + "tracing", + "url", + "walkdir", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "packedvec" +version = "1.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69e0a534dd2e6aefce319af62a0aa0066a76bdfcec0201dfe02df226bc9ec70" +dependencies = [ + "num-traits", + "serde", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "parquet" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f8cf58b29782a7add991f655ff42929e31a7859f5319e53db9e39a714cb113c" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64", + "brotli", + "bytes", + "chrono", + "flate2", + "futures", + "half", + "hashbrown 0.15.5", + "lz4_flex", + "num", + "num-bigint", + "object_store", + "paste", + "seq-macro", + "snap", + "thrift", + "tokio", + "twox-hash 1.6.3", + "zstd", + "zstd-sys", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "promql-parser" +version = "0.9.0" +source = "git+https://github.com/ProjectASAP/promql-parser?branch=asap#c51beafb361af4cc95ed62ae377862c660ceb757" +dependencies = [ + "cfgrammar", + "chrono", + "lazy_static", + "lrlex", + "lrpar", + "regex", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.11.1", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.11.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "snap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" + +[[package]] +name = "sparsevec" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68b4a8ce3045f0fe173fb5ae3c6b7dcfbec02bfa650bb8618b2301f52af0134d" +dependencies = [ + "num-traits", + "packedvec", + "serde", + "vob", +] + +[[package]] +name = "sqlparser" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fe11944a61da0da3f592e19a45ebe5ab92dc14a779907ff1f08fbb797bfefc7" +dependencies = [ + "log", + "sqlparser_derive", +] + +[[package]] +name = "sqlparser_derive" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thrift" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" +dependencies = [ + "byteorder", + "integer-encoding", + "ordered-float", +] + +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "libc", + "num-conv", + "num_threads", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "pin-project-lite", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "twox-hash" +version = "1.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" +dependencies = [ + "cfg-if", + "static_assertions", +] + +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + +[[package]] +name = "typenum" +version = "1.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" +dependencies = [ + "getrandom 0.4.2", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "vergen" +version = "8.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2990d9ea5967266ea0ccf413a4aa5c42a93dbcfda9cb49a97de6931726b12566" +dependencies = [ + "anyhow", + "rustversion", + "time", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "vob" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc936b5a7202a703aeaf7ce05e7931db2e0c8126813f97db3e9e06d867b0bb38" +dependencies = [ + "num-traits", + "serde", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.3+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags 2.11.1", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "web-sys" +version = "0.3.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] -name = "proc-macro2" -version = "1.0.106" +name = "windows-interface" +version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ - "unicode-ident", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "promql-parser" -version = "0.9.0" -source = "git+https://github.com/ProjectASAP/promql-parser?branch=asap#c51beafb361af4cc95ed62ae377862c660ceb757" +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" dependencies = [ - "cfgrammar", - "chrono", - "lazy_static", - "lrlex", - "lrpar", - "regex", + "windows-link", ] [[package]] -name = "quote" -version = "1.0.45" +name = "windows-strings" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" dependencies = [ - "proc-macro2", + "windows-link", ] [[package]] -name = "regex" -version = "1.12.3" +name = "windows-sys" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", + "windows-link", ] [[package]] -name = "regex-automata" -version = "0.4.14" +name = "windows-targets" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] -name = "regex-syntax" -version = "0.8.10" +name = "windows_aarch64_gnullvm" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] -name = "rustversion" -version = "1.0.22" +name = "windows_aarch64_msvc" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] -name = "serde" -version = "1.0.228" +name = "windows_i686_gnu" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" -dependencies = [ - "serde_core", - "serde_derive", -] +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] -name = "serde_core" -version = "1.0.228" +name = "windows_i686_gnullvm" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" -dependencies = [ - "serde_derive", -] +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] -name = "serde_derive" -version = "1.0.228" +name = "windows_i686_msvc" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] -name = "serde_json" -version = "1.0.149" +name = "windows_x86_64_gnu" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" -dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", -] +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] -name = "shlex" -version = "1.3.0" +name = "windows_x86_64_gnullvm" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] -name = "slab" -version = "0.4.12" +name = "windows_x86_64_msvc" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] -name = "sparsevec" -version = "0.2.2" +name = "wit-bindgen" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b4a8ce3045f0fe173fb5ae3c6b7dcfbec02bfa650bb8618b2301f52af0134d" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" dependencies = [ - "num-traits", - "packedvec", - "serde", - "vob", + "wit-bindgen-rust-macro", ] [[package]] -name = "static_assertions" -version = "1.1.0" +name = "wit-bindgen" +version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] -name = "syn" -version = "2.0.117" +name = "wit-bindgen-core" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", + "anyhow", + "heck", + "wit-parser", ] [[package]] -name = "thiserror" -version = "2.0.18" +name = "wit-bindgen-rust" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ - "thiserror-impl", + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", ] [[package]] -name = "thiserror-impl" -version = "2.0.18" +name = "wit-bindgen-rust-macro" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" dependencies = [ + "anyhow", + "prettyplease", "proc-macro2", "quote", "syn", + "wit-bindgen-core", + "wit-bindgen-rust", ] [[package]] -name = "time" -version = "0.3.47" +name = "wit-component" +version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ - "deranged", - "itoa", - "libc", - "num-conv", - "num_threads", - "powerfmt", - "serde_core", - "time-core", - "time-macros", + "anyhow", + "bitflags 2.11.1", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", ] [[package]] -name = "time-core" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" - -[[package]] -name = "time-macros" -version = "0.2.27" +name = "wit-parser" +version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ - "num-conv", - "time-core", + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", ] [[package]] -name = "unicode-ident" -version = "1.0.24" +name = "writeable" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] -name = "unicode-width" -version = "0.2.2" +name = "xz2" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] [[package]] -name = "vergen" -version = "8.3.2" +name = "yoke" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2990d9ea5967266ea0ccf413a4aa5c42a93dbcfda9cb49a97de6931726b12566" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" dependencies = [ - "anyhow", - "rustversion", - "time", + "stable_deref_trait", + "yoke-derive", + "zerofrom", ] [[package]] -name = "vob" -version = "3.0.6" +name = "yoke-derive" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc936b5a7202a703aeaf7ce05e7931db2e0c8126813f97db3e9e06d867b0bb38" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ - "num-traits", - "serde", + "proc-macro2", + "quote", + "syn", + "synstructure", ] [[package]] -name = "wasm-bindgen" -version = "0.2.121" +name = "zerocopy" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", - "wasm-bindgen-shared", + "zerocopy-derive", ] [[package]] -name = "wasm-bindgen-macro" -version = "0.2.121" +name = "zerocopy-derive" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ + "proc-macro2", "quote", - "wasm-bindgen-macro-support", + "syn", ] [[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.121" +name = "zerofrom" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" dependencies = [ - "bumpalo", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", + "zerofrom-derive", ] [[package]] -name = "wasm-bindgen-shared" -version = "0.2.121" +name = "zerofrom-derive" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ - "unicode-ident", + "proc-macro2", + "quote", + "syn", + "synstructure", ] [[package]] -name = "windows-core" -version = "0.62.2" +name = "zerotrie" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" dependencies = [ - "windows-implement", - "windows-interface", - "windows-link", - "windows-result", - "windows-strings", + "displaydoc", + "yoke", + "zerofrom", ] [[package]] -name = "windows-implement" -version = "0.60.2" +name = "zerovec" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ - "proc-macro2", - "quote", - "syn", + "yoke", + "zerofrom", + "zerovec-derive", ] [[package]] -name = "windows-interface" -version = "0.59.3" +name = "zerovec-derive" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", @@ -694,31 +3228,35 @@ dependencies = [ ] [[package]] -name = "windows-link" -version = "0.2.1" +name = "zmij" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" [[package]] -name = "windows-result" -version = "0.4.1" +name = "zstd" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" dependencies = [ - "windows-link", + "zstd-safe", ] [[package]] -name = "windows-strings" -version = "0.5.1" +name = "zstd-safe" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" dependencies = [ - "windows-link", + "zstd-sys", ] [[package]] -name = "zmij" -version = "1.0.21" +name = "zstd-sys" +version = "2.0.13+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/crates/lower/Cargo.toml b/crates/lower/Cargo.toml index e026b92e..710d8bb6 100644 --- a/crates/lower/Cargo.toml +++ b/crates/lower/Cargo.toml @@ -9,3 +9,8 @@ asap-control-core = { path = "../core" } # upstream untouched; the `asap` branch carries our local grammar/function # additions (see THIRD_PARTY.md). promql-parser = { git = "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/ProjectASAP/promql-parser", branch = "asap" } +# SQL front end: parse + plan via DataFusion, then lower its LogicalPlan to L2. +datafusion = "43" + +[dev-dependencies] +tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } diff --git a/crates/lower/src/error.rs b/crates/lower/src/error.rs index 9f377a2c..9a17c8f7 100644 --- a/crates/lower/src/error.rs +++ b/crates/lower/src/error.rs @@ -8,7 +8,8 @@ pub enum LoweringError { UnsupportedFunction(String), /// A PromQL aggregation operator (`sum`, `topk`, …) not supported. UnsupportedAggregateOp(String), - /// A structural PromQL feature (offset, `@`, `without` w/o catalog, …) not supported. + /// A structural feature (PromQL offset/`@`/`without`; SQL JOIN/subquery/…) + /// not supported in this version. UnsupportedFeature(String), /// A required function / aggregator argument was missing. MissingArgument(String), @@ -18,6 +19,18 @@ pub enum LoweringError { WrongLanguage(String), /// The L2→L3 converter failed (name resolution against the bound schema). Convert(asap_control_core::intent_algebra::ConvertError), + + // ── SQL front end (DataFusion) ─────────────────────────────────────────── + /// DataFusion failed to parse / plan the SQL query. + DataFusion(datafusion::error::DataFusionError), + /// A table referenced by the query is absent from the catalog. + TableNotFound(String), + /// A SQL aggregate function not supported in this version. + UnsupportedAggregate(String), + /// A SQL scalar expression that could not be lowered. + InvalidExpression(String), + /// The SQL dialect is not supported (only DataFusionSQL is implemented). + UnsupportedDialect(String), } impl fmt::Display for LoweringError { @@ -31,6 +44,11 @@ impl fmt::Display for LoweringError { Self::InvalidParameter(m) => write!(f, "invalid parameter: {m}"), Self::WrongLanguage(l) => write!(f, "unsupported query language: {l}"), Self::Convert(e) => write!(f, "L2→L3 conversion failed: {e}"), + Self::DataFusion(e) => write!(f, "DataFusion error: {e}"), + Self::TableNotFound(t) => write!(f, "table not found in catalog: {t}"), + Self::UnsupportedAggregate(n) => write!(f, "unsupported aggregate: {n}"), + Self::InvalidExpression(m) => write!(f, "invalid expression: {m}"), + Self::UnsupportedDialect(d) => write!(f, "unsupported SQL dialect: {d}"), } } } @@ -42,3 +60,9 @@ impl From for LoweringError { Self::Convert(e) } } + +impl From for LoweringError { + fn from(e: datafusion::error::DataFusionError) -> Self { + Self::DataFusion(e) + } +} diff --git a/crates/lower/src/lib.rs b/crates/lower/src/lib.rs index c369f03c..c38dcb65 100644 --- a/crates/lower/src/lib.rs +++ b/crates/lower/src/lib.rs @@ -1,21 +1,24 @@ //! L1→L3 lowering passes for the ASAP controller core. //! -//! PromQL flows through three layers, all ending at the canonical intent -//! algebra: L1 parse (`promql-parser`), L2 per-language tree -//! ([`relational::QueryExpr`](asap_control_core::intent_algebra::relational)), -//! and the L2→L3 conversion ([`convert_root`]) that runs the -//! [`Binder`](asap_control_core::intent_algebra::Binder) and folds the -//! single-statistic sketchable aggregate into canonical shapes. +//! Both front ends end at the canonical intent algebra via the same L2→L3 +//! [`convert_root`]: PromQL parses with `promql-parser`, SQL parses + plans with +//! DataFusion. Each emits the per-language +//! [`relational::QueryExpr`](asap_control_core::intent_algebra::relational); the +//! shared converter runs the [`Binder`](asap_control_core::intent_algebra::Binder) +//! for positional name resolution and folds single-statistic sketchable +//! aggregates into canonical shapes. pub mod error; pub mod promql; +pub mod sql; use asap_control_core::intent_algebra::{convert_root, QueryExpr}; use asap_control_core::types::AccuracyTarget; -use asap_control_core::workload::{QueryLanguage, QueryWorkload}; +use asap_control_core::workload::{QueryLanguage, QueryWorkload, SqlDialect}; pub use error::LoweringError; pub use promql::PromqlLowerer; +pub use sql::{SqlCatalog, SqlLowerer}; /// Lower a single PromQL query string to the canonical L3 `QueryExpr`. /// @@ -59,3 +62,65 @@ pub fn lower_promql_batch(workload: &QueryWorkload) -> Vec Result { + let l2 = SqlLowerer::new(catalog).lower(query).await?; + let l3 = convert_root(&l2, &accuracy)?; + Ok(l3) +} + +/// Lower every SQL batch entry in `workload` to a `QueryExpr`. +/// +/// One `Result` per entry — errors are per-query, not fatal for the batch. +/// Returns `WrongLanguage` for every entry if the workload is not SQL, and +/// `UnsupportedDialect` for non-DataFusion SQL dialects. +pub async fn lower_sql_batch( + workload: &QueryWorkload, + catalog: &SqlCatalog, +) -> Vec> { + let entries = match &workload.query_batch { + Some(e) if !e.is_empty() => e, + _ => return vec![], + }; + + // `DataFusion` is a legacy alias for `SQL(DataFusionSQL)`; accept both. + if !matches!( + workload.language, + QueryLanguage::SQL(_) | QueryLanguage::DataFusion + ) { + let lang = format!("{:?}", workload.language); + return entries + .iter() + .map(|_| Err(LoweringError::WrongLanguage(lang.clone()))) + .collect(); + } + if let QueryLanguage::SQL(dialect) = &workload.language { + if !matches!(dialect, SqlDialect::DataFusionSQL) { + let d = format!("{dialect:?}"); + return entries + .iter() + .map(|_| Err(LoweringError::UnsupportedDialect(d.clone()))) + .collect(); + } + } + + let mut results = Vec::with_capacity(entries.len()); + for entry in entries { + let accuracy = entry + .requirements + .as_ref() + .and_then(|r| r.accuracy.clone()) + .unwrap_or(AccuracyTarget::Exact); + results.push(lower_sql(&entry.query.0, catalog, accuracy).await); + } + results +} diff --git a/crates/lower/src/schema_pass.rs b/crates/lower/src/schema_pass.rs deleted file mode 100644 index 1ec5b0b1..00000000 --- a/crates/lower/src/schema_pass.rs +++ /dev/null @@ -1,166 +0,0 @@ -use std::sync::Arc; - -use asap_control_core::intent_algebra::expr::QueryExpr; -use asap_control_core::intent_algebra::schema::{HasSchema, L3Schema, SchemaCatalog}; -use asap_control_core::intent_algebra::L3Node; - -/// Recursively populate the `schema` field on every node in a `QueryExpr` tree. -/// -/// The lowerer creates every node with an empty schema (`make_node`). This -/// pass walks the tree bottom-up, computing each node's output schema from -/// its children's schemas and the catalog, and returns a fully typed -/// `Arc` tree. -pub fn populate_schemas(expr: QueryExpr, catalog: &SchemaCatalog) -> Arc { - let (rebuilt, child_schemas) = rebuild(expr, catalog); - let refs: Vec<&L3Schema> = child_schemas.iter().collect(); - let schema = rebuilt.output_schema(&refs, catalog); - Arc::new(L3Node { - expr: rebuilt, - schema, - }) -} - -/// Recursively rebuild the expression tree with populated child nodes. -/// Returns `(rebuilt_expr, child_output_schemas)` so the caller can pass -/// those schemas to `output_schema`. -fn rebuild(expr: QueryExpr, catalog: &SchemaCatalog) -> (QueryExpr, Vec) { - use QueryExpr::*; - - // Helper: process one child Arc → fresh Arc with schema set. - let proc = |node: Arc| populate_schemas(node.expr.clone(), catalog); - - match expr { - // Leaf: schema comes from the catalog, no child schemas needed. - Scan { .. } => (expr, vec![]), - - Filter { child, pred } => { - let c = proc(child); - let cs = c.schema.clone(); - (Filter { child: c, pred }, vec![cs]) - } - Project { child, cols } => { - let c = proc(child); - let cs = c.schema.clone(); - (Project { child: c, cols }, vec![cs]) - } - Aggregate { - child, - by, - aggs, - having, - output_names, - } => { - let c = proc(child); - let cs = c.schema.clone(); - ( - Aggregate { - child: c, - by, - aggs, - having, - output_names, - }, - vec![cs], - ) - } - Sort { child, keys } => { - let c = proc(child); - let cs = c.schema.clone(); - (Sort { child: c, keys }, vec![cs]) - } - Limit { child, n, offset } => { - let c = proc(child); - let cs = c.schema.clone(); - ( - Limit { - child: c, - n, - offset, - }, - vec![cs], - ) - } - Distinct { child, cols } => { - let c = proc(child); - let cs = c.schema.clone(); - (Distinct { child: c, cols }, vec![cs]) - } - Partition { child, keys } => { - let c = proc(child); - let cs = c.schema.clone(); - (Partition { child: c, keys }, vec![cs]) - } - TimeWindow { - child, - kind, - size, - slide, - } => { - let c = proc(child); - let cs = c.schema.clone(); - ( - TimeWindow { - child: c, - kind, - size, - slide, - }, - vec![cs], - ) - } - WindowFunc { - child, - func, - args, - partition_by, - order_by, - frame, - } => { - let c = proc(child); - let cs = c.schema.clone(); - ( - WindowFunc { - child: c, - func, - args, - partition_by, - order_by, - frame, - }, - vec![cs], - ) - } - SetOp { - kind, - all, - left, - right, - } => { - let l = proc(left); - let r = proc(right); - let ls = l.schema.clone(); - let rs = r.schema.clone(); - ( - SetOp { - kind, - all, - left: l, - right: r, - }, - vec![ls, rs], - ) - } - Merge { children } => { - let new_children: Vec> = children.into_iter().map(proc).collect(); - let schemas: Vec = new_children.iter().map(|c| c.schema.clone()).collect(); - ( - Merge { - children: new_children, - }, - schemas, - ) - } - // Unimplemented variants: return as-is; output_schema will todo!() if called. - other => (other, vec![]), - } -} diff --git a/crates/lower/src/sql/expr.rs b/crates/lower/src/sql/expr.rs index f7a8ddaa..c040b769 100644 --- a/crates/lower/src/sql/expr.rs +++ b/crates/lower/src/sql/expr.rs @@ -1,7 +1,6 @@ use datafusion::logical_expr::{BinaryExpr, Expr, Operator}; -use asap_control_core::intent_algebra::expr::ColumnRef; -use asap_control_core::intent_algebra::{ArithOp, CompareOp, L3Expr, L3Scalar}; +use asap_control_core::intent_algebra::{ArithOp, ColumnRef, CompareOp, L3Expr, L3Scalar}; use crate::error::LoweringError; @@ -22,24 +21,11 @@ pub(super) fn split_conjuncts(expr: &Expr) -> Vec<&Expr> { } } -/// Translate a slice of DataFusion `Expr`s (non-time conjuncts) into a single -/// `L3Expr`. A single element is returned as-is; multiple elements are wrapped -/// in `L3Expr::BoolAnd`. -pub(super) fn conjuncts_to_l3expr(conjuncts: Vec<&Expr>) -> Result { - let parts: Result, _> = conjuncts.iter().map(|e| df_expr_to_l3(e)).collect(); - let mut parts = parts?; - if parts.len() == 1 { - Ok(parts.pop().unwrap()) - } else { - Ok(L3Expr::BoolAnd(parts)) - } -} - /// Translate a DataFusion `Expr` to an `L3Expr`. /// Returns `UnsupportedFeature` for anything not needed in v1. pub(super) fn df_expr_to_l3(expr: &Expr) -> Result { match expr { - Expr::Column(col) => Ok(L3Expr::Column(ColumnRef(col.name.clone()))), + Expr::Column(col) => Ok(L3Expr::Column(ColumnRef::Named(col.name.clone()))), Expr::Literal(sv) => scalar_value_to_l3(sv).map(L3Expr::Literal), diff --git a/crates/lower/src/sql/mod.rs b/crates/lower/src/sql/mod.rs index 4b85a129..bf3f8e2d 100644 --- a/crates/lower/src/sql/mod.rs +++ b/crates/lower/src/sql/mod.rs @@ -1,73 +1,84 @@ +//! SQL → Layer-2 relational lowering. +//! +//! Parses SQL via DataFusion (over the catalog's registered tables), then walks +//! the unoptimized `LogicalPlan` and emits the language-independent +//! [`relational::QueryExpr`](asap_control_core::intent_algebra::relational) that +//! [`convert_root`](asap_control_core::intent_algebra::convert_root) lowers to +//! canonical L3. Positional column identity, accuracy threading, and the +//! window-over-aggregate fold all happen in that converter — this front end +//! only interprets SQL semantics into the shared L2 algebra. + use std::sync::Arc; use datafusion::common::ScalarValue; use datafusion::datasource::MemTable; -use datafusion::logical_expr::{self, Distinct, Expr, LogicalPlan, WindowFunctionDefinition}; +use datafusion::logical_expr::{self, Distinct, Expr, LogicalPlan}; use datafusion::prelude::SessionContext; -use asap_control_core::intent_algebra::expr::{ - AggIntent, ColumnRef, GroupKey, L3Node, Predicate, ProjectItem, QueryExpr, SetOpKind, SortKey, - Source, TableRef, WindowFuncKind, +use asap_control_core::intent_algebra::relational::{ + AggFunc, AggItem, QueryExpr as L2, SourceSpec, }; -use asap_control_core::intent_algebra::schema::{L3Schema, SchemaCatalog, TableSchema}; -use asap_control_core::intent_algebra::{L3Expr, L3Scalar}; -use asap_control_core::types::AccuracyTarget; +use asap_control_core::intent_algebra::{ColumnRef, ProjectItem, SetOpKind, SortKey}; use crate::error::LoweringError; mod expr; -mod time; mod types; -use self::expr::{conjuncts_to_l3expr, df_expr_to_l3, split_conjuncts}; -use self::time::extract_time_range_from_conjuncts; -use self::types::table_schema_to_arrow; +pub use types::SqlCatalog; + +use self::expr::df_expr_to_l3; +use self::types::schema_to_arrow; +/// Lowers SQL strings to the Layer-2 [`relational::QueryExpr`] over a table +/// [`SqlCatalog`]. Call [`convert_root`](asap_control_core::intent_algebra::convert_root) +/// on the result for canonical L3. pub struct SqlLowerer<'a> { - catalog: &'a SchemaCatalog, - accuracy: AccuracyTarget, + catalog: &'a SqlCatalog, } impl<'a> SqlLowerer<'a> { - pub fn new(catalog: &'a SchemaCatalog, accuracy: AccuracyTarget) -> Self { - Self { catalog, accuracy } + pub fn new(catalog: &'a SqlCatalog) -> Self { + Self { catalog } } - pub async fn lower(&self, sql: &str) -> Result { + /// Parse + lower a SQL query to Layer-2 relational form. + pub async fn lower(&self, sql: &str) -> Result { let ctx = self.build_context()?; let df = ctx.sql(sql).await?; let plan = df.into_unoptimized_plan(); self.lower_plan(&plan) } + /// Register the catalog tables (empty Arrow `MemTable`s) so DataFusion can + /// resolve table/column references during planning. fn build_context(&self) -> Result { let ctx = SessionContext::new(); - for (name, table_schema) in &self.catalog.tables { - let arrow_schema = Arc::new(table_schema_to_arrow(table_schema)); + for (name, schema) in &self.catalog.tables { + let arrow_schema = Arc::new(schema_to_arrow(schema)); let mem_table = MemTable::try_new(arrow_schema, vec![])?; ctx.register_table(name.as_str(), Arc::new(mem_table))?; } Ok(ctx) } - fn lower_plan(&self, plan: &LogicalPlan) -> Result { + fn lower_plan(&self, plan: &LogicalPlan) -> Result { match plan { LogicalPlan::TableScan(scan) => self.lower_table_scan(scan), - LogicalPlan::Filter(filter) => self.lower_filter(filter), + LogicalPlan::Filter(filter) => Ok(L2::Filter { + pred: df_expr_to_l3(&filter.predicate)?, + input: Box::new(self.lower_plan(&filter.input)?), + }), LogicalPlan::Projection(proj) => self.lower_projection(proj), LogicalPlan::Aggregate(agg) => self.lower_aggregate(agg), LogicalPlan::Sort(sort) => self.lower_sort(sort), LogicalPlan::Limit(limit) => self.lower_limit(limit), - LogicalPlan::Window(window) => self.lower_window(window), LogicalPlan::Distinct(d) => match d { Distinct::On(_) => Err(LoweringError::UnsupportedFeature("DISTINCT ON".into())), - Distinct::All(input) => { - let child = self.lower_plan(input)?; - Ok(QueryExpr::Distinct { - child: make_untyped_node(child), - cols: vec![], - }) - } + Distinct::All(input) => Ok(L2::Distinct { + cols: vec![], + input: Box::new(self.lower_plan(input)?), + }), }, LogicalPlan::Union(u) => { // Fold n inputs left-associatively into SetOp { Union, all: true }. @@ -77,21 +88,22 @@ impl<'a> SqlLowerer<'a> { .ok_or_else(|| LoweringError::InvalidExpression("empty union".into()))?; let first_expr = self.lower_plan(first)?; iter.try_fold(first_expr, |left, right_plan| { - let right = self.lower_plan(right_plan)?; - Ok(QueryExpr::SetOp { + Ok(L2::SetOp { kind: SetOpKind::Union, all: true, - left: make_untyped_node(left), - right: make_untyped_node(right), + left: Box::new(left), + right: Box::new(self.lower_plan(right_plan)?), }) }) } + LogicalPlan::Window(_) => Err(LoweringError::UnsupportedFeature( + "SQL window functions (no L3 analytic-window node yet)".into(), + )), LogicalPlan::Join(_) => Err(LoweringError::UnsupportedFeature("JOIN".into())), LogicalPlan::Subquery(_) => Err(LoweringError::UnsupportedFeature("subquery".into())), LogicalPlan::SubqueryAlias(alias) => { - // Simple table alias (wraps only a TableScan or another alias) is - // transparent. A derived table (wraps Projection, Aggregate, etc.) - // is an inline-view subquery — unsupported in v1. + // A bare table alias is transparent; a derived table (inline view) + // is unsupported in v1. match alias.input.as_ref() { LogicalPlan::TableScan(_) | LogicalPlan::SubqueryAlias(_) => { self.lower_plan(&alias.input) @@ -108,87 +120,28 @@ impl<'a> SqlLowerer<'a> { } } - fn lower_table_scan(&self, scan: &logical_expr::TableScan) -> Result { + /// Table leaf — carries the catalog's resolved schema so the L2→L3 Binder + /// has positional column identity. Projection pushdown is left to the + /// enclosing `Project` (DataFusion's unoptimized plan sets no projection). + fn lower_table_scan(&self, scan: &logical_expr::TableScan) -> Result { let table_name = scan.table_name.to_string(); - let table_schema = self + let schema = self .catalog .tables .get(&table_name) .ok_or_else(|| LoweringError::TableNotFound(table_name.clone()))?; - let columns = projection_columns(scan, table_schema); - Ok(QueryExpr::Scan { - source: Source::Table { - table_ref: TableRef(table_name), - columns, - time_range: None, - }, - predicates: vec![], - }) + Ok(L2::Source(SourceSpec::with_schema( + table_name, + schema.clone(), + ))) } - fn lower_filter(&self, filter: &logical_expr::Filter) -> Result { - // Walk the full filter chain to find a TableScan at any depth, then - // collect predicates from all stacked filters (including the outermost). - let (inner_preds, maybe_scan) = collect_filter_chain(&filter.input); - - if let Some(scan) = maybe_scan { - let table_name = scan.table_name.to_string(); - if let Some(schema) = self.catalog.tables.get(&table_name) { - if let Some(time_col) = &schema.time_column { - if let Err(e) = schema.validate() { - return Err(LoweringError::InvalidExpression(format!( - "catalog table '{table_name}': {e}" - ))); - } - // Merge outermost predicate + all inner filter predicates into one - // flat conjunct list, then classify for time-range extraction. - let all_conjuncts: Vec<&Expr> = std::iter::once(&filter.predicate) - .chain(inner_preds) - .flat_map(|p| split_conjuncts(p)) - .collect(); - let (time_range, non_time) = - extract_time_range_from_conjuncts(all_conjuncts, time_col); - let columns = projection_columns(scan, schema); - let scan_expr = QueryExpr::Scan { - source: Source::Table { - table_ref: TableRef(table_name), - columns, - time_range, - }, - predicates: vec![], - }; - return if non_time.is_empty() { - Ok(scan_expr) - } else { - let pred_expr = conjuncts_to_l3expr(non_time)?; - Ok(QueryExpr::Filter { - child: make_untyped_node(scan_expr), - pred: Predicate(pred_expr), - }) - }; - } - } - } - - let pred_expr = df_expr_to_l3(&filter.predicate)?; - let child = self.lower_plan(&filter.input)?; - Ok(QueryExpr::Filter { - child: make_untyped_node(child), - pred: Predicate(pred_expr), - }) - } - - fn lower_projection( - &self, - proj: &logical_expr::Projection, - ) -> Result { - // SELECT * — all wildcards means "no column constraint". Pass through - // without a Project wrapper; an empty Scan.columns list means "all columns". + fn lower_projection(&self, proj: &logical_expr::Projection) -> Result { + // SELECT * — no column constraint; pass through without a Project. if proj.expr.iter().any(|e| matches!(e, Expr::Wildcard { .. })) { return self.lower_plan(&proj.input); } - - let child = self.lower_plan(&proj.input)?; + let input = Box::new(self.lower_plan(&proj.input)?); let cols = proj .expr .iter() @@ -200,66 +153,38 @@ impl<'a> SqlLowerer<'a> { _ => df_expr_to_l3(e).map(|expr| ProjectItem { expr, alias: None }), }) .collect::, _>>()?; - - // DataFusion's unoptimized plan never sets TableScan.projection, so we - // derive the Scan's column list from the enclosing projection instead. - let child = push_columns_into_scan(child, &cols); - - Ok(QueryExpr::Project { - child: make_untyped_node(child), - cols, - }) + Ok(L2::Project { cols, input }) } - fn lower_aggregate(&self, agg: &logical_expr::Aggregate) -> Result { - let child = self.lower_plan(&agg.input)?; - let by = agg + fn lower_aggregate(&self, agg: &logical_expr::Aggregate) -> Result { + let input = Box::new(self.lower_plan(&agg.input)?); + let keys = agg .group_expr .iter() - .map(expr_to_group_key) + .map(expr_to_group_name) .collect::, _>>()?; let aggs = agg .aggr_expr .iter() - .map(|e| self.lower_agg_expr(e)) + .map(lower_agg_item) .collect::, _>>()?; - // Use DataFusion's own aggregate output schema for column names — the same - // schema the enclosing Projection was built against when it wrote its column - // references (e.g. "MIN(metrics.ts)"). The first n_groups fields are the - // GROUP BY columns; the remaining fields are the aggregate outputs. - // TODO: output_names couples core's Aggregate IR to DataFusion's internal - // naming convention. Cleaner boundary: emit a Project on top of every - // Aggregate that renames DataFusion's names to user-visible aliases, so - // Aggregate.output_names can be removed and column resolution lives in Project. - let n_groups = agg.group_expr.len(); - let output_names: Vec = agg - .schema - .fields() - .iter() - .skip(n_groups) - .take(agg.aggr_expr.len()) - .map(|f| f.name().to_string()) - .collect(); - Ok(QueryExpr::Aggregate { - child: make_untyped_node(child), - by, + Ok(L2::Aggregate { + keys, aggs, having: None, - output_names, + input, }) } - fn lower_sort(&self, sort: &logical_expr::Sort) -> Result { - // TopK: Sort with a constant LIMIT folded in, all keys descending, on an Aggregate. - // Note: Sort.fetch is Option; Limit.fetch is Option>. + fn lower_sort(&self, sort: &logical_expr::Sort) -> Result { + // TopK: Sort with a folded LIMIT, all keys descending, over an Aggregate. if let Some(k) = sort.fetch { if sort.expr.iter().all(|s| !s.asc) { if let Some(agg) = find_aggregate(strip_projections_and_aliases(&sort.input)) { - return self.lower_as_topk(agg, k); + return self.lower_as_topk(agg, k as u64); } } } - let child = self.lower_plan(&sort.input)?; let keys = sort .expr .iter() @@ -271,267 +196,138 @@ impl<'a> SqlLowerer<'a> { }) }) .collect::, _>>()?; - Ok(QueryExpr::Sort { - child: make_untyped_node(child), + Ok(L2::Sort { keys, + input: Box::new(self.lower_plan(&sort.input)?), }) } - fn lower_limit(&self, limit: &logical_expr::Limit) -> Result { - // TopK: Limit on top of Sort on top of Aggregate, all sort keys DESC. + fn lower_limit(&self, limit: &logical_expr::Limit) -> Result { + // TopK: Limit over Sort over Aggregate, all sort keys DESC, no OFFSET. if let Some(k) = eval_fetch(&limit.fetch) { - if eval_fetch(&limit.skip).unwrap_or(0) > 0 { - return Err(LoweringError::UnsupportedFeature( - "LIMIT ... OFFSET is not supported with ORDER BY ... DESC aggregates (TopK)" - .into(), - )); - } - let inner = strip_aliases(&limit.input); - if let LogicalPlan::Sort(sort) = inner { - if sort.expr.iter().all(|s| !s.asc) { - if let Some(agg) = find_aggregate(strip_projections_and_aliases(&sort.input)) { - return self.lower_as_topk(agg, k); + if eval_fetch(&limit.skip).unwrap_or(0) == 0 { + if let LogicalPlan::Sort(sort) = strip_aliases(&limit.input) { + if sort.expr.iter().all(|s| !s.asc) { + if let Some(agg) = + find_aggregate(strip_projections_and_aliases(&sort.input)) + { + return self.lower_as_topk(agg, k as u64); + } } } } } - let child = self.lower_plan(&limit.input)?; - Ok(QueryExpr::Limit { - child: make_untyped_node(child), - n: eval_fetch(&limit.fetch).map(|v| v as u64), + Ok(L2::Limit { + n: eval_fetch(&limit.fetch).unwrap_or(usize::MAX) as u64, offset: eval_fetch(&limit.skip).unwrap_or(0) as u64, + input: Box::new(self.lower_plan(&limit.input)?), }) } - fn lower_as_topk( - &self, - agg: &logical_expr::Aggregate, - k: usize, - ) -> Result { - let child = self.lower_plan(&agg.input)?; + fn lower_as_topk(&self, agg: &logical_expr::Aggregate, k: u64) -> Result { let by = agg .group_expr .iter() - .map(expr_to_col_ref) + .map(expr_to_group_name) .collect::, _>>()?; - Ok(QueryExpr::Aggregate { - child: make_untyped_node(child), - by: vec![], - aggs: vec![AggIntent::TopK { - k, - by, - accuracy: self.accuracy.clone(), - }], - having: None, - output_names: vec![], + Ok(L2::TopK { + k, + by, + input: Box::new(self.lower_plan(&agg.input)?), }) } +} - fn lower_window(&self, window: &logical_expr::Window) -> Result { - if window.window_expr.len() > 1 { - return Err(LoweringError::UnsupportedFeature(format!( - "multiple window functions in one Window plan node (got {}); split into separate nodes", - window.window_expr.len() - ))); - } - let child = self.lower_plan(&window.input)?; - let first = window - .window_expr - .first() - .ok_or_else(|| LoweringError::InvalidExpression("empty window expressions".into()))?; - if let Expr::WindowFunction(wf) = first { - let func = lower_window_func_kind(&wf.fun)?; - let mut args = wf - .args - .iter() - .map(df_expr_to_l3) - .collect::, _>>()?; +// ── Aggregate / group-key helpers ─────────────────────────────────────────────── - // For NthValue, extract N from args[1] and keep only the column (args[0]). - let func = if matches!(func, WindowFuncKind::NthValue(None)) { - let n = match args.get(1) { - Some(L3Expr::Literal(L3Scalar::Int64(n))) if *n > 0 => *n as u64, - other => { - return Err(LoweringError::InvalidExpression(format!( - "NthValue requires a positive integer literal as second arg, got: {other:?}" - ))) - } - }; - args.truncate(1); - WindowFuncKind::NthValue(Some(n)) - } else { - func +/// Map a DataFusion aggregate expression to a relational [`AggItem`]. The +/// L2→L3 converter resolves the input column to a positional id and applies the +/// workload accuracy target — so this only picks the `AggFunc` + input column. +fn lower_agg_item(expr: &Expr) -> Result { + match expr { + Expr::Alias(a) => lower_agg_item(&a.expr), + Expr::AggregateFunction(agg_fn) => { + let name = agg_fn.func.name().to_lowercase(); + let (func, col) = match name.as_str() { + "count" if agg_fn.distinct => (AggFunc::CountDistinct, agg_col_ref(&agg_fn.args)), + "count" => (AggFunc::Count, ColumnRef::Wildcard), + "sum" => (AggFunc::Sum, agg_col_ref(&agg_fn.args)), + "min" => (AggFunc::Min, agg_col_ref(&agg_fn.args)), + "max" => (AggFunc::Max, agg_col_ref(&agg_fn.args)), + "avg" | "mean" => (AggFunc::Avg, agg_col_ref(&agg_fn.args)), + "stddev" | "stddev_samp" => ( + AggFunc::StdDev { population: false }, + agg_col_ref(&agg_fn.args), + ), + "stddev_pop" => ( + AggFunc::StdDev { population: true }, + agg_col_ref(&agg_fn.args), + ), + "var" | "variance" | "var_samp" => ( + AggFunc::Variance { population: false }, + agg_col_ref(&agg_fn.args), + ), + "var_pop" => ( + AggFunc::Variance { population: true }, + agg_col_ref(&agg_fn.args), + ), + "approx_percentile_cont" | "percentile_cont" => ( + AggFunc::Quantile(extract_percentile_q(&agg_fn.args)?), + agg_col_ref(&agg_fn.args), + ), + "approx_distinct" => (AggFunc::CountDistinct, agg_col_ref(&agg_fn.args)), + _ => return Err(LoweringError::UnsupportedAggregate(name)), }; - debug_assert!( - !matches!(func, WindowFuncKind::NthValue(None)), - "NthValue sentinel not resolved; lower_window has a bug" - ); - - let partition_by = wf - .partition_by - .iter() - .map(expr_to_group_key) - .collect::, _>>()?; - // In DataFusion 43, WindowFunction.order_by is Vec. - let order_by = wf - .order_by - .iter() - .map(|s| { - df_expr_to_l3(&s.expr).map(|expr| SortKey { - expr, - ascending: s.asc, - nulls_first: s.nulls_first, - }) - }) - .collect::, _>>()?; - return Ok(QueryExpr::WindowFunc { - child: make_untyped_node(child), + Ok(AggItem { + alias: name, func, - args, - partition_by, - order_by, - frame: None, - }); + col, + distinct: agg_fn.distinct, + }) } - Err(LoweringError::UnsupportedFeature( - "unexpected non-WindowFunction expr in Window plan node".into(), - )) + _ => Err(LoweringError::UnsupportedAggregate(format!("{expr:?}"))), } +} - fn lower_agg_expr(&self, expr: &Expr) -> Result { - match expr { - Expr::AggregateFunction(agg_fn) => { - let name = agg_fn.func.name().to_lowercase(); - match name.as_str() { - "count" if agg_fn.distinct => Ok(AggIntent::Cardinality { - accuracy: self.accuracy.clone(), - }), - "count" => Ok(AggIntent::Count { - accuracy: self.accuracy.clone(), - }), - "sum" => Ok(AggIntent::Sum { - col: agg_col(&agg_fn.args), - }), - "min" => Ok(AggIntent::Min { - col: agg_col(&agg_fn.args), - }), - "max" => Ok(AggIntent::Max { - col: agg_col(&agg_fn.args), - }), - "avg" | "mean" => Ok(AggIntent::Avg { - col: agg_col(&agg_fn.args), - }), - "stddev" | "stddev_samp" => Ok(AggIntent::Stddev { - col: agg_col(&agg_fn.args), - population: false, - }), - "stddev_pop" => Ok(AggIntent::Stddev { - col: agg_col(&agg_fn.args), - population: true, - }), - "approx_percentile_cont" | "percentile_cont" => { - let q = extract_percentile_q(&agg_fn.args)?; - Ok(AggIntent::Quantile { - q, - accuracy: self.accuracy.clone(), - }) - } - "approx_distinct" => Ok(AggIntent::Cardinality { - accuracy: self.accuracy.clone(), - }), - _ => Err(LoweringError::UnsupportedAggregate(name)), - } - } - Expr::Alias(alias) => self.lower_agg_expr(&alias.expr), - _ => Err(LoweringError::UnsupportedAggregate(format!("{expr:?}"))), +/// The aggregated input column. `COUNT(*)` and non-column arguments yield +/// `Wildcard`; a bare/aliased/cast column yields its name. +fn agg_col_ref(args: &[Expr]) -> ColumnRef { + fn col_name(e: &Expr) -> Option { + match e { + Expr::Column(c) => Some(c.name.clone()), + Expr::Alias(a) => col_name(&a.expr), + Expr::Cast(c) => col_name(&c.expr), + _ => None, } } + match args.first().and_then(col_name) { + Some(name) => ColumnRef::Named(name), + None => ColumnRef::Wildcard, + } } -// ── Free helpers ────────────────────────────────────────────────────────────── - -/// Map a DataFusion `TableScan.projection` (column index list) back to -/// `ColumnRef` names from the catalog schema. -/// Returns an empty `Vec` when the projection is absent (full scan / `SELECT *`). -fn projection_columns(scan: &logical_expr::TableScan, schema: &TableSchema) -> Vec { - match &scan.projection { - Some(indices) => indices - .iter() - .filter_map(|&i| schema.columns.get(i)) - .map(|c| ColumnRef(c.name.clone())) - .collect(), - None => vec![], +fn expr_to_group_name(expr: &Expr) -> Result { + match expr { + Expr::Column(col) => Ok(col.name.clone()), + Expr::Alias(a) => expr_to_group_name(&a.expr), + other => Err(LoweringError::UnsupportedFeature(format!( + "non-column GROUP BY expression: {other}" + ))), } } -/// If `child` (or a Filter wrapping it) contains a `Scan` with an empty -/// column list, populate it from the columns referenced in `cols`. -/// DataFusion's unoptimized plan never sets `TableScan.projection`, so this -/// compensates without requiring optimizer passes that could alter other -/// plan-node shapes our lowerer depends on. -/// -/// Handled topologies: `Project → Scan` and `Project → Filter → Scan`. -/// -/// **Known gap**: `Project → Aggregate → * → Scan` is NOT handled. Aggregate -/// lowering does not call this function, so `Scan.columns` stays empty in any -/// topology where an Aggregate sits between the Project and the Scan. Any -/// downstream stage that uses `Scan.columns` for pruning or cost estimation -/// will see an unconstrained (full) scan in those cases. TODO: propagate -/// column refs through the Aggregate child when implementing column-pruning. -fn push_columns_into_scan(child: QueryExpr, cols: &[ProjectItem]) -> QueryExpr { - match child { - // Recurse through Filter so that Project → Filter → Scan works. - QueryExpr::Filter { child: inner, pred } => { - let updated = push_columns_into_scan(inner.expr.clone(), cols); - QueryExpr::Filter { - child: Arc::new(L3Node { - expr: updated, - schema: inner.schema.clone(), - }), - pred, - } - } - QueryExpr::Scan { - source: - Source::Table { - table_ref, - columns, - time_range, - }, - predicates, - } if columns.is_empty() => { - let mut seen = std::collections::HashSet::::new(); - let col_refs: Vec = cols - .iter() - .flat_map(|item| item.expr.columns_referenced()) - .filter(|&c| seen.insert(c.0.clone())) - .cloned() - .collect(); - QueryExpr::Scan { - source: Source::Table { - table_ref, - columns: col_refs, - time_range, - }, - predicates, - } - } - other => other, +fn extract_percentile_q(args: &[Expr]) -> Result { + match args.get(1) { + Some(Expr::Literal(ScalarValue::Float64(Some(q)))) => Ok(*q), + Some(Expr::Literal(ScalarValue::Float32(Some(q)))) => Ok(*q as f64), + _ => Err(LoweringError::InvalidExpression( + "percentile value must be a float literal (2nd arg)".into(), + )), } } -fn make_untyped_node(expr: QueryExpr) -> Arc { - Arc::new(L3Node { - expr, - schema: L3Schema { - fields: vec![], - time_index: None, - }, - }) -} +// ── LogicalPlan navigation helpers ────────────────────────────────────────────── -/// Evaluate a constant fetch/skip expression to a `usize`. -/// Returns `None` for parametric (non-literal) fetch expressions. fn eval_fetch(expr_opt: &Option>) -> Option { expr_opt.as_ref().and_then(|e| match e.as_ref() { Expr::Literal(ScalarValue::Int64(Some(v))) if *v >= 0 => Some(*v as usize), @@ -548,8 +344,7 @@ fn strip_aliases(plan: &LogicalPlan) -> &LogicalPlan { } } -/// Strip Projection and SubqueryAlias for TopK pattern-matching only. -/// Do NOT use when building the output tree. +/// Strip Projection + SubqueryAlias for TopK pattern-matching only. fn strip_projections_and_aliases(plan: &LogicalPlan) -> &LogicalPlan { match plan { LogicalPlan::SubqueryAlias(a) => strip_projections_and_aliases(&a.input), @@ -566,156 +361,3 @@ fn find_aggregate(plan: &LogicalPlan) -> Option<&logical_expr::Aggregate> { _ => None, } } - -fn expr_to_group_key(expr: &Expr) -> Result { - match expr { - Expr::Column(col) => Ok(GroupKey(col.name.clone())), - Expr::Alias(a) => expr_to_group_key(&a.expr), - other => Err(LoweringError::UnsupportedFeature(format!( - "non-column GROUP BY expression: {other}" - ))), - } -} - -fn expr_to_col_ref(expr: &Expr) -> Result { - match expr { - Expr::Column(col) => Ok(ColumnRef(col.name.clone())), - Expr::Alias(a) => expr_to_col_ref(&a.expr), - other => Err(LoweringError::UnsupportedFeature(format!( - "non-column reference in TopK by-list: {other}" - ))), - } -} - -/// Extract the aggregated column name from aggregate function args. -/// Returns `None` for wildcards (`COUNT(*)`) and non-column expressions. -fn agg_col(args: &[Expr]) -> Option { - match args.first() { - Some(Expr::Column(col)) => Some(ColumnRef(col.name.clone())), - Some(Expr::Alias(a)) => match a.expr.as_ref() { - Expr::Column(col) => Some(ColumnRef(col.name.clone())), - _ => None, - }, - Some(Expr::Cast(c)) => match c.expr.as_ref() { - Expr::Column(col) => Some(ColumnRef(col.name.clone())), - _ => None, - }, - Some(Expr::Wildcard { .. }) | None => None, - _ => None, - } -} - -fn extract_percentile_q(args: &[Expr]) -> Result { - let val = args.get(1).ok_or_else(|| { - LoweringError::InvalidExpression("percentile requires 2 arguments".into()) - })?; - match val { - Expr::Literal(ScalarValue::Float64(Some(q))) => Ok(*q), - Expr::Literal(ScalarValue::Float32(Some(q))) => Ok(*q as f64), - _ => Err(LoweringError::InvalidExpression( - "percentile value must be a float literal".into(), - )), - } -} - -/// Walk a `Filter(Filter(...(TableScan)))` chain. -/// Returns `(predicates_from_inner_filters, Some(scan))` when a TableScan is -/// found at any depth, or `(vec![], None)` if a non-Filter non-Scan node is -/// reached first. The outermost filter's predicate is NOT included — the -/// caller adds it. -fn collect_filter_chain(plan: &LogicalPlan) -> (Vec<&Expr>, Option<&logical_expr::TableScan>) { - let plan = strip_aliases(plan); - match plan { - LogicalPlan::TableScan(scan) => (vec![], Some(scan)), - LogicalPlan::Filter(f) => { - let (mut inner_preds, maybe_scan) = collect_filter_chain(&f.input); - if maybe_scan.is_some() { - inner_preds.push(&f.predicate); - } - (inner_preds, maybe_scan) - } - _ => (vec![], None), - } -} - -fn lower_window_func_kind(fun: &WindowFunctionDefinition) -> Result { - match fun { - // In DataFusion 43 most ranking/nav window functions are WindowUDF. - WindowFunctionDefinition::WindowUDF(udf) => match udf.name().to_lowercase().as_str() { - "row_number" => Ok(WindowFuncKind::RowNumber), - "rank" => Ok(WindowFuncKind::Rank), - "dense_rank" => Ok(WindowFuncKind::DenseRank), - "lag" => Ok(WindowFuncKind::Lag), - "lead" => Ok(WindowFuncKind::Lead), - "first_value" => Ok(WindowFuncKind::FirstValue), - "last_value" => Ok(WindowFuncKind::LastValue), - // NthValue(None) is a sentinel; lower_window extracts the real N from args. - "nth_value" => Ok(WindowFuncKind::NthValue(None)), - other => Err(LoweringError::UnsupportedFeature(format!( - "window fn: {other}" - ))), - }, - WindowFunctionDefinition::AggregateUDF(udf) => match udf.name().to_lowercase().as_str() { - "sum" => Ok(WindowFuncKind::Sum), - "avg" | "mean" => Ok(WindowFuncKind::Avg), - "count" => Ok(WindowFuncKind::Count), - "min" => Ok(WindowFuncKind::Min), - "max" => Ok(WindowFuncKind::Max), - other => Err(LoweringError::UnsupportedFeature(format!( - "window agg: {other}" - ))), - }, - // In DataFusion 43, BuiltInWindowFunction covers FirstValue, LastValue, NthValue. - // NthValue(None) is a sentinel; the real N is extracted from args in lower_window. - WindowFunctionDefinition::BuiltInWindowFunction(biwf) => { - use datafusion::logical_expr::BuiltInWindowFunction; - match biwf { - BuiltInWindowFunction::FirstValue => Ok(WindowFuncKind::FirstValue), - BuiltInWindowFunction::LastValue => Ok(WindowFuncKind::LastValue), - BuiltInWindowFunction::NthValue => Ok(WindowFuncKind::NthValue(None)), - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - // ── collect_filter_chain unit tests (Fix 3) ─────────────────────────────── - - fn empty_scan() -> LogicalPlan { - use datafusion::arrow::datatypes::{DataType, Field, Schema}; - use datafusion::logical_expr::builder::LogicalTableSource; - use datafusion::logical_expr::LogicalPlanBuilder; - - let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); - let source = Arc::new(LogicalTableSource::new(schema)); - LogicalPlanBuilder::scan("t", source, None) - .unwrap() - .build() - .unwrap() - } - - #[test] - fn collect_filter_chain_finds_scan_at_depth_zero() { - let scan = empty_scan(); - let (preds, maybe_scan) = collect_filter_chain(&scan); - assert!(maybe_scan.is_some(), "should find the TableScan"); - assert!(preds.is_empty(), "no inner predicates at depth zero"); - } - - #[test] - fn collect_filter_chain_returns_none_for_non_scan() { - use datafusion::common::DFSchema; - use datafusion::logical_expr::EmptyRelation; - - let empty = LogicalPlan::EmptyRelation(EmptyRelation { - produce_one_row: false, - schema: Arc::new(DFSchema::empty()), - }); - let (preds, maybe_scan) = collect_filter_chain(&empty); - assert!(maybe_scan.is_none()); - assert!(preds.is_empty()); - } -} diff --git a/crates/lower/src/sql/time.rs b/crates/lower/src/sql/time.rs deleted file mode 100644 index 58efaa12..00000000 --- a/crates/lower/src/sql/time.rs +++ /dev/null @@ -1,288 +0,0 @@ -use datafusion::common::ScalarValue; -use datafusion::logical_expr::{BinaryExpr, Expr, Operator}; - -use asap_control_core::intent_algebra::expr::TimeRange; - -/// Core: classify a pre-split list of conjuncts into time bounds + residual. -pub(super) fn extract_time_range_from_conjuncts<'a>( - conjuncts: Vec<&'a Expr>, - time_col: &str, -) -> (Option, Vec<&'a Expr>) { - let mut start_ms: Option = None; - let mut end_ms: Option = None; - let mut non_time: Vec<&'a Expr> = vec![]; - - for c in conjuncts { - match classify_time_pred(c, time_col) { - TimeClass::Start(ms) => { - start_ms = Some(start_ms.map_or(ms, |s: i64| s.max(ms))); - } - TimeClass::End(ms) => { - end_ms = Some(end_ms.map_or(ms, |e: i64| e.min(ms))); - } - TimeClass::Both(lo, hi) => { - start_ms = Some(start_ms.map_or(lo, |s: i64| s.max(lo))); - end_ms = Some(end_ms.map_or(hi, |e: i64| e.min(hi))); - } - TimeClass::NonTime => non_time.push(c), - } - } - - let range = if start_ms.is_some() || end_ms.is_some() { - Some(TimeRange { start_ms, end_ms }) - } else { - None - }; - (range, non_time) -} - -/// Convenience wrapper: split a single expression then classify conjuncts. -#[cfg(test)] -fn extract_time_range<'a>(expr: &'a Expr, time_col: &str) -> (Option, Vec<&'a Expr>) { - use super::expr::split_conjuncts; - extract_time_range_from_conjuncts(split_conjuncts(expr), time_col) -} - -enum TimeClass { - Start(i64), - End(i64), - /// BETWEEN low AND high on the time column — contributes both bounds at once. - Both(i64, i64), - NonTime, -} - -fn classify_time_pred(expr: &Expr, time_col: &str) -> TimeClass { - match expr { - // `ts BETWEEN low AND high` — contributes both a start and end bound. - // `ts NOT BETWEEN …` cannot be expressed as a contiguous TimeRange; treat as non-time. - Expr::Between(b) if !b.negated && is_time_col(&b.expr, time_col) => { - match (expr_to_ms(&b.low), expr_to_ms(&b.high)) { - (Some(lo), Some(hi)) => TimeClass::Both(lo, hi), - _ => TimeClass::NonTime, - } - } - - Expr::BinaryExpr(BinaryExpr { left, op, right }) => { - let (col_is_left, val_expr): (bool, &Expr) = if is_time_col(left, time_col) { - (true, right) - } else if is_time_col(right, time_col) { - (false, left) - } else { - return TimeClass::NonTime; - }; - let Some(ms) = expr_to_ms(val_expr) else { - return TimeClass::NonTime; - }; - match (op, col_is_left) { - (Operator::Gt | Operator::GtEq, true) | (Operator::Lt | Operator::LtEq, false) => { - TimeClass::Start(ms) - } - (Operator::Lt | Operator::LtEq, true) | (Operator::Gt | Operator::GtEq, false) => { - TimeClass::End(ms) - } - // Eq (exact timestamp equality) and all other operators cannot be - // expressed as a contiguous half-open range, so leave them as - // regular Filter predicates rather than time-range bounds. - _ => TimeClass::NonTime, - } - } - - _ => TimeClass::NonTime, - } -} - -fn is_time_col(expr: &Expr, time_col: &str) -> bool { - match expr { - Expr::Column(col) => col.name == time_col, - Expr::Cast(c) => is_time_col(&c.expr, time_col), - _ => false, - } -} - -fn expr_to_ms(expr: &Expr) -> Option { - match expr { - Expr::Literal(sv) => scalar_to_ms(sv), - Expr::Cast(c) => expr_to_ms(&c.expr), - Expr::TryCast(c) => expr_to_ms(&c.expr), - _ => None, - } -} - -/// Round `v` to the nearest millisecond and return it as `i64`. -/// Returns `None` if `v` is non-finite or outside the `i64` range. -fn float_to_ms(v: f64) -> Option { - let rounded = v.round(); - // i64::MAX as f64 rounds up to 2^63, which overflows i64 on cast. - // Use strict less-than for the upper bound. - if rounded.is_finite() && rounded >= i64::MIN as f64 && rounded < i64::MAX as f64 { - Some(rounded as i64) - } else { - None - } -} - -fn scalar_to_ms(sv: &ScalarValue) -> Option { - match sv { - ScalarValue::Int64(Some(v)) => Some(*v), - ScalarValue::Int32(Some(v)) => Some(*v as i64), - ScalarValue::Float64(Some(v)) => float_to_ms(*v), - ScalarValue::Float32(Some(v)) => float_to_ms(*v as f64), - ScalarValue::TimestampMillisecond(Some(ms), _) => Some(*ms), - ScalarValue::TimestampNanosecond(Some(ns), _) => Some(*ns / 1_000_000), - ScalarValue::TimestampMicrosecond(Some(us), _) => Some(*us / 1_000), - ScalarValue::TimestampSecond(Some(s), _) => Some(*s * 1_000), - _ => None, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use datafusion::common::ScalarValue; - use datafusion::logical_expr::{BinaryExpr, Expr, Operator}; - - fn col(name: &str) -> Expr { - Expr::Column(datafusion::common::Column::new_unqualified(name)) - } - fn int(v: i64) -> Expr { - Expr::Literal(ScalarValue::Int64(Some(v))) - } - fn float(v: f64) -> Expr { - Expr::Literal(ScalarValue::Float64(Some(v))) - } - fn bin(left: Expr, op: Operator, right: Expr) -> Expr { - Expr::BinaryExpr(BinaryExpr { - left: Box::new(left), - op, - right: Box::new(right), - }) - } - fn and(l: Expr, r: Expr) -> Expr { - bin(l, Operator::And, r) - } - - #[test] - fn col_left_gt_lower_bound() { - let expr = bin(col("ts"), Operator::Gt, int(1000)); - let (range, non_time) = extract_time_range(&expr, "ts"); - assert_eq!( - range, - Some(TimeRange { - start_ms: Some(1000), - end_ms: None - }) - ); - assert!(non_time.is_empty()); - } - - #[test] - fn col_right_lt_is_start_bound() { - // `1000 < ts` ≡ `ts > 1000` - let expr = bin(int(1000), Operator::Lt, col("ts")); - let (range, non_time) = extract_time_range(&expr, "ts"); - assert_eq!( - range, - Some(TimeRange { - start_ms: Some(1000), - end_ms: None - }) - ); - assert!(non_time.is_empty()); - } - - #[test] - fn col_right_gt_is_end_bound() { - // `2000 > ts` ≡ `ts < 2000` - let expr = bin(int(2000), Operator::Gt, col("ts")); - let (range, non_time) = extract_time_range(&expr, "ts"); - assert_eq!( - range, - Some(TimeRange { - start_ms: None, - end_ms: Some(2000) - }) - ); - assert!(non_time.is_empty()); - } - - #[test] - fn overlapping_repeated_bounds_tighten() { - // `ts > 500 AND ts > 1000` → start = 1000 (tighter) - let expr = and( - bin(col("ts"), Operator::Gt, int(500)), - bin(col("ts"), Operator::Gt, int(1000)), - ); - let (range, _) = extract_time_range(&expr, "ts"); - assert_eq!(range.unwrap().start_ms, Some(1000)); - } - - #[test] - fn overlapping_end_bounds_tighten() { - // `ts < 2000 AND ts < 1500` → end = 1500 (tighter) - let expr = and( - bin(col("ts"), Operator::Lt, int(2000)), - bin(col("ts"), Operator::Lt, int(1500)), - ); - let (range, _) = extract_time_range(&expr, "ts"); - assert_eq!(range.unwrap().end_ms, Some(1500)); - } - - #[test] - fn between_contributes_both_bounds() { - use datafusion::logical_expr::Between; - let expr = Expr::Between(Between { - expr: Box::new(col("ts")), - negated: false, - low: Box::new(int(1000)), - high: Box::new(int(2000)), - }); - let (range, non_time) = extract_time_range(&expr, "ts"); - assert_eq!( - range, - Some(TimeRange { - start_ms: Some(1000), - end_ms: Some(2000) - }) - ); - assert!(non_time.is_empty()); - } - - #[test] - fn not_between_is_non_time() { - use datafusion::logical_expr::Between; - let expr = Expr::Between(Between { - expr: Box::new(col("ts")), - negated: true, - low: Box::new(int(1000)), - high: Box::new(int(2000)), - }); - let (range, non_time) = extract_time_range(&expr, "ts"); - assert!(range.is_none()); - assert_eq!(non_time.len(), 1); - } - - #[test] - fn float_literal_extracted_as_ms() { - let expr = bin(col("ts"), Operator::Gt, float(1_000_000.0)); - let (range, non_time) = extract_time_range(&expr, "ts"); - assert_eq!( - range, - Some(TimeRange { - start_ms: Some(1_000_000), - end_ms: None - }) - ); - assert!(non_time.is_empty()); - } - - #[test] - fn non_time_conjunct_passes_through() { - let expr = and( - bin(col("ts"), Operator::Gt, int(1000)), - bin(col("value"), Operator::Gt, int(0)), - ); - let (range, non_time) = extract_time_range(&expr, "ts"); - assert!(range.is_some()); - assert_eq!(non_time.len(), 1); - } -} diff --git a/crates/lower/src/sql/types.rs b/crates/lower/src/sql/types.rs index 5f124891..21ba90d5 100644 --- a/crates/lower/src/sql/types.rs +++ b/crates/lower/src/sql/types.rs @@ -1,13 +1,41 @@ -use std::sync::Arc; +//! Type bridges between DataFusion's Arrow types and the L3 `DataType`, plus +//! the SQL table catalog used to register tables with DataFusion and to carry +//! resolved leaf schemas into the relational L2 tree. -use datafusion::arrow::datatypes::{DataType as ArrowDataType, Field, Fields, Schema, TimeUnit}; +use std::collections::HashMap; + +use datafusion::arrow::datatypes::{ + DataType as ArrowDataType, Field, Fields, Schema as ArrowSchema, +}; use datafusion::common::ScalarValue; -use asap_control_core::intent_algebra::schema::{L3DataType, TableSchema}; +use asap_control_core::intent_algebra::schema::{Column, DataType, Schema}; use asap_control_core::intent_algebra::L3Scalar; use crate::error::LoweringError; +/// Table catalog for SQL lowering: table name → resolved L3 [`Schema`]. +/// +/// Used twice: to register Arrow-backed `MemTable`s so DataFusion can resolve +/// `SELECT … FROM t`, and to attach each table's schema to the relational +/// `SourceSpec` so the L2→L3 Binder/converter has positional column identity. +#[derive(Debug, Clone, Default)] +pub struct SqlCatalog { + pub tables: HashMap, +} + +impl SqlCatalog { + pub fn new() -> Self { + Self::default() + } + + /// Builder: register `name` with its resolved L3 schema. + pub fn with_table(mut self, name: impl Into, schema: Schema) -> Self { + self.tables.insert(name.into(), schema); + self + } +} + pub(super) fn scalar_value_to_l3(sv: &ScalarValue) -> Result { match sv { ScalarValue::Int64(Some(v)) => Ok(L3Scalar::Int64(*v)), @@ -24,7 +52,6 @@ pub(super) fn scalar_value_to_l3(sv: &ScalarValue) -> Result Ok(L3Scalar::Boolean(*b)), - // Typed nulls and untyped null both become L3Scalar::Null _ if sv.is_null() => Ok(L3Scalar::Null), _ => Err(LoweringError::InvalidExpression(format!( "unsupported scalar: {sv:?}" @@ -32,53 +59,42 @@ pub(super) fn scalar_value_to_l3(sv: &ScalarValue) -> Result Result { +/// Arrow → L3 `DataType` (used for `CAST` targets). L3 is deliberately narrow. +pub(super) fn arrow_to_l3(dt: &ArrowDataType) -> Result { match dt { ArrowDataType::Int64 | ArrowDataType::Int32 | ArrowDataType::Int16 - | ArrowDataType::Int8 => Ok(L3DataType::Int64), - ArrowDataType::Float64 | ArrowDataType::Float32 => Ok(L3DataType::Float64), - ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 => Ok(L3DataType::Utf8), - ArrowDataType::Boolean => Ok(L3DataType::Boolean), - ArrowDataType::Timestamp(_, _) => Ok(L3DataType::Timestamp), - ArrowDataType::Duration(_) => Ok(L3DataType::Duration), + | ArrowDataType::Int8 => Ok(DataType::Int64), + ArrowDataType::Float64 | ArrowDataType::Float32 => Ok(DataType::Float64), + ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 => Ok(DataType::Utf8), + ArrowDataType::Boolean => Ok(DataType::Bool), + ArrowDataType::Timestamp(_, _) => Ok(DataType::Timestamp), other => Err(LoweringError::UnsupportedFeature(format!( - "Arrow type in cast: {other:?}" + "Arrow type: {other:?}" ))), } } -pub(super) fn table_schema_to_arrow(schema: &TableSchema) -> Schema { +/// L3 `DataType` → Arrow (for registering catalog tables with DataFusion). +pub(super) fn l3_to_arrow(dt: &DataType) -> ArrowDataType { + match dt { + DataType::Int64 => ArrowDataType::Int64, + DataType::Float64 => ArrowDataType::Float64, + DataType::Utf8 => ArrowDataType::Utf8, + DataType::Bool => ArrowDataType::Boolean, + DataType::Timestamp => { + ArrowDataType::Timestamp(datafusion::arrow::datatypes::TimeUnit::Millisecond, None) + } + } +} + +/// Build an Arrow schema from an L3 [`Schema`] (column name + type + nullability). +pub(super) fn schema_to_arrow(schema: &Schema) -> ArrowSchema { let fields: Fields = schema .columns .iter() - .map(|c| Field::new(&c.name, l3_to_arrow(&c.data_type), c.nullable)) + .map(|c: &Column| Field::new(&c.name, l3_to_arrow(&c.dtype), c.nullable)) .collect(); - Schema::new(fields) -} - -pub(super) fn l3_to_arrow(dt: &L3DataType) -> ArrowDataType { - match dt { - L3DataType::Int64 => ArrowDataType::Int64, - L3DataType::Float64 => ArrowDataType::Float64, - L3DataType::Utf8 => ArrowDataType::Utf8, - L3DataType::Boolean => ArrowDataType::Boolean, - L3DataType::Timestamp => ArrowDataType::Timestamp(TimeUnit::Millisecond, None), - L3DataType::Duration => ArrowDataType::Duration(TimeUnit::Millisecond), - L3DataType::Map(k, v) => ArrowDataType::Map( - Arc::new(Field::new( - "entries", - ArrowDataType::Struct(Fields::from(vec![ - Field::new("key", l3_to_arrow(k), false), - Field::new("value", l3_to_arrow(v), true), - ])), - false, - )), - false, - ), - L3DataType::List(item) => { - ArrowDataType::List(Arc::new(Field::new("item", l3_to_arrow(item), true))) - } - } + ArrowSchema::new(fields) } diff --git a/crates/lower/tests/sql_lowering.rs b/crates/lower/tests/sql_lowering.rs new file mode 100644 index 00000000..4b687f14 --- /dev/null +++ b/crates/lower/tests/sql_lowering.rs @@ -0,0 +1,114 @@ +//! End-to-end SQL → L2 → canonical L3 lowering tests (positional IR). +//! +//! Validates the re-targeted DataFusion front end: SQL parses + plans, lowers to +//! the relational L2 algebra, and the shared `convert_root` produces positional +//! canonical L3 (the same converter the PromQL path uses). + +use asap_control_core::intent_algebra::schema::{Column, DataType, Schema}; +use asap_control_core::intent_algebra::{AggIntent, QueryExpr, Source}; +use asap_control_core::types::AccuracyTarget; +use asap_control_lower::{lower_sql, SqlCatalog}; + +fn col(name: &str, dtype: DataType) -> Column { + Column { + name: name.into(), + dtype, + nullable: false, + } +} + +/// `metrics(ts, service, latency, bytes)` — column positions 0..3. +fn catalog() -> SqlCatalog { + SqlCatalog::new().with_table( + "metrics", + Schema::with_time_index( + vec![ + col("ts", DataType::Timestamp), + col("service", DataType::Utf8), + col("latency", DataType::Float64), + col("bytes", DataType::Int64), + ], + 0, + vec![], + ), + ) +} + +async fn lower(sql: &str) -> QueryExpr { + lower_sql(sql, &catalog(), AccuracyTarget::Exact) + .await + .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")) +} + +/// Find the first `Aggregate` node anywhere in the tree. +fn find_aggregate(qe: &QueryExpr) -> Option<(&Vec, &Vec)> { + match qe { + QueryExpr::Aggregate { by, aggs, .. } => Some((by, aggs)), + QueryExpr::Project { child, .. } + | QueryExpr::Filter { child, .. } + | QueryExpr::Window { child, .. } + | QueryExpr::Partition { child, .. } + | QueryExpr::Distinct { child, .. } + | QueryExpr::Sort { child, .. } + | QueryExpr::Limit { child, .. } + | QueryExpr::Subquery { child, .. } => find_aggregate(child), + _ => None, + } +} + +#[tokio::test] +async fn select_star_with_where_folds_predicate_onto_scan() { + // SELECT * elides the projection; WHERE folds onto the Scan predicates. + let qe = lower("SELECT * FROM metrics WHERE service = 'api'").await; + let QueryExpr::Scan { + source, predicates, .. + } = &qe + else { + panic!("expected Scan at root, got {qe:?}"); + }; + assert!(matches!(source, Source::Table { table_ref } if table_ref == "metrics")); + assert_eq!(predicates.len(), 1, "WHERE clause folded onto the scan"); +} + +#[tokio::test] +async fn multi_aggregate_group_by_binds_columns_positionally() { + // SUM(bytes)=col 3, AVG(latency)=col 2, GROUP BY service=col 1. + let qe = lower("SELECT service, SUM(bytes), AVG(latency) FROM metrics GROUP BY service").await; + let (by, aggs) = find_aggregate(&qe).expect("expected an Aggregate in the tree"); + assert_eq!(by, &vec![1], "GROUP BY service → column 1"); + assert!( + aggs.contains(&AggIntent::Sum { col: Some(3) }), + "SUM(bytes) → Sum{{col:3}}, got {aggs:?}" + ); + assert!( + aggs.contains(&AggIntent::Avg { col: Some(2) }), + "AVG(latency) → Avg{{col:2}}, got {aggs:?}" + ); +} + +#[tokio::test] +async fn count_star_is_count_intent() { + let qe = lower("SELECT COUNT(*) FROM metrics").await; + let (by, aggs) = find_aggregate(&qe).expect("expected an Aggregate"); + assert!(by.is_empty()); + assert!(matches!(aggs.as_slice(), [AggIntent::Count { .. }])); +} + +#[tokio::test] +async fn count_distinct_is_cardinality() { + let qe = lower("SELECT COUNT(DISTINCT service) FROM metrics").await; + let (_, aggs) = find_aggregate(&qe).expect("expected an Aggregate"); + assert!(matches!(aggs.as_slice(), [AggIntent::Cardinality { .. }])); +} + +#[tokio::test] +async fn unsupported_join_is_rejected_not_mislowered() { + // The front end declines JOIN rather than silently dropping a side. + let res = lower_sql( + "SELECT a.service FROM metrics a JOIN metrics b ON a.service = b.service", + &catalog(), + AccuracyTarget::Exact, + ) + .await; + assert!(res.is_err(), "JOIN should be rejected in v1"); +} From bdca7e1eb02041c360d1944e698fcaaa241c0b2d Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 26 May 2026 10:40:34 -0600 Subject: [PATCH 22/40] =?UTF-8?q?docs+chore:=20mark=20#4=E2=87=84#5=20reco?= =?UTF-8?q?nciliation=20done;=20fix=20stale=20L4=20doc=20comment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - intent-algebra-reconciliation.md: note the intra-repo (#4⇄#5) reconciliation is complete — both front ends lower onto one positional IR — so the ASAP⇄control_plane plan is now the unblocked next step. - sketch_algebra/expr.rs: doc comment said Logical(Rc); the type is Logical(Box) (L3Node no longer exists). Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/core/src/sketch_algebra/expr.rs | 5 +++-- docs/intent-algebra-reconciliation.md | 8 ++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/crates/core/src/sketch_algebra/expr.rs b/crates/core/src/sketch_algebra/expr.rs index 0cc343f1..116ade73 100644 --- a/crates/core/src/sketch_algebra/expr.rs +++ b/crates/core/src/sketch_algebra/expr.rs @@ -22,10 +22,11 @@ pub struct L4Node { /// Sketch-bound IR produced by L4 optimizer rules. L4 rules selectively /// replace logical aggregates and joins in the L3 `QueryExpr` with their /// sketch-bound counterparts; everything not rewritten passes through as -/// `Logical(Rc)`. +/// `Logical(Box)`. /// /// Traversing from the root node yields a DAG; shared sub-expressions appear -/// as multiple `Rc` references to the same `L4Node` or `L3Node`. +/// as multiple `Rc` references to the same `L4Node` (L3 fan-in is expressed +/// via `QueryExpr`'s own `LetBinding`/`Ref`). #[derive(Debug, Clone)] pub enum SummaryExpr { /// Any L3 node that no L4 rule rewrote (e.g. `Filter`, `Project`, `Sort`). diff --git a/docs/intent-algebra-reconciliation.md b/docs/intent-algebra-reconciliation.md index f0084459..de35153e 100644 --- a/docs/intent-algebra-reconciliation.md +++ b/docs/intent-algebra-reconciliation.md @@ -1,5 +1,13 @@ # `intent_algebra` reconciliation plan (ASAPController ⇄ ASAPQuery-backend) +> **Status (prerequisite done):** the *intra-repo* reconciliation — PR #4's +> name-based SQL L3 ⇄ PR #5's positional L3 — is complete on `feat/promql-l1-l3`. +> Both front ends (PromQL + SQL) now lower onto **one positional IR** via the +> shared `convert_root`: `expr_ir` is the SQL∪PromQL scalar superset, `AggIntent` +> carries `col: Option`, the converter is bottom-up schema-aware, and +> the DataFusion front end emits relational L2. The plan below (ASAP ⇄ +> control_plane) is the *next* step, now unblocked. + ASAPController's `crates/core/src/intent_algebra` is a slimmed, refactored fork of the canonical L3 IR in `ASAPQuery-backend/control_plane/src/intent_algebra`. This is the first concrete step of the L4/L5 consolidation: produce **one** From 6ae087b49bcb260e3f30a776f450aebd1cfb7eb7 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 26 May 2026 11:20:38 -0600 Subject: [PATCH 23/40] =?UTF-8?q?feat(sql):=20wire=20JOIN=20front=20end=20?= =?UTF-8?q?(LogicalPlan::Join=20=E2=86=92=20relational=20L2=20Join)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The L2→L3 converter already supported joins (concatenated-schema derivation + positional binding); this connects the SQL front end to it. - lower_join: maps DataFusion JoinType (Inner/Left/Right/Full) → JoinKind; builds the L2 join predicate from the equijoin `on` pairs (left = right) AND-ed with any non-equi `filter`. Semi/anti/mark joins are rejected (no L3 counterpart yet). - Tests: INNER JOIN lowers to Join over two Scans; GROUP BY a right-table column over a join binds against the concatenated schema (region→col 5, bytes→col 3); IN-subquery (semi-join) rejected. 109 tests green. Note: the L3 join predicate stays name-based (unqualified ColumnRef), so a self-join's same-named keys are ambiguous — a qualified-column follow-up. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/lower/src/sql/mod.rs | 52 +++++++++++++- crates/lower/tests/sql_lowering.rs | 112 ++++++++++++++++++++++++----- 2 files changed, 142 insertions(+), 22 deletions(-) diff --git a/crates/lower/src/sql/mod.rs b/crates/lower/src/sql/mod.rs index bf3f8e2d..66115cf7 100644 --- a/crates/lower/src/sql/mod.rs +++ b/crates/lower/src/sql/mod.rs @@ -12,13 +12,15 @@ use std::sync::Arc; use datafusion::common::ScalarValue; use datafusion::datasource::MemTable; -use datafusion::logical_expr::{self, Distinct, Expr, LogicalPlan}; +use datafusion::logical_expr::{self, Distinct, Expr, JoinType, LogicalPlan}; use datafusion::prelude::SessionContext; use asap_control_core::intent_algebra::relational::{ AggFunc, AggItem, QueryExpr as L2, SourceSpec, }; -use asap_control_core::intent_algebra::{ColumnRef, ProjectItem, SetOpKind, SortKey}; +use asap_control_core::intent_algebra::{ + ColumnRef, CompareOp, JoinKind, L3Expr, ProjectItem, SetOpKind, SortKey, +}; use crate::error::LoweringError; @@ -99,7 +101,7 @@ impl<'a> SqlLowerer<'a> { LogicalPlan::Window(_) => Err(LoweringError::UnsupportedFeature( "SQL window functions (no L3 analytic-window node yet)".into(), )), - LogicalPlan::Join(_) => Err(LoweringError::UnsupportedFeature("JOIN".into())), + LogicalPlan::Join(join) => self.lower_join(join), LogicalPlan::Subquery(_) => Err(LoweringError::UnsupportedFeature("subquery".into())), LogicalPlan::SubqueryAlias(alias) => { // A bare table alias is transparent; a derived table (inline view) @@ -136,6 +138,50 @@ impl<'a> SqlLowerer<'a> { ))) } + /// ⋈ — equijoin. The `on` key pairs become `left = right` comparisons, + /// AND-ed with any non-equi `filter`, into the L2 join predicate. The L2→L3 + /// converter derives the concatenated output schema; the join predicate + /// stays name-based (like a `WHERE`). Semi/anti/mark joins have no L3 + /// counterpart yet and are rejected. + fn lower_join(&self, join: &logical_expr::Join) -> Result { + let kind = match join.join_type { + JoinType::Inner => JoinKind::Inner, + JoinType::Left => JoinKind::Left, + JoinType::Right => JoinKind::Right, + JoinType::Full => JoinKind::Full, + other => { + return Err(LoweringError::UnsupportedFeature(format!( + "join type: {other:?}" + ))) + } + }; + let mut conjuncts = join + .on + .iter() + .map(|(l, r)| { + Ok(L3Expr::Compare { + left: Box::new(df_expr_to_l3(l)?), + op: CompareOp::Eq, + right: Box::new(df_expr_to_l3(r)?), + }) + }) + .collect::, LoweringError>>()?; + if let Some(filter) = &join.filter { + conjuncts.push(df_expr_to_l3(filter)?); + } + let pred = match conjuncts.len() { + 0 => None, + 1 => Some(conjuncts.pop().unwrap()), + _ => Some(L3Expr::BoolAnd(conjuncts)), + }; + Ok(L2::Join { + kind, + pred, + left: Box::new(self.lower_plan(&join.left)?), + right: Box::new(self.lower_plan(&join.right)?), + }) + } + fn lower_projection(&self, proj: &logical_expr::Projection) -> Result { // SELECT * — no column constraint; pass through without a Project. if proj.expr.iter().any(|e| matches!(e, Expr::Wildcard { .. })) { diff --git a/crates/lower/tests/sql_lowering.rs b/crates/lower/tests/sql_lowering.rs index 4b687f14..073665e2 100644 --- a/crates/lower/tests/sql_lowering.rs +++ b/crates/lower/tests/sql_lowering.rs @@ -5,7 +5,7 @@ //! canonical L3 (the same converter the PromQL path uses). use asap_control_core::intent_algebra::schema::{Column, DataType, Schema}; -use asap_control_core::intent_algebra::{AggIntent, QueryExpr, Source}; +use asap_control_core::intent_algebra::{AggIntent, JoinKind, QueryExpr, Source}; use asap_control_core::types::AccuracyTarget; use asap_control_lower::{lower_sql, SqlCatalog}; @@ -17,21 +17,29 @@ fn col(name: &str, dtype: DataType) -> Column { } } -/// `metrics(ts, service, latency, bytes)` — column positions 0..3. +/// `metrics(ts, service, latency, bytes)` + `hosts(service, region)`. fn catalog() -> SqlCatalog { - SqlCatalog::new().with_table( - "metrics", - Schema::with_time_index( - vec![ - col("ts", DataType::Timestamp), + SqlCatalog::new() + .with_table( + "metrics", + Schema::with_time_index( + vec![ + col("ts", DataType::Timestamp), + col("service", DataType::Utf8), + col("latency", DataType::Float64), + col("bytes", DataType::Int64), + ], + 0, + vec![], + ), + ) + .with_table( + "hosts", + Schema::new(vec![ col("service", DataType::Utf8), - col("latency", DataType::Float64), - col("bytes", DataType::Int64), - ], - 0, - vec![], - ), - ) + col("region", DataType::Utf8), + ]), + ) } async fn lower(sql: &str) -> QueryExpr { @@ -40,7 +48,7 @@ async fn lower(sql: &str) -> QueryExpr { .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")) } -/// Find the first `Aggregate` node anywhere in the tree. +/// Find the first `Aggregate` node along the single-child spine. fn find_aggregate(qe: &QueryExpr) -> Option<(&Vec, &Vec)> { match qe { QueryExpr::Aggregate { by, aggs, .. } => Some((by, aggs)), @@ -56,6 +64,23 @@ fn find_aggregate(qe: &QueryExpr) -> Option<(&Vec, &Vec)> { } } +/// Find the first `Join` node along the single-child spine. +fn find_join(qe: &QueryExpr) -> Option<&QueryExpr> { + match qe { + QueryExpr::Join { .. } => Some(qe), + QueryExpr::Project { child, .. } + | QueryExpr::Filter { child, .. } + | QueryExpr::Aggregate { child, .. } + | QueryExpr::Window { child, .. } + | QueryExpr::Partition { child, .. } + | QueryExpr::Distinct { child, .. } + | QueryExpr::Sort { child, .. } + | QueryExpr::Limit { child, .. } + | QueryExpr::Subquery { child, .. } => find_join(child), + _ => None, + } +} + #[tokio::test] async fn select_star_with_where_folds_predicate_onto_scan() { // SELECT * elides the projection; WHERE folds onto the Scan predicates. @@ -102,13 +127,62 @@ async fn count_distinct_is_cardinality() { } #[tokio::test] -async fn unsupported_join_is_rejected_not_mislowered() { - // The front end declines JOIN rather than silently dropping a side. +async fn inner_join_lowers_to_join_over_two_scans() { + // INNER JOIN over two distinct tables → L3 Join with both leaves as Scans. + let qe = lower( + "SELECT metrics.bytes, hosts.region \ + FROM metrics JOIN hosts ON metrics.service = hosts.service", + ) + .await; + let join = find_join(&qe).expect("expected a Join in the tree"); + let QueryExpr::Join { + kind, left, right, .. + } = join + else { + unreachable!("find_join only returns Join"); + }; + assert_eq!(*kind, JoinKind::Inner); + assert!(matches!(left.as_ref(), QueryExpr::Scan { .. })); + assert!(matches!(right.as_ref(), QueryExpr::Scan { .. })); +} + +#[tokio::test] +async fn aggregate_over_join_binds_against_concatenated_schema() { + // GROUP BY a right-table column over a join: the key must resolve against + // the concatenated schema, exercising the bottom-up converter end to end. + // Two aggregates → the multi-agg path, which carries GROUP BY keys as + // positional `Aggregate.by` (the single-agg path folds them into Partition). + let qe = lower( + "SELECT hosts.region, SUM(metrics.bytes), COUNT(*) \ + FROM metrics JOIN hosts ON metrics.service = hosts.service \ + GROUP BY hosts.region", + ) + .await; + let (by, aggs) = find_aggregate(&qe).expect("expected an Aggregate over the join"); + // metrics(ts,service,latency,bytes) ++ hosts(service,region) → + // region is column 5, bytes is column 3 of the concatenated schema. + assert_eq!( + by, + &vec![5], + "GROUP BY hosts.region → concatenated column 5" + ); + assert!( + aggs.contains(&AggIntent::Sum { col: Some(3) }), + "SUM(metrics.bytes) → Sum{{col:3}}, got {aggs:?}" + ); +} + +#[tokio::test] +async fn semi_join_is_rejected_not_mislowered() { + // No L3 counterpart for semi/anti joins yet → reject rather than mislower. let res = lower_sql( - "SELECT a.service FROM metrics a JOIN metrics b ON a.service = b.service", + "SELECT service FROM metrics WHERE service IN (SELECT service FROM hosts)", &catalog(), AccuracyTarget::Exact, ) .await; - assert!(res.is_err(), "JOIN should be rejected in v1"); + assert!( + res.is_err(), + "semi-join / IN-subquery should be rejected in v1" + ); } From d96e032d664512c54bcb6cc7e883e493ac678016 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 26 May 2026 11:29:33 -0600 Subject: [PATCH 24/40] feat(core+sql): thread aggregate output names so Project-over-Aggregate resolves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the output_names gap (PR #5 item 5). DataFusion names aggregate outputs in its plan schema (e.g. "sum(metrics.bytes)") and the enclosing Projection references them by those names; the L3 Aggregate named its outputs synthetically ("sum"), so the Project's column refs didn't resolve and fell back to Utf8. - query_expr: Aggregate gains output_names: Vec (parallel to aggs); a non-empty entry overrides the synthetic AggIntent::output_column name. Schema derivation (+ output_schema_for_aggregate helper) honors it. - lower: convert threads AggItem.alias → output_names in all three Aggregate constructions (fused / multi-agg / topk). - sql: lower_aggregate sets each AggItem.alias to DataFusion's aggregate output field name (schema.fields() past the group cols) — the names the Projection uses. - promql: aggregate alias is now empty (was "value") → keeps intent-keyed output names ("sum", "quantile_0_99", …); PromQL output naming unchanged. - cse: carry output_names through the shared-producer rewrite. Test: SELECT SUM(bytes), AVG(latency) → the root Projection's output schema now resolves to Int64 / Float64 (was the Utf8 fallback). 110 tests green, clippy -D warnings clean, fmt clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../src/intent_algebra/column_resolution.rs | 18 ++++++++++--- crates/core/src/intent_algebra/cse.rs | 5 ++++ crates/core/src/intent_algebra/lower.rs | 10 +++++--- crates/core/src/intent_algebra/query_expr.rs | 25 ++++++++++++++++--- crates/lower/src/promql.rs | 8 ++++-- crates/lower/src/sql/mod.rs | 23 +++++++++++++++-- crates/lower/tests/sql_lowering.rs | 23 +++++++++++++++++ 7 files changed, 97 insertions(+), 15 deletions(-) diff --git a/crates/core/src/intent_algebra/column_resolution.rs b/crates/core/src/intent_algebra/column_resolution.rs index 9d25272e..38306ed4 100644 --- a/crates/core/src/intent_algebra/column_resolution.rs +++ b/crates/core/src/intent_algebra/column_resolution.rs @@ -101,7 +101,12 @@ pub fn resolve_named_keys(keys: &[String], schema: &Schema) -> Result Schema { +pub fn output_schema_for_aggregate( + input: &Schema, + by: &[ColumnId], + aggs: &[AggIntent], + output_names: &[String], +) -> Schema { let mut out_cols: Vec = Vec::with_capacity(by.len() + aggs.len()); for &id in by { if let Some(c) = input.columns.get(id) { @@ -119,12 +124,16 @@ pub fn output_schema_for_aggregate(input: &Schema, by: &[ColumnId], aggs: &[AggI dtype: DataType::Float64, nullable: false, }); - for intent in aggs { + for (i, intent) in aggs.iter().enumerate() { let in_col = intent .input_col() .and_then(|id| input.columns.get(id)) .unwrap_or(&probe); - out_cols.push(intent.output_column(in_col)); + let mut out = intent.output_column(in_col); + if let Some(name) = output_names.get(i).filter(|s| !s.is_empty()) { + out.name = name.clone(); + } + out_cols.push(out); } let unique_keys = if by.is_empty() { Vec::new() @@ -171,7 +180,8 @@ mod tests { dtype: DataType::Utf8, nullable: false, }); - let out = output_schema_for_aggregate(&input, &[2usize], &[AggIntent::Sum { col: None }]); + let out = + output_schema_for_aggregate(&input, &[2usize], &[AggIntent::Sum { col: None }], &[]); assert_eq!(out.columns.len(), 2); // host, sum assert_eq!(out.columns[0].name, "host"); assert_eq!(out.columns[1].name, "sum"); diff --git a/crates/core/src/intent_algebra/cse.rs b/crates/core/src/intent_algebra/cse.rs index 54e17e55..1a04085d 100644 --- a/crates/core/src/intent_algebra/cse.rs +++ b/crates/core/src/intent_algebra/cse.rs @@ -89,11 +89,13 @@ pub fn dedupe_subtrees(roots: Vec<(QueryId, QueryExpr)>) -> CseWorkloadPlan { QueryExpr::Aggregate { by, aggs, + output_names, having, child, } if *child == shared_expr => QueryExpr::Aggregate { by, aggs, + output_names, having, child: Box::new(QueryExpr::Ref { name: binding_name.clone(), @@ -162,6 +164,7 @@ mod tests { q: 0.99, accuracy: AccuracyTarget::Epsilon(0.01), }], + output_names: vec![], having: None, child: Box::new(windowed_scan()), }; @@ -178,6 +181,7 @@ mod tests { q, accuracy: AccuracyTarget::Epsilon(0.01), }], + output_names: vec![], having: None, child: Box::new(windowed_scan()), }; @@ -219,6 +223,7 @@ mod tests { let mk = || QueryExpr::Aggregate { by: vec![], aggs: vec![AggIntent::Sum { col: None }], + output_names: vec![], having: None, child: Box::new(scan_no_uk.clone()), }; diff --git a/crates/core/src/intent_algebra/lower.rs b/crates/core/src/intent_algebra/lower.rs index 70e15901..71e77a95 100644 --- a/crates/core/src/intent_algebra/lower.rs +++ b/crates/core/src/intent_algebra/lower.rs @@ -110,6 +110,7 @@ pub fn convert( let aggregate = CQueryExpr::Aggregate { by: Vec::new(), aggs: vec![intent], + output_names: vec![aggs[0].alias.clone()], having: None, child: Box::new(agg_child), }; @@ -150,6 +151,7 @@ pub fn convert( CQueryExpr::Aggregate { by, aggs: intents, + output_names: aggs.iter().map(|item| item.alias.clone()).collect(), having: having.clone().map(Predicate), child: Box::new(child), } @@ -198,6 +200,7 @@ pub fn convert( k: *k as usize, accuracy: acc.clone(), }], + output_names: vec![], having: None, child: Box::new(child), } @@ -425,10 +428,11 @@ mod tests { ] ); - // Output schema types each reducer off its own input column. + // Output schema types each reducer off its own input column, and names + // it from the AggItem alias (threaded via Aggregate.output_names). let out = l3.output_schema().unwrap(); - assert_eq!(out.columns[0], col("sum", DataType::Int64)); // SUM(bytes:Int64) - assert_eq!(out.columns[1], col("avg", DataType::Float64)); // AVG(latency)→Float64 + assert_eq!(out.columns[0], col("total_bytes", DataType::Int64)); // SUM(bytes:Int64) + assert_eq!(out.columns[1], col("avg_latency", DataType::Float64)); // AVG(latency)→Float64 } /// PromQL's single sample-value reducer stays `col: None`. diff --git a/crates/core/src/intent_algebra/query_expr.rs b/crates/core/src/intent_algebra/query_expr.rs index 4368a418..1d7375b8 100644 --- a/crates/core/src/intent_algebra/query_expr.rs +++ b/crates/core/src/intent_algebra/query_expr.rs @@ -237,6 +237,14 @@ pub enum QueryExpr { Aggregate { by: Vec, aggs: Vec, + /// Output column names parallel to `aggs`. A non-empty entry overrides + /// the synthetic intent-keyed name — SQL threads DataFusion's generated + /// name (e.g. `"sum(metrics.bytes)"`) here so an enclosing `Project` + /// resolves the aggregate output by the name it references. An empty + /// entry (or empty vec) falls back to `AggIntent::output_column`'s name + /// (PromQL's convention). + #[serde(default)] + output_names: Vec, #[serde(default)] having: Option, child: Box, @@ -333,7 +341,11 @@ impl QueryExpr { QueryExpr::Window { child, .. } => child.output_schema_in(scope), QueryExpr::Aggregate { - by, aggs, child, .. + by, + aggs, + output_names, + child, + .. } => { let in_schema = child.output_schema_in(scope)?; let mut out_cols: Vec = Vec::with_capacity(by.len() + aggs.len()); @@ -361,13 +373,18 @@ impl QueryExpr { }); // Each reducer types off its own input column (`SUM(bytes)` vs // `AVG(latency)` in one node); `None` falls back to the sample- - // value probe (PromQL's single-column convention). - for intent in aggs { + // value probe (PromQL's single-column convention). A non-empty + // `output_names[i]` overrides the synthetic output column name. + for (i, intent) in aggs.iter().enumerate() { let in_col = intent .input_col() .and_then(|id| in_schema.columns.get(id)) .unwrap_or(&probe); - out_cols.push(intent.output_column(in_col)); + let mut out = intent.output_column(in_col); + if let Some(name) = output_names.get(i).filter(|s| !s.is_empty()) { + out.name = name.clone(); + } + out_cols.push(out); } let unique_keys = if by.is_empty() { Vec::new() diff --git a/crates/lower/src/promql.rs b/crates/lower/src/promql.rs index d3c2a5ac..b859a2bd 100644 --- a/crates/lower/src/promql.rs +++ b/crates/lower/src/promql.rs @@ -398,7 +398,9 @@ fn windowed_aggregate(inner: Inner, keys: Vec, func: AggFunc) -> L2 { L2::Aggregate { keys, aggs: vec![AggItem { - alias: "value".into(), + // Empty alias → the converter keeps PromQL's intent-keyed output + // names ("sum", "quantile_0_99", …) instead of overriding them. + alias: String::new(), func, col: ColumnRef::SampleValue, distinct: false, @@ -415,7 +417,9 @@ fn outer_aggregate(keys: Vec, func: AggFunc, input: L2) -> L2 { L2::Aggregate { keys, aggs: vec![AggItem { - alias: "value".into(), + // Empty alias → the converter keeps PromQL's intent-keyed output + // names ("sum", "quantile_0_99", …) instead of overriding them. + alias: String::new(), func, col: ColumnRef::SampleValue, distinct: false, diff --git a/crates/lower/src/sql/mod.rs b/crates/lower/src/sql/mod.rs index 66115cf7..e89c680a 100644 --- a/crates/lower/src/sql/mod.rs +++ b/crates/lower/src/sql/mod.rs @@ -209,11 +209,30 @@ impl<'a> SqlLowerer<'a> { .iter() .map(expr_to_group_name) .collect::, _>>()?; + // DataFusion names the aggregate outputs in its own schema (e.g. + // "sum(metrics.bytes)") — the same names the enclosing Projection + // references. The schema is [group fields …, aggregate fields …], so + // skip the group fields and thread the rest as L2 aliases → L3 + // `Aggregate.output_names`, letting that Projection resolve them. + let out_names: Vec = agg + .schema + .fields() + .iter() + .skip(agg.group_expr.len()) + .map(|f| f.name().to_string()) + .collect(); let aggs = agg .aggr_expr .iter() - .map(lower_agg_item) - .collect::, _>>()?; + .enumerate() + .map(|(i, e)| { + let mut item = lower_agg_item(e)?; + if let Some(name) = out_names.get(i) { + item.alias = name.clone(); + } + Ok(item) + }) + .collect::, LoweringError>>()?; Ok(L2::Aggregate { keys, aggs, diff --git a/crates/lower/tests/sql_lowering.rs b/crates/lower/tests/sql_lowering.rs index 073665e2..1d85006e 100644 --- a/crates/lower/tests/sql_lowering.rs +++ b/crates/lower/tests/sql_lowering.rs @@ -111,6 +111,29 @@ async fn multi_aggregate_group_by_binds_columns_positionally() { ); } +#[tokio::test] +async fn projection_over_aggregate_resolves_output_types_via_output_names() { + // The enclosing Projection references the aggregates by DataFusion's + // generated names (e.g. "sum(metrics.bytes)"); output_names threads those + // onto the L3 Aggregate so the Project resolves real types — not the Utf8 + // fallback that an unresolved column would get. + let qe = lower("SELECT SUM(bytes), AVG(latency) FROM metrics").await; + let schema = qe + .output_schema() + .expect("root projection schema derivation"); + assert_eq!(schema.columns.len(), 2); + assert_eq!( + schema.columns[0].dtype, + DataType::Int64, + "SUM(bytes:Int64) resolves to Int64, not the Utf8 fallback" + ); + assert_eq!( + schema.columns[1].dtype, + DataType::Float64, + "AVG(latency) resolves to Float64" + ); +} + #[tokio::test] async fn count_star_is_count_intent() { let qe = lower("SELECT COUNT(*) FROM metrics").await; From f9458d63f6eaf900484bef19d6097694dab9bcc3 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 26 May 2026 11:35:44 -0600 Subject: [PATCH 25/40] feat(core): route tabular GROUP BY through positional Aggregate.by (key in schema) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The single-aggregate fused path wraps GROUP BY keys in a name-based Partition (the PromQL streaming-sketch canonical shape) whose output schema is the child's — so the group key is not an output column. That's correct for time series but wrong for SQL, where `SELECT k, agg(...) GROUP BY k` projects `k`. The fused path is now gated on a *time-series* leaf (`relational::leaf_is_tabular` = the leftmost Source carries a resolved schema, i.e. Source::Table). Tabular single-agg GROUP BY falls through to the positional `Aggregate.by` path, so the key lands in the output schema and the enclosing SELECT projection resolves it. PromQL's fused-Partition shape is unchanged. - relational: factor out `leaf_source`; add `leaf_is_tabular`; `source_name` now delegates to `leaf_source`. Test: `SELECT service, SUM(bytes) GROUP BY service` → Aggregate.by=[1], Sum{col:3}, and the root projection schema is [service:Utf8, sum:Int64]. 111 tests green, clippy -D warnings clean, fmt clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/core/src/intent_algebra/lower.rs | 14 ++++++---- crates/core/src/intent_algebra/relational.rs | 28 +++++++++++++++----- crates/lower/tests/sql_lowering.rs | 21 +++++++++++++++ 3 files changed, 51 insertions(+), 12 deletions(-) diff --git a/crates/core/src/intent_algebra/lower.rs b/crates/core/src/intent_algebra/lower.rs index 71e77a95..7f2c91d5 100644 --- a/crates/core/src/intent_algebra/lower.rs +++ b/crates/core/src/intent_algebra/lower.rs @@ -87,11 +87,15 @@ pub fn convert( having, input, } => { - // Single-statistic aggregate (no HAVING) fuses: a `Window` input - // becomes `Window { Aggregate { by: [] } }`; GROUP BY keys wrap the - // result in a `Partition`. The reducer's input column resolves - // against the aggregate's *direct* input (the scan under any window). - if aggs.len() == 1 && having.is_none() { + // Single-statistic aggregate (no HAVING) over a *time-series* leaf + // fuses: a `Window` input becomes `Window { Aggregate { by: [] } }`; + // GROUP BY keys wrap the result in a `Partition` (the streaming + // sketch canonical shape). Tabular (SQL) GROUP BY instead falls + // through to the positional `Aggregate.by` path below, so the group + // keys land in the output schema (a SELECT projects them). The + // reducer's input column resolves against the aggregate's *direct* + // input (the scan under any window). + if aggs.len() == 1 && having.is_none() && !input.leaf_is_tabular() { let (agg_input_l2, window): (&LQueryExpr, Option<(_, _)>) = match input.as_ref() { LQueryExpr::Window { duration, diff --git a/crates/core/src/intent_algebra/relational.rs b/crates/core/src/intent_algebra/relational.rs index 01d4d4c9..3b8bf6c0 100644 --- a/crates/core/src/intent_algebra/relational.rs +++ b/crates/core/src/intent_algebra/relational.rs @@ -223,10 +223,10 @@ impl QueryExpr { } } - /// Outermost metric/table name from the first `Source` leaf. - pub fn source_name(&self) -> Option<&str> { + /// The leftmost `Source` leaf. + pub fn leaf_source(&self) -> Option<&SourceSpec> { match self { - QueryExpr::Source(s) => Some(&s.name), + QueryExpr::Source(s) => Some(s), QueryExpr::Filter { input, .. } | QueryExpr::Project { input, .. } | QueryExpr::Aggregate { input, .. } @@ -236,13 +236,27 @@ impl QueryExpr { | QueryExpr::TopK { input, .. } | QueryExpr::Sort { input, .. } | QueryExpr::Limit { input, .. } - | QueryExpr::PromQLSubquery { input, .. } => input.source_name(), - QueryExpr::Merge { inputs } => inputs.first()?.source_name(), + | QueryExpr::PromQLSubquery { input, .. } => input.leaf_source(), + QueryExpr::Merge { inputs } => inputs.first()?.leaf_source(), QueryExpr::Join { left, .. } | QueryExpr::SetOp { left, .. } - | QueryExpr::BinaryOp { lhs: left, .. } => left.source_name(), - QueryExpr::LetBinding { body, .. } => body.source_name(), + | QueryExpr::BinaryOp { lhs: left, .. } => left.leaf_source(), + QueryExpr::LetBinding { body, .. } => body.leaf_source(), QueryExpr::Ref(_) => None, } } + + /// Outermost metric/table name from the first `Source` leaf. + pub fn source_name(&self) -> Option<&str> { + self.leaf_source().map(|s| s.name.as_str()) + } + + /// Whether the leftmost `Source` leaf carries a resolved schema — i.e. it is + /// a SQL table (`Source::Table`). Time-series (PromQL) leaves return + /// `false`. The converter uses this to keep the time-series fused-Partition + /// canonical shape for PromQL while routing tabular GROUP BY through a + /// positional `Aggregate.by` (so group keys land in the output schema). + pub fn leaf_is_tabular(&self) -> bool { + self.leaf_source().is_some_and(|s| s.schema.is_some()) + } } diff --git a/crates/lower/tests/sql_lowering.rs b/crates/lower/tests/sql_lowering.rs index 1d85006e..658f7ca4 100644 --- a/crates/lower/tests/sql_lowering.rs +++ b/crates/lower/tests/sql_lowering.rs @@ -134,6 +134,27 @@ async fn projection_over_aggregate_resolves_output_types_via_output_names() { ); } +#[tokio::test] +async fn single_agg_group_by_keeps_key_in_output_schema() { + // A tabular single-aggregate GROUP BY routes through the positional + // Aggregate.by path (not the PromQL fused-Partition shape), so the group + // key is a real output column the enclosing SELECT projection resolves. + let qe = lower("SELECT service, SUM(bytes) FROM metrics GROUP BY service").await; + let (by, aggs) = find_aggregate(&qe).expect("expected an Aggregate (not a Partition)"); + assert_eq!(by, &vec![1], "GROUP BY service → Aggregate.by column 1"); + assert!(matches!(aggs.as_slice(), [AggIntent::Sum { col: Some(3) }])); + + // Both the group key and the aggregate resolve in the root projection schema. + let schema = qe.output_schema().expect("root projection schema"); + assert_eq!(schema.columns.len(), 2); + assert_eq!( + schema.columns[0].dtype, + DataType::Utf8, + "service is in the output" + ); + assert_eq!(schema.columns[1].dtype, DataType::Int64, "SUM(bytes)"); +} + #[tokio::test] async fn count_star_is_count_intent() { let qe = lower("SELECT COUNT(*) FROM metrics").await; From e97eaaf30ee9bcb2e91ec1301e413751fd2a7226 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 26 May 2026 12:29:54 -0600 Subject: [PATCH 26/40] refactor(core): split scalar IR into L2Expr (names) + L3Expr (positional) [3a] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 3a of going fully positional (option B): the canonical L3 scalar expression is now positional, so column identity in filters/projections/sort keys/join predicates is unambiguous (no name lookup downstream). - expr_ir: two expression types — L2Expr (Column(ColumnRef), front-end-emitted) and L3Expr (Column(ColumnId), canonical). Shared L3Scalar/CompareOp/ArithOp. - relational L2: Filter/Aggregate.having/Join.pred use L2Expr; new L2ProjectItem / L2SortKey for Project/Sort. - query_expr L3: Predicate/ProjectItem/SortKey carry positional L3Expr; infer_expr_type + default_proj_name work on Column(ColumnId). - converter: new column_resolution::resolve_expr maps L2Expr→L3Expr against the in-scope schema; applied to scan predicates, filter, project, sort, having, and the join predicate (against the concatenated left++right schema). Binder now also seeds filter/project/sort/having/join column names into the usage-derived PromQL leaf so they resolve. - resolve_column_ref: SampleValue falls back to the sole non-timestamp column when "value" is absent (an aggregate renames it, e.g. topk over avg). - front ends emit L2Expr (promql matchers; sql df_expr_to_l2). Self-join duplicate-name disambiguation (qualifiers) is the 3b follow-up; for now such names resolve to the first match. 111 tests green, clippy/fmt clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/core/src/intent_algebra/binder.rs | 36 +++- .../src/intent_algebra/column_resolution.rs | 81 ++++++++- crates/core/src/intent_algebra/expr_ir.rs | 171 +++++++++++++++--- crates/core/src/intent_algebra/lower.rs | 127 +++++++++---- crates/core/src/intent_algebra/mod.rs | 4 +- crates/core/src/intent_algebra/query_expr.rs | 71 ++++---- crates/core/src/intent_algebra/relational.rs | 32 +++- crates/lower/src/promql.rs | 28 +-- crates/lower/src/sql/expr.rs | 82 ++++----- crates/lower/src/sql/mod.rs | 26 ++- crates/lower/tests/promql_lowering.rs | 13 +- 11 files changed, 479 insertions(+), 192 deletions(-) diff --git a/crates/core/src/intent_algebra/binder.rs b/crates/core/src/intent_algebra/binder.rs index 9f594db2..5d066235 100644 --- a/crates/core/src/intent_algebra/binder.rs +++ b/crates/core/src/intent_algebra/binder.rs @@ -11,6 +11,8 @@ //! `SchemaCatalog` is future work; the `Binder` pass does not change when it //! lands, only the catalog impl swaps. +use crate::intent_algebra::expr_ir::L2Expr; +use crate::intent_algebra::query_expr::ColumnRef; use crate::intent_algebra::relational::QueryExpr as LQueryExpr; use crate::intent_algebra::schema::{Column, DataType, Schema}; @@ -110,14 +112,42 @@ fn default_leaf_columns() -> Vec { ] } -/// Collect every distinct group-key name the converter resolves positionally: -/// `Aggregate.keys`, `TopK.by`, and `Partition.keys`. +/// Collect every distinct column name the converter resolves positionally: +/// group keys (`Aggregate.keys`, `TopK.by`, `Partition.keys`) **and** the +/// columns referenced by name in filter / having / project / sort / join +/// expressions (e.g. a PromQL label matcher `m{env="prod"}` references `env`). +/// The Binder seeds these into the usage-derived leaf so positional resolution +/// downstream is total. fn collect_referenced_columns(tree: &LQueryExpr) -> Vec { + fn named(expr: &L2Expr, out: &mut Vec) { + for c in expr.columns_referenced() { + if let ColumnRef::Named(n) = c { + out.push(n.clone()); + } + } + } let mut out: Vec = Vec::new(); tree.walk(&mut |node| match node { - LQueryExpr::Aggregate { keys, .. } => out.extend(keys.iter().cloned()), + LQueryExpr::Aggregate { keys, having, .. } => { + out.extend(keys.iter().cloned()); + if let Some(h) = having { + named(h, &mut out); + } + } LQueryExpr::TopK { by, .. } => out.extend(by.iter().cloned()), LQueryExpr::Partition { keys, .. } => out.extend(keys.keys().iter().cloned()), + LQueryExpr::Filter { pred, .. } => named(pred, &mut out), + LQueryExpr::Project { cols, .. } => { + for item in cols { + named(&item.expr, &mut out); + } + } + LQueryExpr::Sort { keys, .. } => { + for k in keys { + named(&k.expr, &mut out); + } + } + LQueryExpr::Join { pred: Some(p), .. } => named(p, &mut out), _ => {} }); out.sort(); diff --git a/crates/core/src/intent_algebra/column_resolution.rs b/crates/core/src/intent_algebra/column_resolution.rs index 38306ed4..1bde6e06 100644 --- a/crates/core/src/intent_algebra/column_resolution.rs +++ b/crates/core/src/intent_algebra/column_resolution.rs @@ -9,6 +9,7 @@ use thiserror::Error; use crate::intent_algebra::agg_intent::AggIntent; +use crate::intent_algebra::expr_ir::{L2Expr, L3Expr}; use crate::intent_algebra::query_expr::ColumnRef; use crate::intent_algebra::relational::QueryExpr; use crate::intent_algebra::schema::{Column, ColumnId, DataType, Schema}; @@ -64,13 +65,19 @@ pub fn resolve_column_ref(col: &ColumnRef, schema: &Schema) -> Result { - schema - .column_id("value") - .ok_or_else(|| ResolveError::NoSampleValue { - available: schema.columns.iter().map(|c| c.name.clone()).collect(), - }) - } + ColumnRef::SampleValue => schema + .column_id("value") + .or_else(|| { + // After an aggregate the sample value is renamed (e.g. "avg"); + // fall back to the sole non-timestamp column when unambiguous. + let non_ts: Vec = (0..schema.columns.len()) + .filter(|&i| Some(i) != schema.time_index) + .collect(); + (non_ts.len() == 1).then(|| non_ts[0]) + }) + .ok_or_else(|| ResolveError::NoSampleValue { + available: schema.columns.iter().map(|c| c.name.clone()).collect(), + }), ColumnRef::Wildcard => Err(ResolveError::WildcardNotPositional), } } @@ -83,6 +90,66 @@ pub fn resolve_column_refs( cols.iter().map(|c| resolve_column_ref(c, schema)).collect() } +/// Resolve a Layer-2 [`L2Expr`] (name-based) into a positional [`L3Expr`] by +/// resolving every column reference against `schema`. Structural otherwise. +pub fn resolve_expr(expr: &L2Expr, schema: &Schema) -> Result { + let boxed = |e: &L2Expr| -> Result, ResolveError> { + Ok(Box::new(resolve_expr(e, schema)?)) + }; + let each = |es: &[L2Expr]| -> Result, ResolveError> { + es.iter().map(|e| resolve_expr(e, schema)).collect() + }; + Ok(match expr { + L2Expr::Column(c) => L3Expr::Column(resolve_column_ref(c, schema)?), + L2Expr::Literal(s) => L3Expr::Literal(s.clone()), + L2Expr::Compare { left, op, right } => L3Expr::Compare { + left: boxed(left)?, + op: op.clone(), + right: boxed(right)?, + }, + L2Expr::BoolAnd(v) => L3Expr::BoolAnd(each(v)?), + L2Expr::BoolOr(v) => L3Expr::BoolOr(each(v)?), + L2Expr::Not(e) => L3Expr::Not(boxed(e)?), + L2Expr::IsNull(e) => L3Expr::IsNull(boxed(e)?), + L2Expr::IsNotNull(e) => L3Expr::IsNotNull(boxed(e)?), + L2Expr::Cast { expr, to, try_cast } => L3Expr::Cast { + expr: boxed(expr)?, + to: to.clone(), + try_cast: *try_cast, + }, + L2Expr::InList { + expr, + list, + negated, + } => L3Expr::InList { + expr: boxed(expr)?, + list: each(list)?, + negated: *negated, + }, + L2Expr::FunctionCall { name, args } => L3Expr::FunctionCall { + name: name.clone(), + args: each(args)?, + }, + L2Expr::Arith { op, left, right } => L3Expr::Arith { + op: op.clone(), + left: boxed(left)?, + right: boxed(right)?, + }, + L2Expr::Case { + operand, + branches, + else_expr, + } => L3Expr::Case { + operand: operand.as_deref().map(&boxed).transpose()?, + branches: branches + .iter() + .map(|(w, t)| Ok((resolve_expr(w, schema)?, resolve_expr(t, schema)?))) + .collect::, ResolveError>>()?, + else_expr: else_expr.as_deref().map(&boxed).transpose()?, + }, + }) +} + /// Resolve a list of named GROUP BY keys (`Aggregate.keys`) to `ColumnId`s. pub fn resolve_named_keys(keys: &[String], schema: &Schema) -> Result, ResolveError> { keys.iter() diff --git a/crates/core/src/intent_algebra/expr_ir.rs b/crates/core/src/intent_algebra/expr_ir.rs index f0180930..6959971b 100644 --- a/crates/core/src/intent_algebra/expr_ir.rs +++ b/crates/core/src/intent_algebra/expr_ir.rs @@ -1,19 +1,25 @@ //! Language-independent scalar expression IR. //! -//! Used for filter predicates (PromQL label matchers, SQL `WHERE` conjuncts) -//! and projection / sort-key expressions. Carried by both the Layer-2 -//! `relational` IR and the canonical L3 `query_expr` IR so the predicate -//! representation is identical across the lowering boundary. +//! There are **two** scalar expression types, mirroring the lowering boundary: //! -//! The variant set is the **union** of what the two front ends need: PromQL -//! contributes `Regex` / `NotRegex` (`=~` / `!~`); SQL contributes arithmetic, -//! `CASE`, `IN`, `CAST`, `IS [NOT] NULL`, scalar function calls, and the -//! `LIKE` / `ILIKE` comparison family. +//! - [`L2Expr`] — name-based ([`Column(ColumnRef)`](ColumnRef)). The per-language +//! front ends emit it (PromQL label matchers, SQL `WHERE` / projection / +//! sort-key expressions) on the Layer-2 `relational` tree. +//! - [`L3Expr`] — **positional** ([`Column(ColumnId)`](ColumnId)). The canonical +//! L3 `query_expr` tree carries it; the converter resolves every `L2Expr` +//! column reference against the in-scope schema to produce it, so L3 column +//! identity is unambiguous (no name collisions across a join). +//! +//! Both share the same shape and the scalar/operator vocabulary +//! ([`L3Scalar`], [`CompareOp`], [`ArithOp`]) — the **union** of what the two +//! front ends need: PromQL contributes `Regex` / `NotRegex` (`=~` / `!~`); SQL +//! contributes arithmetic, `CASE`, `IN`, `CAST`, `IS [NOT] NULL`, scalar +//! function calls, and the `LIKE` / `ILIKE` comparison family. use serde::{Deserialize, Serialize}; use super::query_expr::ColumnRef; -use super::schema::DataType; +use super::schema::{ColumnId, DataType}; /// A typed scalar constant. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -25,7 +31,7 @@ pub enum L3Scalar { Null, } -/// Binary comparison operators for `L3Expr::Compare`. +/// Binary comparison operators. /// /// `Regex` / `NotRegex` carry PromQL/RE2 regex-match semantics (`=~` / `!~`): /// the right-hand side is a regular-expression pattern, not a literal value. @@ -52,7 +58,7 @@ pub enum CompareOp { NotRegex, } -/// Binary arithmetic operators for `L3Expr::Arith`. +/// Binary arithmetic operators. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum ArithOp { Add, @@ -62,16 +68,136 @@ pub enum ArithOp { Mod, } -/// A scalar expression. Flat conjunctions (`BoolAnd`) / disjunctions -/// (`BoolOr`) make per-conjunct selectivity estimation and label-matcher -/// lowering straightforward without recursive descent. +/// Layer-2 (name-based) scalar expression. Front ends emit this; the converter +/// resolves it into a positional [`L3Expr`]. Flat conjunctions (`BoolAnd`) / +/// disjunctions (`BoolOr`) make per-conjunct selectivity estimation and +/// label-matcher lowering straightforward without recursive descent. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub enum L3Expr { +pub enum L2Expr { /// Reference to a named column / label. Column(ColumnRef), /// A constant literal value. Literal(L3Scalar), /// `left op right` — binary comparison. + Compare { + left: Box, + op: CompareOp, + right: Box, + }, + /// Flat conjunction (logical AND). An empty list is vacuously true. + BoolAnd(Vec), + /// Flat disjunction (logical OR). An empty list is vacuously false. + BoolOr(Vec), + /// Logical NOT. + Not(Box), + /// `expr IS NULL`. + IsNull(Box), + /// `expr IS NOT NULL`. + IsNotNull(Box), + /// `CAST(expr AS to)`; `try_cast` for SQL `TRY_CAST` (NULL on failure). + Cast { + expr: Box, + to: DataType, + try_cast: bool, + }, + /// `expr [NOT] IN (v1, v2, …)`. + InList { + expr: Box, + list: Vec, + negated: bool, + }, + /// Scalar function call, e.g. `LOWER(col)`, `ABS(x)`. + FunctionCall { name: String, args: Vec }, + /// Binary arithmetic: `left op right`. + Arith { + op: ArithOp, + left: Box, + right: Box, + }, + /// SQL `CASE` (both searched and simple forms). `operand` present for the + /// simple form (`CASE expr WHEN …`), absent for searched. + Case { + operand: Option>, + branches: Vec<(L2Expr, L2Expr)>, + else_expr: Option>, + }, +} + +impl L2Expr { + /// If this expression is a `BoolAnd`, return its elements; otherwise a + /// single-element slice containing `self`. + pub fn conjuncts(&self) -> &[L2Expr] { + match self { + L2Expr::BoolAnd(v) => v.as_slice(), + _ => std::slice::from_ref(self), + } + } + + /// If this expression is a `BoolOr`, return its elements; otherwise a + /// single-element slice containing `self`. + pub fn disjuncts(&self) -> &[L2Expr] { + match self { + L2Expr::BoolOr(v) => v.as_slice(), + _ => std::slice::from_ref(self), + } + } + + /// Recursively collect every `ColumnRef` referenced anywhere in this + /// expression. Used by the Binder to seed usage-derived leaf schemas. + pub fn columns_referenced(&self) -> Vec<&ColumnRef> { + match self { + L2Expr::Column(c) => vec![c], + L2Expr::Literal(_) => vec![], + L2Expr::Compare { left, right, .. } | L2Expr::Arith { left, right, .. } => { + let mut v = left.columns_referenced(); + v.extend(right.columns_referenced()); + v + } + L2Expr::BoolAnd(parts) | L2Expr::BoolOr(parts) => { + parts.iter().flat_map(|e| e.columns_referenced()).collect() + } + L2Expr::Not(e) | L2Expr::IsNull(e) | L2Expr::IsNotNull(e) => e.columns_referenced(), + L2Expr::Cast { expr, .. } => expr.columns_referenced(), + L2Expr::InList { expr, list, .. } => { + let mut v = expr.columns_referenced(); + v.extend(list.iter().flat_map(|e| e.columns_referenced())); + v + } + L2Expr::FunctionCall { args, .. } => { + args.iter().flat_map(|e| e.columns_referenced()).collect() + } + L2Expr::Case { + operand, + branches, + else_expr, + } => { + let mut v = vec![]; + if let Some(op) = operand { + v.extend(op.columns_referenced()); + } + for (when, then) in branches { + v.extend(when.columns_referenced()); + v.extend(then.columns_referenced()); + } + if let Some(e) = else_expr { + v.extend(e.columns_referenced()); + } + v + } + } + } +} + +/// Canonical L3 (positional) scalar expression. Same shape as [`L2Expr`] but +/// column references are positional [`ColumnId`]s resolved against the in-scope +/// schema, so identity is unambiguous across joins / duplicate names. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum L3Expr { + /// Positional reference to a column in the in-scope schema. + Column(ColumnId), + /// A constant literal value. + Literal(L3Scalar), + /// `left op right` — binary comparison. Compare { left: Box, op: CompareOp, @@ -87,8 +213,7 @@ pub enum L3Expr { IsNull(Box), /// `expr IS NOT NULL`. IsNotNull(Box), - /// `CAST(expr AS to)`. `try_cast` is `true` for SQL `TRY_CAST`, which - /// returns `NULL` on conversion failure instead of raising an error. + /// `CAST(expr AS to)`; `try_cast` for SQL `TRY_CAST` (NULL on failure). Cast { expr: Box, to: DataType, @@ -108,9 +233,7 @@ pub enum L3Expr { left: Box, right: Box, }, - /// SQL `CASE` expression (both searched and simple forms). - /// `operand` is present for simple CASE (`CASE expr WHEN ...`), - /// absent for searched CASE (`CASE WHEN condition THEN ...`). + /// SQL `CASE` (both searched and simple forms). Case { operand: Option>, branches: Vec<(L3Expr, L3Expr)>, @@ -137,11 +260,11 @@ impl L3Expr { } } - /// Recursively collect every `ColumnRef` referenced anywhere in this - /// expression. Used by L4 for column-lineage and selectivity estimation. - pub fn columns_referenced(&self) -> Vec<&ColumnRef> { + /// Recursively collect every positional [`ColumnId`] referenced anywhere in + /// this expression. Used by L4 for column-lineage and selectivity. + pub fn columns_referenced(&self) -> Vec { match self { - L3Expr::Column(c) => vec![c], + L3Expr::Column(id) => vec![*id], L3Expr::Literal(_) => vec![], L3Expr::Compare { left, right, .. } | L3Expr::Arith { left, right, .. } => { let mut v = left.columns_referenced(); diff --git a/crates/core/src/intent_algebra/lower.rs b/crates/core/src/intent_algebra/lower.rs index 7f2c91d5..2ddfe22d 100644 --- a/crates/core/src/intent_algebra/lower.rs +++ b/crates/core/src/intent_algebra/lower.rs @@ -14,11 +14,12 @@ use thiserror::Error; use crate::intent_algebra::agg_intent::AggIntent; use crate::intent_algebra::binder::Binder; -use crate::intent_algebra::column_resolution::{resolve_named_keys, ResolveError}; +use crate::intent_algebra::column_resolution::{resolve_expr, resolve_named_keys, ResolveError}; +use crate::intent_algebra::expr_ir::{L2Expr, L3Expr, L3Scalar}; use crate::intent_algebra::names::BindingName; use crate::intent_algebra::query_expr::{ - ColumnRef, PartitionKeys as CPartitionKeys, Predicate, QueryExpr as CQueryExpr, Source, - WindowKind, + ColumnRef, PartitionKeys as CPartitionKeys, Predicate, ProjectItem, QueryExpr as CQueryExpr, + SortKey, Source, WindowKind, }; use crate::intent_algebra::relational::{AggFunc, QueryExpr as LQueryExpr, SourceSpec}; use crate::intent_algebra::schema::{ColumnId, Schema}; @@ -62,7 +63,7 @@ pub fn convert( acc: &AccuracyTarget, ) -> Result { Ok(match legacy { - LQueryExpr::Source(spec) => scan(spec, fallback, Vec::new()), + LQueryExpr::Source(spec) => scan(spec, fallback, &[])?, LQueryExpr::Ref(name) => CQueryExpr::Ref { name: BindingName::new(name.clone()), @@ -70,15 +71,17 @@ pub fn convert( // Fold label matchers / pushed-down predicates directly onto the Scan // when the immediate child is a `Source`; otherwise emit a `Filter`. + // Predicate column refs resolve positionally against the input schema. LQueryExpr::Filter { pred, input } => match input.as_ref() { - LQueryExpr::Source(spec) => { - let predicates = pred.conjuncts().iter().cloned().map(Predicate).collect(); - scan(spec, fallback, predicates) + LQueryExpr::Source(spec) => scan(spec, fallback, pred.conjuncts())?, + other => { + let child = convert(other, fallback, acc)?; + let child_schema = child.output_schema()?; + CQueryExpr::Filter { + pred: Predicate(resolve_expr(pred, &child_schema)?), + child: Box::new(child), + } } - other => CQueryExpr::Filter { - pred: Predicate(pred.clone()), - child: Box::new(convert(other, fallback, acc)?), - }, }, LQueryExpr::Aggregate { @@ -152,11 +155,17 @@ pub fn convert( agg_func_to_intent(&item.func, acc, resolve_agg_col(&item.col, &child_schema)) }) .collect(); + let having = having + .as_ref() + .map(|h| -> Result { + Ok(Predicate(resolve_expr(h, &child_schema)?)) + }) + .transpose()?; CQueryExpr::Aggregate { by, aggs: intents, output_names: aggs.iter().map(|item| item.alias.clone()).collect(), - having: having.clone().map(Predicate), + having, child: Box::new(child), } } @@ -176,13 +185,25 @@ pub fn convert( child: Box::new(convert(input, fallback, acc)?), }, - // π — column refs in the project items resolve by name against the - // child schema during L3 schema derivation, so the conversion is a - // structural pass-through of the (shared) `ProjectItem` list. - LQueryExpr::Project { cols, input } => CQueryExpr::Project { - cols: cols.clone(), - child: Box::new(convert(input, fallback, acc)?), - }, + // π — resolve each project item's expression to positional against the + // child's schema. + LQueryExpr::Project { cols, input } => { + let child = convert(input, fallback, acc)?; + let child_schema = child.output_schema()?; + let cols = cols + .iter() + .map(|item| -> Result { + Ok(ProjectItem { + alias: item.alias.clone(), + expr: resolve_expr(&item.expr, &child_schema)?, + }) + }) + .collect::, _>>()?; + CQueryExpr::Project { + cols, + child: Box::new(child), + } + } LQueryExpr::Partition { keys, input } => CQueryExpr::Partition { keys: keys.clone(), @@ -222,17 +243,26 @@ pub fn convert( pred, left, right, - } => CQueryExpr::Join { - kind: kind.clone(), - pred: Predicate(pred.clone().unwrap_or( - crate::intent_algebra::expr_ir::L3Expr::Literal( - crate::intent_algebra::expr_ir::L3Scalar::Boolean(true), - ), - )), - // Each branch is bound independently — see the `BinaryOp` arm. - left: Box::new(convert_root(left, acc)?), - right: Box::new(convert_root(right, acc)?), - }, + } => { + // Each branch is bound independently (different leaves / label sets). + let left = convert_root(left, acc)?; + let right = convert_root(right, acc)?; + // The join predicate resolves against the concatenated left++right + // schema (the Join's own output shape), so left refs land at + // 0..left_len and right refs at left_len.. . + let mut concat = left.output_schema()?; + concat.columns.extend(right.output_schema()?.columns); + let pred = match pred { + Some(p) => Predicate(resolve_expr(p, &concat)?), + None => Predicate(L3Expr::Literal(L3Scalar::Boolean(true))), + }; + CQueryExpr::Join { + kind: kind.clone(), + pred, + left: Box::new(left), + right: Box::new(right), + } + } LQueryExpr::SetOp { kind, @@ -246,10 +276,24 @@ pub fn convert( right: Box::new(convert_root(right, acc)?), }, - LQueryExpr::Sort { keys, input } => CQueryExpr::Sort { - keys: keys.clone(), - child: Box::new(convert(input, fallback, acc)?), - }, + LQueryExpr::Sort { keys, input } => { + let child = convert(input, fallback, acc)?; + let child_schema = child.output_schema()?; + let keys = keys + .iter() + .map(|k| -> Result { + Ok(SortKey { + expr: resolve_expr(&k.expr, &child_schema)?, + ascending: k.ascending, + nulls_first: k.nulls_first, + }) + }) + .collect::, _>>()?; + CQueryExpr::Sort { + keys, + child: Box::new(child), + } + } LQueryExpr::Limit { n, offset, input } => CQueryExpr::Limit { n: *n as usize, @@ -295,7 +339,12 @@ pub fn convert( /// Build a canonical `Scan`. A schema-bearing [`SourceSpec`] (SQL table) emits /// a `Source::Table` carrying that resolved schema; a schema-less one (PromQL) /// emits a `Source::TimeSeries` carrying the Binder's usage-derived `fallback`. -fn scan(spec: &SourceSpec, fallback: &Schema, predicates: Vec) -> CQueryExpr { +/// The L2 predicate conjuncts are resolved positionally against the leaf schema. +fn scan( + spec: &SourceSpec, + fallback: &Schema, + pred_conjuncts: &[L2Expr], +) -> Result { let (source, schema) = match &spec.schema { Some(s) => ( Source::Table { @@ -310,11 +359,15 @@ fn scan(spec: &SourceSpec, fallback: &Schema, predicates: Vec) -> CQu fallback.clone(), ), }; - CQueryExpr::Scan { + let predicates = pred_conjuncts + .iter() + .map(|e| -> Result { Ok(Predicate(resolve_expr(e, &schema)?)) }) + .collect::, _>>()?; + Ok(CQueryExpr::Scan { source, predicates, schema, - } + }) } /// Resolve a Layer-2 aggregate-input [`ColumnRef`] to a positional input diff --git a/crates/core/src/intent_algebra/mod.rs b/crates/core/src/intent_algebra/mod.rs index 169cd0c3..96309d9d 100644 --- a/crates/core/src/intent_algebra/mod.rs +++ b/crates/core/src/intent_algebra/mod.rs @@ -27,10 +27,10 @@ pub use agg_intent::{ pub use binder::{Binder, SchemaCatalog, UsageDerivedCatalog}; pub use column_resolution::{ infer_schema_for_root, infer_source_schema, output_schema_for_aggregate, resolve_column_ref, - resolve_column_refs, resolve_named_keys, ResolveError, + resolve_column_refs, resolve_expr, resolve_named_keys, ResolveError, }; pub use cse::{dedupe_subtrees, CseWorkloadPlan}; -pub use expr_ir::{ArithOp, CompareOp, L3Expr, L3Scalar}; +pub use expr_ir::{ArithOp, CompareOp, L2Expr, L3Expr, L3Scalar}; pub use lower::{convert, convert_root, ConvertError}; pub use names::{BindingName, QueryId}; pub use query_expr::{ diff --git a/crates/core/src/intent_algebra/query_expr.rs b/crates/core/src/intent_algebra/query_expr.rs index 1d7375b8..c27e53b1 100644 --- a/crates/core/src/intent_algebra/query_expr.rs +++ b/crates/core/src/intent_algebra/query_expr.rs @@ -421,21 +421,20 @@ impl QueryExpr { // is re-found by name. QueryExpr::Project { cols, child } => { let in_schema = child.output_schema_in(scope)?; - let columns: Vec = cols - .iter() - .enumerate() - .map(|(i, item)| { - let (dtype, nullable) = infer_expr_type(&item.expr, &in_schema); - Column { - name: item - .alias - .clone() - .unwrap_or_else(|| default_proj_name(&item.expr, i)), - dtype, - nullable, - } - }) - .collect(); + let columns: Vec = + cols.iter() + .enumerate() + .map(|(i, item)| { + let (dtype, nullable) = infer_expr_type(&item.expr, &in_schema); + Column { + name: item.alias.clone().unwrap_or_else(|| { + default_proj_name(&item.expr, i, &in_schema) + }), + dtype, + nullable, + } + }) + .collect(); let time_index = columns.iter().position(|c| c.name == "ts"); Ok(Schema { columns, @@ -515,17 +514,11 @@ impl QueryExpr { /// (the L4/emit layer refines with a real function/type registry). fn infer_expr_type(expr: &L3Expr, schema: &Schema) -> (DataType, bool) { match expr { - L3Expr::Column(ColumnRef::Named(name)) => schema - .column_id(name) - .and_then(|id| schema.columns.get(id)) - .map(|c| (c.dtype.clone(), c.nullable)) - .unwrap_or((DataType::Utf8, true)), - L3Expr::Column(ColumnRef::SampleValue) => schema - .column_id("value") - .and_then(|id| schema.columns.get(id)) + L3Expr::Column(id) => schema + .columns + .get(*id) .map(|c| (c.dtype.clone(), c.nullable)) - .unwrap_or((DataType::Float64, false)), - L3Expr::Column(ColumnRef::Wildcard) => (DataType::Float64, false), + .unwrap_or((DataType::Float64, true)), L3Expr::Literal(s) => match s { L3Scalar::Int64(_) => (DataType::Int64, false), L3Scalar::Float64(_) => (DataType::Float64, false), @@ -570,11 +563,14 @@ fn infer_expr_type(expr: &L3Expr, schema: &Schema) -> (DataType, bool) { } /// Default output-column name for a projection item with no explicit alias: -/// a bare column keeps its name; anything else gets a positional `col_{i}`. -fn default_proj_name(expr: &L3Expr, idx: usize) -> String { +/// a bare column keeps its (schema) name; anything else gets `col_{i}`. +fn default_proj_name(expr: &L3Expr, idx: usize, schema: &Schema) -> String { match expr { - L3Expr::Column(ColumnRef::Named(n)) => n.clone(), - L3Expr::Column(ColumnRef::SampleValue) => "value".to_string(), + L3Expr::Column(id) => schema + .columns + .get(*id) + .map(|c| c.name.clone()) + .unwrap_or_else(|| format!("col_{idx}")), _ => format!("col_{idx}"), } } @@ -643,25 +639,25 @@ mod tests { ); let q = QueryExpr::Project { cols: vec![ - // bare column passthrough keeps its name + type + // bare column passthrough keeps its (schema) name + type: host=col 1 ProjectItem { alias: None, - expr: L3Expr::Column(ColumnRef::Named("host".into())), + expr: L3Expr::Column(1), }, - // arithmetic over the sample value → Float64 + // arithmetic over value (col 2) → Float64 ProjectItem { alias: Some("dbl".into()), expr: L3Expr::Arith { op: ArithOp::Add, - left: Box::new(L3Expr::Column(ColumnRef::SampleValue)), - right: Box::new(L3Expr::Column(ColumnRef::SampleValue)), + left: Box::new(L3Expr::Column(2)), + right: Box::new(L3Expr::Column(2)), }, }, // comparison → Bool (nullable under 3-valued logic) ProjectItem { alias: Some("flag".into()), expr: L3Expr::Compare { - left: Box::new(L3Expr::Column(ColumnRef::SampleValue)), + left: Box::new(L3Expr::Column(2)), op: CompareOp::Gt, right: Box::new(L3Expr::Literal(L3Scalar::Float64(0.0))), }, @@ -691,13 +687,14 @@ mod tests { ); let q = QueryExpr::Project { cols: vec![ + // value=col 1, ts=col 0 ProjectItem { alias: None, - expr: L3Expr::Column(ColumnRef::SampleValue), + expr: L3Expr::Column(1), }, ProjectItem { alias: None, - expr: L3Expr::Column(ColumnRef::Named("ts".into())), + expr: L3Expr::Column(0), }, ], child: Box::new(child), diff --git a/crates/core/src/intent_algebra/relational.rs b/crates/core/src/intent_algebra/relational.rs index 3b8bf6c0..af65593c 100644 --- a/crates/core/src/intent_algebra/relational.rs +++ b/crates/core/src/intent_algebra/relational.rs @@ -9,12 +9,26 @@ use std::time::Duration; -use super::expr_ir::L3Expr; -pub use super::query_expr::{ - BinaryOpKind, ColumnRef, PartitionKeys, ProjectItem, SortKey, VectorMatch, -}; +use super::expr_ir::L2Expr; +pub use super::query_expr::{BinaryOpKind, ColumnRef, PartitionKeys, VectorMatch}; use super::schema::Schema; +/// SELECT-list item at Layer 2 — a name-based [`L2Expr`] + optional alias. +/// (`query_expr::ProjectItem` is the positional L3 sibling.) +#[derive(Debug, Clone, PartialEq)] +pub struct L2ProjectItem { + pub alias: Option, + pub expr: L2Expr, +} + +/// ORDER BY key at Layer 2 — a name-based [`L2Expr`] + direction. +#[derive(Debug, Clone, PartialEq)] +pub struct L2SortKey { + pub expr: L2Expr, + pub ascending: bool, + pub nulls_first: bool, +} + /// Base relation / metric stream source. #[derive(Debug, Clone, PartialEq)] pub struct SourceSpec { @@ -97,12 +111,12 @@ pub enum QueryExpr { Ref(String), /// σ — row-level filter (WHERE / PromQL label matchers). - Filter { pred: L3Expr, input: Box }, + Filter { pred: L2Expr, input: Box }, /// π — projection / SELECT list (SQL). Column refs in `cols` resolve by /// name against the child schema during conversion. Project { - cols: Vec, + cols: Vec, input: Box, }, @@ -110,7 +124,7 @@ pub enum QueryExpr { Aggregate { keys: Vec, aggs: Vec, - having: Option, + having: Option, input: Box, }, @@ -142,7 +156,7 @@ pub enum QueryExpr { Join { kind: super::query_expr::JoinKind, - pred: Option, + pred: Option, left: Box, right: Box, }, @@ -154,7 +168,7 @@ pub enum QueryExpr { }, Sort { - keys: Vec, + keys: Vec, input: Box, }, Limit { diff --git a/crates/lower/src/promql.rs b/crates/lower/src/promql.rs index b859a2bd..7d0e2491 100644 --- a/crates/lower/src/promql.rs +++ b/crates/lower/src/promql.rs @@ -38,12 +38,12 @@ use promql_parser::parser::{ }; use asap_control_core::intent_algebra::query_expr::{ - BinaryOpKind, ColumnRef, GroupSide, SortKey, VectorGrouping, VectorMatch, VectorMatchKind, + BinaryOpKind, ColumnRef, GroupSide, VectorGrouping, VectorMatch, VectorMatchKind, }; use asap_control_core::intent_algebra::relational::{ - AggFunc, AggItem, QueryExpr as L2, SourceSpec, + AggFunc, AggItem, L2SortKey, QueryExpr as L2, SourceSpec, }; -use asap_control_core::intent_algebra::{CompareOp, L3Expr, L3Scalar}; +use asap_control_core::intent_algebra::{CompareOp, L2Expr, L3Scalar}; use crate::error::LoweringError; @@ -87,7 +87,7 @@ enum InnerFunc { struct Inner { metric: String, - matchers: Vec, + matchers: Vec, window: Option, func: Option, } @@ -364,8 +364,8 @@ fn build(inner: Inner, keys: Vec, outer: Outer) -> Result { }; let base = windowed_aggregate(inner, keys, func); let sorted = L2::Sort { - keys: vec![SortKey { - expr: L3Expr::Column(ColumnRef::SampleValue), + keys: vec![L2SortKey { + expr: L2Expr::Column(ColumnRef::SampleValue), ascending: !descending, nulls_first: false, }], @@ -443,7 +443,7 @@ fn window_scan(inner: Inner) -> L2 { } } -fn filtered_source(metric: String, matchers: Vec) -> L2 { +fn filtered_source(metric: String, matchers: Vec) -> L2 { let source = L2::Source(SourceSpec::new(metric)); if matchers.is_empty() { source @@ -451,7 +451,7 @@ fn filtered_source(metric: String, matchers: Vec) -> L2 { let pred = if matchers.len() == 1 { matchers.into_iter().next().unwrap() } else { - L3Expr::BoolAnd(matchers) + L2Expr::BoolAnd(matchers) }; L2::Filter { pred, @@ -512,7 +512,7 @@ fn resolve_group(agg: &AggregateExpr) -> Result> { // ── Free helpers ────────────────────────────────────────────────────────────── -fn vs_parts(vs: &VectorSelector) -> Result<(String, Vec)> { +fn vs_parts(vs: &VectorSelector) -> Result<(String, Vec)> { // `offset` / `@` shift the evaluation/lookback time. The intent algebra has // no representation for either, so silently lowering them (as if absent) // would change the query's meaning. Reject rather than mislower. @@ -543,21 +543,21 @@ fn vs_parts(vs: &VectorSelector) -> Result<(String, Vec)> { Ok((metric, matchers)) } -fn matcher_to_l3expr(m: &Matcher) -> L3Expr { +fn matcher_to_l3expr(m: &Matcher) -> L2Expr { let op = match &m.op { MatchOp::Equal => CompareOp::Eq, MatchOp::NotEqual => CompareOp::Ne, MatchOp::Re(_) => CompareOp::Regex, MatchOp::NotRe(_) => CompareOp::NotRegex, }; - L3Expr::Compare { - left: Box::new(L3Expr::Column(ColumnRef::Named(m.name.clone()))), + L2Expr::Compare { + left: Box::new(L2Expr::Column(ColumnRef::Named(m.name.clone()))), op, - right: Box::new(L3Expr::Literal(L3Scalar::Utf8(m.value.clone()))), + right: Box::new(L2Expr::Literal(L3Scalar::Utf8(m.value.clone()))), } } -fn extract_matrix(expr: &Expr) -> Result<(String, Vec, Duration)> { +fn extract_matrix(expr: &Expr) -> Result<(String, Vec, Duration)> { match expr { Expr::MatrixSelector(ms) => { let (metric, matchers) = vs_parts(&ms.vs)?; diff --git a/crates/lower/src/sql/expr.rs b/crates/lower/src/sql/expr.rs index c040b769..dd7cfa6d 100644 --- a/crates/lower/src/sql/expr.rs +++ b/crates/lower/src/sql/expr.rs @@ -1,6 +1,6 @@ use datafusion::logical_expr::{BinaryExpr, Expr, Operator}; -use asap_control_core::intent_algebra::{ArithOp, ColumnRef, CompareOp, L3Expr, L3Scalar}; +use asap_control_core::intent_algebra::{ArithOp, ColumnRef, CompareOp, L2Expr, L3Scalar}; use crate::error::LoweringError; @@ -21,26 +21,26 @@ pub(super) fn split_conjuncts(expr: &Expr) -> Vec<&Expr> { } } -/// Translate a DataFusion `Expr` to an `L3Expr`. +/// Translate a DataFusion `Expr` to an `L2Expr`. /// Returns `UnsupportedFeature` for anything not needed in v1. -pub(super) fn df_expr_to_l3(expr: &Expr) -> Result { +pub(super) fn df_expr_to_l2(expr: &Expr) -> Result { match expr { - Expr::Column(col) => Ok(L3Expr::Column(ColumnRef::Named(col.name.clone()))), + Expr::Column(col) => Ok(L2Expr::Column(ColumnRef::Named(col.name.clone()))), - Expr::Literal(sv) => scalar_value_to_l3(sv).map(L3Expr::Literal), + Expr::Literal(sv) => scalar_value_to_l3(sv).map(L2Expr::Literal), - Expr::Alias(a) => df_expr_to_l3(&a.expr), + Expr::Alias(a) => df_expr_to_l2(&a.expr), Expr::BinaryExpr(BinaryExpr { left, op, right }) => match op { Operator::And => { let parts = split_conjuncts(expr); - let l3_parts: Result, _> = parts.iter().map(|e| df_expr_to_l3(e)).collect(); - Ok(L3Expr::BoolAnd(l3_parts?)) + let l3_parts: Result, _> = parts.iter().map(|e| df_expr_to_l2(e)).collect(); + Ok(L2Expr::BoolAnd(l3_parts?)) } Operator::Or => { let parts = split_disjuncts(expr); - let l3_parts: Result, _> = parts.iter().map(|e| df_expr_to_l3(e)).collect(); - Ok(L3Expr::BoolOr(l3_parts?)) + let l3_parts: Result, _> = parts.iter().map(|e| df_expr_to_l2(e)).collect(); + Ok(L2Expr::BoolOr(l3_parts?)) } Operator::Eq => compare(left, CompareOp::Eq, right), Operator::NotEq => compare(left, CompareOp::Ne, right), @@ -77,13 +77,13 @@ pub(super) fn df_expr_to_l3(expr: &Expr) -> Result { // Unary minus: negate literals directly; wrap others in -1 * x. Expr::Negative(inner) => { - let inner_l3 = df_expr_to_l3(inner)?; + let inner_l3 = df_expr_to_l2(inner)?; match inner_l3 { - L3Expr::Literal(L3Scalar::Int64(v)) => Ok(L3Expr::Literal(L3Scalar::Int64(-v))), - L3Expr::Literal(L3Scalar::Float64(v)) => Ok(L3Expr::Literal(L3Scalar::Float64(-v))), - other => Ok(L3Expr::Arith { + L2Expr::Literal(L3Scalar::Int64(v)) => Ok(L2Expr::Literal(L3Scalar::Int64(-v))), + L2Expr::Literal(L3Scalar::Float64(v)) => Ok(L2Expr::Literal(L3Scalar::Float64(-v))), + other => Ok(L2Expr::Arith { op: ArithOp::Mul, - left: Box::new(L3Expr::Literal(L3Scalar::Int64(-1))), + left: Box::new(L2Expr::Literal(L3Scalar::Int64(-1))), right: Box::new(other), }), } @@ -94,35 +94,35 @@ pub(super) fn df_expr_to_l3(expr: &Expr) -> Result { let operand = c .expr .as_ref() - .map(|e| df_expr_to_l3(e).map(Box::new)) + .map(|e| df_expr_to_l2(e).map(Box::new)) .transpose()?; let branches = c .when_then_expr .iter() - .map(|(when, then)| Ok((df_expr_to_l3(when)?, df_expr_to_l3(then)?))) + .map(|(when, then)| Ok((df_expr_to_l2(when)?, df_expr_to_l2(then)?))) .collect::, LoweringError>>()?; let else_expr = c .else_expr .as_ref() - .map(|e| df_expr_to_l3(e).map(Box::new)) + .map(|e| df_expr_to_l2(e).map(Box::new)) .transpose()?; - Ok(L3Expr::Case { + Ok(L2Expr::Case { operand, branches, else_expr, }) } - Expr::Not(inner) => Ok(L3Expr::Not(Box::new(df_expr_to_l3(inner)?))), + Expr::Not(inner) => Ok(L2Expr::Not(Box::new(df_expr_to_l2(inner)?))), - Expr::IsNull(inner) => Ok(L3Expr::IsNull(Box::new(df_expr_to_l3(inner)?))), + Expr::IsNull(inner) => Ok(L2Expr::IsNull(Box::new(df_expr_to_l2(inner)?))), - Expr::IsNotNull(inner) => Ok(L3Expr::IsNotNull(Box::new(df_expr_to_l3(inner)?))), + Expr::IsNotNull(inner) => Ok(L2Expr::IsNotNull(Box::new(df_expr_to_l2(inner)?))), Expr::Cast(c) => { - let inner = df_expr_to_l3(&c.expr)?; + let inner = df_expr_to_l2(&c.expr)?; let to = arrow_to_l3(&c.data_type)?; - Ok(L3Expr::Cast { + Ok(L2Expr::Cast { expr: Box::new(inner), to, try_cast: false, @@ -131,9 +131,9 @@ pub(super) fn df_expr_to_l3(expr: &Expr) -> Result { // TRY_CAST returns NULL on conversion failure; preserve that semantic. Expr::TryCast(c) => { - let inner = df_expr_to_l3(&c.expr)?; + let inner = df_expr_to_l2(&c.expr)?; let to = arrow_to_l3(&c.data_type)?; - Ok(L3Expr::Cast { + Ok(L2Expr::Cast { expr: Box::new(inner), to, try_cast: true, @@ -141,9 +141,9 @@ pub(super) fn df_expr_to_l3(expr: &Expr) -> Result { } Expr::InList(il) => { - let expr = df_expr_to_l3(&il.expr)?; - let list: Result, _> = il.list.iter().map(df_expr_to_l3).collect(); - Ok(L3Expr::InList { + let expr = df_expr_to_l2(&il.expr)?; + let list: Result, _> = il.list.iter().map(df_expr_to_l2).collect(); + Ok(L2Expr::InList { expr: Box::new(expr), list: list?, negated: il.negated, @@ -159,15 +159,15 @@ pub(super) fn df_expr_to_l3(expr: &Expr) -> Result { // NOT BETWEEN: invert each side let lt = compare(&b.expr, CompareOp::Lt, &b.low)?; let gt = compare(&b.expr, CompareOp::Gt, &b.high)?; - Ok(L3Expr::BoolOr(vec![lt, gt])) + Ok(L2Expr::BoolOr(vec![lt, gt])) } else { - Ok(L3Expr::BoolAnd(vec![x_low, x_high])) + Ok(L2Expr::BoolAnd(vec![x_low, x_high])) } } Expr::ScalarFunction(sf) => { - let args: Result, _> = sf.args.iter().map(df_expr_to_l3).collect(); - Ok(L3Expr::FunctionCall { + let args: Result, _> = sf.args.iter().map(df_expr_to_l2).collect(); + Ok(L2Expr::FunctionCall { name: sf.func.name().to_string(), args: args?, }) @@ -180,19 +180,19 @@ pub(super) fn df_expr_to_l3(expr: &Expr) -> Result { } } -pub(super) fn compare(left: &Expr, op: CompareOp, right: &Expr) -> Result { - Ok(L3Expr::Compare { - left: Box::new(df_expr_to_l3(left)?), +pub(super) fn compare(left: &Expr, op: CompareOp, right: &Expr) -> Result { + Ok(L2Expr::Compare { + left: Box::new(df_expr_to_l2(left)?), op, - right: Box::new(df_expr_to_l3(right)?), + right: Box::new(df_expr_to_l2(right)?), }) } -pub(super) fn arith(left: &Expr, op: ArithOp, right: &Expr) -> Result { - Ok(L3Expr::Arith { +pub(super) fn arith(left: &Expr, op: ArithOp, right: &Expr) -> Result { + Ok(L2Expr::Arith { op, - left: Box::new(df_expr_to_l3(left)?), - right: Box::new(df_expr_to_l3(right)?), + left: Box::new(df_expr_to_l2(left)?), + right: Box::new(df_expr_to_l2(right)?), }) } diff --git a/crates/lower/src/sql/mod.rs b/crates/lower/src/sql/mod.rs index e89c680a..23f8634a 100644 --- a/crates/lower/src/sql/mod.rs +++ b/crates/lower/src/sql/mod.rs @@ -16,11 +16,9 @@ use datafusion::logical_expr::{self, Distinct, Expr, JoinType, LogicalPlan}; use datafusion::prelude::SessionContext; use asap_control_core::intent_algebra::relational::{ - AggFunc, AggItem, QueryExpr as L2, SourceSpec, -}; -use asap_control_core::intent_algebra::{ - ColumnRef, CompareOp, JoinKind, L3Expr, ProjectItem, SetOpKind, SortKey, + AggFunc, AggItem, L2ProjectItem, L2SortKey, QueryExpr as L2, SourceSpec, }; +use asap_control_core::intent_algebra::{ColumnRef, CompareOp, JoinKind, L2Expr, SetOpKind}; use crate::error::LoweringError; @@ -29,7 +27,7 @@ mod types; pub use types::SqlCatalog; -use self::expr::df_expr_to_l3; +use self::expr::df_expr_to_l2; use self::types::schema_to_arrow; /// Lowers SQL strings to the Layer-2 [`relational::QueryExpr`] over a table @@ -68,7 +66,7 @@ impl<'a> SqlLowerer<'a> { match plan { LogicalPlan::TableScan(scan) => self.lower_table_scan(scan), LogicalPlan::Filter(filter) => Ok(L2::Filter { - pred: df_expr_to_l3(&filter.predicate)?, + pred: df_expr_to_l2(&filter.predicate)?, input: Box::new(self.lower_plan(&filter.input)?), }), LogicalPlan::Projection(proj) => self.lower_projection(proj), @@ -159,20 +157,20 @@ impl<'a> SqlLowerer<'a> { .on .iter() .map(|(l, r)| { - Ok(L3Expr::Compare { - left: Box::new(df_expr_to_l3(l)?), + Ok(L2Expr::Compare { + left: Box::new(df_expr_to_l2(l)?), op: CompareOp::Eq, - right: Box::new(df_expr_to_l3(r)?), + right: Box::new(df_expr_to_l2(r)?), }) }) .collect::, LoweringError>>()?; if let Some(filter) = &join.filter { - conjuncts.push(df_expr_to_l3(filter)?); + conjuncts.push(df_expr_to_l2(filter)?); } let pred = match conjuncts.len() { 0 => None, 1 => Some(conjuncts.pop().unwrap()), - _ => Some(L3Expr::BoolAnd(conjuncts)), + _ => Some(L2Expr::BoolAnd(conjuncts)), }; Ok(L2::Join { kind, @@ -192,11 +190,11 @@ impl<'a> SqlLowerer<'a> { .expr .iter() .map(|e| match e { - Expr::Alias(a) => df_expr_to_l3(&a.expr).map(|expr| ProjectItem { + Expr::Alias(a) => df_expr_to_l2(&a.expr).map(|expr| L2ProjectItem { expr, alias: Some(a.name.clone()), }), - _ => df_expr_to_l3(e).map(|expr| ProjectItem { expr, alias: None }), + _ => df_expr_to_l2(e).map(|expr| L2ProjectItem { expr, alias: None }), }) .collect::, _>>()?; Ok(L2::Project { cols, input }) @@ -254,7 +252,7 @@ impl<'a> SqlLowerer<'a> { .expr .iter() .map(|s| { - df_expr_to_l3(&s.expr).map(|expr| SortKey { + df_expr_to_l2(&s.expr).map(|expr| L2SortKey { expr, ascending: s.asc, nulls_first: s.nulls_first, diff --git a/crates/lower/tests/promql_lowering.rs b/crates/lower/tests/promql_lowering.rs index 3ac06cb1..3445418b 100644 --- a/crates/lower/tests/promql_lowering.rs +++ b/crates/lower/tests/promql_lowering.rs @@ -3,8 +3,8 @@ use std::time::Duration; use asap_control_core::intent_algebra::{ - AggIntent, BinaryOpKind, ColumnRef, CompareOp, L3Expr, L3Scalar, PartitionKeys, QueryExpr, - Source, WindowKind, + AggIntent, BinaryOpKind, CompareOp, L3Expr, L3Scalar, PartitionKeys, QueryExpr, Source, + WindowKind, }; use asap_control_core::types::AccuracyTarget; use asap_control_core::workload::{ @@ -40,14 +40,19 @@ fn bare_selector_is_scan_with_predicates() { #[test] fn regex_matcher_lowers_to_regex_compareop() { let qe = lower(r#"http_requests_total{path=~"/api/.*"}"#); - let QueryExpr::Scan { predicates, .. } = &qe else { + let QueryExpr::Scan { + predicates, schema, .. + } = &qe + else { panic!("expected Scan, got {qe:?}"); }; let L3Expr::Compare { left, op, right } = &predicates[0].0 else { panic!("expected Compare, got {:?}", predicates[0].0); }; assert_eq!(*op, CompareOp::Regex); - assert!(matches!(left.as_ref(), L3Expr::Column(ColumnRef::Named(n)) if n == "path")); + // The label matcher's column is resolved positionally against the scan schema. + let path_id = schema.column_id("path").expect("path in scan schema"); + assert!(matches!(left.as_ref(), L3Expr::Column(id) if *id == path_id)); assert!(matches!(right.as_ref(), L3Expr::Literal(L3Scalar::Utf8(v)) if v == "/api/.*")); } From 389fa661db9d4e302cf0fb3d3c511eaf2c38b58f Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 27 May 2026 07:25:39 -0600 Subject: [PATCH 27/40] fix(sql): correct three silent-mis-lowering bugs in the SQL aggregate path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by self-review of #5. All produced a tree that silently meant something different from the query. 1. `ORDER BY DESC LIMIT k` over GROUP BY became a frequency heavy-hitter regardless of the ranking expression — `lower_as_topk` discarded the aggregate and emitted AggIntent::TopK. Now gated by `heavy_hitter_topk`: only a single non-DISTINCT COUNT, ranked DESC by that count's output column, becomes TopK (mirrors the PromQL `topk over count_over_time` rule). Every other ranking (SUM/AVG/… or a group key) keeps the real Aggregate under a generic Sort+Limit. (Aliased/ordinal count keys safely fall back to generic.) 2. `SUM/AVG/MIN/MAX/STDDEV/VAR(DISTINCT x)` silently lowered as non-distinct (AggItem.distinct was set but never read). L3 has no distinct value-reducer, so these are now rejected (UnsupportedAggregate) — only COUNT(DISTINCT) maps (to Cardinality). 3. A value reducer over a non-column expression (`SUM(a*b)`) mapped its arg to Wildcard → col:None → reduced an arbitrary probe column. Value reducers now require a real column (`reducer_col`) and are rejected otherwise. Related: `resolve_agg_col` now errors on an unresolved Named column instead of silently returning None (matching resolve_named_keys' strictness). Tests: count-ranked topk → heavy-hitter; AVG-ranked LIMIT keeps the aggregate; SUM(DISTINCT)/SUM(expr) rejected. 115 tests green, clippy -D warnings + fmt clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/core/src/intent_algebra/lower.rs | 30 ++++--- crates/lower/src/sql/mod.rs | 115 +++++++++++++++++++----- crates/lower/tests/sql_lowering.rs | 63 +++++++++++++ 3 files changed, 178 insertions(+), 30 deletions(-) diff --git a/crates/core/src/intent_algebra/lower.rs b/crates/core/src/intent_algebra/lower.rs index 2ddfe22d..61ae5370 100644 --- a/crates/core/src/intent_algebra/lower.rs +++ b/crates/core/src/intent_algebra/lower.rs @@ -112,7 +112,7 @@ pub fn convert( let intent = agg_func_to_intent( &aggs[0].func, acc, - resolve_agg_col(&aggs[0].col, &agg_in_schema), + resolve_agg_col(&aggs[0].col, &agg_in_schema)?, ); let aggregate = CQueryExpr::Aggregate { by: Vec::new(), @@ -151,10 +151,11 @@ pub fn convert( let by = resolve_named_keys(keys, &child_schema)?; let intents = aggs .iter() - .map(|item| { - agg_func_to_intent(&item.func, acc, resolve_agg_col(&item.col, &child_schema)) + .map(|item| -> Result { + let col = resolve_agg_col(&item.col, &child_schema)?; + Ok(agg_func_to_intent(&item.func, acc, col)) }) - .collect(); + .collect::, _>>()?; let having = having .as_ref() .map(|h| -> Result { @@ -371,13 +372,22 @@ fn scan( } /// Resolve a Layer-2 aggregate-input [`ColumnRef`] to a positional input -/// column. `SampleValue` / `Wildcard` carry no specific column → `None` (the -/// PromQL sample-value convention); a named column (`SUM(bytes)`) resolves to -/// its position so the L3 reducer types off the right input. -fn resolve_agg_col(col: &ColumnRef, schema: &Schema) -> Option { +/// column. `SampleValue` / `Wildcard` carry no specific column → `Ok(None)` +/// (the PromQL sample-value / `COUNT(*)` convention); a `Named` column +/// (`SUM(bytes)`) must resolve to its position, else it is an error — silently +/// dropping it to `None` would reduce the wrong column (the schema probe). +fn resolve_agg_col(col: &ColumnRef, schema: &Schema) -> Result, ResolveError> { match col { - ColumnRef::Named(name) => schema.column_id(name), - ColumnRef::SampleValue | ColumnRef::Wildcard => None, + ColumnRef::Named(name) => { + schema + .column_id(name) + .map(Some) + .ok_or_else(|| ResolveError::NotFound { + name: name.clone(), + available: schema.columns.iter().map(|c| c.name.clone()).collect(), + }) + } + ColumnRef::SampleValue | ColumnRef::Wildcard => Ok(None), } } diff --git a/crates/lower/src/sql/mod.rs b/crates/lower/src/sql/mod.rs index 23f8634a..7a7b8730 100644 --- a/crates/lower/src/sql/mod.rs +++ b/crates/lower/src/sql/mod.rs @@ -240,10 +240,13 @@ impl<'a> SqlLowerer<'a> { } fn lower_sort(&self, sort: &logical_expr::Sort) -> Result { - // TopK: Sort with a folded LIMIT, all keys descending, over an Aggregate. + // Heavy-hitter TopK only when ranking DESC by a single COUNT aggregate + // (the frequency sketch the `TopK` intent represents). Any other + // ranking — by a SUM/AVG/… output or a group column — keeps the real + // Aggregate under a generic Sort+Limit so its aggregate isn't discarded. if let Some(k) = sort.fetch { - if sort.expr.iter().all(|s| !s.asc) { - if let Some(agg) = find_aggregate(strip_projections_and_aliases(&sort.input)) { + if let Some(agg) = find_aggregate(strip_projections_and_aliases(&sort.input)) { + if heavy_hitter_topk(sort, agg) { return self.lower_as_topk(agg, k as u64); } } @@ -266,14 +269,13 @@ impl<'a> SqlLowerer<'a> { } fn lower_limit(&self, limit: &logical_expr::Limit) -> Result { - // TopK: Limit over Sort over Aggregate, all sort keys DESC, no OFFSET. + // Heavy-hitter TopK only for a count-ranked Limit-over-Sort-over-Aggregate + // with no OFFSET (see `lower_sort`). Otherwise fall through to Limit+Sort. if let Some(k) = eval_fetch(&limit.fetch) { if eval_fetch(&limit.skip).unwrap_or(0) == 0 { if let LogicalPlan::Sort(sort) = strip_aliases(&limit.input) { - if sort.expr.iter().all(|s| !s.asc) { - if let Some(agg) = - find_aggregate(strip_projections_and_aliases(&sort.input)) - { + if let Some(agg) = find_aggregate(strip_projections_and_aliases(&sort.input)) { + if heavy_hitter_topk(sort, agg) { return self.lower_as_topk(agg, k as u64); } } @@ -311,28 +313,38 @@ fn lower_agg_item(expr: &Expr) -> Result { Expr::Alias(a) => lower_agg_item(&a.expr), Expr::AggregateFunction(agg_fn) => { let name = agg_fn.func.name().to_lowercase(); + // L3 has no DISTINCT modifier for the value reducers; only + // COUNT(DISTINCT) maps (to Cardinality). Reject DISTINCT elsewhere + // rather than silently lowering `SUM(DISTINCT x)` as `SUM(x)`. + if agg_fn.distinct && name != "count" { + return Err(LoweringError::UnsupportedAggregate(format!( + "DISTINCT {name}" + ))); + } + // Value reducers (`reducer_col`) require a real column — `SUM(a*b)` + // is rejected, not silently reduced over a probe column. let (func, col) = match name.as_str() { "count" if agg_fn.distinct => (AggFunc::CountDistinct, agg_col_ref(&agg_fn.args)), "count" => (AggFunc::Count, ColumnRef::Wildcard), - "sum" => (AggFunc::Sum, agg_col_ref(&agg_fn.args)), - "min" => (AggFunc::Min, agg_col_ref(&agg_fn.args)), - "max" => (AggFunc::Max, agg_col_ref(&agg_fn.args)), - "avg" | "mean" => (AggFunc::Avg, agg_col_ref(&agg_fn.args)), + "sum" => (AggFunc::Sum, reducer_col(&name, &agg_fn.args)?), + "min" => (AggFunc::Min, reducer_col(&name, &agg_fn.args)?), + "max" => (AggFunc::Max, reducer_col(&name, &agg_fn.args)?), + "avg" | "mean" => (AggFunc::Avg, reducer_col(&name, &agg_fn.args)?), "stddev" | "stddev_samp" => ( AggFunc::StdDev { population: false }, - agg_col_ref(&agg_fn.args), + reducer_col(&name, &agg_fn.args)?, ), "stddev_pop" => ( AggFunc::StdDev { population: true }, - agg_col_ref(&agg_fn.args), + reducer_col(&name, &agg_fn.args)?, ), "var" | "variance" | "var_samp" => ( AggFunc::Variance { population: false }, - agg_col_ref(&agg_fn.args), + reducer_col(&name, &agg_fn.args)?, ), "var_pop" => ( AggFunc::Variance { population: true }, - agg_col_ref(&agg_fn.args), + reducer_col(&name, &agg_fn.args)?, ), "approx_percentile_cont" | "percentile_cont" => ( AggFunc::Quantile(extract_percentile_q(&agg_fn.args)?), @@ -352,9 +364,9 @@ fn lower_agg_item(expr: &Expr) -> Result { } } -/// The aggregated input column. `COUNT(*)` and non-column arguments yield -/// `Wildcard`; a bare/aliased/cast column yields its name. -fn agg_col_ref(args: &[Expr]) -> ColumnRef { +/// The first aggregate argument's column name (bare / aliased / cast column), +/// or `None` for `*` / a non-column expression. +fn agg_col_name(args: &[Expr]) -> Option { fn col_name(e: &Expr) -> Option { match e { Expr::Column(c) => Some(c.name.clone()), @@ -363,12 +375,28 @@ fn agg_col_ref(args: &[Expr]) -> ColumnRef { _ => None, } } - match args.first().and_then(col_name) { + args.first().and_then(col_name) +} + +/// The aggregated input column. `COUNT(*)` and non-column arguments yield +/// `Wildcard`; a bare/aliased/cast column yields its name. +fn agg_col_ref(args: &[Expr]) -> ColumnRef { + match agg_col_name(args) { Some(name) => ColumnRef::Named(name), None => ColumnRef::Wildcard, } } +/// The single input column of a value reducer (`SUM`/`MIN`/`MAX`/`AVG`/stddev/ +/// variance). Errors if the argument is not a column: L3 reduces a column, not +/// an arbitrary expression (`SUM(a*b)`), so silently picking a probe column +/// would compute the wrong result. +fn reducer_col(name: &str, args: &[Expr]) -> Result { + agg_col_name(args).map(ColumnRef::Named).ok_or_else(|| { + LoweringError::UnsupportedAggregate(format!("{name} over a non-column expression")) + }) +} + fn expr_to_group_name(expr: &Expr) -> Result { match expr { Expr::Column(col) => Ok(col.name.clone()), @@ -389,6 +417,53 @@ fn extract_percentile_q(args: &[Expr]) -> Result { } } +/// True iff `sort` ranks **descending by a single `COUNT` aggregate** of `agg` +/// — the only shape the heavy-hitter (frequency) `TopK` sketch is correct for. +/// +/// Requires `agg` to have exactly one aggregate (a plain, non-`DISTINCT` +/// `COUNT`) and the sole sort key to reference *that* output column (not a +/// group key, a `SUM`/`AVG`/… output, or a multi-aggregate select). Anything +/// else stays a generic `Sort` + `Limit` over the real `Aggregate`, mirroring +/// the PromQL gate (`topk` is heavy-hitter only over `count_over_time`). +fn heavy_hitter_topk(sort: &logical_expr::Sort, agg: &logical_expr::Aggregate) -> bool { + let [key] = sort.expr.as_slice() else { + return false; + }; + if key.asc { + return false; + } + if agg.aggr_expr.len() != 1 || !is_count_aggregate(&agg.aggr_expr[0]) { + return false; + } + // The DESC key must rank by the count's output column, not a group key. The + // aggregate schema is `[group fields …, aggregate fields …]`, so the single + // count output sits at index `group_expr.len()`. + let count_name = agg + .schema + .fields() + .get(agg.group_expr.len()) + .map(|f| f.name().clone()); + column_name(&key.expr) == count_name +} + +/// The referenced column name of a bare/aliased column expression, else `None`. +fn column_name(expr: &Expr) -> Option { + match expr { + Expr::Column(c) => Some(c.name.clone()), + Expr::Alias(a) => column_name(&a.expr), + _ => None, + } +} + +/// Whether `expr` is a plain (non-`DISTINCT`) `COUNT` aggregate. +fn is_count_aggregate(expr: &Expr) -> bool { + match expr { + Expr::Alias(a) => is_count_aggregate(&a.expr), + Expr::AggregateFunction(f) => f.func.name().eq_ignore_ascii_case("count") && !f.distinct, + _ => false, + } +} + // ── LogicalPlan navigation helpers ────────────────────────────────────────────── fn eval_fetch(expr_opt: &Option>) -> Option { diff --git a/crates/lower/tests/sql_lowering.rs b/crates/lower/tests/sql_lowering.rs index 658f7ca4..b1dc0f92 100644 --- a/crates/lower/tests/sql_lowering.rs +++ b/crates/lower/tests/sql_lowering.rs @@ -155,6 +155,69 @@ async fn single_agg_group_by_keeps_key_in_output_schema() { assert_eq!(schema.columns[1].dtype, DataType::Int64, "SUM(bytes)"); } +#[tokio::test] +async fn count_ranked_topk_is_heavy_hitter() { + // `ORDER BY COUNT(*) DESC LIMIT k` over a single COUNT aggregate is the one + // case the heavy-hitter (frequency) sketch is correct for. (The key must + // reference the count output directly; an alias would safely fall back to a + // generic Sort+Limit.) + let qe = lower( + "SELECT service, COUNT(*) FROM metrics GROUP BY service ORDER BY COUNT(*) DESC LIMIT 10", + ) + .await; + let (by, aggs) = find_aggregate(&qe).expect("expected an Aggregate"); + assert_eq!(by, &vec![1], "GROUP BY service → col 1"); + assert!( + matches!(aggs.as_slice(), [AggIntent::TopK { k: 10, .. }]), + "count-ranked topk → heavy-hitter TopK, got {aggs:?}" + ); +} + +#[tokio::test] +async fn non_count_ranked_limit_keeps_the_aggregate() { + // Ranking by AVG (not a count) must NOT become a frequency heavy-hitter — + // the AVG aggregate has to survive as a generic Sort+Limit. + let qe = lower( + "SELECT service, AVG(latency) AS a FROM metrics GROUP BY service ORDER BY a DESC LIMIT 10", + ) + .await; + let (_, aggs) = find_aggregate(&qe).expect("expected an Aggregate"); + assert!( + aggs.iter().any(|a| matches!(a, AggIntent::Avg { .. })), + "AVG must be preserved, got {aggs:?}" + ); + assert!( + !aggs.iter().any(|a| matches!(a, AggIntent::TopK { .. })), + "AVG ranking must not become a frequency heavy-hitter, got {aggs:?}" + ); +} + +#[tokio::test] +async fn distinct_value_reducer_is_rejected_not_dropped() { + // L3 has no distinct-Sum; SUM(DISTINCT x) must be rejected, not silently + // lowered as SUM(x). + let res = lower_sql( + "SELECT SUM(DISTINCT bytes) FROM metrics", + &catalog(), + AccuracyTarget::Exact, + ) + .await; + assert!(res.is_err(), "SUM(DISTINCT ...) should be rejected"); +} + +#[tokio::test] +async fn aggregate_over_non_column_expression_is_rejected() { + // L3 reduces a column, not an arbitrary expression — SUM(bytes + 1) must be + // rejected rather than silently reducing a probe column. + let res = lower_sql( + "SELECT SUM(bytes + 1) FROM metrics", + &catalog(), + AccuracyTarget::Exact, + ) + .await; + assert!(res.is_err(), "SUM() should be rejected"); +} + #[tokio::test] async fn count_star_is_count_intent() { let qe = lower("SELECT COUNT(*) FROM metrics").await; From d9377f9ff057482ab008d0da9c910a46b7cc6ec8 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 27 May 2026 07:33:29 -0600 Subject: [PATCH 28/40] fix(promql+sql): reject malformed range-vector args + out-of-range agg params MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit More self-review (#5) fixes — reject rather than silently mis-lower: #6 extract_matrix no longer descends through an arbitrary Call to find the matrix selector. `rate(abs(m[5m]))` previously lowered as `rate(m[5m])` (wrapper silently stripped); it is now rejected. #7 topk/bottomk k is validated as a non-negative integer (count_param) instead of `as u64` silently truncating `topk(2.7,…)`→2 / saturating negatives→0. #9 quantile φ (PromQL quantile / quantile_over_time / histogram_quantile, and SQL approx_percentile_cont) is validated to be finite and in [0,1] (quantile_param), so φ=NaN/2.0 is rejected rather than producing a bogus intent and `quantile_NaN`/`quantile_1_5` output-column name. Tests: fractional/negative topk k rejected; out-of-range φ rejected across all three quantile forms; function-wrapped range vector rejected. 118 tests green, clippy -D warnings + fmt clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/lower/src/promql.rs | 44 ++++++++++++++++++++++----- crates/lower/src/sql/mod.rs | 21 +++++++++---- crates/lower/tests/promql_lowering.rs | 31 +++++++++++++++++++ 3 files changed, 83 insertions(+), 13 deletions(-) diff --git a/crates/lower/src/promql.rs b/crates/lower/src/promql.rs index 7d0e2491..27046341 100644 --- a/crates/lower/src/promql.rs +++ b/crates/lower/src/promql.rs @@ -140,12 +140,12 @@ fn walk_aggregate(agg: &AggregateExpr) -> Result { let outer = if op == token::T_TOPK { Outer::TopK { - k: num_param(agg)? as u64, + k: count_param(agg)?, descending: true, } } else if op == token::T_BOTTOMK { Outer::TopK { - k: num_param(agg)? as u64, + k: count_param(agg)?, descending: false, } } else if op == token::T_COUNT { @@ -170,7 +170,7 @@ fn walk_aggregate(agg: &AggregateExpr) -> Result { } else if op == token::T_STDVAR { Outer::Plain(OuterIntent::Variance) } else if op == token::T_QUANTILE { - Outer::Plain(OuterIntent::Quantile(num_param(agg)?)) + Outer::Plain(OuterIntent::Quantile(quantile_param(num_param(agg)?)?)) } else { return Err(LoweringError::UnsupportedAggregateOp(format!( "aggregate token {op}" @@ -189,7 +189,7 @@ fn walk_aggregate(agg: &AggregateExpr) -> Result { /// `histogram_quantile(φ, sum by (le) (rate(m_bucket[w])))` pattern, which the /// old "extract the matrix and substitute a bare Quantile" path could not. fn walk_histogram_quantile(call: &Call) -> Result { - let phi = num_arg(call, 0)?; + let phi = quantile_param(num_arg(call, 0)?)?; let inner = walk(arg(call, 1)?)?; Ok(outer_aggregate(vec![], AggFunc::Quantile(phi), inner)) } @@ -289,7 +289,7 @@ fn lower_inner_call(call: &Call) -> Result { }) } "quantile_over_time" => { - let phi = num_arg(call, 0)?; + let phi = quantile_param(num_arg(call, 0)?)?; let (metric, matchers, window) = extract_matrix(arg(call, 1)?)?; Ok(Inner { metric, @@ -564,9 +564,12 @@ fn extract_matrix(expr: &Expr) -> Result<(String, Vec, Duration)> { Ok((metric, matchers, ms.range)) } Expr::Paren(p) => extract_matrix(&p.expr), - Expr::Call(c) => extract_matrix(arg(c, 0)?), + // A range-vector function argument must be a (parenthesised) matrix + // selector. Do NOT descend through an arbitrary `Call` — that would + // silently strip an unsupported wrapper (`rate(deriv(m[5m]))` lowering + // as `rate(m[5m])`). Reject instead. other => Err(LoweringError::UnsupportedFeature(format!( - "expected range-vector argument, got {:?}", + "expected a range-vector (matrix) argument, got {:?}", std::mem::discriminant(other) ))), } @@ -603,6 +606,33 @@ fn num_expr(expr: &Expr) -> Result { } } +/// `topk`/`bottomk` count parameter — a non-negative integer. Rejects +/// fractional / negative / non-finite values rather than silently truncating +/// or saturating them via `as u64` (`topk(2.7, …)` ≠ `topk(2, …)`). +fn count_param(agg: &AggregateExpr) -> Result { + let v = num_param(agg)?; + if v.is_finite() && v >= 0.0 && v.fract() == 0.0 && v <= u64::MAX as f64 { + Ok(v as u64) + } else { + Err(LoweringError::InvalidParameter(format!( + "topk/bottomk k must be a non-negative integer, got {v}" + ))) + } +} + +/// Quantile φ — must be a finite value in `[0, 1]`. Rejects NaN/∞ and +/// out-of-range φ (which would otherwise propagate into a bogus intent and +/// output-column name like `quantile_NaN`). +fn quantile_param(q: f64) -> Result { + if q.is_finite() && (0.0..=1.0).contains(&q) { + Ok(q) + } else { + Err(LoweringError::InvalidParameter(format!( + "quantile φ must be in [0, 1], got {q}" + ))) + } +} + fn binop(id: token::TokenId) -> Result { Ok(if id == token::T_ADD { BinaryOpKind::Add diff --git a/crates/lower/src/sql/mod.rs b/crates/lower/src/sql/mod.rs index 7a7b8730..dd29b803 100644 --- a/crates/lower/src/sql/mod.rs +++ b/crates/lower/src/sql/mod.rs @@ -408,12 +408,21 @@ fn expr_to_group_name(expr: &Expr) -> Result { } fn extract_percentile_q(args: &[Expr]) -> Result { - match args.get(1) { - Some(Expr::Literal(ScalarValue::Float64(Some(q)))) => Ok(*q), - Some(Expr::Literal(ScalarValue::Float32(Some(q)))) => Ok(*q as f64), - _ => Err(LoweringError::InvalidExpression( - "percentile value must be a float literal (2nd arg)".into(), - )), + let q = match args.get(1) { + Some(Expr::Literal(ScalarValue::Float64(Some(q)))) => *q, + Some(Expr::Literal(ScalarValue::Float32(Some(q)))) => *q as f64, + _ => { + return Err(LoweringError::InvalidExpression( + "percentile value must be a float literal (2nd arg)".into(), + )) + } + }; + if q.is_finite() && (0.0..=1.0).contains(&q) { + Ok(q) + } else { + Err(LoweringError::InvalidExpression(format!( + "percentile must be in [0, 1], got {q}" + ))) } } diff --git a/crates/lower/tests/promql_lowering.rs b/crates/lower/tests/promql_lowering.rs index 3445418b..8c0279e4 100644 --- a/crates/lower/tests/promql_lowering.rs +++ b/crates/lower/tests/promql_lowering.rs @@ -423,6 +423,37 @@ fn without_grouping_is_unsupported() { assert!(format!("{err}").contains("without"), "got {err}"); } +// ── parameter validation (reject rather than silently truncate/garble) ────────── + +#[test] +fn fractional_or_negative_topk_k_is_rejected() { + // `as u64` would silently truncate 2.7→2 / saturate -1→0. + assert!(lower_promql("topk(2.7, count_over_time(m[1m]))", AccuracyTarget::Exact).is_err()); + assert!(lower_promql("bottomk(2.5, sum_over_time(m[1m]))", AccuracyTarget::Exact).is_err()); +} + +#[test] +fn out_of_range_quantile_phi_is_rejected() { + // φ outside [0,1] would otherwise yield a bogus `quantile_1_5` column. + assert!(lower_promql("quantile(1.5, up)", AccuracyTarget::Exact).is_err()); + assert!(lower_promql("quantile_over_time(1.5, m[5m])", AccuracyTarget::Exact).is_err()); + assert!(lower_promql( + "histogram_quantile(2.0, rate(b[5m]))", + AccuracyTarget::Exact + ) + .is_err()); +} + +#[test] +fn function_wrapped_range_vector_is_rejected_not_stripped() { + // `rate(abs(m[5m]))` must NOT silently lower as `rate(m[5m])` — the wrapper + // is rejected (here, at parse or in extract_matrix), never stripped. + assert!( + lower_promql("rate(abs(http_requests_total[5m]))", AccuracyTarget::Exact).is_err(), + "function-wrapped range vector should be rejected" + ); +} + // ── accuracy propagation ────────────────────────────────────────────────────── #[test] From 42c8636eca4a9ccca367992e8da3a9236acfa4cd Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 27 May 2026 07:46:28 -0600 Subject: [PATCH 29/40] fix(core+promql): resolve HAVING against agg output schema (#5); cap nesting depth (#8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Final self-review (#5) fixes. #5 The non-fused Aggregate arm resolved HAVING against the aggregate's *input* schema, but HAVING references the aggregate's *output* columns (group keys + agg results). It now resolves against the derived output schema (output_schema_for_aggregate). Latent today (front ends emit having:None; SQL HAVING arrives as a Filter-over-Aggregate) but now correct for when an L2 Aggregate.having is populated. #8 PromQL lowering recursed over the parser AST (walk + mutually-recursive helpers, extract_matrix, lower_inner) with no depth limit — a pathologically nested query could overflow the stack. A bounded `check_depth` pass now rejects nesting beyond MAX_DEPTH (256) up front (the check itself recurses at most MAX_DEPTH frames). SQL nesting is already bounded by DataFusion's parser recursion limit. Tests: HAVING `n` resolves to the count output column (index 2), not the input schema; 300-deep nested parens return an error, not a crash. 120 tests green, clippy -D warnings + fmt clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/core/src/intent_algebra/lower.rs | 68 +++++++++++++++++++++++-- crates/lower/src/promql.rs | 46 +++++++++++++++++ crates/lower/tests/promql_lowering.rs | 9 ++++ 3 files changed, 118 insertions(+), 5 deletions(-) diff --git a/crates/core/src/intent_algebra/lower.rs b/crates/core/src/intent_algebra/lower.rs index 61ae5370..504d99b0 100644 --- a/crates/core/src/intent_algebra/lower.rs +++ b/crates/core/src/intent_algebra/lower.rs @@ -14,7 +14,9 @@ use thiserror::Error; use crate::intent_algebra::agg_intent::AggIntent; use crate::intent_algebra::binder::Binder; -use crate::intent_algebra::column_resolution::{resolve_expr, resolve_named_keys, ResolveError}; +use crate::intent_algebra::column_resolution::{ + output_schema_for_aggregate, resolve_expr, resolve_named_keys, ResolveError, +}; use crate::intent_algebra::expr_ir::{L2Expr, L3Expr, L3Scalar}; use crate::intent_algebra::names::BindingName; use crate::intent_algebra::query_expr::{ @@ -145,27 +147,32 @@ pub fn convert( } // Plain canonical `Aggregate`: multi-agg or HAVING-bearing. Keys + - // per-reducer input columns resolve against the child's schema. + // per-reducer input columns resolve against the child's (input) + // schema; HAVING references the aggregate's *output* columns, so it + // resolves against the derived output schema instead. let child = convert(input, fallback, acc)?; let child_schema = child.output_schema()?; let by = resolve_named_keys(keys, &child_schema)?; - let intents = aggs + let intents: Vec = aggs .iter() .map(|item| -> Result { let col = resolve_agg_col(&item.col, &child_schema)?; Ok(agg_func_to_intent(&item.func, acc, col)) }) .collect::, _>>()?; + let output_names: Vec = aggs.iter().map(|item| item.alias.clone()).collect(); let having = having .as_ref() .map(|h| -> Result { - Ok(Predicate(resolve_expr(h, &child_schema)?)) + let out_schema = + output_schema_for_aggregate(&child_schema, &by, &intents, &output_names); + Ok(Predicate(resolve_expr(h, &out_schema)?)) }) .transpose()?; CQueryExpr::Aggregate { by, aggs: intents, - output_names: aggs.iter().map(|item| item.alias.clone()).collect(), + output_names, having, child: Box::new(child), } @@ -432,6 +439,7 @@ fn agg_func_to_intent(func: &AggFunc, acc: &AccuracyTarget, col: Option > 5` — HAVING references the aggregate + /// OUTPUT column (`n`), absent from the input schema, so it must resolve + /// against the derived output schema `[region(0), tot(1), n(2)]`. + #[test] + fn having_resolves_against_aggregate_output_schema() { + let schema = Schema::new(vec![ + col("region", DataType::Utf8), + col("bytes", DataType::Int64), + ]); + let tree = LQueryExpr::Aggregate { + keys: vec!["region".into()], + aggs: vec![ + AggItem { + alias: "tot".into(), + func: AggFunc::Sum, + col: ColumnRef::Named("bytes".into()), + distinct: false, + }, + AggItem { + alias: "n".into(), + func: AggFunc::Count, + col: ColumnRef::Wildcard, + distinct: false, + }, + ], + having: Some(L2Expr::Compare { + left: Box::new(L2Expr::Column(ColumnRef::Named("n".into()))), + op: CompareOp::Gt, + right: Box::new(L2Expr::Literal(L3Scalar::Int64(5))), + }), + input: Box::new(LQueryExpr::Source(SourceSpec::with_schema("t", schema))), + }; + let l3 = convert(&tree, &Schema::default(), &AccuracyTarget::Exact).unwrap(); + let CQueryExpr::Aggregate { + having: Some(having), + .. + } = &l3 + else { + panic!("expected Aggregate with HAVING, got {l3:?}"); + }; + let L3Expr::Compare { left, .. } = &having.0 else { + panic!("expected Compare HAVING, got {:?}", having.0); + }; + assert_eq!( + **left, + L3Expr::Column(2), + "HAVING `n` resolves to the count output column (index 2), not the input schema" + ); + } } diff --git a/crates/lower/src/promql.rs b/crates/lower/src/promql.rs index 27046341..297a5ef4 100644 --- a/crates/lower/src/promql.rs +++ b/crates/lower/src/promql.rs @@ -92,13 +92,59 @@ struct Inner { func: Option, } +/// Maximum PromQL expression nesting depth the walker accepts. Real queries +/// nest only a handful deep; this bounds the recursive descent (`walk` and the +/// mutually-recursive helpers) so a pathologically nested query is rejected +/// rather than overflowing the stack. +const MAX_DEPTH: usize = 256; + impl PromqlLowerer { pub fn lower(query: &str) -> Result { let ast = parser::parse(query).map_err(LoweringError::Parse)?; + // Reject over-deep nesting up front, so the (mutually-recursive) walk + // below cannot blow the stack. The check itself recurses at most + // `MAX_DEPTH` frames before erroring, so it is bounded too. + check_depth(&ast, MAX_DEPTH)?; walk(&ast) } } +/// Bounded depth check over the parser AST: errors once nesting would exceed +/// `budget` frames, descending into every child expression. +fn check_depth(expr: &Expr, budget: usize) -> Result<()> { + let Some(budget) = budget.checked_sub(1) else { + return Err(LoweringError::UnsupportedFeature(format!( + "query nesting exceeds the {MAX_DEPTH}-level limit" + ))); + }; + match expr { + Expr::Aggregate(a) => { + check_depth(&a.expr, budget)?; + if let Some(p) = &a.param { + check_depth(p, budget)?; + } + } + Expr::Unary(u) => check_depth(&u.expr, budget)?, + Expr::Binary(b) => { + check_depth(&b.lhs, budget)?; + check_depth(&b.rhs, budget)?; + } + Expr::Paren(p) => check_depth(&p.expr, budget)?, + Expr::Subquery(s) => check_depth(&s.expr, budget)?, + Expr::Call(c) => { + for arg in &c.args.args { + check_depth(arg, budget)?; + } + } + Expr::MatrixSelector(_) + | Expr::VectorSelector(_) + | Expr::NumberLiteral(_) + | Expr::StringLiteral(_) + | Expr::Extension(_) => {} + } + Ok(()) +} + fn walk(expr: &Expr) -> Result { match expr { Expr::Aggregate(agg) => walk_aggregate(agg), diff --git a/crates/lower/tests/promql_lowering.rs b/crates/lower/tests/promql_lowering.rs index 8c0279e4..4ea07cf8 100644 --- a/crates/lower/tests/promql_lowering.rs +++ b/crates/lower/tests/promql_lowering.rs @@ -454,6 +454,15 @@ fn function_wrapped_range_vector_is_rejected_not_stripped() { ); } +#[test] +fn pathologically_nested_query_is_rejected_not_stack_overflow() { + // 300 nested parens parse fine but exceed the walker's depth limit (256); + // it must return an error, not overflow the stack. + let q = format!("{}m{}", "(".repeat(300), ")".repeat(300)); + let err = lower_promql(&q, AccuracyTarget::Exact).unwrap_err(); + assert!(format!("{err}").contains("nesting"), "got {err}"); +} + // ── accuracy propagation ────────────────────────────────────────────────────── #[test] From 6409ef3629976397bb204c05112b669446681697 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 27 May 2026 07:57:35 -0600 Subject: [PATCH 30/40] feat(sql): restore SQL window functions on the positional IR (#11) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-introduces the analytic-window node #4 had (dropped in the re-target), now positional + with the output-name fix #4 left as a TODO. - query_expr: `WindowFuncKind` (RowNumber/Rank/DenseRank/Lag/Lead/FirstValue/ LastValue/NthValue/Sum/Avg/Count/Min/Max) + `WindowFunc { func, args, partition_by: Vec, order_by, output_name, child }`. Schema derivation = child schema + one window-output column typed per func. - relational L2: name-based `WindowFunc { …, partition_by: Vec, … }`; walk/leaf_source updated. - converter: resolves args / partition_by / order_by positionally against the child schema. - sql front end: `lower_window` (re-targeted to L2) + `lower_window_func_kind`; `LogicalPlan::Window` no longer rejected. `output_name` is taken from the Window plan's schema (the name an enclosing Projection references) — fixing #4's hardcoded-name TODO. NthValue's N lifted from the literal 2nd arg. One window function per node; frames not modelled (default frame assumed). Tests: ROW_NUMBER() OVER (PARTITION BY service ORDER BY bytes DESC) → WindowFunc with partition_by=[1], order_by=[Column(3) DESC], Int64 output column resolved in the root projection; SUM(bytes) OVER (…) → WindowFunc{Sum, args:[Column(3)]}. 122 tests green, clippy -D warnings + fmt clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/core/src/intent_algebra/lower.rs | 37 ++++++ crates/core/src/intent_algebra/mod.rs | 2 +- crates/core/src/intent_algebra/query_expr.rs | 77 ++++++++++++ crates/core/src/intent_algebra/relational.rs | 14 ++- crates/lower/src/sql/mod.rs | 123 ++++++++++++++++++- crates/lower/tests/promql_conformance.rs | 1 + crates/lower/tests/sql_lowering.rs | 70 ++++++++++- 7 files changed, 316 insertions(+), 8 deletions(-) diff --git a/crates/core/src/intent_algebra/lower.rs b/crates/core/src/intent_algebra/lower.rs index 504d99b0..22994f8e 100644 --- a/crates/core/src/intent_algebra/lower.rs +++ b/crates/core/src/intent_algebra/lower.rs @@ -325,6 +325,43 @@ pub fn convert( child: Box::new(convert(input, fallback, acc)?), }, + // Analytic window: args / partition-by / order-by resolve positionally + // against the child's output schema. + LQueryExpr::WindowFunc { + func, + args, + partition_by, + order_by, + output_name, + input, + } => { + let child = convert(input, fallback, acc)?; + let child_schema = child.output_schema()?; + let args = args + .iter() + .map(|a| resolve_expr(a, &child_schema)) + .collect::, _>>()?; + let partition_by = resolve_named_keys(partition_by, &child_schema)?; + let order_by = order_by + .iter() + .map(|k| -> Result { + Ok(SortKey { + expr: resolve_expr(&k.expr, &child_schema)?, + ascending: k.ascending, + nulls_first: k.nulls_first, + }) + }) + .collect::, _>>()?; + CQueryExpr::WindowFunc { + func: func.clone(), + args, + partition_by, + order_by, + output_name: output_name.clone(), + child: Box::new(child), + } + } + LQueryExpr::BinaryOp { op, lhs, diff --git a/crates/core/src/intent_algebra/mod.rs b/crates/core/src/intent_algebra/mod.rs index 96309d9d..fd71d2b9 100644 --- a/crates/core/src/intent_algebra/mod.rs +++ b/crates/core/src/intent_algebra/mod.rs @@ -36,6 +36,6 @@ pub use names::{BindingName, QueryId}; pub use query_expr::{ BinaryOpKind, BindingScope, ColumnRef, DataModel, GroupSide, JoinKind, PartitionKeys, Predicate, ProjectItem, QueryExpr, QueryExprError, SetOpKind, SortKey, Source, VectorGrouping, - VectorMatch, VectorMatchKind, WindowKind, + VectorMatch, VectorMatchKind, WindowFuncKind, WindowKind, }; pub use schema::{cse_reuse_is_legal, Column, ColumnId, CseError, DataType, Schema}; diff --git a/crates/core/src/intent_algebra/query_expr.rs b/crates/core/src/intent_algebra/query_expr.rs index c27e53b1..b595f2c4 100644 --- a/crates/core/src/intent_algebra/query_expr.rs +++ b/crates/core/src/intent_algebra/query_expr.rs @@ -162,6 +162,27 @@ pub enum SetOpKind { Except, } +/// SQL analytic window function (`fn(...) OVER (…)`). Distinct from a streaming +/// time `Window`: this is an analytic frame over already-materialised rows. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WindowFuncKind { + RowNumber, + Rank, + DenseRank, + Lag, + Lead, + FirstValue, + LastValue, + /// `NTH_VALUE(expr, n)` — `n` is resolved from the (literal) 2nd argument. + NthValue(Option), + Sum, + Avg, + Count, + Min, + Max, +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct SortKey { pub expr: L3Expr, @@ -313,6 +334,22 @@ pub enum QueryExpr { child: Box, }, + /// SQL analytic window function: `func(args) OVER (PARTITION BY … ORDER BY …)`. + /// Output schema = child schema + one column named `output_name` (the name + /// the enclosing `Project` references). Window frames are not modelled yet. + WindowFunc { + func: WindowFuncKind, + /// Operand expressions (`LAG(value)` → `[Column(value_id)]`); empty for + /// the rank-only functions (`ROW_NUMBER`/`RANK`/`DENSE_RANK`). + args: Vec, + partition_by: Vec, + order_by: Vec, + /// The output column's name — DataFusion's window-expr field name, so a + /// `Project` above resolves it (cf. `Aggregate.output_names`). + output_name: String, + child: Box, + }, + /// Arithmetic / comparison / boolean composition (PromQL binary ops). BinaryOp { op: BinaryOpKind, @@ -503,6 +540,46 @@ impl QueryExpr { unique_keys: Vec::new(), }) } + // ψ-analytic — child schema + one appended window-output column. + QueryExpr::WindowFunc { + func, + args, + output_name, + child, + .. + } => { + let mut out = child.output_schema_in(scope)?; + // First operand's (dtype, nullable) from the child schema, owned + // so the borrow ends before we append. + let arg = args.first().and_then(|a| match a { + L3Expr::Column(id) => out.columns.get(*id), + _ => None, + }); + let arg_dtype = || arg.map_or(DataType::Float64, |c| c.dtype.clone()); + let (dtype, nullable) = match func { + WindowFuncKind::RowNumber + | WindowFuncKind::Rank + | WindowFuncKind::DenseRank + | WindowFuncKind::Count => (DataType::Int64, false), + WindowFuncKind::Sum | WindowFuncKind::Avg => (DataType::Float64, true), + // Navigation funcs: arg type, nullable (boundary rows are NULL). + WindowFuncKind::Lag + | WindowFuncKind::Lead + | WindowFuncKind::FirstValue + | WindowFuncKind::LastValue + | WindowFuncKind::NthValue(_) => (arg_dtype(), true), + WindowFuncKind::Min | WindowFuncKind::Max => { + (arg_dtype(), arg.is_none_or(|c| c.nullable)) + } + }; + out.columns.push(Column { + name: output_name.clone(), + dtype, + nullable, + }); + Ok(out) + } + QueryExpr::BinaryOp { lhs, .. } => lhs.output_schema_in(scope), } } diff --git a/crates/core/src/intent_algebra/relational.rs b/crates/core/src/intent_algebra/relational.rs index af65593c..7c8091b3 100644 --- a/crates/core/src/intent_algebra/relational.rs +++ b/crates/core/src/intent_algebra/relational.rs @@ -10,7 +10,7 @@ use std::time::Duration; use super::expr_ir::L2Expr; -pub use super::query_expr::{BinaryOpKind, ColumnRef, PartitionKeys, VectorMatch}; +pub use super::query_expr::{BinaryOpKind, ColumnRef, PartitionKeys, VectorMatch, WindowFuncKind}; use super::schema::Schema; /// SELECT-list item at Layer 2 — a name-based [`L2Expr`] + optional alias. @@ -190,6 +190,16 @@ pub enum QueryExpr { input: Box, }, + /// SQL analytic window function `func(args) OVER (PARTITION BY … ORDER BY …)`. + WindowFunc { + func: WindowFuncKind, + args: Vec, + partition_by: Vec, + order_by: Vec, + output_name: String, + input: Box, + }, + /// Binary op between two instant-vector expressions (PromQL `+`, `/`, …). BinaryOp { op: BinaryOpKind, @@ -214,6 +224,7 @@ impl QueryExpr { | QueryExpr::TopK { input, .. } | QueryExpr::Sort { input, .. } | QueryExpr::Limit { input, .. } + | QueryExpr::WindowFunc { input, .. } | QueryExpr::PromQLSubquery { input, .. } => input.walk(f), QueryExpr::Merge { inputs } => { for i in inputs { @@ -250,6 +261,7 @@ impl QueryExpr { | QueryExpr::TopK { input, .. } | QueryExpr::Sort { input, .. } | QueryExpr::Limit { input, .. } + | QueryExpr::WindowFunc { input, .. } | QueryExpr::PromQLSubquery { input, .. } => input.leaf_source(), QueryExpr::Merge { inputs } => inputs.first()?.leaf_source(), QueryExpr::Join { left, .. } diff --git a/crates/lower/src/sql/mod.rs b/crates/lower/src/sql/mod.rs index dd29b803..57431e59 100644 --- a/crates/lower/src/sql/mod.rs +++ b/crates/lower/src/sql/mod.rs @@ -12,13 +12,17 @@ use std::sync::Arc; use datafusion::common::ScalarValue; use datafusion::datasource::MemTable; -use datafusion::logical_expr::{self, Distinct, Expr, JoinType, LogicalPlan}; +use datafusion::logical_expr::{ + self, Distinct, Expr, JoinType, LogicalPlan, WindowFunctionDefinition, +}; use datafusion::prelude::SessionContext; use asap_control_core::intent_algebra::relational::{ AggFunc, AggItem, L2ProjectItem, L2SortKey, QueryExpr as L2, SourceSpec, }; -use asap_control_core::intent_algebra::{ColumnRef, CompareOp, JoinKind, L2Expr, SetOpKind}; +use asap_control_core::intent_algebra::{ + ColumnRef, CompareOp, JoinKind, L2Expr, L3Scalar, SetOpKind, WindowFuncKind, +}; use crate::error::LoweringError; @@ -96,9 +100,7 @@ impl<'a> SqlLowerer<'a> { }) }) } - LogicalPlan::Window(_) => Err(LoweringError::UnsupportedFeature( - "SQL window functions (no L3 analytic-window node yet)".into(), - )), + LogicalPlan::Window(window) => self.lower_window(window), LogicalPlan::Join(join) => self.lower_join(join), LogicalPlan::Subquery(_) => Err(LoweringError::UnsupportedFeature("subquery".into())), LogicalPlan::SubqueryAlias(alias) => { @@ -180,6 +182,80 @@ impl<'a> SqlLowerer<'a> { }) } + /// `func(args) OVER (PARTITION BY … ORDER BY …)`. One window function per + /// plan node; window frames are not modelled yet (default frame assumed). + fn lower_window(&self, window: &logical_expr::Window) -> Result { + if window.window_expr.len() > 1 { + return Err(LoweringError::UnsupportedFeature(format!( + "multiple window functions in one plan node (got {}); split them", + window.window_expr.len() + ))); + } + let input = Box::new(self.lower_plan(&window.input)?); + let first = window + .window_expr + .first() + .ok_or_else(|| LoweringError::InvalidExpression("empty window expression".into()))?; + let Expr::WindowFunction(wf) = first else { + return Err(LoweringError::InvalidExpression( + "expected a window function in Window plan node".into(), + )); + }; + let func = lower_window_func_kind(&wf.fun)?; + let mut args = wf + .args + .iter() + .map(df_expr_to_l2) + .collect::, _>>()?; + // Nth_value: lift N from the (literal) 2nd arg, keep only the column. + let func = if matches!(func, WindowFuncKind::NthValue(None)) { + let n = match args.get(1) { + Some(L2Expr::Literal(L3Scalar::Int64(n))) if *n > 0 => *n as u64, + other => { + return Err(LoweringError::InvalidExpression(format!( + "NTH_VALUE requires a positive integer literal 2nd arg, got {other:?}" + ))) + } + }; + args.truncate(1); + WindowFuncKind::NthValue(Some(n)) + } else { + func + }; + let partition_by = wf + .partition_by + .iter() + .map(expr_to_group_name) + .collect::, _>>()?; + let order_by = wf + .order_by + .iter() + .map(|s| { + df_expr_to_l2(&s.expr).map(|expr| L2SortKey { + expr, + ascending: s.asc, + nulls_first: s.nulls_first, + }) + }) + .collect::, _>>()?; + // The window plan's schema is `[input fields …, window output]`; the last + // field is the window column's name (what an enclosing Project references). + let output_name = window + .schema + .fields() + .last() + .map(|f| f.name().clone()) + .unwrap_or_else(|| "window".into()); + Ok(L2::WindowFunc { + func, + args, + partition_by, + order_by, + output_name, + input, + }) + } + fn lower_projection(&self, proj: &logical_expr::Projection) -> Result { // SELECT * — no column constraint; pass through without a Project. if proj.expr.iter().any(|e| matches!(e, Expr::Wildcard { .. })) { @@ -508,3 +584,40 @@ fn find_aggregate(plan: &LogicalPlan) -> Option<&logical_expr::Aggregate> { _ => None, } } + +/// Map a DataFusion window-function definition to the L3 [`WindowFuncKind`]. +/// `NthValue` is returned with `None`; `lower_window` fills in `n` from args. +fn lower_window_func_kind(fun: &WindowFunctionDefinition) -> Result { + let unsupported = |what: &str, name: &str| { + LoweringError::UnsupportedFeature(format!("window {what}: {name}")) + }; + match fun { + WindowFunctionDefinition::WindowUDF(udf) => match udf.name().to_lowercase().as_str() { + "row_number" => Ok(WindowFuncKind::RowNumber), + "rank" => Ok(WindowFuncKind::Rank), + "dense_rank" => Ok(WindowFuncKind::DenseRank), + "lag" => Ok(WindowFuncKind::Lag), + "lead" => Ok(WindowFuncKind::Lead), + "first_value" => Ok(WindowFuncKind::FirstValue), + "last_value" => Ok(WindowFuncKind::LastValue), + "nth_value" => Ok(WindowFuncKind::NthValue(None)), + other => Err(unsupported("function", other)), + }, + WindowFunctionDefinition::AggregateUDF(udf) => match udf.name().to_lowercase().as_str() { + "sum" => Ok(WindowFuncKind::Sum), + "avg" | "mean" => Ok(WindowFuncKind::Avg), + "count" => Ok(WindowFuncKind::Count), + "min" => Ok(WindowFuncKind::Min), + "max" => Ok(WindowFuncKind::Max), + other => Err(unsupported("aggregate", other)), + }, + WindowFunctionDefinition::BuiltInWindowFunction(biwf) => { + use datafusion::logical_expr::BuiltInWindowFunction; + match biwf { + BuiltInWindowFunction::FirstValue => Ok(WindowFuncKind::FirstValue), + BuiltInWindowFunction::LastValue => Ok(WindowFuncKind::LastValue), + BuiltInWindowFunction::NthValue => Ok(WindowFuncKind::NthValue(None)), + } + } + } +} diff --git a/crates/lower/tests/promql_conformance.rs b/crates/lower/tests/promql_conformance.rs index 83f1ebf7..def1b16d 100644 --- a/crates/lower/tests/promql_conformance.rs +++ b/crates/lower/tests/promql_conformance.rs @@ -75,6 +75,7 @@ fn collect(e: &QueryExpr, out: &mut Vec) { | QueryExpr::Limit { child, .. } | QueryExpr::Subquery { child, .. } | QueryExpr::Distinct { child, .. } + | QueryExpr::WindowFunc { child, .. } | QueryExpr::Project { child, .. } => collect(child, out), QueryExpr::BinaryOp { lhs, rhs, .. } => { collect(lhs, out); diff --git a/crates/lower/tests/sql_lowering.rs b/crates/lower/tests/sql_lowering.rs index b1dc0f92..1b8823a1 100644 --- a/crates/lower/tests/sql_lowering.rs +++ b/crates/lower/tests/sql_lowering.rs @@ -5,7 +5,9 @@ //! canonical L3 (the same converter the PromQL path uses). use asap_control_core::intent_algebra::schema::{Column, DataType, Schema}; -use asap_control_core::intent_algebra::{AggIntent, JoinKind, QueryExpr, Source}; +use asap_control_core::intent_algebra::{ + AggIntent, JoinKind, L3Expr, QueryExpr, Source, WindowFuncKind, +}; use asap_control_core::types::AccuracyTarget; use asap_control_lower::{lower_sql, SqlCatalog}; @@ -293,3 +295,69 @@ async fn semi_join_is_rejected_not_mislowered() { "semi-join / IN-subquery should be rejected in v1" ); } + +/// Find the first `WindowFunc` node along the single-child spine. +fn find_windowfunc(qe: &QueryExpr) -> Option<&QueryExpr> { + match qe { + QueryExpr::WindowFunc { .. } => Some(qe), + QueryExpr::Project { child, .. } + | QueryExpr::Filter { child, .. } + | QueryExpr::Aggregate { child, .. } + | QueryExpr::Window { child, .. } + | QueryExpr::Partition { child, .. } + | QueryExpr::Distinct { child, .. } + | QueryExpr::Sort { child, .. } + | QueryExpr::Limit { child, .. } + | QueryExpr::Subquery { child, .. } => find_windowfunc(child), + _ => None, + } +} + +#[tokio::test] +async fn window_function_lowers_to_positional_windowfunc() { + // ROW_NUMBER() OVER (PARTITION BY service ORDER BY bytes DESC). + let qe = lower( + "SELECT service, ROW_NUMBER() OVER (PARTITION BY service ORDER BY bytes DESC) \ + FROM metrics", + ) + .await; + let win = find_windowfunc(&qe).expect("expected a WindowFunc node"); + let QueryExpr::WindowFunc { + func, + partition_by, + order_by, + .. + } = win + else { + unreachable!("find_windowfunc only returns WindowFunc"); + }; + assert_eq!(*func, WindowFuncKind::RowNumber); + assert_eq!(partition_by, &vec![1], "PARTITION BY service → col 1"); + assert_eq!(order_by.len(), 1); + assert_eq!( + order_by[0].expr, + L3Expr::Column(3), + "ORDER BY bytes → col 3" + ); + assert!(!order_by[0].ascending, "DESC"); + + // The window output column is appended to the schema (Int64 for ROW_NUMBER), + // and the enclosing projection resolves it (output_name threading). + let schema = qe.output_schema().expect("root schema"); + assert!( + schema.columns.iter().any(|c| c.dtype == DataType::Int64), + "row_number output column present, got {:?}", + schema.columns + ); +} + +#[tokio::test] +async fn window_aggregate_lowers_to_windowfunc() { + let qe = lower("SELECT service, SUM(bytes) OVER (PARTITION BY service) FROM metrics").await; + let win = find_windowfunc(&qe).expect("expected a WindowFunc node"); + let QueryExpr::WindowFunc { func, args, .. } = win else { + unreachable!(); + }; + assert_eq!(*func, WindowFuncKind::Sum); + assert_eq!(args, &vec![L3Expr::Column(3)], "SUM(bytes) → arg col 3"); +} From 61533c60c439445d823ca9cbe1d264b47d267458 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 27 May 2026 08:14:51 -0600 Subject: [PATCH 31/40] fix(sql): alias-qualify schema columns so join keys disambiguate (issue #7) A join over columns that share a name (`metrics.service = hosts.service`, or any self-join) previously collapsed both refs onto the first matching position, because `ColumnRef` resolution was purely name-based. Carry the table/alias qualifier through the schema and resolve it: - `Column` gains `table: Option` (+ `Column::new` / `with_table`); `#[serde(default)]` keeps the field backward-compatible. - `Schema::column_id_qualified(table, name)` matches on both, with the bare-name lookup as fallback for unqualified schemas. - `ColumnRef::Qualified { table, name }` (emitted by `df_expr_to_l2` from DataFusion's relation qualifier) resolves via the qualified lookup. - The SQL front end qualifies a scan's columns with the table name, and a `SubqueryAlias` over a table re-qualifies them with the alias, so a self-join's two sides are distinguishable. Tests: `metrics JOIN hosts ON metrics.service = hosts.service` binds to distinct positions [1,4]; `metrics a JOIN metrics b ON a.service = b.service` binds to [1,5]. Full workspace green (124 tests), clippy + fmt clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/core/src/intent_algebra/agg_intent.rs | 12 +--- crates/core/src/intent_algebra/binder.rs | 36 ++-------- .../src/intent_algebra/column_resolution.rs | 35 ++++------ crates/core/src/intent_algebra/cse.rs | 6 +- crates/core/src/intent_algebra/lower.rs | 14 ++-- crates/core/src/intent_algebra/query_expr.rs | 52 ++++++-------- crates/core/src/intent_algebra/schema.rs | 41 +++++++++-- crates/lower/src/sql/expr.rs | 10 ++- crates/lower/src/sql/mod.rs | 38 +++++++--- crates/lower/tests/sql_lowering.rs | 70 +++++++++++++++++-- 10 files changed, 193 insertions(+), 121 deletions(-) diff --git a/crates/core/src/intent_algebra/agg_intent.rs b/crates/core/src/intent_algebra/agg_intent.rs index b6ca2d8f..05a0a58e 100644 --- a/crates/core/src/intent_algebra/agg_intent.rs +++ b/crates/core/src/intent_algebra/agg_intent.rs @@ -144,11 +144,7 @@ impl AggIntent { } fn col(name: &str, dtype: DataType, nullable: bool) -> Column { - Column { - name: name.into(), - dtype, - nullable, - } + Column::new(name, dtype, nullable) } /// `0.99` → `"0_99"`, `0.5` → `"0_5"`. Used by `Quantile` output naming so @@ -227,11 +223,7 @@ mod tests { use crate::intent_algebra::schema::{Column, DataType}; fn c(name: &str, dtype: DataType) -> Column { - Column { - name: name.into(), - dtype, - nullable: false, - } + Column::new(name, dtype, false) } #[test] diff --git a/crates/core/src/intent_algebra/binder.rs b/crates/core/src/intent_algebra/binder.rs index 5d066235..ad74faa8 100644 --- a/crates/core/src/intent_algebra/binder.rs +++ b/crates/core/src/intent_algebra/binder.rs @@ -79,11 +79,7 @@ impl Binder { // Append one column per referenced-but-unknown name (group keys etc.). for name in collect_referenced_columns(tree) { if !columns.iter().any(|c| c.name == name) { - columns.push(Column { - name, - dtype: DataType::Utf8, - nullable: true, - }); + columns.push(Column::new(name, DataType::Utf8, true)); } } @@ -99,16 +95,8 @@ impl Binder { /// The conventional PromQL leaf shape: `(ts: Timestamp, value: Float64)`. fn default_leaf_columns() -> Vec { vec![ - Column { - name: "ts".into(), - dtype: DataType::Timestamp, - nullable: false, - }, - Column { - name: "value".into(), - dtype: DataType::Float64, - nullable: false, - }, + Column::new("ts", DataType::Timestamp, false), + Column::new("value", DataType::Float64, false), ] } @@ -191,21 +179,9 @@ mod tests { fn columns_for(&self, source: &str) -> Option> { (source == "known").then(|| { vec![ - Column { - name: "ts".into(), - dtype: DataType::Timestamp, - nullable: false, - }, - Column { - name: "value".into(), - dtype: DataType::Float64, - nullable: false, - }, - Column { - name: "datacenter".into(), - dtype: DataType::Utf8, - nullable: false, - }, + Column::new("ts", DataType::Timestamp, false), + Column::new("value", DataType::Float64, false), + Column::new("datacenter", DataType::Utf8, false), ] }) } diff --git a/crates/core/src/intent_algebra/column_resolution.rs b/crates/core/src/intent_algebra/column_resolution.rs index 1bde6e06..35baa74c 100644 --- a/crates/core/src/intent_algebra/column_resolution.rs +++ b/crates/core/src/intent_algebra/column_resolution.rs @@ -32,16 +32,8 @@ pub enum ResolveError { pub fn infer_source_schema(_metric_or_table: &str) -> Schema { Schema::with_time_index( vec![ - Column { - name: "ts".into(), - dtype: DataType::Timestamp, - nullable: false, - }, - Column { - name: "value".into(), - dtype: DataType::Float64, - nullable: false, - }, + Column::new("ts", DataType::Timestamp, false), + Column::new("value", DataType::Float64, false), ], 0, Vec::new(), @@ -65,6 +57,15 @@ pub fn resolve_column_ref(col: &ColumnRef, schema: &Schema) -> Result schema + .column_id_qualified(table, name) + .or_else(|| schema.column_id(name)) + .ok_or_else(|| ResolveError::NotFound { + name: format!("{table}.{name}"), + available: schema.columns.iter().map(|c| c.name.clone()).collect(), + }), ColumnRef::SampleValue => schema .column_id("value") .or_else(|| { @@ -186,11 +187,7 @@ pub fn output_schema_for_aggregate( let probe = value_col_idx .and_then(|i| input.columns.get(i)) .cloned() - .unwrap_or(Column { - name: "value".into(), - dtype: DataType::Float64, - nullable: false, - }); + .unwrap_or_else(|| Column::new("value", DataType::Float64, false)); for (i, intent) in aggs.iter().enumerate() { let in_col = intent .input_col() @@ -242,11 +239,9 @@ mod tests { #[test] fn aggregate_strips_time_and_keeps_unique_keys() { let mut input = infer_source_schema("m"); - input.columns.push(Column { - name: "host".into(), - dtype: DataType::Utf8, - nullable: false, - }); + input + .columns + .push(Column::new("host", DataType::Utf8, false)); let out = output_schema_for_aggregate(&input, &[2usize], &[AggIntent::Sum { col: None }], &[]); assert_eq!(out.columns.len(), 2); // host, sum diff --git a/crates/core/src/intent_algebra/cse.rs b/crates/core/src/intent_algebra/cse.rs index 1a04085d..9d5401e1 100644 --- a/crates/core/src/intent_algebra/cse.rs +++ b/crates/core/src/intent_algebra/cse.rs @@ -122,11 +122,7 @@ mod tests { use std::time::Duration; fn col(name: &str, dtype: DataType) -> Column { - Column { - name: name.into(), - dtype, - nullable: false, - } + Column::new(name, dtype, false) } fn ts_scan() -> QueryExpr { diff --git a/crates/core/src/intent_algebra/lower.rs b/crates/core/src/intent_algebra/lower.rs index 22994f8e..c7ed2be4 100644 --- a/crates/core/src/intent_algebra/lower.rs +++ b/crates/core/src/intent_algebra/lower.rs @@ -431,6 +431,14 @@ fn resolve_agg_col(col: &ColumnRef, schema: &Schema) -> Result, available: schema.columns.iter().map(|c| c.name.clone()).collect(), }) } + ColumnRef::Qualified { table, name } => schema + .column_id_qualified(table, name) + .or_else(|| schema.column_id(name)) + .map(Some) + .ok_or_else(|| ResolveError::NotFound { + name: format!("{table}.{name}"), + available: schema.columns.iter().map(|c| c.name.clone()).collect(), + }), ColumnRef::SampleValue | ColumnRef::Wildcard => Ok(None), } } @@ -484,11 +492,7 @@ mod tests { use crate::intent_algebra::schema::{Column, DataType, Schema}; fn col(name: &str, dtype: DataType) -> Column { - Column { - name: name.into(), - dtype, - nullable: false, - } + Column::new(name, dtype, false) } /// A SQL-shaped `SELECT SUM(bytes), AVG(latency) FROM t` lowers each diff --git a/crates/core/src/intent_algebra/query_expr.rs b/crates/core/src/intent_algebra/query_expr.rs index b595f2c4..c51e9e19 100644 --- a/crates/core/src/intent_algebra/query_expr.rs +++ b/crates/core/src/intent_algebra/query_expr.rs @@ -70,6 +70,13 @@ impl Source { #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum ColumnRef { Named(String), + /// Table-qualified reference (`t.col` / `alias.col`). Resolved by + /// `(table, name)` so a column name shared across a join (`a.k` vs `b.k`) + /// binds to the correct side. + Qualified { + table: String, + name: String, + }, /// The implicit metric sample value (PromQL — always the series value). SampleValue, /// All rows / COUNT(*). @@ -403,11 +410,7 @@ impl QueryExpr { let probe = value_col_idx .and_then(|i| in_schema.columns.get(i)) .cloned() - .unwrap_or(Column { - name: "value".into(), - dtype: DataType::Float64, - nullable: false, - }); + .unwrap_or_else(|| Column::new("value", DataType::Float64, false)); // Each reducer types off its own input column (`SUM(bytes)` vs // `AVG(latency)` in one node); `None` falls back to the sample- // value probe (PromQL's single-column convention). A non-empty @@ -458,20 +461,18 @@ impl QueryExpr { // is re-found by name. QueryExpr::Project { cols, child } => { let in_schema = child.output_schema_in(scope)?; - let columns: Vec = - cols.iter() - .enumerate() - .map(|(i, item)| { - let (dtype, nullable) = infer_expr_type(&item.expr, &in_schema); - Column { - name: item.alias.clone().unwrap_or_else(|| { - default_proj_name(&item.expr, i, &in_schema) - }), - dtype, - nullable, - } - }) - .collect(); + let columns: Vec = cols + .iter() + .enumerate() + .map(|(i, item)| { + let (dtype, nullable) = infer_expr_type(&item.expr, &in_schema); + let name = item + .alias + .clone() + .unwrap_or_else(|| default_proj_name(&item.expr, i, &in_schema)); + Column::new(name, dtype, nullable) + }) + .collect(); let time_index = columns.iter().position(|c| c.name == "ts"); Ok(Schema { columns, @@ -572,11 +573,8 @@ impl QueryExpr { (arg_dtype(), arg.is_none_or(|c| c.nullable)) } }; - out.columns.push(Column { - name: output_name.clone(), - dtype, - nullable, - }); + out.columns + .push(Column::new(output_name.clone(), dtype, nullable)); Ok(out) } @@ -678,11 +676,7 @@ mod tests { use crate::intent_algebra::expr_ir::{ArithOp, CompareOp}; fn col(name: &str, dtype: DataType, nullable: bool) -> Column { - Column { - name: name.into(), - dtype, - nullable, - } + Column::new(name, dtype, nullable) } fn scan( diff --git a/crates/core/src/intent_algebra/schema.rs b/crates/core/src/intent_algebra/schema.rs index 0a895cce..ea63846b 100644 --- a/crates/core/src/intent_algebra/schema.rs +++ b/crates/core/src/intent_algebra/schema.rs @@ -42,6 +42,30 @@ pub struct Column { /// Whether NULL values are allowed in this column. PromQL value /// columns are non-nullable; SQL columns inherit their DDL nullability. pub nullable: bool, + /// Optional table/alias qualifier (SQL `t.col` / `t AS a` → `a`). Travels + /// with the column through joins so a `ColumnRef::Qualified` can pick the + /// right side when both carry the same `name`. `None` for PromQL labels and + /// unqualified columns. + #[serde(default)] + pub table: Option, +} + +impl Column { + /// An unqualified column (`table = None`). + pub fn new(name: impl Into, dtype: DataType, nullable: bool) -> Self { + Self { + name: name.into(), + dtype, + nullable, + table: None, + } + } + + /// This column re-qualified under `table` (e.g. by a `SubqueryAlias`). + pub fn with_table(mut self, table: impl Into) -> Self { + self.table = Some(table.into()); + self + } } /// L3 column data types. Deliberately narrow: no sketch state at this @@ -117,11 +141,20 @@ impl Schema { } } - /// Look up a column by name. `None` if not present. + /// Look up a column by name (first match). `None` if not present. pub fn column_id(&self, name: &str) -> Option { self.columns.iter().position(|c| c.name == name) } + /// Look up a column by `(table, name)` qualifier — disambiguates columns + /// that share a `name` across a join (`a.k` vs `b.k`). `None` if no column + /// has both that qualifier and name. + pub fn column_id_qualified(&self, table: &str, name: &str) -> Option { + self.columns + .iter() + .position(|c| c.name == name && c.table.as_deref() == Some(table)) + } + /// Whether this schema has *any* provable unique key. The CSE pass /// reads this to decide whether two `Ref` consumers can safely share /// a producer (see `design.md` §6 line ~1284 + the unit test in @@ -209,11 +242,7 @@ mod tests { use super::*; fn col(name: &str, dtype: DataType) -> Column { - Column { - name: name.into(), - dtype, - nullable: false, - } + Column::new(name, dtype, false) } /// `cse_reuse_is_legal` accepts a producer schema with at least one diff --git a/crates/lower/src/sql/expr.rs b/crates/lower/src/sql/expr.rs index dd7cfa6d..d11d9a6a 100644 --- a/crates/lower/src/sql/expr.rs +++ b/crates/lower/src/sql/expr.rs @@ -25,7 +25,15 @@ pub(super) fn split_conjuncts(expr: &Expr) -> Vec<&Expr> { /// Returns `UnsupportedFeature` for anything not needed in v1. pub(super) fn df_expr_to_l2(expr: &Expr) -> Result { match expr { - Expr::Column(col) => Ok(L2Expr::Column(ColumnRef::Named(col.name.clone()))), + // Preserve DataFusion's relation qualifier so a column name shared + // across a join (`a.k` vs `b.k`) resolves to the correct side. + Expr::Column(col) => Ok(L2Expr::Column(match &col.relation { + Some(rel) => ColumnRef::Qualified { + table: rel.to_string(), + name: col.name.clone(), + }, + None => ColumnRef::Named(col.name.clone()), + })), Expr::Literal(sv) => scalar_value_to_l3(sv).map(L2Expr::Literal), diff --git a/crates/lower/src/sql/mod.rs b/crates/lower/src/sql/mod.rs index 57431e59..87eee6b4 100644 --- a/crates/lower/src/sql/mod.rs +++ b/crates/lower/src/sql/mod.rs @@ -20,6 +20,7 @@ use datafusion::prelude::SessionContext; use asap_control_core::intent_algebra::relational::{ AggFunc, AggItem, L2ProjectItem, L2SortKey, QueryExpr as L2, SourceSpec, }; +use asap_control_core::intent_algebra::schema::Schema; use asap_control_core::intent_algebra::{ ColumnRef, CompareOp, JoinKind, L2Expr, L3Scalar, SetOpKind, WindowFuncKind, }; @@ -104,12 +105,14 @@ impl<'a> SqlLowerer<'a> { LogicalPlan::Join(join) => self.lower_join(join), LogicalPlan::Subquery(_) => Err(LoweringError::UnsupportedFeature("subquery".into())), LogicalPlan::SubqueryAlias(alias) => { - // A bare table alias is transparent; a derived table (inline view) - // is unsupported in v1. + // An alias over a table re-qualifies the scan's columns with the + // alias (so `a.col` / `b.col` in a self-join disambiguate). A + // derived table (inline view) is unsupported in v1. match alias.input.as_ref() { - LogicalPlan::TableScan(_) | LogicalPlan::SubqueryAlias(_) => { - self.lower_plan(&alias.input) + LogicalPlan::TableScan(scan) => { + self.scan_source(&scan.table_name.to_string(), &alias.alias.to_string()) } + LogicalPlan::SubqueryAlias(_) => self.lower_plan(&alias.input), _ => Err(LoweringError::UnsupportedFeature( "subquery (inline view / derived table)".into(), )), @@ -126,15 +129,32 @@ impl<'a> SqlLowerer<'a> { /// has positional column identity. Projection pushdown is left to the /// enclosing `Project` (DataFusion's unoptimized plan sets no projection). fn lower_table_scan(&self, scan: &logical_expr::TableScan) -> Result { - let table_name = scan.table_name.to_string(); + let table = scan.table_name.to_string(); + self.scan_source(&table, &table) + } + + /// A `Source` over catalog table `table`, with its columns qualified by + /// `qualifier` (the table name, or an alias from a `SubqueryAlias`) so + /// `Qualified` column refs resolve to the right side across a join. + fn scan_source(&self, table: &str, qualifier: &str) -> Result { let schema = self .catalog .tables - .get(&table_name) - .ok_or_else(|| LoweringError::TableNotFound(table_name.clone()))?; + .get(table) + .ok_or_else(|| LoweringError::TableNotFound(table.to_string()))?; + let qualified = Schema { + columns: schema + .columns + .iter() + .cloned() + .map(|c| c.with_table(qualifier)) + .collect(), + time_index: schema.time_index, + unique_keys: schema.unique_keys.clone(), + }; Ok(L2::Source(SourceSpec::with_schema( - table_name, - schema.clone(), + table.to_string(), + qualified, ))) } diff --git a/crates/lower/tests/sql_lowering.rs b/crates/lower/tests/sql_lowering.rs index 1b8823a1..a54df409 100644 --- a/crates/lower/tests/sql_lowering.rs +++ b/crates/lower/tests/sql_lowering.rs @@ -6,17 +6,13 @@ use asap_control_core::intent_algebra::schema::{Column, DataType, Schema}; use asap_control_core::intent_algebra::{ - AggIntent, JoinKind, L3Expr, QueryExpr, Source, WindowFuncKind, + AggIntent, CompareOp, JoinKind, L3Expr, QueryExpr, Source, WindowFuncKind, }; use asap_control_core::types::AccuracyTarget; use asap_control_lower::{lower_sql, SqlCatalog}; fn col(name: &str, dtype: DataType) -> Column { - Column { - name: name.into(), - dtype, - nullable: false, - } + Column::new(name, dtype, false) } /// `metrics(ts, service, latency, bytes)` + `hosts(service, region)`. @@ -255,6 +251,68 @@ async fn inner_join_lowers_to_join_over_two_scans() { assert!(matches!(right.as_ref(), QueryExpr::Scan { .. })); } +/// The two `ColumnId`s an equijoin predicate `Column(l) = Column(r)` binds to, +/// returned sorted so the assertion is independent of left/right ordering. +fn join_eq_columns(join: &QueryExpr) -> [usize; 2] { + let QueryExpr::Join { pred, .. } = join else { + unreachable!("expected a Join"); + }; + let L3Expr::Compare { + left, + op: CompareOp::Eq, + right, + } = &pred.0 + else { + panic!("expected an equijoin Compare, got {:?}", pred.0); + }; + match (left.as_ref(), right.as_ref()) { + (L3Expr::Column(l), L3Expr::Column(r)) => { + let mut cols = [*l, *r]; + cols.sort_unstable(); + cols + } + other => panic!("expected Column = Column, got {other:?}"), + } +} + +#[tokio::test] +async fn join_predicate_disambiguates_shared_column_name() { + // Issue #7: `metrics.service = hosts.service` shares a column name across the + // join. The qualified refs must bind to two *distinct* positions in the + // concatenated schema, not collapse onto the first `service`. + // metrics(ts,service,latency,bytes) ++ hosts(service,region) + // → metrics.service = col 1, hosts.service = col 4. + let qe = lower( + "SELECT metrics.bytes, hosts.region \ + FROM metrics JOIN hosts ON metrics.service = hosts.service", + ) + .await; + let join = find_join(&qe).expect("expected a Join in the tree"); + assert_eq!( + join_eq_columns(join), + [1, 4], + "join key must bind to distinct positions, not the same `service`" + ); +} + +#[tokio::test] +async fn self_join_disambiguates_via_aliases() { + // A self-join shares *every* column name; the alias qualifiers (`a`/`b`) are + // the only way to tell the two `service` columns apart. + // metrics ++ metrics → a.service = col 1, b.service = col 5 (4 cols/side). + let qe = lower( + "SELECT a.bytes, b.latency \ + FROM metrics a JOIN metrics b ON a.service = b.service", + ) + .await; + let join = find_join(&qe).expect("expected a self-Join in the tree"); + assert_eq!( + join_eq_columns(join), + [1, 5], + "self-join keys must bind to distinct sides" + ); +} + #[tokio::test] async fn aggregate_over_join_binds_against_concatenated_schema() { // GROUP BY a right-table column over a join: the key must resolve against From e0deef9e185bac51115bedee65b8b8eabd8b49a9 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 27 May 2026 08:33:16 -0600 Subject: [PATCH 32/40] fix(promql): reject unary negation instead of silently dropping the sign MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `-expr` flips the sign of every sample (`-rate(...)` negates the rate), but the PromQL walker passed `Expr::Unary` straight through to its operand, computing `+expr` — a wrong result, not a clean gap. `promql_parser::UnaryExpr` is built only by negation (`Neg`): unary `+` folds to identity and `-` folds into a negated `NumberLiteral`, so a `Unary` node always wraps a vector expression needing a sign flip. The L2 PromQL path has no negate/scalar node to model this (`walk` rejects bare scalar operands, so there's no `-1 * x` form), so reject it as an `UnsupportedFeature` gap — consistent with how offset/`@`/`without`/`group` are handled. This reclassifies ~25 corpus queries from silently-mislowered to cleanly-rejected (testdata lowered 574→549, still above the regression tripwire); SQL's `Expr::Negative` path is unaffected (it models `-1 * x`). Adds `unary_negation_is_rejected__GAP` to the conformance suite. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/lower/src/promql.rs | 10 +++++++++- crates/lower/tests/promql_conformance.rs | 12 ++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/crates/lower/src/promql.rs b/crates/lower/src/promql.rs index 297a5ef4..f9c55154 100644 --- a/crates/lower/src/promql.rs +++ b/crates/lower/src/promql.rs @@ -152,7 +152,15 @@ fn walk(expr: &Expr) -> Result { Expr::Call(call) => build(lower_inner_call(call)?, vec![], Outer::None), Expr::Binary(bin) => walk_binary(bin), Expr::Paren(p) => walk(&p.expr), - Expr::Unary(u) => walk(&u.expr), + // `UnaryExpr` is built only by negation (`Neg`); unary `+` is folded to + // identity and `-` to a negated `NumberLiteral`, so this always + // wraps a vector expression whose samples must be sign-flipped. The L2 + // PromQL path has no scalar/negate node to express that (there's no + // `-1 * x`, since `walk` rejects bare scalar operands), so reject it + // rather than silently dropping the sign and computing `+expr`. + Expr::Unary(_) => Err(LoweringError::UnsupportedFeature( + "unary negation (`-expr`): no negate/scalar node in the L2 PromQL path".into(), + )), Expr::Subquery(sq) => Ok(L2::PromQLSubquery { range: sq.range, resolution: sq.step, diff --git a/crates/lower/tests/promql_conformance.rs b/crates/lower/tests/promql_conformance.rs index def1b16d..7f4d3684 100644 --- a/crates/lower/tests/promql_conformance.rs +++ b/crates/lower/tests/promql_conformance.rs @@ -418,6 +418,18 @@ fn vector_comparison_filters() { assert!(matches!(&qe, QueryExpr::BinaryOp { op, .. } if *op == BinaryOpKind::Gt)); } +#[test] +fn unary_negation_is_rejected__GAP() { + // SEMANTICS (PromQL): `-expr` flips the sign of every sample (and `-rate(…)` + // negates the rate). With no negate/scalar node in the L2 PromQL path we + // can't model that, so it's rejected rather than silently lowered as `+expr` + // (which would compute the wrong result). `-` folds into the + // literal at parse time and is caught by the bare-scalar rejection instead. + let _ = rejected("-rate(http_errors_total[5m])"); + let _ = rejected("-some_metric"); + let _ = rejected("-metric_a or -metric_b"); +} + #[test] fn scalar_literal_operand_is_rejected__GAP() { // SEMANTICS (PromQL): `v > 10*1024*1024` filters by a scalar threshold. From 1f1f9086dbbfcacc82051582f4fb6251efff00df Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 27 May 2026 08:33:16 -0600 Subject: [PATCH 33/40] fix(lower): drop "PromQL" from the shared UnsupportedFeature label MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `LoweringError::UnsupportedFeature` is raised by both front ends (PromQL offset/`@`/`without`/negation; SQL join type/subquery/derived table), but its Display hardcoded "unsupported PromQL feature: {m}" — so a SQL user saw "unsupported PromQL feature: subquery". Make the label neutral; the message string already carries the specifics. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/lower/src/error.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/lower/src/error.rs b/crates/lower/src/error.rs index 9a17c8f7..4de1c304 100644 --- a/crates/lower/src/error.rs +++ b/crates/lower/src/error.rs @@ -39,7 +39,10 @@ impl fmt::Display for LoweringError { Self::Parse(e) => write!(f, "PromQL parse error: {e}"), Self::UnsupportedFunction(n) => write!(f, "unsupported PromQL function: {n}"), Self::UnsupportedAggregateOp(n) => write!(f, "unsupported PromQL aggregate op: {n}"), - Self::UnsupportedFeature(m) => write!(f, "unsupported PromQL feature: {m}"), + // Raised by both front ends (PromQL offset/`@`/`without`/negation; + // SQL join type/subquery/derived table), so keep the label neutral — + // the message string carries the specifics. + Self::UnsupportedFeature(m) => write!(f, "unsupported feature: {m}"), Self::MissingArgument(m) => write!(f, "missing argument: {m}"), Self::InvalidParameter(m) => write!(f, "invalid parameter: {m}"), Self::WrongLanguage(l) => write!(f, "unsupported query language: {l}"), From e0e1b47f2fe90dbe76c2f405d2bd5b661c1bc963 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 27 May 2026 08:40:28 -0600 Subject: [PATCH 34/40] test: add regression tests for the review fixes (#7 qualifiers, unary, #2, #3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Locks behavior introduced by the recent fixes, +5 tests (125→130): - promql_conformance: nested unary negation propagates rejection (`a - -b`, `sum(-x)`), not just top-level; and count→Cardinality threads the AccuracyTarget (Exact stays exact, Epsilon carried) — pins review #2. - sql_lowering: a qualified WHERE on the *duplicated* join column (`WHERE hosts.service = ...`) binds to the qualified position (4), not the first `service` (1) — extends the #7 disambiguation past the join key. - error.rs: `UnsupportedFeature` Display is language-neutral (no "PromQL") — pins the #3 fix without depending on DataFusion plan shapes. - schema.rs: `Column.table` deserializes to `None` when the key is absent (the `#[serde(default)]` backward-compat contract), and a qualified column round-trips. Full workspace green (130 tests), clippy -D warnings + fmt clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/core/src/intent_algebra/schema.rs | 20 ++++++++++++ crates/lower/src/error.rs | 15 +++++++++ crates/lower/tests/promql_conformance.rs | 36 +++++++++++++++++++++ crates/lower/tests/sql_lowering.rs | 40 ++++++++++++++++++++++++ 4 files changed, 111 insertions(+) diff --git a/crates/core/src/intent_algebra/schema.rs b/crates/core/src/intent_algebra/schema.rs index ea63846b..235afb72 100644 --- a/crates/core/src/intent_algebra/schema.rs +++ b/crates/core/src/intent_algebra/schema.rs @@ -358,4 +358,24 @@ mod tests { let back: Schema = serde_json::from_str(&json).unwrap(); assert_eq!(s, back); } + + #[test] + fn column_table_defaults_to_none_when_absent() { + // `Column.table` is `#[serde(default)]` so schemas serialized before the + // qualifier field existed still deserialize (to `table: None`) instead + // of erroring. Drop the key from a serialized column to simulate that. + let mut v = serde_json::to_value(col("svc", DataType::Utf8)).unwrap(); + assert!(v.as_object_mut().unwrap().remove("table").is_some()); + let back: Column = serde_json::from_value(v).unwrap(); + assert_eq!(back, col("svc", DataType::Utf8)); + assert!(back.table.is_none()); + } + + #[test] + fn qualified_column_serde_roundtrip() { + let c = col("service", DataType::Utf8).with_table("hosts"); + let back: Column = serde_json::from_str(&serde_json::to_string(&c).unwrap()).unwrap(); + assert_eq!(back, c); + assert_eq!(back.table.as_deref(), Some("hosts")); + } } diff --git a/crates/lower/src/error.rs b/crates/lower/src/error.rs index 4de1c304..e22002f5 100644 --- a/crates/lower/src/error.rs +++ b/crates/lower/src/error.rs @@ -69,3 +69,18 @@ impl From for LoweringError { Self::DataFusion(e) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unsupported_feature_label_is_language_neutral() { + // `UnsupportedFeature` is raised by both front ends, so its Display must + // not hardcode "PromQL" — a SQL user rejecting a subquery shouldn't see + // "unsupported PromQL feature: subquery". + let msg = LoweringError::UnsupportedFeature("subquery".into()).to_string(); + assert_eq!(msg, "unsupported feature: subquery"); + assert!(!msg.contains("PromQL"), "got: {msg}"); + } +} diff --git a/crates/lower/tests/promql_conformance.rs b/crates/lower/tests/promql_conformance.rs index 7f4d3684..5c156ee1 100644 --- a/crates/lower/tests/promql_conformance.rs +++ b/crates/lower/tests/promql_conformance.rs @@ -428,6 +428,42 @@ fn unary_negation_is_rejected__GAP() { let _ = rejected("-rate(http_errors_total[5m])"); let _ = rejected("-some_metric"); let _ = rejected("-metric_a or -metric_b"); + // Negation nested inside a larger expression propagates the rejection, + // rather than lowering the rest with the inner sign silently dropped. + let _ = rejected("http_requests_total - -http_errors_total"); + let _ = rejected("sum(-node_cpu_seconds_total)"); +} + +#[test] +fn count_maps_to_cardinality_and_inherits_accuracy() { + // SEMANTICS (review #2): PromQL `count by (...)` counts distinct series → the + // `Cardinality` intent. The workload's AccuracyTarget threads onto it: + // `Exact` stays exact (no silent HLL substitution); an approximate target is + // carried through for L4 to honor. This pins the intentional count→Cardinality + // mapping and its accuracy gating. + let exact = lower_promql("count by (job) (up)", AccuracyTarget::Exact).unwrap(); + assert!( + has(&exact, |i| matches!( + i, + AggIntent::Cardinality { + accuracy: AccuracyTarget::Exact + } + )), + "count→Cardinality must stay Exact under AccuracyTarget::Exact, got {:?}", + intents(&exact) + ); + + let approx = lower_promql("count by (job) (up)", AccuracyTarget::Epsilon(0.01)).unwrap(); + assert!( + has(&approx, |i| matches!( + i, + AggIntent::Cardinality { + accuracy: AccuracyTarget::Epsilon(e) + } if (*e - 0.01).abs() < 1e-9 + )), + "count→Cardinality must carry the approximate target, got {:?}", + intents(&approx) + ); } #[test] diff --git a/crates/lower/tests/sql_lowering.rs b/crates/lower/tests/sql_lowering.rs index a54df409..868e5057 100644 --- a/crates/lower/tests/sql_lowering.rs +++ b/crates/lower/tests/sql_lowering.rs @@ -79,6 +79,22 @@ fn find_join(qe: &QueryExpr) -> Option<&QueryExpr> { } } +/// The first `Filter` node along the single-child spine. +fn find_filter(qe: &QueryExpr) -> Option<&QueryExpr> { + match qe { + QueryExpr::Filter { .. } => Some(qe), + QueryExpr::Project { child, .. } + | QueryExpr::Aggregate { child, .. } + | QueryExpr::Window { child, .. } + | QueryExpr::Partition { child, .. } + | QueryExpr::Distinct { child, .. } + | QueryExpr::Sort { child, .. } + | QueryExpr::Limit { child, .. } + | QueryExpr::Subquery { child, .. } => find_filter(child), + _ => None, + } +} + #[tokio::test] async fn select_star_with_where_folds_predicate_onto_scan() { // SELECT * elides the projection; WHERE folds onto the Scan predicates. @@ -313,6 +329,30 @@ async fn self_join_disambiguates_via_aliases() { ); } +#[tokio::test] +async fn qualified_where_over_join_resolves_to_right_side() { + // Issue #7 beyond the join key: a WHERE on the *duplicated* column name + // (`service` exists on both sides) must bind to the qualified side, not the + // first match. metrics.service = col 1, hosts.service = col 4 → `hosts.service` + // must resolve to 4. (Unoptimized plan keeps the Filter above the Join — no + // predicate pushdown — so it binds against the concatenated schema.) + let qe = lower( + "SELECT metrics.bytes FROM metrics JOIN hosts ON metrics.service = hosts.service \ + WHERE hosts.service = 'api'", + ) + .await; + let filter = find_filter(&qe).expect("expected a Filter over the join"); + let QueryExpr::Filter { pred, .. } = filter else { + unreachable!("find_filter only returns Filter"); + }; + assert!( + matches!(&pred.0, L3Expr::Compare { left, op: CompareOp::Eq, .. } + if matches!(left.as_ref(), L3Expr::Column(4))), + "hosts.service must bind to concatenated position 4 (not the first `service`), got {:?}", + pred.0 + ); +} + #[tokio::test] async fn aggregate_over_join_binds_against_concatenated_schema() { // GROUP BY a right-table column over a join: the key must resolve against From fe5d0a27b76551318ea7c588ad51476fe1bfc4ee Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 27 May 2026 08:48:01 -0600 Subject: [PATCH 35/40] refactor(core): unify the duplicated operator vocabularies (review A) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `BinaryOpKind` (query-level `BinaryOp`) and the scalar IR's `ArithOp`/`CompareOp` each defined Add/Sub/Mul/Div/Mod, Eq/Ne/Lt/Le/Gt/Ge, and Like/Regex twice — and had already drifted (`CompareOp` had ILike/NotILike that `BinaryOpKind` lacked; only `BinaryOpKind` had a `Display`). So a comparison had two representations and its rendering depended on which copy you held. `BinaryOpKind` now *reuses* the scalar ops — `Arith(ArithOp)` / `Compare(CompareOp)` — and keeps only the PromQL-vector ops with no scalar counterpart (And/Or/Unless/ Pow/Atan2). `Display` lives once on `ArithOp`/`CompareOp` and `BinaryOpKind` delegates, so every operator has exactly one spelling and one rendering. `binop` (PromQL token→op) and the affected tests updated; no behavior change. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/core/src/intent_algebra/expr_ir.rs | 31 +++++++++ crates/core/src/intent_algebra/query_expr.rs | 67 ++++++++------------ crates/lower/src/promql.rs | 24 +++---- crates/lower/tests/promql_conformance.rs | 10 +-- crates/lower/tests/promql_lowering.rs | 6 +- 5 files changed, 78 insertions(+), 60 deletions(-) diff --git a/crates/core/src/intent_algebra/expr_ir.rs b/crates/core/src/intent_algebra/expr_ir.rs index 6959971b..423a3c50 100644 --- a/crates/core/src/intent_algebra/expr_ir.rs +++ b/crates/core/src/intent_algebra/expr_ir.rs @@ -58,6 +58,25 @@ pub enum CompareOp { NotRegex, } +impl std::fmt::Display for CompareOp { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + CompareOp::Eq => "==", + CompareOp::Ne => "!=", + CompareOp::Lt => "<", + CompareOp::Le => "<=", + CompareOp::Gt => ">", + CompareOp::Ge => ">=", + CompareOp::Like => "LIKE", + CompareOp::NotLike => "NOT LIKE", + CompareOp::ILike => "ILIKE", + CompareOp::NotILike => "NOT ILIKE", + CompareOp::Regex => "=~", + CompareOp::NotRegex => "!~", + }) + } +} + /// Binary arithmetic operators. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum ArithOp { @@ -68,6 +87,18 @@ pub enum ArithOp { Mod, } +impl std::fmt::Display for ArithOp { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + ArithOp::Add => "+", + ArithOp::Sub => "-", + ArithOp::Mul => "*", + ArithOp::Div => "/", + ArithOp::Mod => "%", + }) + } +} + /// Layer-2 (name-based) scalar expression. Front ends emit this; the converter /// resolves it into a positional [`L3Expr`]. Flat conjunctions (`BoolAnd`) / /// disjunctions (`BoolOr`) make per-conjunct selectivity estimation and diff --git a/crates/core/src/intent_algebra/query_expr.rs b/crates/core/src/intent_algebra/query_expr.rs index c51e9e19..79f91f85 100644 --- a/crates/core/src/intent_algebra/query_expr.rs +++ b/crates/core/src/intent_algebra/query_expr.rs @@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize}; use thiserror::Error; use super::agg_intent::AggIntent; -use super::expr_ir::{L3Expr, L3Scalar}; +use super::expr_ir::{ArithOp, CompareOp, L3Expr, L3Scalar}; use super::names::BindingName; use super::schema::{Column, ColumnId, DataType, Schema}; @@ -101,55 +101,40 @@ impl PartitionKeys { } } +/// Operator on the query-level `BinaryOp` node. Reuses the scalar IR's +/// [`ArithOp`] / [`CompareOp`] so every arithmetic/comparison operator has +/// exactly one representation (and one `Display`) across the IR; the remaining +/// variants are PromQL vector-set / power ops with no scalar-IR counterpart. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum BinaryOpKind { - Add, - Sub, - Mul, - Div, - Mod, - Pow, - Atan2, - Eq, - Ne, - Lt, - Le, - Gt, - Ge, + /// Arithmetic — `Add/Sub/Mul/Div/Mod` (shared with `L3Expr::Arith`). + Arith(ArithOp), + /// Comparison — `Eq/Ne/Lt/Le/Gt/Ge` + `Like/ILike/Regex` family (shared + /// with `L3Expr::Compare`). + Compare(CompareOp), + /// PromQL logical-set intersection (`and`). And, + /// PromQL logical-set union (`or`). Or, + /// PromQL logical-set complement (`unless`). Unless, - Like, - NotLike, - Regex, - NotRegex, + /// Exponentiation (`^`) — PromQL vector op, no scalar-IR counterpart. + Pow, + /// `atan2` — PromQL vector op, no scalar-IR counterpart. + Atan2, } impl std::fmt::Display for BinaryOpKind { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let s = match self { - BinaryOpKind::Add => "+", - BinaryOpKind::Sub => "-", - BinaryOpKind::Mul => "*", - BinaryOpKind::Div => "/", - BinaryOpKind::Mod => "%", - BinaryOpKind::Pow => "^", - BinaryOpKind::Atan2 => "atan2", - BinaryOpKind::Eq => "==", - BinaryOpKind::Ne => "!=", - BinaryOpKind::Lt => "<", - BinaryOpKind::Le => "<=", - BinaryOpKind::Gt => ">", - BinaryOpKind::Ge => ">=", - BinaryOpKind::And => "AND", - BinaryOpKind::Or => "OR", - BinaryOpKind::Unless => "unless", - BinaryOpKind::Like => "LIKE", - BinaryOpKind::NotLike => "NOT LIKE", - BinaryOpKind::Regex => "=~", - BinaryOpKind::NotRegex => "!~", - }; - f.write_str(s) + match self { + BinaryOpKind::Arith(op) => write!(f, "{op}"), + BinaryOpKind::Compare(op) => write!(f, "{op}"), + BinaryOpKind::And => f.write_str("AND"), + BinaryOpKind::Or => f.write_str("OR"), + BinaryOpKind::Unless => f.write_str("unless"), + BinaryOpKind::Pow => f.write_str("^"), + BinaryOpKind::Atan2 => f.write_str("atan2"), + } } } diff --git a/crates/lower/src/promql.rs b/crates/lower/src/promql.rs index f9c55154..77fc8562 100644 --- a/crates/lower/src/promql.rs +++ b/crates/lower/src/promql.rs @@ -43,7 +43,7 @@ use asap_control_core::intent_algebra::query_expr::{ use asap_control_core::intent_algebra::relational::{ AggFunc, AggItem, L2SortKey, QueryExpr as L2, SourceSpec, }; -use asap_control_core::intent_algebra::{CompareOp, L2Expr, L3Scalar}; +use asap_control_core::intent_algebra::{ArithOp, CompareOp, L2Expr, L3Scalar}; use crate::error::LoweringError; @@ -689,31 +689,31 @@ fn quantile_param(q: f64) -> Result { fn binop(id: token::TokenId) -> Result { Ok(if id == token::T_ADD { - BinaryOpKind::Add + BinaryOpKind::Arith(ArithOp::Add) } else if id == token::T_SUB { - BinaryOpKind::Sub + BinaryOpKind::Arith(ArithOp::Sub) } else if id == token::T_MUL { - BinaryOpKind::Mul + BinaryOpKind::Arith(ArithOp::Mul) } else if id == token::T_DIV { - BinaryOpKind::Div + BinaryOpKind::Arith(ArithOp::Div) } else if id == token::T_MOD { - BinaryOpKind::Mod + BinaryOpKind::Arith(ArithOp::Mod) } else if id == token::T_POW { BinaryOpKind::Pow } else if id == token::T_ATAN2 { BinaryOpKind::Atan2 } else if id == token::T_EQLC { - BinaryOpKind::Eq + BinaryOpKind::Compare(CompareOp::Eq) } else if id == token::T_NEQ { - BinaryOpKind::Ne + BinaryOpKind::Compare(CompareOp::Ne) } else if id == token::T_LSS { - BinaryOpKind::Lt + BinaryOpKind::Compare(CompareOp::Lt) } else if id == token::T_LTE { - BinaryOpKind::Le + BinaryOpKind::Compare(CompareOp::Le) } else if id == token::T_GTR { - BinaryOpKind::Gt + BinaryOpKind::Compare(CompareOp::Gt) } else if id == token::T_GTE { - BinaryOpKind::Ge + BinaryOpKind::Compare(CompareOp::Ge) } else if id == token::T_LAND { BinaryOpKind::And } else if id == token::T_LOR { diff --git a/crates/lower/tests/promql_conformance.rs b/crates/lower/tests/promql_conformance.rs index 5c156ee1..dc9c7bf7 100644 --- a/crates/lower/tests/promql_conformance.rs +++ b/crates/lower/tests/promql_conformance.rs @@ -34,7 +34,7 @@ use std::time::Duration; use asap_control_core::intent_algebra::{ - AggIntent, BinaryOpKind, PartitionKeys, QueryExpr, Source, + AggIntent, ArithOp, BinaryOpKind, CompareOp, PartitionKeys, QueryExpr, Source, }; use asap_control_core::types::AccuracyTarget; use asap_control_lower::{lower_promql, LoweringError}; @@ -388,7 +388,7 @@ fn vector_arithmetic() { let QueryExpr::BinaryOp { op, .. } = &qe else { panic!("expected BinaryOp, got {qe:?}"); }; - assert_eq!(*op, BinaryOpKind::Add); + assert_eq!(*op, BinaryOpKind::Arith(ArithOp::Add)); } #[test] @@ -402,7 +402,7 @@ fn on_matching_with_group_left() { else { panic!("expected BinaryOp, got {qe:?}"); }; - assert_eq!(*op, BinaryOpKind::Div); + assert_eq!(*op, BinaryOpKind::Arith(ArithOp::Div)); let vm = vector_match.as_ref().expect("on(...) group_left present"); assert_eq!(vm.labels, vec!["instance".to_string(), "job".to_string()]); assert!( @@ -415,7 +415,9 @@ fn on_matching_with_group_left() { fn vector_comparison_filters() { // SEMANTICS: `>` between two vectors keeps the LHS series where it holds. let qe = ok("go_goroutines > go_threads"); - assert!(matches!(&qe, QueryExpr::BinaryOp { op, .. } if *op == BinaryOpKind::Gt)); + assert!( + matches!(&qe, QueryExpr::BinaryOp { op, .. } if *op == BinaryOpKind::Compare(CompareOp::Gt)) + ); } #[test] diff --git a/crates/lower/tests/promql_lowering.rs b/crates/lower/tests/promql_lowering.rs index 4ea07cf8..d3d1e273 100644 --- a/crates/lower/tests/promql_lowering.rs +++ b/crates/lower/tests/promql_lowering.rs @@ -3,8 +3,8 @@ use std::time::Duration; use asap_control_core::intent_algebra::{ - AggIntent, BinaryOpKind, CompareOp, L3Expr, L3Scalar, PartitionKeys, QueryExpr, Source, - WindowKind, + AggIntent, ArithOp, BinaryOpKind, CompareOp, L3Expr, L3Scalar, PartitionKeys, QueryExpr, + Source, WindowKind, }; use asap_control_core::types::AccuracyTarget; use asap_control_core::workload::{ @@ -355,7 +355,7 @@ fn binary_op_division() { let QueryExpr::BinaryOp { op, lhs, rhs, .. } = &qe else { panic!("expected BinaryOp, got {qe:?}"); }; - assert_eq!(*op, BinaryOpKind::Div); + assert_eq!(*op, BinaryOpKind::Arith(ArithOp::Div)); assert!( matches!(lhs.as_ref(), QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Rate { .. }])) ); From 1a16657d93dc85bfb05289d25cc22ca0253d1901 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 27 May 2026 08:53:41 -0600 Subject: [PATCH 36/40] refactor(core): remove name-based residue from the positional L3 IR (review E) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two leftovers contradicted the "fully positional L3" thesis: 1. `QueryExpr::Distinct.cols` was `Vec` (name-based, resolved lazily in `output_schema_in`) unlike every other L3 column reference. It's now `Vec`; the converter resolves the L2 dedup keys against the child schema up front (`resolve_column_refs`), and `output_schema_in` just adds them as a unique key. 2. `ColumnRef` (incl. the `SampleValue`/`Wildcard` front-end conventions) lived in the L3 `query_expr` module though it's purely an L2 / name-based concept that the converter always resolves away. Moved it to `expr_ir` next to `L2Expr` (its only structural user), breaking the query_expr⇄expr_ir import cycle; re-exported from the crate root so external paths are unchanged. No behavior change (SQL's `Distinct::All` still emits empty `cols`). Adds a test pinning `SELECT DISTINCT` → `Distinct { cols: Vec }`. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/core/src/intent_algebra/binder.rs | 2 +- .../src/intent_algebra/column_resolution.rs | 2 +- crates/core/src/intent_algebra/expr_ir.rs | 21 +++++++++- crates/core/src/intent_algebra/lower.rs | 23 +++++++---- crates/core/src/intent_algebra/mod.rs | 6 +-- crates/core/src/intent_algebra/query_expr.rs | 39 ++++--------------- crates/core/src/intent_algebra/relational.rs | 4 +- crates/lower/src/promql.rs | 4 +- crates/lower/tests/sql_lowering.rs | 13 +++++++ 9 files changed, 64 insertions(+), 50 deletions(-) diff --git a/crates/core/src/intent_algebra/binder.rs b/crates/core/src/intent_algebra/binder.rs index ad74faa8..bdc9287b 100644 --- a/crates/core/src/intent_algebra/binder.rs +++ b/crates/core/src/intent_algebra/binder.rs @@ -11,8 +11,8 @@ //! `SchemaCatalog` is future work; the `Binder` pass does not change when it //! lands, only the catalog impl swaps. +use crate::intent_algebra::expr_ir::ColumnRef; use crate::intent_algebra::expr_ir::L2Expr; -use crate::intent_algebra::query_expr::ColumnRef; use crate::intent_algebra::relational::QueryExpr as LQueryExpr; use crate::intent_algebra::schema::{Column, DataType, Schema}; diff --git a/crates/core/src/intent_algebra/column_resolution.rs b/crates/core/src/intent_algebra/column_resolution.rs index 35baa74c..ffdab9b9 100644 --- a/crates/core/src/intent_algebra/column_resolution.rs +++ b/crates/core/src/intent_algebra/column_resolution.rs @@ -9,8 +9,8 @@ use thiserror::Error; use crate::intent_algebra::agg_intent::AggIntent; +use crate::intent_algebra::expr_ir::ColumnRef; use crate::intent_algebra::expr_ir::{L2Expr, L3Expr}; -use crate::intent_algebra::query_expr::ColumnRef; use crate::intent_algebra::relational::QueryExpr; use crate::intent_algebra::schema::{Column, ColumnId, DataType, Schema}; diff --git a/crates/core/src/intent_algebra/expr_ir.rs b/crates/core/src/intent_algebra/expr_ir.rs index 423a3c50..d49b2a03 100644 --- a/crates/core/src/intent_algebra/expr_ir.rs +++ b/crates/core/src/intent_algebra/expr_ir.rs @@ -18,9 +18,28 @@ use serde::{Deserialize, Serialize}; -use super::query_expr::ColumnRef; use super::schema::{ColumnId, DataType}; +/// A name-based column reference. This is an L2 / front-end concept — the +/// converter resolves every `ColumnRef` into a positional [`ColumnId`], so it +/// does not appear in the L3 [`QueryExpr`](super::query_expr::QueryExpr). It +/// includes the two PromQL-conventional synthetic columns. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum ColumnRef { + Named(String), + /// Table-qualified reference (`t.col` / `alias.col`). Resolved by + /// `(table, name)` so a column name shared across a join (`a.k` vs `b.k`) + /// binds to the correct side. + Qualified { + table: String, + name: String, + }, + /// The implicit metric sample value (PromQL — always the series value). + SampleValue, + /// All rows / COUNT(*). + Wildcard, +} + /// A typed scalar constant. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum L3Scalar { diff --git a/crates/core/src/intent_algebra/lower.rs b/crates/core/src/intent_algebra/lower.rs index c7ed2be4..ef605ff1 100644 --- a/crates/core/src/intent_algebra/lower.rs +++ b/crates/core/src/intent_algebra/lower.rs @@ -15,13 +15,14 @@ use thiserror::Error; use crate::intent_algebra::agg_intent::AggIntent; use crate::intent_algebra::binder::Binder; use crate::intent_algebra::column_resolution::{ - output_schema_for_aggregate, resolve_expr, resolve_named_keys, ResolveError, + output_schema_for_aggregate, resolve_column_refs, resolve_expr, resolve_named_keys, + ResolveError, }; -use crate::intent_algebra::expr_ir::{L2Expr, L3Expr, L3Scalar}; +use crate::intent_algebra::expr_ir::{ColumnRef, L2Expr, L3Expr, L3Scalar}; use crate::intent_algebra::names::BindingName; use crate::intent_algebra::query_expr::{ - ColumnRef, PartitionKeys as CPartitionKeys, Predicate, ProjectItem, QueryExpr as CQueryExpr, - SortKey, Source, WindowKind, + PartitionKeys as CPartitionKeys, Predicate, ProjectItem, QueryExpr as CQueryExpr, SortKey, + Source, WindowKind, }; use crate::intent_algebra::relational::{AggFunc, QueryExpr as LQueryExpr, SourceSpec}; use crate::intent_algebra::schema::{ColumnId, Schema}; @@ -218,10 +219,16 @@ pub fn convert( child: Box::new(convert(input, fallback, acc)?), }, - LQueryExpr::Distinct { cols, input } => CQueryExpr::Distinct { - cols: cols.clone(), - child: Box::new(convert(input, fallback, acc)?), - }, + LQueryExpr::Distinct { cols, input } => { + // Resolve the L2 (name-based) dedup keys to positional ids against + // the converted child's schema, like every other L3 column ref. + let child = convert(input, fallback, acc)?; + let cols = resolve_column_refs(cols, &child.output_schema()?)?; + CQueryExpr::Distinct { + cols, + child: Box::new(child), + } + } LQueryExpr::TopK { k, by, input } => { let child = convert(input, fallback, acc)?; diff --git a/crates/core/src/intent_algebra/mod.rs b/crates/core/src/intent_algebra/mod.rs index fd71d2b9..4b64544c 100644 --- a/crates/core/src/intent_algebra/mod.rs +++ b/crates/core/src/intent_algebra/mod.rs @@ -30,12 +30,12 @@ pub use column_resolution::{ resolve_column_refs, resolve_expr, resolve_named_keys, ResolveError, }; pub use cse::{dedupe_subtrees, CseWorkloadPlan}; -pub use expr_ir::{ArithOp, CompareOp, L2Expr, L3Expr, L3Scalar}; +pub use expr_ir::{ArithOp, ColumnRef, CompareOp, L2Expr, L3Expr, L3Scalar}; pub use lower::{convert, convert_root, ConvertError}; pub use names::{BindingName, QueryId}; pub use query_expr::{ - BinaryOpKind, BindingScope, ColumnRef, DataModel, GroupSide, JoinKind, PartitionKeys, - Predicate, ProjectItem, QueryExpr, QueryExprError, SetOpKind, SortKey, Source, VectorGrouping, + BinaryOpKind, BindingScope, DataModel, GroupSide, JoinKind, PartitionKeys, Predicate, + ProjectItem, QueryExpr, QueryExprError, SetOpKind, SortKey, Source, VectorGrouping, VectorMatch, VectorMatchKind, WindowFuncKind, WindowKind, }; pub use schema::{cse_reuse_is_legal, Column, ColumnId, CseError, DataType, Schema}; diff --git a/crates/core/src/intent_algebra/query_expr.rs b/crates/core/src/intent_algebra/query_expr.rs index 79f91f85..7a4e2207 100644 --- a/crates/core/src/intent_algebra/query_expr.rs +++ b/crates/core/src/intent_algebra/query_expr.rs @@ -65,24 +65,6 @@ impl Source { } } -/// A column reference by name, or one of the two PromQL-conventional -/// synthetic columns. -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub enum ColumnRef { - Named(String), - /// Table-qualified reference (`t.col` / `alias.col`). Resolved by - /// `(table, name)` so a column name shared across a join (`a.k` vs `b.k`) - /// binds to the correct side. - Qualified { - table: String, - name: String, - }, - /// The implicit metric sample value (PromQL — always the series value). - SampleValue, - /// All rows / COUNT(*). - Wildcard, -} - /// Grouping key set (`by (...)` / `without (...)`). #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum PartitionKeys { @@ -278,9 +260,10 @@ pub enum QueryExpr { keys: PartitionKeys, child: Box, }, - /// δ — SQL `DISTINCT` / row deduplication. + /// δ — SQL `DISTINCT` / row deduplication. Positional like every other L3 + /// column reference; empty = dedup on all columns (`SELECT DISTINCT *`). Distinct { - cols: Vec, + cols: Vec, child: Box, }, /// ⊕ — exact union of sub-results from independent stages / shards. @@ -467,18 +450,10 @@ impl QueryExpr { } QueryExpr::Distinct { cols, child } => { - let in_schema = child.output_schema_in(scope)?; - let mut out = in_schema.clone(); - let mut key_ids: Vec = Vec::with_capacity(cols.len()); - for c in cols { - if let ColumnRef::Named(name) = c { - if let Some(id) = in_schema.column_id(name) { - key_ids.push(id); - } - } - } - if !key_ids.is_empty() { - out.add_unique_key(key_ids); + let mut out = child.output_schema_in(scope)?; + // Deduplicating on `cols` makes them a unique key of the result. + if !cols.is_empty() { + out.add_unique_key(cols.clone()); } Ok(out) } diff --git a/crates/core/src/intent_algebra/relational.rs b/crates/core/src/intent_algebra/relational.rs index 7c8091b3..8f168f93 100644 --- a/crates/core/src/intent_algebra/relational.rs +++ b/crates/core/src/intent_algebra/relational.rs @@ -9,8 +9,8 @@ use std::time::Duration; -use super::expr_ir::L2Expr; -pub use super::query_expr::{BinaryOpKind, ColumnRef, PartitionKeys, VectorMatch, WindowFuncKind}; +pub use super::expr_ir::{ColumnRef, L2Expr}; +pub use super::query_expr::{BinaryOpKind, PartitionKeys, VectorMatch, WindowFuncKind}; use super::schema::Schema; /// SELECT-list item at Layer 2 — a name-based [`L2Expr`] + optional alias. diff --git a/crates/lower/src/promql.rs b/crates/lower/src/promql.rs index 77fc8562..46df9b8c 100644 --- a/crates/lower/src/promql.rs +++ b/crates/lower/src/promql.rs @@ -38,12 +38,12 @@ use promql_parser::parser::{ }; use asap_control_core::intent_algebra::query_expr::{ - BinaryOpKind, ColumnRef, GroupSide, VectorGrouping, VectorMatch, VectorMatchKind, + BinaryOpKind, GroupSide, VectorGrouping, VectorMatch, VectorMatchKind, }; use asap_control_core::intent_algebra::relational::{ AggFunc, AggItem, L2SortKey, QueryExpr as L2, SourceSpec, }; -use asap_control_core::intent_algebra::{ArithOp, CompareOp, L2Expr, L3Scalar}; +use asap_control_core::intent_algebra::{ArithOp, ColumnRef, CompareOp, L2Expr, L3Scalar}; use crate::error::LoweringError; diff --git a/crates/lower/tests/sql_lowering.rs b/crates/lower/tests/sql_lowering.rs index 868e5057..d0d99f5b 100644 --- a/crates/lower/tests/sql_lowering.rs +++ b/crates/lower/tests/sql_lowering.rs @@ -247,6 +247,19 @@ async fn count_distinct_is_cardinality() { assert!(matches!(aggs.as_slice(), [AggIntent::Cardinality { .. }])); } +#[tokio::test] +async fn select_distinct_lowers_to_distinct_with_positional_cols() { + // SELECT DISTINCT → a `Distinct` node whose `cols` are positional ColumnIds + // (not name-based ColumnRefs). DataFusion's `Distinct::All` dedups on every + // column, so `cols` is empty here — but the field type is now `Vec`. + let qe = lower("SELECT DISTINCT service FROM metrics").await; + let QueryExpr::Distinct { cols, .. } = &qe else { + panic!("expected a Distinct at the root, got {qe:?}"); + }; + let _: &Vec = cols; // compile-time: positional ids, not ColumnRefs + assert!(cols.is_empty(), "DISTINCT * dedups on all columns"); +} + #[tokio::test] async fn inner_join_lowers_to_join_over_two_scans() { // INNER JOIN over two distinct tables → L3 Join with both leaves as Scans. From e1b309d011f703966bcbf3b302e391fa6ab4e5d0 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 27 May 2026 09:12:58 -0600 Subject: [PATCH 37/40] feat(core): converge PromQL grouped aggregates onto positional Aggregate.by (review D) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The headline non-canonicality: PromQL `sum by (job) (…)` parked its group key in a name-based `Partition { keys: By(["job"]) }` over an `Aggregate { by: [] }`, while SQL `GROUP BY job` produced `Aggregate { by: [] }`. Two unrelated shapes for the same work (blocking workload CSE/#6), and worse — `Partition` is documented as a *droppable* L5 sharding hint, yet it was the **only** place the PromQL group key lived, so an L5 consumer honoring that docstring would compute a global aggregate. Root cause: in the two-level `sum by(x)(rate(m[w]))` shape, `x` exists only in the leaf Scan schema; the inner `Rate` aggregate (`by: []`) dropped it, so the outer `Sum` had no positional column to group on. Fix: model range-vector functions as what they are — **per-series, label-preserving** reductions (one value per series, all labels retained, the sample value replaced and kept named `value`). Then the outer cross-series aggregate resolves its keys positionally into `Aggregate.by`, exactly like SQL. - `AggIntent::is_per_series()` (Rate/Increase); `output_schema_in`'s Aggregate arm branches per-series reductions to label-preserving schema derivation. - The converter's fused path resolves group keys to `Aggregate.by` for non-windowed reductions (instant selectors + per-series rate/increase). A *windowed* reduction here is per-series (e.g. `avg_over_time`) — its keys belong to an enclosing level — so it keeps the legacy name-based `Partition` fallback rather than folding them into a per-series `by` (which would also break downstream sample-value resolution, e.g. `topk by(h)(avg_over_time(…))`). So instant-vector and rate/increase grouped aggregates (the dominant + all tested patterns) now emit the same positional shape as SQL; grouped `*_over_time` still falls back to `Partition` pending a follow-up (its window-hoisting interaction). Tests updated to the positional shape; adds a unit test pinning rate's label-preserving schema. Full suite green (132). Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/core/src/intent_algebra/agg_intent.rs | 9 +++ crates/core/src/intent_algebra/lower.rs | 20 ++++++- crates/core/src/intent_algebra/query_expr.rs | 61 ++++++++++++++++++++ crates/lower/tests/promql_conformance.rs | 53 +++++++++++------ crates/lower/tests/promql_lowering.rs | 28 +++++---- 5 files changed, 138 insertions(+), 33 deletions(-) diff --git a/crates/core/src/intent_algebra/agg_intent.rs b/crates/core/src/intent_algebra/agg_intent.rs index 05a0a58e..73922f41 100644 --- a/crates/core/src/intent_algebra/agg_intent.rs +++ b/crates/core/src/intent_algebra/agg_intent.rs @@ -99,6 +99,15 @@ impl AggIntent { } } + /// Whether this is a *per-series* reduction — it reduces a single series' + /// samples over its range window (one value out per series), so it does + /// **not** collapse across series and every label column is preserved. + /// `rate`/`increase` carry their window in the intent. (Cross-series + /// reductions like `sum`/`avg` over a series set return `false`.) + pub fn is_per_series(&self) -> bool { + matches!(self, Self::Rate { .. } | Self::Increase { .. }) + } + /// The positional input column this intent reduces, if it carries one. /// `None` = the synthetic time-series sample value (PromQL) or an /// argument-less aggregate (`Count` / `Cardinality` / `TopK`). Used by diff --git a/crates/core/src/intent_algebra/lower.rs b/crates/core/src/intent_algebra/lower.rs index ef605ff1..9f431d16 100644 --- a/crates/core/src/intent_algebra/lower.rs +++ b/crates/core/src/intent_algebra/lower.rs @@ -117,8 +117,24 @@ pub fn convert( acc, resolve_agg_col(&aggs[0].col, &agg_in_schema)?, ); + // Resolve the group keys positionally against the aggregate's + // input so the grouping lives in `Aggregate.by` — the *same* + // shape SQL produces. Only when this is a *non-windowed* + // reduction: an instant aggregate (`sum by (job) (m)`) or a + // cross-series reduction over a label-preserving `rate`/ + // `increase` (`sum by (job) (rate(m[w]))`), where the key is in + // scope. A *windowed* reduction here is per-series (e.g. + // `avg_over_time`) — its keys belong to an enclosing level, so + // keep the legacy name-based `Partition` marker instead of + // folding them into a per-series `by`. + let by = if window.is_none() { + resolve_named_keys(keys, &agg_in_schema).unwrap_or_default() + } else { + Vec::new() + }; + let grouped_positionally = by.len() == keys.len(); let aggregate = CQueryExpr::Aggregate { - by: Vec::new(), + by, aggs: vec![intent], output_names: vec![aggs[0].alias.clone()], having: None, @@ -137,7 +153,7 @@ pub fn convert( }, None => aggregate, }; - return Ok(if keys.is_empty() { + return Ok(if grouped_positionally { sketch } else { CQueryExpr::Partition { diff --git a/crates/core/src/intent_algebra/query_expr.rs b/crates/core/src/intent_algebra/query_expr.rs index 7a4e2207..b58d877e 100644 --- a/crates/core/src/intent_algebra/query_expr.rs +++ b/crates/core/src/intent_algebra/query_expr.rs @@ -360,6 +360,30 @@ impl QueryExpr { .. } => { let in_schema = child.output_schema_in(scope)?; + + // Per-series range reduction (`rate`/`increase`): one value out + // per series, so it is *label-preserving* — every label column + // survives and only the sample value is replaced (kept named + // `value`, the PromQL convention). This is what lets an outer + // cross-series `Aggregate.by` resolve its group keys positionally + // over `sum by (job) (rate(...))`. + if by.is_empty() && !aggs.is_empty() && aggs.iter().all(|a| a.is_per_series()) { + let value_idx = in_schema.column_id("value").or_else(|| { + (0..in_schema.columns.len()).find(|&i| Some(i) != in_schema.time_index) + }); + let mut columns = in_schema.columns.clone(); + if let Some(vi) = value_idx { + let mut out = aggs[0].output_column(&columns[vi]); + out.name = "value".into(); + columns[vi] = out; + } + return Ok(Schema { + columns, + time_index: in_schema.time_index, + unique_keys: in_schema.unique_keys.clone(), + }); + } + let mut out_cols: Vec = Vec::with_capacity(by.len() + aggs.len()); for &id in by { let c = @@ -706,6 +730,43 @@ mod tests { assert!(s.unique_keys.is_empty()); } + #[test] + fn per_series_rate_preserves_labels() { + // A per-series range reduction (`rate`) is label-preserving: it produces + // one value per series, so every label survives and only the sample + // value is replaced (kept named `value`). This is what lets an outer + // cross-series `Aggregate.by` group on those labels positionally. + let child = scan( + vec![ + col("ts", DataType::Timestamp, false), + col("value", DataType::Float64, false), + col("job", DataType::Utf8, true), + ], + Some(0), + vec![], + ); + let rate = QueryExpr::Aggregate { + by: vec![], + aggs: vec![AggIntent::Rate { + window: Duration::from_secs(300), + }], + output_names: vec![], + having: None, + child: Box::new(child), + }; + let s = rate.output_schema().unwrap(); + assert_eq!( + s.columns + .iter() + .map(|c| c.name.as_str()) + .collect::>(), + vec!["ts", "value", "job"], + "rate preserves all labels; only the sample value is replaced" + ); + assert_eq!(s.time_index, Some(0)); + assert!(s.column_id("job").is_some(), "label survives the reduction"); + } + #[test] fn project_keeps_time_index_when_ts_passed_through() { let child = scan( diff --git a/crates/lower/tests/promql_conformance.rs b/crates/lower/tests/promql_conformance.rs index dc9c7bf7..32a29b33 100644 --- a/crates/lower/tests/promql_conformance.rs +++ b/crates/lower/tests/promql_conformance.rs @@ -34,7 +34,7 @@ use std::time::Duration; use asap_control_core::intent_algebra::{ - AggIntent, ArithOp, BinaryOpKind, CompareOp, PartitionKeys, QueryExpr, Source, + AggIntent, ArithOp, BinaryOpKind, CompareOp, QueryExpr, Source, }; use asap_control_core::types::AccuracyTarget; use asap_control_lower::{lower_promql, LoweringError}; @@ -210,18 +210,25 @@ fn sum_collapses_all_series() { } #[test] -fn sum_by_preserves_dimensions_as_partition() { - // SEMANTICS: `by(job,instance)` keeps those labels; grouping rides on - // Partition. Keys are canonicalised (sorted), so order is normalised. +fn sum_by_groups_via_positional_aggregate() { + // SEMANTICS: `by(job,instance)` keeps those labels; the grouping lives on a + // positional `Aggregate.by` — the same shape SQL `GROUP BY` produces (not a + // name-based Partition). Binder leaf = [ts, value, instance, job] (referenced + // keys appended sorted), so the keys resolve to columns [2, 3]. let qe = ok("sum by(job, instance) (node_filesystem_size_bytes)"); - let QueryExpr::Partition { keys, .. } = &qe else { - panic!("expected Partition for `by(...)`, got {qe:?}"); + let QueryExpr::Aggregate { + by, aggs, child, .. + } = &qe + else { + panic!("expected positional Aggregate for `by(...)`, got {qe:?}"); }; assert_eq!( - *keys, - PartitionKeys::By(vec!["instance".into(), "job".into()]) + by, + &vec![2, 3], + "group keys resolve to positional ColumnIds" ); - assert!(has(&qe, |i| matches!(i, AggIntent::Sum { .. }))); + assert!(matches!(aggs.as_slice(), [AggIntent::Sum { .. }])); + assert!(matches!(child.as_ref(), QueryExpr::Scan { .. })); } #[test] @@ -288,17 +295,22 @@ fn sum_of_rate_is_two_levels() { #[test] fn sum_by_of_rate_groups_outer_level() { + // Outer cross-series Sum grouped on positional `Aggregate.by` over the + // label-preserving inner Rate. Leaf = [ts, value, instance] → by = [2]. let qe = ok("sum by(instance) (rate(node_network_receive_bytes_total[5m]))"); - let QueryExpr::Partition { keys, child } = &qe else { - panic!("expected Partition, got {qe:?}"); + let QueryExpr::Aggregate { + by, aggs, child, .. + } = &qe + else { + panic!("expected outer Aggregate grouped by instance, got {qe:?}"); }; - assert_eq!(*keys, PartitionKeys::By(vec!["instance".into()])); - // Partition → Sum → Rate. + assert_eq!(by, &vec![2]); + assert!(matches!(aggs.as_slice(), [AggIntent::Sum { .. }])); + // child is the inner per-series Rate aggregate. assert!(matches!( child.as_ref(), - QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Sum { .. }]) + QueryExpr::Aggregate { aggs, .. } if matches!(aggs.as_slice(), [AggIntent::Rate { .. }]) )); - assert!(has(&qe, |i| matches!(i, AggIntent::Rate { .. }))); } // ───────────────────────────────────────────────────────────────────────────── @@ -371,10 +383,13 @@ fn histogram_quantile_over_sum_by_le_preserves_le_grouping() { panic!("expected outer Aggregate{{Quantile}}, got {qe:?}"); }; assert!(matches!(aggs.as_slice(), [AggIntent::Quantile { .. }])); - assert!(matches!( - child.as_ref(), - QueryExpr::Partition { keys, .. } if *keys == PartitionKeys::By(vec!["le".into()]) - )); + // `sum by(le)` now survives as a positional Aggregate (by = [2], `le`), over + // the inner Rate — no name-based Partition. + let QueryExpr::Aggregate { by, aggs, .. } = child.as_ref() else { + panic!("expected `sum by(le)` as a positional Aggregate, got {child:?}"); + }; + assert_eq!(by, &vec![2]); + assert!(matches!(aggs.as_slice(), [AggIntent::Sum { .. }])); } // ───────────────────────────────────────────────────────────────────────────── diff --git a/crates/lower/tests/promql_lowering.rs b/crates/lower/tests/promql_lowering.rs index d3d1e273..d99fbd4e 100644 --- a/crates/lower/tests/promql_lowering.rs +++ b/crates/lower/tests/promql_lowering.rs @@ -168,10 +168,13 @@ fn histogram_quantile_over_sum_by_le_preserves_grouping() { panic!("expected outer Aggregate{{Quantile}}, got {qe:?}"); }; assert!(matches!(aggs.as_slice(), [AggIntent::Quantile { q, .. }] if (*q - 0.99).abs() < 1e-9)); - assert!( - matches!(child.as_ref(), QueryExpr::Partition { keys, .. } if *keys == PartitionKeys::By(vec!["le".into()])), - "expected `sum by (le)` to survive as a Partition under the quantile, got {child:?}" - ); + // `sum by (le)` survives as a positional Aggregate (by = [2], `le`) over the + // inner Rate — no name-based Partition. + let QueryExpr::Aggregate { by, aggs, .. } = child.as_ref() else { + panic!("expected `sum by (le)` as a positional Aggregate, got {child:?}"); + }; + assert_eq!(by, &vec![2]); + assert!(matches!(aggs.as_slice(), [AggIntent::Sum { .. }])); } // ── rate / increase carry their own window (no Window node) ───────────────────── @@ -223,16 +226,17 @@ fn sum_over_rate_keeps_both_levels() { #[test] fn sum_by_over_rate_groups_the_outer_sum() { - // `sum by (job) (rate(...))`: the grouping belongs to the OUTER sum, landing - // on a Partition that wraps the two-level aggregate. + // `sum by (job) (rate(...))`: the grouping belongs to the OUTER sum and lands + // on a positional `Aggregate.by` (the same shape SQL produces) over the + // label-preserving inner Rate. Leaf = [ts, value, job] → by = [2]. let qe = lower("sum by (job) (rate(http_requests_total[5m]))"); - let QueryExpr::Partition { keys, child } = &qe else { - panic!("expected Partition by job, got {qe:?}"); - }; - assert_eq!(*keys, PartitionKeys::By(vec!["job".into()])); - let QueryExpr::Aggregate { aggs, child, .. } = child.as_ref() else { - panic!("expected Aggregate{{Sum}} under Partition, got {child:?}"); + let QueryExpr::Aggregate { + by, aggs, child, .. + } = &qe + else { + panic!("expected outer Aggregate grouped by job, got {qe:?}"); }; + assert_eq!(by, &vec![2]); assert!(matches!(aggs.as_slice(), [AggIntent::Sum { .. }])); assert!(matches!( child.as_ref(), From 3356bdfa311ed7e27a6d483f00491795efa51980 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 27 May 2026 09:25:17 -0600 Subject: [PATCH 38/40] fix(core): qualify L2 group keys so GROUP BY/PARTITION BY/TopK disambiguate joins (L2 review #1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The issue-#7 qualifier fix reached scalar predicates but not the grouping channels: `Aggregate.keys`, `TopK.by`, and `WindowFunc.partition_by` were plain `Vec`, and `expr_to_group_name` discarded the relation qualifier, so `resolve_named_keys` resolved by first-match `column_id`. On a self-join, `GROUP BY a.k` and `GROUP BY b.k` both bound to column 0 → `b.k` silently grouped by `a.k` (wrong result). Same bug class #7 closed for predicates, still open for keys. Fix: the three group-key channels now carry `ColumnRef` (qualified-capable) like the scalar path, and resolve via `resolve_column_refs` → `column_id_qualified` with the bare-name fallback. `expr_to_group_ref` (SQL) preserves `col.relation`; PromQL emits `ColumnRef::Named` (labels have no qualifier). The Binder seeds the bare names; `resolve_named_keys` is removed (subsumed by `resolve_column_refs`). Confirmed by a self-join test: `GROUP BY b.service` → `Aggregate.by = [5]`, `GROUP BY a.service` → `[1]`. Full suite green (133), clippy + fmt clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/core/src/intent_algebra/binder.rs | 18 +++++++--- .../src/intent_algebra/column_resolution.rs | 26 ++++----------- crates/core/src/intent_algebra/lower.rs | 33 ++++++++++++++----- crates/core/src/intent_algebra/mod.rs | 2 +- crates/core/src/intent_algebra/relational.rs | 12 ++++--- crates/lower/src/promql.rs | 13 ++++---- crates/lower/src/sql/mod.rs | 21 ++++++++---- crates/lower/tests/sql_lowering.rs | 31 +++++++++++++++++ 8 files changed, 104 insertions(+), 52 deletions(-) diff --git a/crates/core/src/intent_algebra/binder.rs b/crates/core/src/intent_algebra/binder.rs index bdc9287b..0004e327 100644 --- a/crates/core/src/intent_algebra/binder.rs +++ b/crates/core/src/intent_algebra/binder.rs @@ -106,23 +106,31 @@ fn default_leaf_columns() -> Vec { /// expressions (e.g. a PromQL label matcher `m{env="prod"}` references `env`). /// The Binder seeds these into the usage-derived leaf so positional resolution /// downstream is total. +/// Push a `ColumnRef`'s bare name (the schema-seedable identifier). `Qualified` +/// collapses to its `name`; `SampleValue`/`Wildcard` carry no name. +fn push_ref_name(c: &ColumnRef, out: &mut Vec) { + match c { + ColumnRef::Named(n) => out.push(n.clone()), + ColumnRef::Qualified { name, .. } => out.push(name.clone()), + ColumnRef::SampleValue | ColumnRef::Wildcard => {} + } +} + fn collect_referenced_columns(tree: &LQueryExpr) -> Vec { fn named(expr: &L2Expr, out: &mut Vec) { for c in expr.columns_referenced() { - if let ColumnRef::Named(n) = c { - out.push(n.clone()); - } + push_ref_name(c, out); } } let mut out: Vec = Vec::new(); tree.walk(&mut |node| match node { LQueryExpr::Aggregate { keys, having, .. } => { - out.extend(keys.iter().cloned()); + keys.iter().for_each(|k| push_ref_name(k, &mut out)); if let Some(h) = having { named(h, &mut out); } } - LQueryExpr::TopK { by, .. } => out.extend(by.iter().cloned()), + LQueryExpr::TopK { by, .. } => by.iter().for_each(|k| push_ref_name(k, &mut out)), LQueryExpr::Partition { keys, .. } => out.extend(keys.keys().iter().cloned()), LQueryExpr::Filter { pred, .. } => named(pred, &mut out), LQueryExpr::Project { cols, .. } => { diff --git a/crates/core/src/intent_algebra/column_resolution.rs b/crates/core/src/intent_algebra/column_resolution.rs index ffdab9b9..7fc87fa5 100644 --- a/crates/core/src/intent_algebra/column_resolution.rs +++ b/crates/core/src/intent_algebra/column_resolution.rs @@ -1,10 +1,10 @@ //! Schema-driven column resolution for the Layer-2 `relational` IR. //! -//! The Layer-2 IR uses `ColumnRef::Named(String)` / `Aggregate.keys: -//! Vec`; the canonical IR uses positional [`ColumnId`] resolved -//! against a per-node [`Schema`]. These helpers bridge the two — the -//! [`Binder`](super::binder) builds the schema, and [`resolve_named_keys`] -//! turns the L2 names into positional ids. +//! The Layer-2 IR uses `ColumnRef` (name-based, optionally table-qualified); +//! the canonical IR uses positional [`ColumnId`] resolved against a per-node +//! [`Schema`]. These helpers bridge the two — the [`Binder`](super::binder) +//! builds the schema, and [`resolve_column_refs`] turns the L2 refs (group +//! keys, dedup columns) into positional ids, qualifier-aware. use thiserror::Error; @@ -151,24 +151,10 @@ pub fn resolve_expr(expr: &L2Expr, schema: &Schema) -> Result Result, ResolveError> { - keys.iter() - .map(|name| { - schema - .column_id(name) - .ok_or_else(|| ResolveError::NotFound { - name: name.clone(), - available: schema.columns.iter().map(|c| c.name.clone()).collect(), - }) - }) - .collect() -} - /// Output schema produced by an `Aggregate { by, aggs }` over `input`. /// Mirrors `QueryExpr::output_schema_in`'s `Aggregate` arm; out-of-range `by` /// ids are silently dropped (callers needing the strict check resolve `by` -/// via [`resolve_named_keys`], which surfaces `NotFound`). +/// via [`resolve_column_refs`], which surfaces `NotFound`). pub fn output_schema_for_aggregate( input: &Schema, by: &[ColumnId], diff --git a/crates/core/src/intent_algebra/lower.rs b/crates/core/src/intent_algebra/lower.rs index 9f431d16..5464ae49 100644 --- a/crates/core/src/intent_algebra/lower.rs +++ b/crates/core/src/intent_algebra/lower.rs @@ -15,8 +15,7 @@ use thiserror::Error; use crate::intent_algebra::agg_intent::AggIntent; use crate::intent_algebra::binder::Binder; use crate::intent_algebra::column_resolution::{ - output_schema_for_aggregate, resolve_column_refs, resolve_expr, resolve_named_keys, - ResolveError, + output_schema_for_aggregate, resolve_column_refs, resolve_expr, ResolveError, }; use crate::intent_algebra::expr_ir::{ColumnRef, L2Expr, L3Expr, L3Scalar}; use crate::intent_algebra::names::BindingName; @@ -128,7 +127,7 @@ pub fn convert( // keep the legacy name-based `Partition` marker instead of // folding them into a per-series `by`. let by = if window.is_none() { - resolve_named_keys(keys, &agg_in_schema).unwrap_or_default() + resolve_column_refs(keys, &agg_in_schema).unwrap_or_default() } else { Vec::new() }; @@ -156,8 +155,11 @@ pub fn convert( return Ok(if grouped_positionally { sketch } else { + // Fallback (windowed per-series reduction): keep the legacy + // name-based `Partition`. These keys are PromQL labels + // (unqualified), so the bare names suffice. CQueryExpr::Partition { - keys: CPartitionKeys::By(keys.clone()), + keys: CPartitionKeys::By(ref_names(keys)), child: Box::new(sketch), } }); @@ -169,7 +171,7 @@ pub fn convert( // resolves against the derived output schema instead. let child = convert(input, fallback, acc)?; let child_schema = child.output_schema()?; - let by = resolve_named_keys(keys, &child_schema)?; + let by = resolve_column_refs(keys, &child_schema)?; let intents: Vec = aggs .iter() .map(|item| -> Result { @@ -249,7 +251,7 @@ pub fn convert( LQueryExpr::TopK { k, by, input } => { let child = convert(input, fallback, acc)?; let child_schema = child.output_schema()?; - let by = resolve_named_keys(by, &child_schema)?; + let by = resolve_column_refs(by, &child_schema)?; CQueryExpr::Aggregate { by, aggs: vec![AggIntent::TopK { @@ -364,7 +366,7 @@ pub fn convert( .iter() .map(|a| resolve_expr(a, &child_schema)) .collect::, _>>()?; - let partition_by = resolve_named_keys(partition_by, &child_schema)?; + let partition_by = resolve_column_refs(partition_by, &child_schema)?; let order_by = order_by .iter() .map(|k| -> Result { @@ -438,6 +440,19 @@ fn scan( }) } +/// Bare names of a `ColumnRef` list, dropping `SampleValue`/`Wildcard`. Used +/// only for the legacy name-based `Partition` fallback (PromQL labels, which +/// are unqualified — `Qualified` collapses to its bare `name`). +fn ref_names(refs: &[ColumnRef]) -> Vec { + refs.iter() + .filter_map(|c| match c { + ColumnRef::Named(n) => Some(n.clone()), + ColumnRef::Qualified { name, .. } => Some(name.clone()), + ColumnRef::SampleValue | ColumnRef::Wildcard => None, + }) + .collect() +} + /// Resolve a Layer-2 aggregate-input [`ColumnRef`] to a positional input /// column. `SampleValue` / `Wildcard` carry no specific column → `Ok(None)` /// (the PromQL sample-value / `COUNT(*)` convention); a `Named` column @@ -631,7 +646,7 @@ mod tests { right: Box::new(meta), }; let tree = LQueryExpr::Aggregate { - keys: vec!["region".into()], + keys: vec![ColumnRef::Named("region".into())], aggs: vec![ AggItem { alias: "tot".into(), @@ -676,7 +691,7 @@ mod tests { col("bytes", DataType::Int64), ]); let tree = LQueryExpr::Aggregate { - keys: vec!["region".into()], + keys: vec![ColumnRef::Named("region".into())], aggs: vec![ AggItem { alias: "tot".into(), diff --git a/crates/core/src/intent_algebra/mod.rs b/crates/core/src/intent_algebra/mod.rs index 4b64544c..9ee324d5 100644 --- a/crates/core/src/intent_algebra/mod.rs +++ b/crates/core/src/intent_algebra/mod.rs @@ -27,7 +27,7 @@ pub use agg_intent::{ pub use binder::{Binder, SchemaCatalog, UsageDerivedCatalog}; pub use column_resolution::{ infer_schema_for_root, infer_source_schema, output_schema_for_aggregate, resolve_column_ref, - resolve_column_refs, resolve_expr, resolve_named_keys, ResolveError, + resolve_column_refs, resolve_expr, ResolveError, }; pub use cse::{dedupe_subtrees, CseWorkloadPlan}; pub use expr_ir::{ArithOp, ColumnRef, CompareOp, L2Expr, L3Expr, L3Scalar}; diff --git a/crates/core/src/intent_algebra/relational.rs b/crates/core/src/intent_algebra/relational.rs index 8f168f93..8f167e0c 100644 --- a/crates/core/src/intent_algebra/relational.rs +++ b/crates/core/src/intent_algebra/relational.rs @@ -120,9 +120,11 @@ pub enum QueryExpr { input: Box, }, - /// γ + α — GROUP BY (`keys`) followed by aggregate functions. + /// γ + α — GROUP BY (`keys`) followed by aggregate functions. Keys are + /// `ColumnRef` (not bare strings) so a table-qualified key (`b.k`) resolves + /// to the correct join side, matching the scalar-predicate path. Aggregate { - keys: Vec, + keys: Vec, aggs: Vec, having: Option, input: Box, @@ -145,10 +147,10 @@ pub enum QueryExpr { cols: Vec, input: Box, }, - /// τ — heavy-hitter top-k. `by` are the grouping keys. + /// τ — heavy-hitter top-k. `by` are the grouping keys (qualified-capable). TopK { k: u64, - by: Vec, + by: Vec, input: Box, }, /// ⊕ — merge sub-results from independent branches. @@ -194,7 +196,7 @@ pub enum QueryExpr { WindowFunc { func: WindowFuncKind, args: Vec, - partition_by: Vec, + partition_by: Vec, order_by: Vec, output_name: String, input: Box, diff --git a/crates/lower/src/promql.rs b/crates/lower/src/promql.rs index 46df9b8c..ccd1fed1 100644 --- a/crates/lower/src/promql.rs +++ b/crates/lower/src/promql.rs @@ -368,7 +368,7 @@ fn lower_inner_call(call: &Call) -> Result { /// Assemble the Layer-2 tree from a lowered inner vector, the resolved group /// keys, and the enclosing aggregator shape. -fn build(inner: Inner, keys: Vec, outer: Outer) -> Result { +fn build(inner: Inner, keys: Vec, outer: Outer) -> Result { match outer { Outer::None => match &inner.func { None => Ok(filtered_source(inner.metric, inner.matchers)), @@ -437,7 +437,7 @@ fn build(inner: Inner, keys: Vec, outer: Outer) -> Result { /// `Aggregate{keys, [func]}` over `[Window{w}] → Filter(Source)`. Rate/Increase /// carry their own window in the func, so no `Window` node is emitted. -fn windowed_aggregate(inner: Inner, keys: Vec, func: AggFunc) -> L2 { +fn windowed_aggregate(inner: Inner, keys: Vec, func: AggFunc) -> L2 { let skip_window = matches!(func, AggFunc::Rate { .. } | AggFunc::Increase { .. }); let window = inner.window; let base = filtered_source(inner.metric, inner.matchers); @@ -467,7 +467,7 @@ fn windowed_aggregate(inner: Inner, keys: Vec, func: AggFunc) -> L2 { /// `Aggregate{keys, [func]}` directly over an existing L2 subtree — the OUTER /// level of a two-level aggregation such as `sum(rate(…))` or the /// `Aggregate{[Quantile]}` that wraps a `histogram_quantile` argument. -fn outer_aggregate(keys: Vec, func: AggFunc, input: L2) -> L2 { +fn outer_aggregate(keys: Vec, func: AggFunc, input: L2) -> L2 { L2::Aggregate { keys, aggs: vec![AggItem { @@ -544,16 +544,17 @@ fn outer_func(o: &OuterIntent) -> AggFunc { /// Resolve `by(labels)` into a key list. `without(...)` needs the metric's /// full label set, which the usage-derived schema model doesn't carry, so it /// is rejected (a registry-backed `SchemaCatalog` would lift this). -fn resolve_group(agg: &AggregateExpr) -> Result> { +fn resolve_group(agg: &AggregateExpr) -> Result> { match &agg.modifier { None => Ok(vec![]), Some(LabelModifier::Include(ls)) => { // Grouping labels are a set: `by (a, b)` ≡ `by (b, a)`. Canonicalise - // so equivalent groupings lower to identical keys. + // so equivalent groupings lower to identical keys. PromQL labels have + // no table qualifier → `ColumnRef::Named`. let mut keys = ls.labels.clone(); keys.sort(); keys.dedup(); - Ok(keys) + Ok(keys.into_iter().map(ColumnRef::Named).collect()) } Some(LabelModifier::Exclude(_)) => Err(LoweringError::UnsupportedFeature( "`without(...)` grouping requires a registry-backed catalog of the \ diff --git a/crates/lower/src/sql/mod.rs b/crates/lower/src/sql/mod.rs index 87eee6b4..e7ff24f1 100644 --- a/crates/lower/src/sql/mod.rs +++ b/crates/lower/src/sql/mod.rs @@ -245,7 +245,7 @@ impl<'a> SqlLowerer<'a> { let partition_by = wf .partition_by .iter() - .map(expr_to_group_name) + .map(expr_to_group_ref) .collect::, _>>()?; let order_by = wf .order_by @@ -301,7 +301,7 @@ impl<'a> SqlLowerer<'a> { let keys = agg .group_expr .iter() - .map(expr_to_group_name) + .map(expr_to_group_ref) .collect::, _>>()?; // DataFusion names the aggregate outputs in its own schema (e.g. // "sum(metrics.bytes)") — the same names the enclosing Projection @@ -389,7 +389,7 @@ impl<'a> SqlLowerer<'a> { let by = agg .group_expr .iter() - .map(expr_to_group_name) + .map(expr_to_group_ref) .collect::, _>>()?; Ok(L2::TopK { k, @@ -493,10 +493,19 @@ fn reducer_col(name: &str, args: &[Expr]) -> Result { }) } -fn expr_to_group_name(expr: &Expr) -> Result { +fn expr_to_group_ref(expr: &Expr) -> Result { match expr { - Expr::Column(col) => Ok(col.name.clone()), - Expr::Alias(a) => expr_to_group_name(&a.expr), + // Preserve the relation qualifier so a GROUP BY / PARTITION BY key over a + // join (`b.k` vs `a.k`) resolves to the correct side — the same rule the + // scalar predicate path uses (`df_expr_to_l2`). + Expr::Column(col) => Ok(match &col.relation { + Some(rel) => ColumnRef::Qualified { + table: rel.to_string(), + name: col.name.clone(), + }, + None => ColumnRef::Named(col.name.clone()), + }), + Expr::Alias(a) => expr_to_group_ref(&a.expr), other => Err(LoweringError::UnsupportedFeature(format!( "non-column GROUP BY expression: {other}" ))), diff --git a/crates/lower/tests/sql_lowering.rs b/crates/lower/tests/sql_lowering.rs index d0d99f5b..e37e81c4 100644 --- a/crates/lower/tests/sql_lowering.rs +++ b/crates/lower/tests/sql_lowering.rs @@ -366,6 +366,37 @@ async fn qualified_where_over_join_resolves_to_right_side() { ); } +#[tokio::test] +async fn self_join_group_by_disambiguates_via_qualifier() { + // L2 group-key qualifier fix: GROUP BY on the *duplicated* column over a + // self-join must bind to the qualified side, not first-match. metrics ⋈ + // metrics → a.service = col 1, b.service = col 5. (Without qualified keys, + // both `GROUP BY a.service` and `GROUP BY b.service` collapsed to col 1.) + let qe_b = lower( + "SELECT b.service, COUNT(*) FROM metrics a JOIN metrics b \ + ON a.service = b.service GROUP BY b.service", + ) + .await; + let (by, _) = find_aggregate(&qe_b).expect("expected an Aggregate over the self-join"); + assert_eq!( + by, + &vec![5], + "GROUP BY b.service binds to the b side (col 5)" + ); + + let qe_a = lower( + "SELECT a.service, COUNT(*) FROM metrics a JOIN metrics b \ + ON a.service = b.service GROUP BY a.service", + ) + .await; + let (by, _) = find_aggregate(&qe_a).expect("expected an Aggregate over the self-join"); + assert_eq!( + by, + &vec![1], + "GROUP BY a.service binds to the a side (col 1)" + ); +} + #[tokio::test] async fn aggregate_over_join_binds_against_concatenated_schema() { // GROUP BY a right-table column over a join: the key must resolve against From 958f7869c18eea897e3b482d2ad149179dcb6727 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 27 May 2026 09:35:33 -0600 Subject: [PATCH 39/40] =?UTF-8?q?refactor(core):=20L2=20hygiene=20?= =?UTF-8?q?=E2=80=94=20drop=20dead=20AggItem.distinct,=20align=20alias,=20?= =?UTF-8?q?mark=20reserved=20nodes=20(L2=20review=20#3/#4/#6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - #4: remove `AggItem.distinct` — a write-only field the converter never read, redundant with `AggFunc::CountDistinct` (COUNT(DISTINCT) sets the func; value DISTINCT is rejected up front). It encoded "distinct" a second way, consumed zero ways. - #6: `AggItem.alias` is now `Option` (was a `""` sentinel), matching its sibling `L2ProjectItem.alias`. The converter maps `None → ""` for L3's `output_names` sentinel; PromQL emits `None`, SQL `Some(name)`. - #3: doc-mark the L2 nodes no front end produces as **Reserved** — `Ref`/`LetBinding` (CSE is L3), `Merge` (UNION→SetOp), L2 `Partition` (PromQL emits `Aggregate.keys`), and `PartitionKeys::Without` (rejected up front) — so the dead converter arms read as intentional, not oversights. No behavior change. Full suite green (133), clippy + fmt clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/core/src/intent_algebra/lower.rs | 28 +++++++++----------- crates/core/src/intent_algebra/query_expr.rs | 3 +++ crates/core/src/intent_algebra/relational.rs | 17 +++++++++--- crates/lower/src/promql.rs | 10 +++---- crates/lower/src/sql/mod.rs | 5 ++-- 5 files changed, 34 insertions(+), 29 deletions(-) diff --git a/crates/core/src/intent_algebra/lower.rs b/crates/core/src/intent_algebra/lower.rs index 5464ae49..2238788d 100644 --- a/crates/core/src/intent_algebra/lower.rs +++ b/crates/core/src/intent_algebra/lower.rs @@ -135,7 +135,7 @@ pub fn convert( let aggregate = CQueryExpr::Aggregate { by, aggs: vec![intent], - output_names: vec![aggs[0].alias.clone()], + output_names: vec![aggs[0].alias.clone().unwrap_or_default()], having: None, child: Box::new(agg_child), }; @@ -179,7 +179,10 @@ pub fn convert( Ok(agg_func_to_intent(&item.func, acc, col)) }) .collect::, _>>()?; - let output_names: Vec = aggs.iter().map(|item| item.alias.clone()).collect(); + let output_names: Vec = aggs + .iter() + .map(|item| item.alias.clone().unwrap_or_default()) + .collect(); let having = having .as_ref() .map(|h| -> Result { @@ -552,16 +555,14 @@ mod tests { keys: vec![], aggs: vec![ AggItem { - alias: "total_bytes".into(), + alias: Some("total_bytes".into()), func: AggFunc::Sum, col: ColumnRef::Named("bytes".into()), - distinct: false, }, AggItem { - alias: "avg_latency".into(), + alias: Some("avg_latency".into()), func: AggFunc::Avg, col: ColumnRef::Named("latency".into()), - distinct: false, }, ], having: None, @@ -603,10 +604,9 @@ mod tests { let tree = LQueryExpr::Aggregate { keys: vec![], aggs: vec![AggItem { - alias: "value".into(), + alias: Some("value".into()), func: AggFunc::Sum, col: ColumnRef::SampleValue, - distinct: false, }], having: None, input: Box::new(LQueryExpr::Source(SourceSpec::new("m"))), @@ -649,16 +649,14 @@ mod tests { keys: vec![ColumnRef::Named("region".into())], aggs: vec![ AggItem { - alias: "tot".into(), + alias: Some("tot".into()), func: AggFunc::Sum, col: ColumnRef::Named("bytes".into()), - distinct: false, }, AggItem { - alias: "n".into(), + alias: Some("n".into()), func: AggFunc::Count, col: ColumnRef::Wildcard, - distinct: false, }, ], having: None, @@ -694,16 +692,14 @@ mod tests { keys: vec![ColumnRef::Named("region".into())], aggs: vec![ AggItem { - alias: "tot".into(), + alias: Some("tot".into()), func: AggFunc::Sum, col: ColumnRef::Named("bytes".into()), - distinct: false, }, AggItem { - alias: "n".into(), + alias: Some("n".into()), func: AggFunc::Count, col: ColumnRef::Wildcard, - distinct: false, }, ], having: Some(L2Expr::Compare { diff --git a/crates/core/src/intent_algebra/query_expr.rs b/crates/core/src/intent_algebra/query_expr.rs index b58d877e..d8604028 100644 --- a/crates/core/src/intent_algebra/query_expr.rs +++ b/crates/core/src/intent_algebra/query_expr.rs @@ -69,6 +69,9 @@ impl Source { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum PartitionKeys { By(Vec), + /// **Reserved**: PromQL `without(...)` is rejected up front (the + /// usage-derived schema can't enumerate the label complement), so no front + /// end produces this variant yet. Without(Vec), } diff --git a/crates/core/src/intent_algebra/relational.rs b/crates/core/src/intent_algebra/relational.rs index 8f167e0c..9fbcdc50 100644 --- a/crates/core/src/intent_algebra/relational.rs +++ b/crates/core/src/intent_algebra/relational.rs @@ -63,10 +63,11 @@ impl SourceSpec { /// One aggregate function in a GROUP BY / AGGREGATE node. #[derive(Debug, Clone, PartialEq)] pub struct AggItem { - pub alias: String, + /// Output alias (`None` = use the intent's conventional name). Matches + /// `L2ProjectItem.alias`'s convention — no `""` sentinel. + pub alias: Option, pub func: AggFunc, pub col: ColumnRef, - pub distinct: bool, } /// Layer-2 aggregate functions. Mapped to canonical [`AggIntent`] by @@ -107,7 +108,9 @@ pub enum AggFunc { pub enum QueryExpr { /// A named metric stream or table — the outermost leaf. Source(SourceSpec), - /// Reference to a CTE / let-binding by name. + /// Reference to a CTE / let-binding by name. **Reserved**: no front end + /// emits `Ref`/`LetBinding` yet (CSE runs on L3); the converter arm exists + /// for forward-compatibility (e.g. PromQL recording rules). Ref(String), /// σ — row-level filter (WHERE / PromQL label matchers). @@ -138,6 +141,8 @@ pub enum QueryExpr { }, /// Partition the stream by key-tuple (`by (dims)` / `without (dims)`). + /// **Reserved**: PromQL `by(...)` emits `Aggregate.keys` (the converter + /// synthesizes any L3 `Partition`); no front end emits an L2 `Partition`. Partition { keys: PartitionKeys, input: Box, @@ -153,7 +158,9 @@ pub enum QueryExpr { by: Vec, input: Box, }, - /// ⊕ — merge sub-results from independent branches. + /// ⊕ — merge sub-results from independent branches. **Reserved**: SQL + /// `UNION` lowers to `SetOp`; no front end emits `Merge` yet (reserved for + /// sharded / fan-in plans). Merge { inputs: Vec }, Join { @@ -179,6 +186,8 @@ pub enum QueryExpr { input: Box, }, + /// **Reserved**: see [`Ref`](Self::Ref) — no front end emits `LetBinding` + /// yet (DAG fan-in / CSE is expressed on L3). LetBinding { name: String, expr: Box, diff --git a/crates/lower/src/promql.rs b/crates/lower/src/promql.rs index ccd1fed1..654149c4 100644 --- a/crates/lower/src/promql.rs +++ b/crates/lower/src/promql.rs @@ -452,12 +452,11 @@ fn windowed_aggregate(inner: Inner, keys: Vec, func: AggFunc) -> L2 { L2::Aggregate { keys, aggs: vec![AggItem { - // Empty alias → the converter keeps PromQL's intent-keyed output + // None alias → the converter keeps PromQL's intent-keyed output // names ("sum", "quantile_0_99", …) instead of overriding them. - alias: String::new(), + alias: None, func, col: ColumnRef::SampleValue, - distinct: false, }], having: None, input: Box::new(input), @@ -471,12 +470,11 @@ fn outer_aggregate(keys: Vec, func: AggFunc, input: L2) -> L2 { L2::Aggregate { keys, aggs: vec![AggItem { - // Empty alias → the converter keeps PromQL's intent-keyed output + // None alias → the converter keeps PromQL's intent-keyed output // names ("sum", "quantile_0_99", …) instead of overriding them. - alias: String::new(), + alias: None, func, col: ColumnRef::SampleValue, - distinct: false, }], having: None, input: Box::new(input), diff --git a/crates/lower/src/sql/mod.rs b/crates/lower/src/sql/mod.rs index e7ff24f1..a06353ec 100644 --- a/crates/lower/src/sql/mod.rs +++ b/crates/lower/src/sql/mod.rs @@ -322,7 +322,7 @@ impl<'a> SqlLowerer<'a> { .map(|(i, e)| { let mut item = lower_agg_item(e)?; if let Some(name) = out_names.get(i) { - item.alias = name.clone(); + item.alias = Some(name.clone()); } Ok(item) }) @@ -450,10 +450,9 @@ fn lower_agg_item(expr: &Expr) -> Result { _ => return Err(LoweringError::UnsupportedAggregate(name)), }; Ok(AggItem { - alias: name, + alias: Some(name), func, col, - distinct: agg_fn.distinct, }) } _ => Err(LoweringError::UnsupportedAggregate(format!("{expr:?}"))), From a7f9c83a259558bcc4b6d308ffc5584d51e80ef7 Mon Sep 17 00:00:00 2001 From: zz_y Date: Wed, 27 May 2026 09:38:02 -0600 Subject: [PATCH 40/40] refactor(core): unify L2Expr/L3Expr into one generic Expr (L2 review #2, scalar) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two scalar IRs were byte-identical 13-variant enums differing only in the column-reference type, with duplicated `conjuncts`/`disjuncts`/`columns_referenced` — maintained twice. Collapse them into a single `Expr`: - `type L2Expr = Expr` (name-based, front-end-emitted) - `type L3Expr = Expr` (positional, resolved) The helpers are now one generic `impl Expr` (`columns_referenced` returns `Vec<&C>`, unifying the L2 `&ColumnRef` and L3 owned-`ColumnId` variants — the sole caller is the L2 Binder). Because the aliases preserve variant construction/pattern syntax (`L2Expr::Compare { .. }` etc.), every front end, the converter's `resolve_expr` map, and all tests compile unchanged. This is the clean, low-risk half of the QueryExpr parameterization the review recommended starting with. No behavior change. Full suite green (133), clippy + fmt clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/core/src/intent_algebra/expr_ir.rs | 230 ++++++---------------- crates/core/src/intent_algebra/mod.rs | 2 +- 2 files changed, 66 insertions(+), 166 deletions(-) diff --git a/crates/core/src/intent_algebra/expr_ir.rs b/crates/core/src/intent_algebra/expr_ir.rs index d49b2a03..97a1d990 100644 --- a/crates/core/src/intent_algebra/expr_ir.rs +++ b/crates/core/src/intent_algebra/expr_ir.rs @@ -1,16 +1,17 @@ //! Language-independent scalar expression IR. //! -//! There are **two** scalar expression types, mirroring the lowering boundary: +//! One generic [`Expr`] spans the lowering boundary; the two layers are +//! aliases that differ only in the column-reference type `C`: //! -//! - [`L2Expr`] — name-based ([`Column(ColumnRef)`](ColumnRef)). The per-language -//! front ends emit it (PromQL label matchers, SQL `WHERE` / projection / -//! sort-key expressions) on the Layer-2 `relational` tree. -//! - [`L3Expr`] — **positional** ([`Column(ColumnId)`](ColumnId)). The canonical -//! L3 `query_expr` tree carries it; the converter resolves every `L2Expr` -//! column reference against the in-scope schema to produce it, so L3 column -//! identity is unambiguous (no name collisions across a join). +//! - [`L2Expr`] = `Expr` — name-based. The per-language front ends +//! emit it (PromQL label matchers, SQL `WHERE` / projection / sort-key +//! expressions) on the Layer-2 `relational` tree. +//! - [`L3Expr`] = `Expr` — **positional**. The canonical L3 +//! `query_expr` tree carries it; the converter resolves every `ColumnRef` +//! against the in-scope schema to produce it, so L3 column identity is +//! unambiguous (no name collisions across a join). //! -//! Both share the same shape and the scalar/operator vocabulary +//! `Expr` shares the scalar/operator vocabulary //! ([`L3Scalar`], [`CompareOp`], [`ArithOp`]) — the **union** of what the two //! front ends need: PromQL contributes `Regex` / `NotRegex` (`=~` / `!~`); SQL //! contributes arithmetic, `CASE`, `IN`, `CAST`, `IS [NOT] NULL`, scalar @@ -118,223 +119,122 @@ impl std::fmt::Display for ArithOp { } } -/// Layer-2 (name-based) scalar expression. Front ends emit this; the converter -/// resolves it into a positional [`L3Expr`]. Flat conjunctions (`BoolAnd`) / -/// disjunctions (`BoolOr`) make per-conjunct selectivity estimation and -/// label-matcher lowering straightforward without recursive descent. +/// Scalar expression, generic over its column-reference type `C`. The two +/// lowering layers are aliases over the *same* shape — only the column +/// reference differs — so there is one definition (and one set of helpers) to +/// maintain, and the converter is a structural map that swaps `C`: +/// +/// - [`L2Expr`] = `Expr` — name-based, front-end-emitted. +/// - [`L3Expr`] = `Expr` — positional, resolved against the schema. +/// +/// Flat conjunctions (`BoolAnd`) / disjunctions (`BoolOr`) make per-conjunct +/// selectivity estimation and label-matcher lowering straightforward without +/// recursive descent. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub enum L2Expr { - /// Reference to a named column / label. - Column(ColumnRef), +pub enum Expr { + /// A column reference — `ColumnRef` (L2) or positional `ColumnId` (L3). + Column(C), /// A constant literal value. Literal(L3Scalar), /// `left op right` — binary comparison. Compare { - left: Box, + left: Box>, op: CompareOp, - right: Box, + right: Box>, }, /// Flat conjunction (logical AND). An empty list is vacuously true. - BoolAnd(Vec), + BoolAnd(Vec>), /// Flat disjunction (logical OR). An empty list is vacuously false. - BoolOr(Vec), + BoolOr(Vec>), /// Logical NOT. - Not(Box), + Not(Box>), /// `expr IS NULL`. - IsNull(Box), + IsNull(Box>), /// `expr IS NOT NULL`. - IsNotNull(Box), + IsNotNull(Box>), /// `CAST(expr AS to)`; `try_cast` for SQL `TRY_CAST` (NULL on failure). Cast { - expr: Box, + expr: Box>, to: DataType, try_cast: bool, }, /// `expr [NOT] IN (v1, v2, …)`. InList { - expr: Box, - list: Vec, + expr: Box>, + list: Vec>, negated: bool, }, /// Scalar function call, e.g. `LOWER(col)`, `ABS(x)`. - FunctionCall { name: String, args: Vec }, + FunctionCall { name: String, args: Vec> }, /// Binary arithmetic: `left op right`. Arith { op: ArithOp, - left: Box, - right: Box, + left: Box>, + right: Box>, }, /// SQL `CASE` (both searched and simple forms). `operand` present for the /// simple form (`CASE expr WHEN …`), absent for searched. Case { - operand: Option>, - branches: Vec<(L2Expr, L2Expr)>, - else_expr: Option>, + operand: Option>>, + branches: Vec<(Expr, Expr)>, + else_expr: Option>>, }, } -impl L2Expr { - /// If this expression is a `BoolAnd`, return its elements; otherwise a - /// single-element slice containing `self`. - pub fn conjuncts(&self) -> &[L2Expr] { - match self { - L2Expr::BoolAnd(v) => v.as_slice(), - _ => std::slice::from_ref(self), - } - } - - /// If this expression is a `BoolOr`, return its elements; otherwise a - /// single-element slice containing `self`. - pub fn disjuncts(&self) -> &[L2Expr] { - match self { - L2Expr::BoolOr(v) => v.as_slice(), - _ => std::slice::from_ref(self), - } - } - - /// Recursively collect every `ColumnRef` referenced anywhere in this - /// expression. Used by the Binder to seed usage-derived leaf schemas. - pub fn columns_referenced(&self) -> Vec<&ColumnRef> { - match self { - L2Expr::Column(c) => vec![c], - L2Expr::Literal(_) => vec![], - L2Expr::Compare { left, right, .. } | L2Expr::Arith { left, right, .. } => { - let mut v = left.columns_referenced(); - v.extend(right.columns_referenced()); - v - } - L2Expr::BoolAnd(parts) | L2Expr::BoolOr(parts) => { - parts.iter().flat_map(|e| e.columns_referenced()).collect() - } - L2Expr::Not(e) | L2Expr::IsNull(e) | L2Expr::IsNotNull(e) => e.columns_referenced(), - L2Expr::Cast { expr, .. } => expr.columns_referenced(), - L2Expr::InList { expr, list, .. } => { - let mut v = expr.columns_referenced(); - v.extend(list.iter().flat_map(|e| e.columns_referenced())); - v - } - L2Expr::FunctionCall { args, .. } => { - args.iter().flat_map(|e| e.columns_referenced()).collect() - } - L2Expr::Case { - operand, - branches, - else_expr, - } => { - let mut v = vec![]; - if let Some(op) = operand { - v.extend(op.columns_referenced()); - } - for (when, then) in branches { - v.extend(when.columns_referenced()); - v.extend(then.columns_referenced()); - } - if let Some(e) = else_expr { - v.extend(e.columns_referenced()); - } - v - } - } - } -} +/// Layer-2 (name-based) scalar expression. Front ends emit this; the converter +/// resolves it into a positional [`L3Expr`]. +pub type L2Expr = Expr; -/// Canonical L3 (positional) scalar expression. Same shape as [`L2Expr`] but -/// column references are positional [`ColumnId`]s resolved against the in-scope -/// schema, so identity is unambiguous across joins / duplicate names. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub enum L3Expr { - /// Positional reference to a column in the in-scope schema. - Column(ColumnId), - /// A constant literal value. - Literal(L3Scalar), - /// `left op right` — binary comparison. - Compare { - left: Box, - op: CompareOp, - right: Box, - }, - /// Flat conjunction (logical AND). An empty list is vacuously true. - BoolAnd(Vec), - /// Flat disjunction (logical OR). An empty list is vacuously false. - BoolOr(Vec), - /// Logical NOT. - Not(Box), - /// `expr IS NULL`. - IsNull(Box), - /// `expr IS NOT NULL`. - IsNotNull(Box), - /// `CAST(expr AS to)`; `try_cast` for SQL `TRY_CAST` (NULL on failure). - Cast { - expr: Box, - to: DataType, - try_cast: bool, - }, - /// `expr [NOT] IN (v1, v2, …)`. - InList { - expr: Box, - list: Vec, - negated: bool, - }, - /// Scalar function call, e.g. `LOWER(col)`, `ABS(x)`. - FunctionCall { name: String, args: Vec }, - /// Binary arithmetic: `left op right`. - Arith { - op: ArithOp, - left: Box, - right: Box, - }, - /// SQL `CASE` (both searched and simple forms). - Case { - operand: Option>, - branches: Vec<(L3Expr, L3Expr)>, - else_expr: Option>, - }, -} +/// Canonical L3 (positional) scalar expression — column references are +/// [`ColumnId`]s resolved against the in-scope schema, so identity is +/// unambiguous across joins / duplicate names. +pub type L3Expr = Expr; -impl L3Expr { +impl Expr { /// If this expression is a `BoolAnd`, return its elements; otherwise a /// single-element slice containing `self`. - pub fn conjuncts(&self) -> &[L3Expr] { + pub fn conjuncts(&self) -> &[Expr] { match self { - L3Expr::BoolAnd(v) => v.as_slice(), + Expr::BoolAnd(v) => v.as_slice(), _ => std::slice::from_ref(self), } } /// If this expression is a `BoolOr`, return its elements; otherwise a /// single-element slice containing `self`. - pub fn disjuncts(&self) -> &[L3Expr] { + pub fn disjuncts(&self) -> &[Expr] { match self { - L3Expr::BoolOr(v) => v.as_slice(), + Expr::BoolOr(v) => v.as_slice(), _ => std::slice::from_ref(self), } } - /// Recursively collect every positional [`ColumnId`] referenced anywhere in - /// this expression. Used by L4 for column-lineage and selectivity. - pub fn columns_referenced(&self) -> Vec { + /// Recursively collect every column reference anywhere in this expression. + /// Used by the Binder (L2) to seed usage-derived leaf schemas, and available + /// to L4 (L3) for column-lineage / selectivity. + pub fn columns_referenced(&self) -> Vec<&C> { match self { - L3Expr::Column(id) => vec![*id], - L3Expr::Literal(_) => vec![], - L3Expr::Compare { left, right, .. } | L3Expr::Arith { left, right, .. } => { + Expr::Column(c) => vec![c], + Expr::Literal(_) => vec![], + Expr::Compare { left, right, .. } | Expr::Arith { left, right, .. } => { let mut v = left.columns_referenced(); v.extend(right.columns_referenced()); v } - L3Expr::BoolAnd(parts) | L3Expr::BoolOr(parts) => { + Expr::BoolAnd(parts) | Expr::BoolOr(parts) => { parts.iter().flat_map(|e| e.columns_referenced()).collect() } - L3Expr::Not(e) | L3Expr::IsNull(e) | L3Expr::IsNotNull(e) => e.columns_referenced(), - L3Expr::Cast { expr, .. } => expr.columns_referenced(), - L3Expr::InList { expr, list, .. } => { + Expr::Not(e) | Expr::IsNull(e) | Expr::IsNotNull(e) => e.columns_referenced(), + Expr::Cast { expr, .. } => expr.columns_referenced(), + Expr::InList { expr, list, .. } => { let mut v = expr.columns_referenced(); v.extend(list.iter().flat_map(|e| e.columns_referenced())); v } - L3Expr::FunctionCall { args, .. } => { + Expr::FunctionCall { args, .. } => { args.iter().flat_map(|e| e.columns_referenced()).collect() } - L3Expr::Case { + Expr::Case { operand, branches, else_expr, diff --git a/crates/core/src/intent_algebra/mod.rs b/crates/core/src/intent_algebra/mod.rs index 9ee324d5..9eeb9568 100644 --- a/crates/core/src/intent_algebra/mod.rs +++ b/crates/core/src/intent_algebra/mod.rs @@ -30,7 +30,7 @@ pub use column_resolution::{ resolve_column_refs, resolve_expr, ResolveError, }; pub use cse::{dedupe_subtrees, CseWorkloadPlan}; -pub use expr_ir::{ArithOp, ColumnRef, CompareOp, L2Expr, L3Expr, L3Scalar}; +pub use expr_ir::{ArithOp, ColumnRef, CompareOp, Expr, L2Expr, L3Expr, L3Scalar}; pub use lower::{convert, convert_root, ConvertError}; pub use names::{BindingName, QueryId}; pub use query_expr::{