diff --git a/controller/.gitignore b/controller/.gitignore deleted file mode 100644 index 2f7896d1..00000000 --- a/controller/.gitignore +++ /dev/null @@ -1 +0,0 @@ -target/ diff --git a/controller/Cargo.lock b/controller/Cargo.lock deleted file mode 100644 index 8f8b85c1..00000000 --- a/controller/Cargo.lock +++ /dev/null @@ -1,2922 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "adler2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" - -[[package]] -name = "aho-corasick" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" -dependencies = [ - "memchr", -] - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - -[[package]] -name = "anyhow" -version = "1.0.102" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" - -[[package]] -name = "ar_archive_writer" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eb93bbb63b9c227414f6eb3a0adfddca591a8ce1e9b60661bb08969b87e340b" -dependencies = [ - "object", -] - -[[package]] -name = "async-stream" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" -dependencies = [ - "async-stream-impl", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-stream-impl" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "async-trait" -version = "0.1.89" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - -[[package]] -name = "autocfg" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" - -[[package]] -name = "axum" -version = "0.7.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" -dependencies = [ - "async-trait", - "axum-core", - "base64", - "bytes", - "futures-util", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-util", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "serde_json", - "serde_path_to_error", - "serde_urlencoded", - "sha1", - "sync_wrapper", - "tokio", - "tokio-tungstenite 0.24.0", - "tower 0.5.3", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "axum-core" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http", - "http-body", - "http-body-util", - "mime", - "pin-project-lite", - "rustversion", - "sync_wrapper", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - -[[package]] -name = "bincode" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] - -[[package]] -name = "bitflags" -version = "2.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - -[[package]] -name = "bumpalo" -version = "3.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - -[[package]] -name = "bytes" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" - -[[package]] -name = "cactus" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbc26382d871df4b7442e3df10a9402bf3cf5e55cbd66f12be38861425f0564" - -[[package]] -name = "cc" -version = "1.2.57" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" -dependencies = [ - "find-msvc-tools", - "jobserver", - "libc", - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" - -[[package]] -name = "cfg_aliases" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" - -[[package]] -name = "cfgrammar" -version = "0.13.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe45e18904af7af10e4312df7c97251e98af98c70f42f1f2587aecfcbee56bf" -dependencies = [ - "indexmap 2.13.0", - "lazy_static", - "num-traits", - "regex", - "serde", - "vob", -] - -[[package]] -name = "chrono" -version = "0.4.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" -dependencies = [ - "iana-time-zone", - "js-sys", - "num-traits", - "serde", - "wasm-bindgen", - "windows-link", -] - -[[package]] -name = "controller" -version = "0.1.0" -dependencies = [ - "anyhow", - "axum", - "bytes", - "chrono", - "futures-util", - "http-body-util", - "parking_lot", - "prometheus", - "promql-parser", - "prost", - "prost-build", - "reqwest", - "serde", - "serde_json", - "serde_yaml", - "sqlparser", - "thiserror 1.0.69", - "tokio", - "tokio-stream", - "tokio-tungstenite 0.21.0", - "tonic", - "tonic-build", - "tower 0.4.13", - "tracing", - "tracing-subscriber", - "zstd", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - -[[package]] -name = "cpufeatures" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" -dependencies = [ - "libc", -] - -[[package]] -name = "crc32fast" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "crypto-common" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "data-encoding" -version = "2.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" - -[[package]] -name = "deranged" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" -dependencies = [ - "powerfmt", -] - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "crypto-common", -] - -[[package]] -name = "displaydoc" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "either" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" - -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - -[[package]] -name = "errno" -version = "0.3.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" -dependencies = [ - "libc", - "windows-sys 0.61.2", -] - -[[package]] -name = "fastrand" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" - -[[package]] -name = "filetime" -version = "0.2.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" -dependencies = [ - "cfg-if", - "libc", - "libredox", -] - -[[package]] -name = "find-msvc-tools" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" - -[[package]] -name = "fixedbitset" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" - -[[package]] -name = "flate2" -version = "1.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" -dependencies = [ - "crc32fast", - "miniz_oxide", -] - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "form_urlencoded" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "futures-channel" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" -dependencies = [ - "futures-core", -] - -[[package]] -name = "futures-core" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" - -[[package]] -name = "futures-macro" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "futures-sink" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" - -[[package]] -name = "futures-task" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" - -[[package]] -name = "futures-util" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" -dependencies = [ - "futures-core", - "futures-macro", - "futures-sink", - "futures-task", - "pin-project-lite", - "slab", -] - -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - -[[package]] -name = "getopts" -version = "0.2.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" -dependencies = [ - "unicode-width", -] - -[[package]] -name = "getrandom" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "wasi", - "wasm-bindgen", -] - -[[package]] -name = "getrandom" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "r-efi", - "wasip2", - "wasm-bindgen", -] - -[[package]] -name = "h2" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http", - "indexmap 2.13.0", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - -[[package]] -name = "hashbrown" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - -[[package]] -name = "http" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" -dependencies = [ - "bytes", - "itoa", -] - -[[package]] -name = "http-body" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" -dependencies = [ - "bytes", - "http", -] - -[[package]] -name = "http-body-util" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" -dependencies = [ - "bytes", - "futures-core", - "http", - "http-body", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" - -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - -[[package]] -name = "hyper" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" -dependencies = [ - "atomic-waker", - "bytes", - "futures-channel", - "futures-core", - "h2", - "http", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "pin-utils", - "smallvec", - "tokio", - "want", -] - -[[package]] -name = "hyper-rustls" -version = "0.27.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" -dependencies = [ - "http", - "hyper", - "hyper-util", - "rustls", - "rustls-pki-types", - "tokio", - "tokio-rustls", - "tower-service", - "webpki-roots", -] - -[[package]] -name = "hyper-timeout" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" -dependencies = [ - "hyper", - "hyper-util", - "pin-project-lite", - "tokio", - "tower-service", -] - -[[package]] -name = "hyper-util" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" -dependencies = [ - "base64", - "bytes", - "futures-channel", - "futures-util", - "http", - "http-body", - "hyper", - "ipnet", - "libc", - "percent-encoding", - "pin-project-lite", - "socket2 0.6.3", - "tokio", - "tower-service", - "tracing", -] - -[[package]] -name = "iana-time-zone" -version = "0.1.65" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "log", - "wasm-bindgen", - "windows-core", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - -[[package]] -name = "icu_collections" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" -dependencies = [ - "displaydoc", - "potential_utf", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_locale_core" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" -dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", -] - -[[package]] -name = "icu_normalizer" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" -dependencies = [ - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "zerovec", -] - -[[package]] -name = "icu_normalizer_data" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" - -[[package]] -name = "icu_properties" -version = "2.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" -dependencies = [ - "icu_collections", - "icu_locale_core", - "icu_properties_data", - "icu_provider", - "zerotrie", - "zerovec", -] - -[[package]] -name = "icu_properties_data" -version = "2.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" - -[[package]] -name = "icu_provider" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" -dependencies = [ - "displaydoc", - "icu_locale_core", - "writeable", - "yoke", - "zerofrom", - "zerotrie", - "zerovec", -] - -[[package]] -name = "idna" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" -dependencies = [ - "idna_adapter", - "smallvec", - "utf8_iter", -] - -[[package]] -name = "idna_adapter" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" -dependencies = [ - "icu_normalizer", - "icu_properties", -] - -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", -] - -[[package]] -name = "indexmap" -version = "2.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" -dependencies = [ - "equivalent", - "hashbrown 0.16.1", -] - -[[package]] -name = "ipnet" -version = "2.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" - -[[package]] -name = "iri-string" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a" -dependencies = [ - "memchr", - "serde", -] - -[[package]] -name = "itertools" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" - -[[package]] -name = "jobserver" -version = "0.1.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" -dependencies = [ - "getrandom 0.3.4", - "libc", -] - -[[package]] -name = "js-sys" -version = "0.3.91" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" -dependencies = [ - "once_cell", - "wasm-bindgen", -] - -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - -[[package]] -name = "libc" -version = "0.2.183" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" - -[[package]] -name = "libredox" -version = "0.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ddbf48fd451246b1f8c2610bd3b4ac0cc6e149d89832867093ab69a17194f08" -dependencies = [ - "bitflags", - "libc", - "plain", - "redox_syscall 0.7.3", -] - -[[package]] -name = "linux-raw-sys" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" - -[[package]] -name = "linux-raw-sys" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" - -[[package]] -name = "litemap" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" - -[[package]] -name = "lock_api" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" -dependencies = [ - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" - -[[package]] -name = "lrlex" -version = "0.13.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c71364e868116ee891b0f93559eb9eca5675bec28b22d33c58481e66c3951d7e" -dependencies = [ - "cfgrammar", - "getopts", - "lazy_static", - "lrpar", - "num-traits", - "quote", - "regex", - "regex-syntax", - "serde", - "vergen", -] - -[[package]] -name = "lrpar" -version = "0.13.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51b265a81193d94c92d1c9c715498d6fa505bce3f789ceecb24ab5d6fa2dbc71" -dependencies = [ - "bincode", - "cactus", - "cfgrammar", - "filetime", - "indexmap 2.13.0", - "lazy_static", - "lrtable", - "num-traits", - "packedvec", - "regex", - "serde", - "static_assertions", - "vergen", - "vob", -] - -[[package]] -name = "lrtable" -version = "0.13.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc36d15214ca997a5097845be1f932b7ee6125c36f5c5e55f6c49e027ddeb6de" -dependencies = [ - "cfgrammar", - "fnv", - "num-traits", - "serde", - "sparsevec", - "vob", -] - -[[package]] -name = "lru-slab" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" - -[[package]] -name = "matchers" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" -dependencies = [ - "regex-automata", -] - -[[package]] -name = "matchit" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" - -[[package]] -name = "memchr" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" - -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - -[[package]] -name = "miniz_oxide" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" -dependencies = [ - "adler2", - "simd-adler32", -] - -[[package]] -name = "mio" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" -dependencies = [ - "libc", - "wasi", - "windows-sys 0.61.2", -] - -[[package]] -name = "multimap" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" - -[[package]] -name = "nu-ansi-term" -version = "0.50.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "num-conv" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" - -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", -] - -[[package]] -name = "num_threads" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" -dependencies = [ - "libc", -] - -[[package]] -name = "object" -version = "0.37.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" -dependencies = [ - "memchr", -] - -[[package]] -name = "once_cell" -version = "1.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" - -[[package]] -name = "packedvec" -version = "1.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69e0a534dd2e6aefce319af62a0aa0066a76bdfcec0201dfe02df226bc9ec70" -dependencies = [ - "num-traits", - "serde", -] - -[[package]] -name = "parking_lot" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall 0.5.18", - "smallvec", - "windows-link", -] - -[[package]] -name = "percent-encoding" -version = "2.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" - -[[package]] -name = "petgraph" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" -dependencies = [ - "fixedbitset", - "indexmap 2.13.0", -] - -[[package]] -name = "pin-project" -version = "1.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "pkg-config" -version = "0.3.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" - -[[package]] -name = "plain" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" - -[[package]] -name = "potential_utf" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" -dependencies = [ - "zerovec", -] - -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - -[[package]] -name = "prettyplease" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" -dependencies = [ - "proc-macro2", - "syn", -] - -[[package]] -name = "proc-macro2" -version = "1.0.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "procfs" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "731e0d9356b0c25f16f33b5be79b1c57b562f141ebfcdb0ad8ac2c13a24293b4" -dependencies = [ - "bitflags", - "hex", - "lazy_static", - "procfs-core", - "rustix 0.38.44", -] - -[[package]] -name = "procfs-core" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3554923a69f4ce04c4a754260c338f505ce22642d3830e049a399fc2059a29" -dependencies = [ - "bitflags", - "hex", -] - -[[package]] -name = "prometheus" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1" -dependencies = [ - "cfg-if", - "fnv", - "lazy_static", - "libc", - "memchr", - "parking_lot", - "procfs", - "thiserror 1.0.69", -] - -[[package]] -name = "promql-parser" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df2791a28f8ea7e48f2838999c06d089184d44adb860feab682d45dd190ef718" -dependencies = [ - "cfgrammar", - "chrono", - "lazy_static", - "lrlex", - "lrpar", - "regex", -] - -[[package]] -name = "prost" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-build" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" -dependencies = [ - "heck", - "itertools", - "log", - "multimap", - "once_cell", - "petgraph", - "prettyplease", - "prost", - "prost-types", - "regex", - "syn", - "tempfile", -] - -[[package]] -name = "prost-derive" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "prost-types" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" -dependencies = [ - "prost", -] - -[[package]] -name = "psm" -version = "0.1.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3852766467df634d74f0b2d7819bf8dc483a0eb2e3b0f50f756f9cfe8b0d18d8" -dependencies = [ - "ar_archive_writer", - "cc", -] - -[[package]] -name = "quinn" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" -dependencies = [ - "bytes", - "cfg_aliases", - "pin-project-lite", - "quinn-proto", - "quinn-udp", - "rustc-hash", - "rustls", - "socket2 0.6.3", - "thiserror 2.0.18", - "tokio", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-proto" -version = "0.11.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" -dependencies = [ - "bytes", - "getrandom 0.3.4", - "lru-slab", - "rand 0.9.2", - "ring", - "rustc-hash", - "rustls", - "rustls-pki-types", - "slab", - "thiserror 2.0.18", - "tinyvec", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-udp" -version = "0.5.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" -dependencies = [ - "cfg_aliases", - "libc", - "once_cell", - "socket2 0.6.3", - "tracing", - "windows-sys 0.60.2", -] - -[[package]] -name = "quote" -version = "1.0.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - -[[package]] -name = "rand" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" -dependencies = [ - "rand_chacha 0.9.0", - "rand_core 0.9.5", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core 0.9.5", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.17", -] - -[[package]] -name = "rand_core" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" -dependencies = [ - "getrandom 0.3.4", -] - -[[package]] -name = "recursive" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" -dependencies = [ - "recursive-proc-macro-impl", - "stacker", -] - -[[package]] -name = "recursive-proc-macro-impl" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" -dependencies = [ - "quote", - "syn", -] - -[[package]] -name = "redox_syscall" -version = "0.5.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" -dependencies = [ - "bitflags", -] - -[[package]] -name = "redox_syscall" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16" -dependencies = [ - "bitflags", -] - -[[package]] -name = "regex" -version = "1.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" - -[[package]] -name = "reqwest" -version = "0.12.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" -dependencies = [ - "base64", - "bytes", - "futures-core", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-util", - "js-sys", - "log", - "percent-encoding", - "pin-project-lite", - "quinn", - "rustls", - "rustls-pki-types", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tokio-rustls", - "tower 0.5.3", - "tower-http", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "webpki-roots", -] - -[[package]] -name = "ring" -version = "0.17.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" -dependencies = [ - "cc", - "cfg-if", - "getrandom 0.2.17", - "libc", - "untrusted", - "windows-sys 0.52.0", -] - -[[package]] -name = "rustc-hash" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" - -[[package]] -name = "rustix" -version = "0.38.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", -] - -[[package]] -name = "rustix" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", -] - -[[package]] -name = "rustls" -version = "0.23.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" -dependencies = [ - "once_cell", - "ring", - "rustls-pki-types", - "rustls-webpki", - "subtle", - "zeroize", -] - -[[package]] -name = "rustls-pki-types" -version = "1.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" -dependencies = [ - "web-time", - "zeroize", -] - -[[package]] -name = "rustls-webpki" -version = "0.103.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" -dependencies = [ - "ring", - "rustls-pki-types", - "untrusted", -] - -[[package]] -name = "rustversion" -version = "1.0.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" - -[[package]] -name = "ryu" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" - -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "serde" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" -dependencies = [ - "serde_core", - "serde_derive", -] - -[[package]] -name = "serde_core" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_json" -version = "1.0.149" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" -dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", -] - -[[package]] -name = "serde_path_to_error" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" -dependencies = [ - "itoa", - "serde", - "serde_core", -] - -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "serde_yaml" -version = "0.9.34+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" -dependencies = [ - "indexmap 2.13.0", - "itoa", - "ryu", - "serde", - "unsafe-libyaml", -] - -[[package]] -name = "sha1" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "sharded-slab" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" -dependencies = [ - "lazy_static", -] - -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "signal-hook-registry" -version = "1.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" -dependencies = [ - "errno", - "libc", -] - -[[package]] -name = "simd-adler32" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" - -[[package]] -name = "slab" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" - -[[package]] -name = "smallvec" -version = "1.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" - -[[package]] -name = "socket2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - -[[package]] -name = "socket2" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" -dependencies = [ - "libc", - "windows-sys 0.61.2", -] - -[[package]] -name = "sparsevec" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b4a8ce3045f0fe173fb5ae3c6b7dcfbec02bfa650bb8618b2301f52af0134d" -dependencies = [ - "num-traits", - "packedvec", - "serde", - "vob", -] - -[[package]] -name = "sqlparser" -version = "0.61.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" -dependencies = [ - "log", - "recursive", -] - -[[package]] -name = "stable_deref_trait" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" - -[[package]] -name = "stacker" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d74a23609d509411d10e2176dc2a4346e3b4aea2e7b1869f19fdedbc71c013" -dependencies = [ - "cc", - "cfg-if", - "libc", - "psm", - "windows-sys 0.59.0", -] - -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - -[[package]] -name = "subtle" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" - -[[package]] -name = "syn" -version = "2.0.117" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "sync_wrapper" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" -dependencies = [ - "futures-core", -] - -[[package]] -name = "synstructure" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tempfile" -version = "3.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" -dependencies = [ - "fastrand", - "getrandom 0.3.4", - "once_cell", - "rustix 1.1.4", - "windows-sys 0.61.2", -] - -[[package]] -name = "thiserror" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" -dependencies = [ - "thiserror-impl 1.0.69", -] - -[[package]] -name = "thiserror" -version = "2.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" -dependencies = [ - "thiserror-impl 2.0.18", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "thread_local" -version = "1.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "time" -version = "0.3.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" -dependencies = [ - "deranged", - "itoa", - "libc", - "num-conv", - "num_threads", - "powerfmt", - "serde_core", - "time-core", - "time-macros", -] - -[[package]] -name = "time-core" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" - -[[package]] -name = "time-macros" -version = "0.2.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" -dependencies = [ - "num-conv", - "time-core", -] - -[[package]] -name = "tinystr" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" -dependencies = [ - "displaydoc", - "zerovec", -] - -[[package]] -name = "tinyvec" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "tokio" -version = "1.50.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" -dependencies = [ - "bytes", - "libc", - "mio", - "parking_lot", - "pin-project-lite", - "signal-hook-registry", - "socket2 0.6.3", - "tokio-macros", - "windows-sys 0.61.2", -] - -[[package]] -name = "tokio-macros" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tokio-rustls" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" -dependencies = [ - "rustls", - "tokio", -] - -[[package]] -name = "tokio-stream" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-tungstenite" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c83b561d025642014097b66e6c1bb422783339e0909e4429cde4749d1990bc38" -dependencies = [ - "futures-util", - "log", - "tokio", - "tungstenite 0.21.0", -] - -[[package]] -name = "tokio-tungstenite" -version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" -dependencies = [ - "futures-util", - "log", - "tokio", - "tungstenite 0.24.0", -] - -[[package]] -name = "tokio-util" -version = "0.7.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tonic" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" -dependencies = [ - "async-stream", - "async-trait", - "axum", - "base64", - "bytes", - "flate2", - "h2", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-timeout", - "hyper-util", - "percent-encoding", - "pin-project", - "prost", - "socket2 0.5.10", - "tokio", - "tokio-stream", - "tower 0.4.13", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tonic-build" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9557ce109ea773b399c9b9e5dca39294110b74f1f342cb347a80d1fce8c26a11" -dependencies = [ - "prettyplease", - "proc-macro2", - "prost-build", - "prost-types", - "quote", - "syn", -] - -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "indexmap 1.9.3", - "pin-project", - "pin-project-lite", - "rand 0.8.5", - "slab", - "tokio", - "tokio-util", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" -dependencies = [ - "futures-core", - "futures-util", - "pin-project-lite", - "sync_wrapper", - "tokio", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-http" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" -dependencies = [ - "bitflags", - "bytes", - "futures-util", - "http", - "http-body", - "iri-string", - "pin-project-lite", - "tower 0.5.3", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - -[[package]] -name = "tower-service" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" - -[[package]] -name = "tracing" -version = "0.1.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" -dependencies = [ - "log", - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tracing-core" -version = "0.1.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" -dependencies = [ - "once_cell", - "valuable", -] - -[[package]] -name = "tracing-log" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - -[[package]] -name = "tracing-subscriber" -version = "0.3.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" -dependencies = [ - "matchers", - "nu-ansi-term", - "once_cell", - "regex-automata", - "sharded-slab", - "smallvec", - "thread_local", - "tracing", - "tracing-core", - "tracing-log", -] - -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - -[[package]] -name = "tungstenite" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ef1a641ea34f399a848dea702823bbecfb4c486f911735368f1f137cb8257e1" -dependencies = [ - "byteorder", - "bytes", - "data-encoding", - "http", - "httparse", - "log", - "rand 0.8.5", - "sha1", - "thiserror 1.0.69", - "url", - "utf-8", -] - -[[package]] -name = "tungstenite" -version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" -dependencies = [ - "byteorder", - "bytes", - "data-encoding", - "http", - "httparse", - "log", - "rand 0.8.5", - "sha1", - "thiserror 1.0.69", - "utf-8", -] - -[[package]] -name = "typenum" -version = "1.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" - -[[package]] -name = "unicode-ident" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" - -[[package]] -name = "unicode-width" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" - -[[package]] -name = "unsafe-libyaml" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" - -[[package]] -name = "untrusted" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" - -[[package]] -name = "url" -version = "2.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", - "serde", -] - -[[package]] -name = "utf-8" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" - -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - -[[package]] -name = "valuable" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" - -[[package]] -name = "vergen" -version = "8.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2990d9ea5967266ea0ccf413a4aa5c42a93dbcfda9cb49a97de6931726b12566" -dependencies = [ - "anyhow", - "rustversion", - "time", -] - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "vob" -version = "3.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc936b5a7202a703aeaf7ce05e7931db2e0c8126813f97db3e9e06d867b0bb38" -dependencies = [ - "num-traits", - "serde", -] - -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - -[[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" - -[[package]] -name = "wasip2" -version = "1.0.2+wasi-0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" -dependencies = [ - "wit-bindgen", -] - -[[package]] -name = "wasm-bindgen" -version = "0.2.114" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.64" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" -dependencies = [ - "cfg-if", - "futures-util", - "js-sys", - "once_cell", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.114" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.114" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" -dependencies = [ - "bumpalo", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.114" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "web-sys" -version = "0.3.91" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "web-time" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "webpki-roots" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" -dependencies = [ - "rustls-pki-types", -] - -[[package]] -name = "windows-core" -version = "0.62.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link", - "windows-result", - "windows-strings", -] - -[[package]] -name = "windows-implement" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "windows-interface" -version = "0.59.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "windows-link" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" - -[[package]] -name = "windows-result" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-strings" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", -] - -[[package]] -name = "windows-sys" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - -[[package]] -name = "wit-bindgen" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" - -[[package]] -name = "writeable" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" - -[[package]] -name = "yoke" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" -dependencies = [ - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - -[[package]] -name = "zerocopy" -version = "0.8.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efbb2a062be311f2ba113ce66f697a4dc589f85e78a4aea276200804cea0ed87" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "zerofrom" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - -[[package]] -name = "zeroize" -version = "1.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" - -[[package]] -name = "zerotrie" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", -] - -[[package]] -name = "zerovec" -version = "0.11.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "zmij" -version = "1.0.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" - -[[package]] -name = "zstd" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "7.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" -dependencies = [ - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" -dependencies = [ - "cc", - "pkg-config", -] diff --git a/controller/Cargo.toml b/controller/Cargo.toml deleted file mode 100644 index 7c7a7ec3..00000000 --- a/controller/Cargo.toml +++ /dev/null @@ -1,42 +0,0 @@ -[package] -name = "controller" -version = "0.1.0" -edition = "2021" - -[[bin]] -name = "controller" -path = "src/main.rs" - -[dependencies] -tokio = { version = "1", features = ["full"] } -axum = { version = "0.7", features = ["ws"] } -futures-util = "0.3" -serde = { version = "1", features = ["derive"] } -serde_json = "1" -serde_yaml = "0.9" -reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } -anyhow = "1" -thiserror = "1" -tracing = "0.1" -tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } -chrono = { version = "0.4", features = ["serde"] } -sqlparser = "0.61" -promql-parser = "0.8" -prost = "0.13" -bytes = "1" -zstd = "0.13" -parking_lot = "0.12" -prometheus = { version = "0.13", default-features = false, features = ["process"] } -tonic = { version = "0.12", features = ["gzip"] } -tokio-stream = { version = "0.1", features = ["net"] } - -[dev-dependencies] -tokio = { version = "1", features = ["full", "test-util"] } -tower = { version = "0.4", features = ["util"] } -axum = { version = "0.7", features = ["ws"] } -http-body-util = "0.1" -tokio-tungstenite = "0.21" - -[build-dependencies] -prost-build = "0.13" -tonic-build = "0.12" diff --git a/controller/build.rs b/controller/build.rs deleted file mode 100644 index 04f96967..00000000 --- a/controller/build.rs +++ /dev/null @@ -1,11 +0,0 @@ -fn main() -> Result<(), Box> { - prost_build::compile_protos(&["proto/opamp.proto"], &["proto/"])?; - // asap.runtime.v1.RuntimeSamples service — receives - // PushExporter batches from agents. Must stay in lockstep - // with `sketch-bench/sketch-runtime/proto/feedback.proto`. - tonic_build::configure() - .build_server(true) - .build_client(false) - .compile_protos(&["proto/feedback.proto"], &["proto/"])?; - Ok(()) -} diff --git a/controller/docs/design.md b/controller/docs/design.md deleted file mode 100644 index 28f35ec0..00000000 --- a/controller/docs/design.md +++ /dev/null @@ -1,1722 +0,0 @@ -# ASAPController — design - -> **Mirror.** This file mirrors `docs/design.md` from -> [`ProjectASAP/ASAPController#docs/workload-plan-and-queryspec`](https://github.com/ProjectASAP/ASAPController/tree/docs/workload-plan-and-queryspec/docs) -> as of 2026-05-05. -> -> ASAPController upstream is currently doc-only. The actively-running -> controller implementation lives next to this doc in -> [`controller/src/`](../src/) — the file you're reading describes the -> **target architecture** the implementation is converging toward. -> Where this doc and the current `analyzer::QuerySpec` shape diverge, -> the doc is the design intent. - -Target repo: **`github.com/ProjectASAP/ASAPController`** (currently doc-only). - -Merges three existing codebases into one: - -1. **`DataCollector/controller`** (Rust) — service-shaped; end-to-end data lifecycle planner (collection → transmission → storage → analytics query); OpAMP + HTTP; SLA-driven replanning loop. -2. **`ASAPQuery[-backend]/asap-planner-rs`** (Rust) — CLI-shaped; analytics-query-only planner; YAML-in → YAML-out. Two divergent copies today. -3. **`asap-fusion`** (Rust) — library-shaped; DataFusion operator-level rewrite rules with sketch awareness; multi-query batch fusion is aspirational, executor is a thin wrapper. - -## 1. Goals - -1. **One repo, one workspace.** One `Cargo.toml` at root, one place to file issues, one release cadence. -2. **Common core, pluggable deployment models.** The three existing codebases each solve a *different* planning problem. Factor out what's actually shared; keep the deployment-model-specific parts as crates so each deployment model can evolve independently. -3. **Extensible for future deployment models** (4th, 5th, …). A new deployment model should land as a new crate that implements well-defined traits — not by touching the core or the runtime. -4. **Preserve the two deployment shapes:** - - **Service**: long-running HTTP+OpAMP process that accepts live `QuerySpec`s, replans on SLA violations, pushes configs to agents/backends. (DC controller today.) - - **CLI**: one-shot "read workload YAML → emit `streaming_config.yaml` + `inference_config.yaml`". (`asap-planner-rs` today.) - Both should be thin shells over the same core. -5. **No regression in today's wire contracts.** `POST /api/v1/streaming-config` to ASAPQuery-backend and OpAMP push to agents must keep working byte-for-byte through the migration. The backend's capability-miss callback `ControllerClient.create_plan` must keep working. -6. **Sketch is a primitive, not a mandate.** The optimizer selects among physical alternatives for each logical operator — e.g. `HashJoin` / `SortMergeJoin` / `SketchJoin`, or `SortAgg` / `HashAgg` / `SketchAgg` — the same way a traditional DB optimizer picks a join algorithm. A plan may come back with zero sketch operators when an exact path Pareto-dominates for the query's accuracy target. Sketches are one primitive class the optimizer can reach for; the framework does not privilege them. - -## 2. Non-goals - -- **Not** redesigning the `Plan` IR end-to-end. DC's `algebra/` + fusion's `translator/optimizer/executor` stay where they are semantically; we merge them into a shared `Plan` crate with a clean trait seam, not a green-field rewrite. -- **Not** unifying DataFusion's `LogicalPlan` with DC's custom algebra at the type level. Those are different universes. Deployment models own their IR; core owns the *staged dispatch contract*. -- **Not** in-scope: changing the existing wire protocol between controller and backend. ASAPQuery-backend keeps consuming `streaming_config.yaml` and keeps responding on `/api/v1/plan`. - -## 3. The organizing spine: DC controller's 5-layer pipeline - -DataCollector/controller already documents its query→sketch translation as a 5-layer pipeline (`DataCollector/controller/docs/query-to-sketch-translation.md`). That's the spine of this merger: - -| # | Layer | What it does | Today's locations | -|---|-------|--------------|-------------------| -| 1 | **Query Language** | Parse raw strings (PromQL, SQL, DataFusion, ElasticDSL, …) into a language-specific AST | DC `controller/src/query_parser/{promql,sql,mod}.rs`; asap-planner-rs pulls `promql-parser` + `sqlparser` directly; asap-fusion consumes a pre-built DataFusion `LogicalPlan` (its L1 happens upstream) | -| 2 | **Language Logical Plan** | Per-language algebra tree (`Aggregate` / `Window` / `Filter` / `Sort` / `Limit`) preserving language semantics, **no sketch names, no sketch binding** | DC `controller/src/algebra/lower.rs`; asap-fusion inherits DataFusion's `LogicalPlan` as its L2; asap-planner-rs has no L2 today (uses a template-pattern catalogue) — **Phase 4 builds one** | -| 3 | **Intent algebra** | Language- and deployment-independent IR: `QueryExpr` + `AggIntent`. Describes **intent only** — *what* to compute, with accuracy target. **No sketch type, no sketch parameters, no sketch-bound nodes** (`SketchAgg` / `SketchJoin` / `SketchSubtract` etc. live in the L4 IR `SketchExpr`). **No language-shaped operators** (no `HistogramQuantile`, no `PromQLSubquery` — those are PromQL L2 nodes that lower to data-model-agnostic shapes here). **One canonical form per plan** — no `WindowedAgg` (use `Window` over `Aggregate`). Heavy-hitter intents are first-class (`AggIntent::TopK`) so heavy-hitter sketches bind directly on the intent rather than on a generic `Sort + Limit` shape; generic `Sort + Limit` survives in `QueryExpr` for non-heavy-hitter cases (e.g. `ORDER BY name LIMIT 10`). Every edge carries a typed `Schema`. Data-model-agnostic — `QueryExpr::Scan` wraps a `Source` sum with `TimeSeries` / `Table` / `Join` variants so the same L3 IR covers ASAPQuery's time-series queries and asap-fusion's tabular queries. | DC `controller/src/algebra/{expr,directory}.rs`; asap-fusion's 3-variant `SubPopulationAnalyticsType` maps to a subset; asap-planner-rs's 9-variant `Statistic` maps to a subset — **Phase 4 splits sketch binding out of planner's current fused L3+L4** | -| 4 | **Sketch algebra + optimizer** | Cost-aware algebraic rewrite rules under deployment constraints. **This is where sketch binding happens** — L4 rules take intent-only L3 (`QueryExpr`) and emit the sketch-bound IR (`SketchExpr`). ~12 rules in DC; a smaller targeted subset in planner; `SketchConfigRule` + `HashModeRule` in fusion. | Core provides the **rule engine driver** + `OptimizerRule` trait + a shared rule library + the sketch-bound IR `core::sketch_algebra::SketchExpr`; deployment models **pick** which rules to enable + supply their own deployment constraints. DC `controller/src/algebra/optimizer.rs` (rules); fusion `src/optimizer/rules/`; planner's `map_statistic_to_precompute_operator` | -| 5 | **Physical Execution Plan** | Assign ops to pipeline stages (edge / gateway / backend / object store); produce the deployment-specific artifact (OpAMP YAML, `streaming_config.yaml`, rewritten DataFusion `LogicalPlan`). **Sketch binding is already committed by L4**; L5 is about stage allocation + emission. | Core provides the **stage allocator framework** + `PhysicalPlanner` trait + the sketch catalogue; deployment models supply their own **topology** (3-stage / 1-stage / 0-stage) + their own **emitter** for the output format. DC `controller/src/algebra/{physical,allocator,plan}.rs`; asap-planner-rs `output/generator.rs`; asap-fusion `src/executor/` | - -**The doc's key claim: layers 1–3 are query-language-independent and workload-independent.** That makes them the natural **common core**. Every deployment model reads PromQL (or SQL, or …) the same way, lowers it to the same per-language algebra, and lowers THAT to the same intent-algebra IR (intent only, no sketch binding). - -**L4 and L5 also have substantial common infrastructure.** Initially we assumed deployment models owned L4/L5 wholesale; on closer inspection what's actually deployment-model-specific is *which rules fire* (L4) and *what topology + output format* (L5) — not the rule engine, not the allocator, not the sketch catalogue. Those frameworks belong in core. This makes deployment models significantly thinner: each becomes a small crate that picks rules from a shared library, declares a deployment topology, and writes an emitter. - -### Sketch binding lives in L4, not L3 - -A key clarification after cross-checking the three source repos: **L3 is intent-only**. DC's `AggIntent` names *what* to compute (`Quantile(0.99, ε=0.01)`, `Cardinality(δ=0.001)`, …) without committing to a sketch type. Picking KLL vs DDSketch, CMS vs CMS-with-heap, parameter sizes — all of that is L4's job, driven by deployment constraints. - -More generally: **L1–L3 lower into a logical representation + `AggIntent`; L4 and L5 choose the concrete execution plan.** That choice is a standard physical-operator selection — `HashJoin` vs `SortMergeJoin` vs `SketchJoin`; `SortAgg` vs `HashAgg` vs `SketchAgg`. "Use a sketch" is one option among several; the same L4 rule framework that picks sketch parameters also picks between sketch and non-sketch operators when a rule is registered for the intent. This keeps the existing 5-layer split intact: nothing about the layering presupposes the output contains a sketch. - -Today: -- DC: correctly separated (L3 has `AggIntent`, L4 picks sketch via cost model). -- asap-fusion: correctly separated — `SketchConfigRule` at L4 fills `SketchConfig::NULL` with concrete `CountMinSketch{5,4096}` / `KLL{k=200,m=8}`. -- asap-planner-rs: **L3+L4 fused today**. `map_statistic_to_precompute_operator` jumps from `Statistic` straight to `AggregationType::DatasketchesKLL{k=200}` in one call. **Phase 4 splits this** — `Statistic → AggIntent` at L3, `AggIntent + DeploymentConstraints → AggregationType + SketchParams` at L4. - -### Intent vocabulary: DC's `AggIntent` is a superset; deployment models use subsets - -DC's pre-cleanup superset had ~25 variants; after L3 normalisation (see §6) it's smaller because language-flavored synonyms (`QuantileOverTime → Window + Quantile`) no longer earn their own intent. The post-cleanup core is 9 (`Count, Sum, Min, Max, Quantile, TopK, Cardinality, Rate, Increase`); the long-term ceiling is bounded by genuinely-distinct operations (stddev, variance, approximate-join-cardinality, …), not by language-flavored synonyms. Planner's 9-variant `Statistic` maps directly: `Topk` keeps its own intent (heavy-hitter sketches like SpaceSaving / CMS-with-heap compute it as a single primitive, so the intent earns L3 visibility). Fusion's 3 variants (`Count, Sum, Quantile`) map directly. Adding a new intent (e.g. stddev) is a core change that deployment models opt into. - -### Data-model support: both time-series and tabular - -ASAPQuery-backend / DC controller operate on time-series data (metrics + labels + timestamp); asap-fusion operates on tabular data (DataFusion `LogicalPlan` over relations) that may or may not be time-indexed. These two data models differ fundamentally in their leaf shape (`metric + labels + time` vs. `table + columns`), but they share everything above the leaf — filter semantics, aggregation semantics, sketches themselves. - -Core handles this with: -- **`QueryExpr::Scan { source: Source, ... }`** where `Source` is a sum type (`TimeSeries`, `Table`, `Join`, …). Deployment models' L1→L2→L3 lowering produces the appropriate variant; L4 rules that care about the data model gate on `source.data_model()`. -- **`AggIntent::requires() -> DataModel`** — each intent variant tags whether it's data-model-agnostic (`Count`, `Sum`, `Min`, `Max`, `Quantile`, `Cardinality`), time-series-only (`Rate`, `Increase` — both carry PromQL counter-reset semantics), or tabular-only (future additions for joins, correlated subqueries). -- **Sketches are data-model-agnostic by construction.** KLL / CMS / HLL / DDSketch ingest a stream of values; that stream can come from a time-series window or a table column, the sketch does not know or care. So `BindKllOnQuantile` and siblings work uniformly across both. - -Practically, this means: -- `deployment-model-asapquery` + `deployment-model-asaplifecycle` lower into `QueryExpr` with `Source::TimeSeries` leaves. -- `deployment-model-asapfusion` lowers into `QueryExpr` with `Source::Table` leaves (and, in future, `Source::Join` when it extends to multi-table queries). -- A hypothetical OLAP deployment model that runs approximate queries over tabular data reuses `Source::Table` + the same `AggIntent` subset fusion uses, plus any OLAP-specific intents it adds. - -See §6 `core::intent_algebra` for the concrete type sketches. - -### Scope: start single-query, grow into workload-aware - -The initial implementation can operate on **one query at a time** — L4 picks physical operators per query against per-query constraints, and `CostModel::workload_cost` degenerates to a sum of per-plan costs. This matches today's three source repos (all single-query planners) and is the minimum bar for parity during the migration. - -**Workload-awareness is an extension, not a rewrite.** When ≥2 queries are planned together, `workload_cost` credits shared sub-expressions (sketches, precomputed aggregates) so the planner can pick a plan for `q1` that lets `q2` read its output for free. Nothing in the L1–L5 spine changes — only the cost objective widens and the rule engine gains cross-plan visibility. See §6 `core::cost` and §13 future work. - -### L2 is mandatory; the tree shape is an evolvable contract - -Every deployment model must produce an L2 tree, even when the source language didn't originally come as one. asap-planner-rs's current approach (PromQL pattern catalogue → `IntermediateAggConfig`) skips L2; Phase 4 will reverse-engineer the five PromQL pattern shapes into a `PromqlLogicalPlan` tree so the L1→L2→L3 pipeline is uniform. - -A future deployment model whose source semantics genuinely don't fit a tree (e.g. a constraint-based query language) would motivate revisiting the L2 contract at that time. Until then, L2 = per-language tree, mandatory, no elision. - -### System I/O contract — what the controller takes in, what it emits - -The controller is a **planner**, not an executor. It does not run queries; it decides where each piece of a query runs. The data plane (OTel collectors / ASAPQuery-backend / DataFusion `SessionContext`) runs them. - -**System input — what the controller takes in.** A `QueryWorkload` (one or more `QuerySpec`s) plus deployment context (available executors, their capabilities, current SLA targets, telemetry of recent violations). The `QuerySpec` carries the raw query string in its source language (PromQL / SQL / DataFusion / ElasticDSL) and the accuracy / latency / cost target. Workloads can arrive via four entry points (HTTP `POST /plan`, OpAMP capability-miss callback, YAML file for the CLI shell, query-log replay); they all normalise to `QueryWorkload` before L1. - -**Workload features that the controller cares about, beyond the queries themselves:** -- Execution model: **batch** (one-shot YAML, query-log replay) vs **streaming** (live pipeline that must keep producing results as data arrives). -- Data input source: time-series scrape, tabular relation, query log replay, etc. Drives `Source` variant choice in L3. -- Reuse opportunity: ≥2 queries planned together → `CostModel::workload_cost` credits shared sub-expressions. - -**System output — what the controller emits.** For each registered executor in the deployment, a sub-DAG of the optimized plan plus the configuration that lets that executor run it. Concretely: - -| Output | Consumer | Wire shape | -|---|---|---| -| Per-executor sub-DAG assignment | The executor (edge agent / gateway / backend / DataFusion session) | OpAMP `RemoteConfig` (OTel YAML) / `streaming_config.yaml` POST / rewritten `LogicalPlan` | -| Cut-edges between executors | The transport between executors (OTel pipeline, HTTP, sketch-merge / compute-from-raw over the precompute engine, …) | Implied by the per-executor configs; not a separate artifact | -| Plan ID + provenance metadata | The controller's own `PlanStore` for replan / EXPLAIN / observability | JSON / proto | - -**Premise behind the staged dispatch.** Two distinct concepts: - -- **Stage** (`StageId`) — a categorical *tier* in the data lifecycle (edge / gateway / backend / in-process). The topology declares which stages exist (3-stage / 1-stage / 0-stage). Stages are roles, not instances. -- **Executor** (`Executor`, defined in `core::physical::executor`) — a *concrete runtime instance* that occupies a stage. Carries `id`, `stage: StageId`, `capabilities`, and `address` (OpAMP agent / HTTP endpoint / in-process handle). One stage may have N executors — e.g. a 50-host edge fleet is 50 `Executor`s all with `stage = StageId("edge")`; a singleton backend is one `Executor` at `stage = StageId("backend")`. - -Every stage is in principle capable of running the entire query tree — all stages speak the same physical operators. The controller's job is to decide *which stage* runs *which sub-tree / sub-DAG* under the deployment's constraints (memory budget per stage, network bandwidth between stages, sketch backends available at each stage). A "stage assignment" is a colouring of the L4-bound `SketchExpr` DAG by `StageId`, with sketch-merge / data-shipping nodes inserted on the cut edges. L5's `StageAllocator` does the colouring at stage granularity; the per-deployment-model `PhysicalPlanner` then materialises each stage's sub-DAG into one config per `Executor` at that stage (varying only by per-executor connection / identity details). The executor list comes from `DeploymentConstraints::executors()`. - -This is what makes the topology a *parameter* rather than an axis of code: the same `SketchExpr` plus a different `TopologyDescriptor` produces edge-only / 1-stage / 3-stage / 0-stage placements without rewriting the plan. - -**Symbolic plan vs concrete plan.** L3 `QueryExpr` and L4 `SketchExpr` are symbolic — they describe operations and bindings without committing to *where* anything runs. L5 produces the concrete plan: stage-assigned, executor-targeted, ready to serialize into the executor's configuration format. The split is what lets the controller swap topologies (single-stage backend → three-stage edge/gateway/backend) without re-running L1-L4. - -This drives the core/deployment model split: - -- **`crates/core/`** owns all 5 layers of **shared infrastructure**: L1-3 end-to-end (parsers + lowering + intent-only IR), plus L4's rule engine driver + rule library + cost-model traits, plus L5's stage-allocator framework + `PhysicalPlanner` trait + sketch catalogue. -- **Each deployment model is a thin crate** that: (1) picks which of core's L4 rules to enable + adds any deployment-model-specific rules, (2) declares its deployment topology (how many stages, where data flows), (3) provides an emitter for its output format. That's usually a few hundred lines, not thousands. - -## 4. Principles - -### P1. Core owns shared infrastructure across all 5 layers; deployment models own choices - -See §3. Core is NOT just types + trait stubs — it ships working parsers + lowering passes (L1-3), a rule engine + rule library + cost-model traits (L4 framework), and a stage allocator + physical-plan framework + sketch catalogue (L5 framework). What deployment models plug in is **which rules fire** (picking from core's library + adding their own), **deployment topology** (how many stages; DC=3, query=1, fusion=0), and an **emitter** for the output format. A deployment model that accepts all core's default L4 rules and uses `core::physical::single_stage_topology` is maybe 200 lines of code. - -### P2. Core has no I/O - -No HTTP, no OpAMP, no YAML, no Prometheus scrape, no `tokio::spawn`. Pure algorithms over in-memory data. This is what makes deployment models unit-testable without running the runtime. - -### P3. Runtime is a thin binary, deployment models are libraries - -The `asap-controller` binary is assembled from: runtime (HTTP/OpAMP/replanner/store) + N deployment model crates registered as plugins. Swapping deployment models is a build-time feature flag or a runtime registry entry. No deployment model may reach directly into another. - -### P4. One input boundary, one output boundary - -**Input**: everything that enters the controller (HTTP `QuerySpec`, Prometheus query log replay, YAML workload, capability-miss callback) normalizes into a single `QueryWorkload` type in core — a collection of `QuerySpec`s each feeding L1→L2→L3. - -**Output**: L5 emitters (OpAMP `RemoteConfig`, backend `StreamingConfig` POST, one-shot YAML file, rewritten DataFusion `LogicalPlan`) all implement a `PlanEmitter` trait. Deployment models supply emitter implementations; core doesn't know which emitters exist. - -This is the "extension point for future deployment models" — a new deployment model adds an L4 rule set + an L5 emitter and registers them. - -### P5. No feature-flag spaghetti - -If something must be optional (e.g. OpAMP for deployments that don't run OTel collectors), it's a separate crate, not a `cfg` block in core. - -## 5. Target repo layout - -``` -ASAPController/ -├── Cargo.toml # workspace -├── README.md -├── docs/ -│ ├── design.md # this file -│ ├── migration-plan.md # the companion file -│ ├── deployment-models/ # per-deployment-model design notes -│ └── adr/ # architecture decision records -├── proto/ -│ ├── asap_control.proto # Plan IR + QueryWorkload on the wire -│ └── opamp.proto # vendored from DataCollector -├── crates/ -│ ├── core/ # Shared infrastructure across all 5 layers; no I/O -│ │ ├── query_language/ # L1: per-language parsers — promql/, sql/, datafusion/, elasticdsl/ -│ │ ├── logical_plan/ # L2: per-language algebra tree (Aggregate/Window/Filter/…) -│ │ ├── intent_algebra/ # L3 IR: QueryExpr + AggIntent + Schema/HasSchema (intent only) -│ │ ├── sketch_algebra/ # L4 IR: SketchExpr (sketch-bound — kind + params committed) -│ │ ├── lower/ # L1→L2→L3 lowering passes (one entry per language) -│ │ ├── optimizer/ # L4 framework — produces SketchExpr from QueryExpr: -│ │ │ ├── engine/ # rule driver — fixed-point iteration, cycle detection, priority -│ │ │ ├── trait/ # OptimizerRule + RuleCategory (PushDown / Fusion / Elim / Bind) -│ │ │ ├── rules/ # shared rule library (e.g. sketch-binding rules; stream-vs-batch picker) -│ │ │ └── cost/ # CostModel trait + generic impls (memory budget, accuracy degradation) -│ │ ├── physical/ # L5 framework: -│ │ │ ├── planner_trait/ # PhysicalPlanner trait -│ │ │ ├── stage_allocator/ # generic topology-driven allocator -│ │ │ ├── topology/ # Topology descriptor types (edge/gateway/backend, single, zero) -│ │ │ ├── executor/ # Executor type — concrete runtime instance occupying a stage -│ │ │ └── sketch_catalog/ # candidate sketch types + parameter constraints -│ │ ├── pipeline/ # orchestrates L1→L2→L3→L4→L5, parameterized on deployment model -│ │ ├── workload/ # QueryWorkload wrapper around QuerySpecs -│ │ ├── emit/ # PlanEmitter trait — implemented by deployment model L5 -│ │ ├── registry/ # DeploymentModelRegistry, DeploymentModelId -│ │ └── telemetry/ # tracing macros, metric names (no exporter) -│ ├── runtime/ # service skeleton — HTTP, OpAMP, replanner -│ │ ├── http/ # axum surface — /plan, /replan, /metrics, /status -│ │ ├── opamp/ # WebSocket OpAMP server -│ │ ├── monitor/ # Scraper, Thresholds, Violation -│ │ ├── replan/ # Replanner — SLA + expiry triggers -│ │ ├── store/ # PlanStore, WorkloadStore -│ │ └── backend_client/ # HTTP client (pushes to ASAPQuery-backend, etc.) -│ ├── deployment-model-asaplifecycle/ # thin — picks rules, 3-stage topology, OTel/backend emitters -│ │ ├── rules.rs # L4 rule selection + DC-specific rules (stage-aware push-down) -│ │ ├── topology.rs # 3-stage: edge / gateway / backend -│ │ ├── cost.rs # deployment-model-specific cost impls — delta / online / pareto / tco -│ │ └── emit/ # OpAmpRemoteConfig + AsapqueryBackendConfig emitters -│ ├── deployment-model-asapquery/ # thin — rules, 1-stage topology, YAML emitters + query-log input -│ │ ├── rules.rs # L4 rule selection + sketch-binding rule (split from map_statistic_*) -│ │ ├── topology.rs # 1-stage: backend-only -│ │ ├── emit/ # StreamingConfig.yaml + InferenceConfig.yaml -│ │ ├── query_log/ # extra L1 input: Prometheus query-log replay -│ │ └── schema/ # PromQLSchema discovery from Prometheus (feeds L1) -│ ├── deployment-model-asapfusion/ # thin — DF-flavored rules, 0-stage, in-process emit -│ │ ├── rules.rs # L4 rules for DataFusion LogicalPlan (sketch-aware rewrites) -│ │ ├── topology.rs # 0-stage: in-process -│ │ ├── emit/ # rewritten DataFusion LogicalPlan -│ │ ├── executor/ # DataFusion SessionContext wrapper (library-mode execution) -│ │ └── sketch_support/ # asap_sketchlib-backed rewrites -│ ├── control-proto/ # generated from proto/ (tonic/prost) -│ └── testing/ # test fixtures + harness shared across deployment models -└── bin/ - ├── asap-controller/ # long-running service with all deployment models - │ └── main.rs # axum + OpAMP + replanner + registered deployment models - ├── asap-query/ # one-shot CLI for deployment-model-asapquery (what asap-planner-rs is today) - │ └── main.rs # clap — read workload YAML, emit two YAMLs - ├── asap-lifecycle/ # OPTIONAL: standalone service with only deployment-model-asaplifecycle - │ └── main.rs # slimmer image — no DataFusion, no query YAML emitters - └── asap-fusion-bench/ # OPTIONAL: benchmark harness over deployment-model-asapfusion - └── main.rs # criterion entry; used by researchers -``` - -**Per-deployment-model standalone binaries** are first-class. Each `bin//` is a thin shell that `use`s only the deployment model crates it needs — so `bin/asap-lifecycle/` doesn't pull `datafusion` into its dep tree, and `bin/asap-fusion-bench/` doesn't pull `axum`/`opamp`. Feature flags on the workspace root let you `cargo build -p asap-lifecycle` and get a minimal binary. - -### Why three deployment model crates today, not one monolithic `deployment-models/` - -Each deployment model has a different problem shape: - -| | **lifecycle** | **query** | **fusion** | -|---|---|---|---| -| Input | workload + live metrics | workload YAML / query log | DataFusion `LogicalPlan` | -| Decision unit | end-to-end staged pipeline | per-aggregation YAML | per-operator rewrite | -| Output | OpAMP OTel config + `StreamingConfig` YAML | `streaming_config.yaml` + `inference_config.yaml` | rewritten `LogicalPlan` | -| Trigger | QuerySpec, SLA violation, expiry | one-shot CLI invocation | a DataFusion session constructing a query | -| Cost model | accuracy × latency × $ staged | single-query accuracy/latency | operator selectivity / sketch feasibility | - -Mashing them into one crate means the union of all their dependencies (sqlparser + promql-parser + DataFusion + OpAMP proto + Prometheus client) bleeds into every downstream user. Separating them means a user who only needs `deployment-model-asapfusion` (an offline query-optimization benchmark, say) can depend on it without pulling OpAMP. - -## 6. Core crate details (layers 1–3 + driver) - -Core is not a trait-stubs library. It ships real L1/L2/L3 code lifted from DC's `controller/src/query_parser/` + `controller/src/algebra/` and exposes a small set of traits for L4/L5 plugin points. - -### `core::query_language` — Layer 1 - -Per-language parsers, one module each: - -- `promql/` — wraps `promql-parser`. Input: `&str` PromQL. Output: `PromqlAst`. -- `sql/` — wraps `sqlparser`. Input: `&str` SQL. Output: `SqlAst`. -- `datafusion/` — wraps DataFusion's own parser. Output: `DfAst`. -- `elasticdsl/` — wraps `elastic_dsl_utilities`. Output: `EsAst`. - -Each returns a language-flavored AST type. No sketch awareness. - -> **Implementation status (Phase D).** L1 lives in -> `controller/src/query_language/` with a `Language` trait -> (`fn id() -> QueryLanguage; fn parse(&str) -> Result`) -> + a `LanguageAst` sum type (one variant per `QueryLanguage`). The -> PromQL backend (`query_language::promql::PromQLLanguage`) wraps the -> existing `query_parser::{parse_query, parse_query_expr}` entry points -> — no parsing logic was duplicated; `PromQLAst` bundles the algebra -> tree (`QueryExpr`) and the flat `ParsedQuery` summary the legacy -> analyzer expects. The `Sql`, `DataFusion`, and `ElasticDsl` -> backends are stubbed: they implement `Language` but `parse` returns -> `ParseError::Unimplemented(...)`. This keeps the type system uniform -> for the orchestrator while the DC build ships PromQL only. L2 lives -> in `controller/src/language_logical_plan/`: `LanguageLogicalPlan` -> mirrors `LanguageAst` with one variant per language; PromQL's L2 IS -> the existing `QueryExpr` tree (rebadged + paired with a flat -> `LanguageLogicalPlanSummary` projection). `lower_to_logical_plan` -> is the L1 → L2 pass; non-PromQL variants surface -> `LoweringError::UnsupportedLanguage` cleanly. The legacy -> `query_parser::parse_query` / `parse_query_expr` entry points stay -> unchanged for back-compat. - -### `core::logical_plan` — Layer 2 - -A **per-language** algebra tree — one `enum LogicalPlan` per language. Preserves language-specific semantics (PromQL instant vs range vector, SQL window frames, Elastic buckets) that would be lossy to collapse this early. Types are symmetric: `Aggregate { AggFunc }`, `Window`, `Filter`, `Sort`, `Limit`. No sketch names yet. - -### `core::intent_algebra` — Layer 3 - -The language- and deployment-independent IR. **Pure intent at this layer**: no language-specific operators (no `HistogramQuantile`, no `PromQLSubquery` — those are L2 PromQL nodes), no sketch types, no sketch parameters, no physical operator choice. Data-model-agnostic: supports both **time-series** inputs (ASAPQuery-backend, DC lifecycle) and **tabular** inputs (asap-fusion, future OLAP deployment models) via a `Source` sum type inside `QueryExpr::Scan`. - -#### Design rules for L3 - -1. **One canonical form per plan.** No redundant variants whose semantics decompose into other variants. `Window` over `Aggregate` is the canonical windowed-aggregate shape; there is no separate `WindowedAgg`. This keeps L4 rule matching unambiguous (a rule fires on one shape, not on N synonyms). The exception is when an "intent" is its own physical primitive: heavy-hitter top-k is served by sketches (SpaceSaving, CMS-with-heap) as a single operation, so `AggIntent::TopK` is a first-class intent at L3 — distinct from the generic `Sort + Limit` operator pair, which still appears in `QueryExpr` for non-heavy-hitter cases (e.g. `ORDER BY name LIMIT 10`). -2. **Language-orthogonal.** No PromQL-shaped or SQL-shaped operators leak into L3. `HistogramQuantile` is a PromQL artifact (it consumes Prometheus's specific bucketed-histogram exposition format and produces a quantile from buckets) — it lives in `core::logical_plan::promql` (L2) and lowers to bucket reads + a regular `Quantile` intent over those bucket counts. `PromQLSubquery` (`[range:resolution]`) is a *driver* construct — it asks the engine to evaluate the inner expression at N timestamps — and is expanded by the PromQL L1→L2 lowering into a set of independent queries, not preserved as an L3 DAG node. -3. **Intent at L3, sketch at L4.** L3 carries `AggIntent` ("compute a quantile to ε=0.01 accuracy"). The choice between `HashAgg` / `SortAgg` / `SketchAgg(KLL{k=200})` is made by L4 cost-aware rules, not encoded in L3. -4. **One physical-choice node per logical operator.** L3 has `Aggregate` (logical) and `Join` (logical). L4 produces the sketch-bound physical alternatives (`SketchAgg`, `SketchJoin`, `SketchSubtract`, `SketchDelete`, `SketchEstimate`, `SketchMerge`) into an extended IR — see "L4 sketch-bound IR" below. Mixing logical and physical at L3 (the previous draft did this with `SketchAgg` / `JoinSketch` at L3) creates ambiguity about which layer owns which decision. -5. **DAG, not tree.** Edges between nodes carry typed schemas (see "Schema flow" below). A node's output schema is a function of its inputs and parameters and is verifiable independently of the surrounding context. The shape is a DAG, not a tree, because **a producer node can have multiple parents that share its precomputed intermediate state** — instead of recomputing the same sub-expression once per consumer, the consumers fan in to a single node and read its output. Three sources of fan-in: - 1. **Explicit, in-query.** SQL CTEs (`WITH name AS (expr) SELECT ... FROM name JOIN name AS n2 ON ...`) and PromQL recording rules name a sub-expression and reference it N times; each reference becomes a `QueryExpr::Ref(name)` parent of the named producer. - 2. **L4 reuse rules.** When two queries planned together both need (e.g.) a p99 quantile of the same series, the optimizer can introduce a shared sketch node whose output feeds both — even though neither query author wrote a CTE. - 3. **L4 stage / shard structure.** A pre-aggregate computed once on the edge can feed multiple downstream gateway-stage operators. - - `CostModel::workload_cost` credits a shared producer's build cost once across all consumers, which is what makes reuse a Pareto win. Tree IRs lose this — they have to duplicate the producer for every consumer. - -```rust -pub enum QueryExpr { - // ── Base relations ──────────────────────────────────────────────────── - /// A metric stream / table / join. Outermost leaf. - Scan { source: Source, predicates: Vec }, - /// Reference to a CTE / let-binding by name; resolved at plan time. - Ref(String), - - // ── Filtering & projection ──────────────────────────────────────────── - /// σ — row-level filter (WHERE / PromQL label matchers). - Filter { child: Box, pred: Predicate }, - /// π — column projection (SELECT list). - Project { child: Box, cols: Vec }, - - // ── Aggregation (logical, intent-only) ──────────────────────────────── - /// γ + α — GROUP BY + aggregate intents, with optional HAVING. - /// `aggs` carry `AggIntent`; concrete sketch / non-sketch operator - /// is chosen by L4 and lives in the L4-extended IR (`SketchExpr`). - Aggregate { child: Box, by: Vec, - aggs: Vec, having: Option }, - - // ── Time / streaming windows ────────────────────────────────────────── - /// ψ — tumbling / sliding / session window over the time axis. Defines - /// the lifecycle (flush / reset bounds) of any aggregate in its sub-tree / sub-DAG. - /// PromQL `[5m]` and streaming windows lower here. SQL `OVER (...)` - /// analytic frames are a different node — see `WindowFunc` below. - Window { child: Box, kind: WindowKind, - size: Duration, slide: Option }, - - // ── Distributed-execution structure ─────────────────────────────────── - /// Partition the stream by key tuple (`GROUP BY` / PromQL `by (dims)`). - Partition { child: Box, keys: PartitionKeys }, - /// δ — SQL `DISTINCT` / row deduplication on `cols`. - Distinct { child: Box, cols: Vec }, - /// ⊕ — union of sub-results from independent stages or shards (the - /// exact-merge case). Sketch unions are a separate node in `SketchExpr` - /// because they carry sketch-family / params type constraints. - Merge { children: Vec }, - - // ── Joins (logical) ─────────────────────────────────────────────────── - /// Logical join. L4 picks the physical alternative — `HashJoin` / - /// `SortMergeJoin` / `SketchJoin` (e.g. KMV / theta-sketch / join-sample) - /// — based on selectivity, memory budget, and accuracy target. The - /// sketch-aware variant lives in `SketchExpr::SketchJoin`. - Join { kind: JoinKind, left: Box, right: Box, - pred: Option }, - - // ── Set operators ───────────────────────────────────────────────────── - /// UNION / INTERSECT / EXCEPT, with or without ALL. - SetOp { kind: SetOpKind, all: bool, - left: Box, right: Box }, - - // ── Ordering & limiting ─────────────────────────────────────────────── - /// Generic order-by — survives L3 for non-heavy-hitter cases - /// (`ORDER BY name LIMIT 10`, `ORDER BY ts DESC LIMIT 1`). - Sort { child: Box, keys: Vec }, - /// `LIMIT n OFFSET k`. The heavy-hitter shape (`ORDER BY count DESC - /// LIMIT k`, PromQL `topk(k, …)`) is recognised at L1→L2→L3 lowering - /// and produces `AggIntent::TopK` rather than generic `Sort + Limit`, - /// so heavy-hitter sketches (SpaceSaving, CMS-with-heap) bind on the - /// intent. Generic `Sort + Limit` flows through unchanged. - Limit { child: Box, n: u64, offset: u64 }, - - // ── Subquery / CTE ──────────────────────────────────────────────────── - Subquery { child: Box, alias: String }, - /// SQL `WITH name AS (expr) IN body`; lowering target for PromQL - /// recording-rule bindings. - LetBinding { name: String, expr: Box, body: Box }, - - // ── Analytic (OVER) window functions ────────────────────────────────── - /// SQL `OVER (PARTITION BY ... ORDER BY ... ROWS BETWEEN ...)`. - /// Distinct from `Window` above — that is a streaming/tumbling window - /// over the time axis; this is an analytic frame over already-grouped rows. - WindowFunc { child: Box, func: WindowFuncKind, - partition_by: Vec, order_by: Vec, - frame: Option }, - - // ── Binary composition ──────────────────────────────────────────────── - /// Arithmetic / comparison / boolean composition between two relational - /// sub-expressions (PromQL binary ops including `and`/`or`/`unless`, - /// SQL boolean composition). - BinaryOp { op: BinaryOpKind, lhs: Box, rhs: Box, - vector_match: Option }, -} - -pub enum WindowKind { Tumbling, Sliding, Session } - -pub enum Source { - /// Time-series input — deployment-model-asapquery / deployment-model-asaplifecycle shape. - TimeSeries { metric: MetricRef, time: TimeRange, labels: LabelFilter }, - /// Tabular input — deployment-model-asapfusion / future-OLAP shape. - Table { table_ref: TableRef, columns: Vec }, - /// Join over Sources composes leaf shapes recursively. - Join { left: Box, right: Box, on: JoinKey }, - // Future: WindowedStream, Subquery — added by deployment models that need them. -} - -pub enum DataModel { TimeSeries, Tabular, Any } - -impl Source { - pub fn data_model(&self) -> DataModel { /* … */ } - /// Output schema produced by this leaf (see "Schema flow"). - pub fn schema(&self, catalog: &SchemaCatalog) -> Schema { /* … */ } -} -``` - -#### What was removed during cleanup, and why - -| Removed | Reason | Replacement | -|---|---|---| -| `TopK { k, by }` *as a `QueryExpr` node* | A `QueryExpr`-level top-k operator collapses two distinct concepts: (a) the *intent* of "compute heavy hitters", which has its own sketch primitive, and (b) the generic operator pair `Sort + Limit`, which doesn't. Splitting them puts heavy-hitter logic at the right level | Heavy-hitter intent → `AggIntent::TopK` (L3, recognised by L1→L2→L3 lowering of `ORDER BY count DESC LIMIT k`, PromQL `topk(k, …)`); generic ordering+limit → `Sort + Limit` `QueryExpr` nodes (unchanged). Both retained — they describe different things | -| `WindowedAgg { intent, window }` | Equivalent to `Window` over `Aggregate`; redundant. Tumbling/sliding kind moves onto `Window::kind` | `Window { kind: …, … } → Aggregate { aggs: [intent] }` | -| `SketchAgg { intent, col }` | Sketch-bound; L3 must be intent-only | `Aggregate { aggs: [intent] }` at L3; L4 emits `SketchExpr::SketchAgg` | -| `JoinSketch { outer, inner, key }` | Sketch-bound physical alternative; L3 must be intent-only. Several papers describe sketch-of-join (KMV, theta, join-sample) — picking one is an L4 cost decision, not an L3 surface choice | `Join { … }` at L3; L4 emits `SketchExpr::SketchJoin` when a `Bind*OnJoin` rule fires | -| `HistogramQuantile { phi }` | PromQL artifact — consumes Prometheus's specific bucketed-histogram format. Language-specific | Lowers in PromQL L1→L2 to bucket reads + `Aggregate { aggs: [Quantile{q: phi, …}] }` over the bucket counts | -| `PromQLSubquery { range, resolution }` | Driver construct — asks the engine to evaluate the inner expression at N timestamps; not a single DAG node | Expanded by PromQL L1→L2 lowering into a set of independent `QueryExpr` instances, not preserved at L3 | -| `Dedup { col }` | Single-column-only spelling of SQL `DISTINCT` | Renamed to `Distinct { cols }`, generalised to N columns | - -#### Schema flow — every L3 edge carries a typed schema - -Every node has a derivable output schema given its input schemas and parameters. The DAG is type-checked: a `Filter` whose predicate references a column not in its child's output schema fails at plan time. - -```rust -pub struct Schema { - pub fields: Vec, - /// Index into `fields` for the time axis, if any. PromQL leaves carry one; - /// SQL leaves may or may not. - pub time_index: Option, - /// Optional metadata for reuse-aware planning. Each inner `Vec` is - /// a set of column indices that together uniquely identify rows; the outer - /// `Vec` allows multiple unique-key sets (e.g. primary key + another unique - /// constraint). - /// - /// **Populated by**: the per-node input/output spec (e.g. `Aggregate { by, .. }` - /// emits `unique_keys = [by]`; `Distinct { cols }` adds `cols`; `Project` - /// carries forward the retained columns; most other nodes pass through). - /// - /// **Consumed by**: `CostModel::workload_cost` only — the reuse-aware path - /// that credits shared sub-expressions across multiple queries. Single-query - /// plans, the `Bind*` rules, push-down rules, and L5 emitters do not read - /// this field. If the reuse path is deferred (see §13 future work), this - /// field is dead weight; it lives here so the metadata is available the - /// moment workload-aware planning lands without requiring an L3-wide - /// schema change. - pub unique_keys: Vec>, -} - -pub struct Field { - pub name: String, - pub dtype: DataType, // Int64 / Float64 / Utf8 / Map / … - pub nullable: bool, -} - -pub trait HasSchema { - fn input_schemas(&self) -> Vec<&Schema>; - fn output_schema(&self, inputs: &[&Schema], cat: &SchemaCatalog) -> Schema; -} -``` - -Per-node input/output spec — the stable contract for L3 nodes (full implementation in `core/src/intent_algebra/schema.rs`). Each row reads independently: every column position, type, and constraint is named explicitly rather than carried by a shorthand like `S` or `L` / `R`. - -| Node | Input schemas | Output schema | -|---|---|---| -| `Scan { source }` | none — leaf node | `source.schema(catalog)` — derived from the source's catalog metadata (TimeSeries metric labels + value + timestamp; or Table columns from `information_schema`; or recursive `Source::Join`) | -| `Ref(name)` | none — pointer node | the output schema of the `LetBinding` whose `name` matches; resolved at plan time | -| `Filter { pred }` | one input schema (the child's output) | the child's input schema unchanged — `Filter` is a row-level refinement; no columns added, removed, or re-typed | -| `Project { cols }` | one input schema | the input schema projected to `cols` — fields filtered and reordered to match `cols`; `time_index` and `unique_keys` carried over for retained columns | -| `Aggregate { by, aggs }` | one input schema | the `by` columns (carried verbatim from input) followed by one new column per entry in `aggs`, each named and typed by `AggIntent::output_type(input_field)`; `unique_keys = [by]` | -| `Window { kind, size, slide }` | one input schema, **must** contain a `time_index` field | the input schema extended with synthetic `window_id` and `window_start` / `window_end` metadata fields | -| `Partition { keys }` | one input schema | the input schema unchanged — logical-only marker; carries a sharding hint for L5's stage allocator | -| `Distinct { cols }` | one input schema | the input schema with `unique_keys` tightened to include `cols`; field types unchanged | -| `Merge` | N input schemas, all union-compatible (same field names, same types, same nullability, same `time_index` position if any) | the first input's schema (representative; checked union-compatible with the rest) | -| `Join { kind, pred }` | two input schemas — left and right children | the concatenation of left's fields and right's fields, minus columns that USING / NATURAL deduplicates; nullability widened on the OUTER side for outer joins | -| `SetOp { kind, all }` | two input schemas, must be union-compatible (as for `Merge`) | the left input's schema | -| `Sort { keys }` | one input schema; every field referenced in `keys` must be present in it | the input schema unchanged | -| `Limit { n, offset }` | one input schema | the input schema unchanged | -| `Subquery { alias }` | one input schema | the input schema with `alias` applied as the table alias to all field names | -| `LetBinding { name, expr, body }` | `expr` produces an intermediate schema bound to `name`; `body` consumes it (and any other in-scope bindings) | the `body`'s output schema | -| `WindowFunc { func, partition_by, order_by, frame }` | one input schema; every field in `partition_by` / `order_by` must be present | the input schema extended with one new column carrying the analytic-function output (named after `func`, typed per `func`) | -| `BinaryOp { op, vector_match }` | two input schemas — left and right operands. PromQL vector-match constraints (`on`/`ignoring` + `group_left`/`group_right`) govern label-set compatibility | for arithmetic/comparison `op`: the left input's schema with the value column re-typed to the result of `op`; for boolean `op` (`and`, `or`, `unless`): the left input's schema with a boolean value column | - -##### Implementation status - -Phase F (in `controller/src/intent_algebra/schema.rs` + `controller/src/intent_algebra/cse.rs`) lands the load-bearing consumer of `Schema::unique_keys`: - -- `cse_reuse_is_legal(producer_schema, consumer_count) -> Result<(), CseError>` — the gatekeeper. Two `QueryExpr::Ref` consumers may share a producer only when the producer's output schema has a non-empty `unique_keys` set and the consumer count is ≥ 2. This is the proof point that `unique_keys` is load-bearing — without it, the deduper conservatively refuses to share and reuse "drops on the floor" (this section, line ~1356). -- `dedupe_subtrees(roots) -> CseWorkloadPlan` — basic implementation of the workload-level CSE pass for the literal "≥2 root queries with identical sub-expressions" case (the batched-queries example, line ~1256). Detects shared `Aggregate` children, gates on `cse_reuse_is_legal`, hoists into a `LetBinding`. Richer detection (alpha-equivalence, schema-merge across compatible-but-not-identical shapes, recursive nested CSE) is downstream — Phase F lands the gate + the basic case so the cost-model side has something to credit. - -The full general CSE algorithm (the optimisation half) remains future work. - -#### DAG schema, DB schema, sketch catalog — three distinct metadata sources - -These three are sometimes conflated and shouldn't be. Only the first two are *schemas* (descriptions of stream / table shape); the sketch catalog is a *registry* of available primitives, not a description of a stream: - -| Source | Where it lives | What it describes | Who reads it | -|---|---|---|---| -| **DAG schema** | On every edge of the L3 / L4 / L5 DAG (`Schema` above) | Columns + types flowing between operators | L4 rules (selectivity estimation, push-down legality), L5 emitter | -| **DB / source schema** | The query target (Prometheus TSDB metric metadata, SQL `information_schema`, DataFusion catalog) | What metrics / tables / columns exist in the data plane, with their types and indexing | `core::lower::*` to resolve names during L1→L2; exposed through a `SchemaCatalog` interface | -| **Sketch catalog** | `core::physical::sketch_catalog` (built at startup; static) | What sketches the runtime can build; what intents each one serves; mergeability, accuracy / confidence guarantees, supported aggregation keys, parameter ranges | L4 binding rules to choose a sketch for an `AggIntent`; L5 to instantiate the sketch | - -L1→L2 lowering reads the **DB schema** to resolve symbols. L3 onward, every edge carries a **DAG schema** that is type-checked locally. L4 binding rules consult the **sketch catalog** to map an intent to a concrete sketch under the deployment's constraints. They are three separate inputs to three distinct decisions. - -#### `AggIntent` — what to compute, not how - -```rust -pub enum AggIntent { - // Data-model-agnostic - Count { accuracy: AccuracyTarget }, - Sum, - Min, Max, - Quantile { q: f64, accuracy: AccuracyTarget }, - /// Heavy-hitter top-k. Distinct from generic `Sort + Limit` because a - /// dedicated sketch primitive (SpaceSaving, CMS-with-heap, Misra-Gries) - /// computes it as a single operation. L1→L2→L3 lowering produces this - /// when it recognises a heavy-hitter shape (`ORDER BY count DESC LIMIT k`, - /// PromQL `topk(k, …)`); other ordering+limit cases stay as - /// `QueryExpr::Sort + QueryExpr::Limit`. - TopK { k: usize, by: Vec, accuracy: AccuracyTarget }, - Cardinality { accuracy: AccuracyTarget }, - - // Time-series streaming derivatives — specific operations, not just - // "Sum / Count over a Window". `Rate` is the per-second average derivative - // computed with PromQL's counter-reset adjustment, not a generic windowed - // mean. Kept distinct because (a) they have counter-reset semantics that - // exact `Sum` does not, and (b) sketch backends specialised for - // derivatives (e.g. delta-set aggregator) bind on these intents directly. - Rate { window: Duration }, - Increase { window: Duration }, - - // Tabular / OLAP — added as deployment models demand - // CorrelatedSubqueryCount { … }, ApproxJoinCardinality { … }, -} - -impl AggIntent { - /// Which data-model this intent semantically requires. L4 rules - /// consult this to skip non-applicable intents (e.g. `Rate` over - /// a `Source::Table` is nonsense). - pub fn requires(&self) -> DataModel { /* … */ } - /// Output column type — used by L3 schema derivation for `Aggregate`. - pub fn output_type(&self, input: &Field) -> DataType { /* … */ } - /// Which sketch families in the catalog can serve this intent. - /// Read by L4 binding rules. - pub fn candidate_sketches(&self) -> &'static [SketchKind] { /* … */ } -} -``` - -**Why `TopK` *is* an intent (and `Sort + Limit` is not collapsed into it).** Heavy-hitter top-k has a dedicated sketch primitive — SpaceSaving / CMS-with-heap / Misra-Gries compute it in a single pass with sub-linear memory. L4 binding rules want to fire on the *intent* "give me the top-k frequent items" rather than on a syntactic shape, the same argument that makes `Quantile` an intent rather than a `Sort` + "pick the φ-th element" pattern. So `AggIntent::TopK` lives at L3. Generic `QueryExpr::Sort + QueryExpr::Limit` *also* survives, because not every order-by-limit query is heavy-hitter (`ORDER BY name LIMIT 10`, `ORDER BY ts DESC LIMIT 1`); these have no sketch alternative and stay as generic operators. The canonical-form invariant is preserved because L1→L2→L3 lowering picks one or the other deterministically based on whether it recognises a heavy-hitter pattern. - -**Why no `QuantileOverTime` intent?** It duplicated `Quantile` over a `Window`. The window — its kind, its size, its slide — is fully captured by the surrounding `Window { … }` node; the quantile *operation* is the same regardless of whether the input was a windowed time-series or a row-grouped table. PromQL's `quantile_over_time(0.99, m[5m])` lowers cleanly to `Window{size=5m} → Aggregate{aggs:[Quantile{q=0.99}]}`. One intent halves the L4 rule surface (one bind rule per operation, not per language-flavor of an operation). - -**Why `Rate` and `Increase` survive that argument.** They are not "Sum / Count over a Window with a different name" — they include PromQL's counter-reset adjustment, which is a non-trivial transformation an exact `Sum` does not perform. They earn distinct intent variants because they parameterise different physical operators (delta-set aggregators bind on these intents directly). If a non-PromQL streaming language has the same notion (e.g. SQL `RATE() OVER (RANGE)`), it lowers to the same intent — the intent vocabulary names the operation, not the language. - -#### Implementation status - -Phase B (this PR) ships the L3 IR in `controller/src/intent_algebra/`: - -- `intent_algebra::AggIntent` — vocabulary above (`Count`, `Sum`, `Min`, `Max`, `Avg`, `Quantile`, `TopK`, `Cardinality`, `Frequency`, `Rate`, `Increase`). -- `intent_algebra::QueryExpr` — variant subset for the DC + PromQL deployment: `Scan`, `Window`, `Aggregate`, `LetBinding`, `Ref`. The remaining variants (`Filter`, `Project`, `Partition`, `Distinct`, `Merge`, `Join`, `SetOp`, `Sort`, `Limit`, `Subquery`, `WindowFunc`, `BinaryOp`) are deferred so each lands with a planner consumer rather than as dead code; adding them is purely additive. -- `intent_algebra::Schema` — typed schema flow with `unique_keys` (the load-bearing CSE-legality field, `cse_substitution_legal_only_with_unique_keys` test pins the invariant). -- `intent_algebra::lower_parsed_query` — `query_parser::ParsedQuery` → `QueryExpr` single-query lowering, exposed as a standalone function. The analyzer is **not** yet wired to emit `QueryExpr`; that wiring is the follow-up phase's job, kept separate so the IR rev and the consumer rev land independently. - -Follow-up phases: - -- **Phase C** — wire `Analyzer::analyze` to also produce a `QueryExpr` alongside `QueryWorkload`, populate `WorkloadPlan::roots` from the lowered roots. -- **Phase D** — workload-level CSE pass (`core::lower::workload::dedupe_subtrees`) that hoists shared sub-DAGs into `WorkloadPlan::bindings`, leaning on `Schema::unique_keys` for legality (the batched-queries example above). -- **Phase E** — L4 `SketchExpr` IR (`core::sketch_algebra`) + L4 binding rules. -- **Phase F** *(this PR)* — `CostModel::workload_cost` + `Schema::unique_keys`-based CSE legality. See the per-section "Implementation status" notes under `core::cost` and the "Schema flow" table below for what shipped / what's deferred. - -### `core::sketch_algebra` — Layer 4 IR (`SketchExpr`) - -L4 binding rules consume L3 `QueryExpr` (in `core::intent_algebra`) and produce `SketchExpr` (in `core::sketch_algebra`). This is the IR L5 emitters consume. The two-IR split — intent-only L3 (`QueryExpr`) and sketch-bound L4 (`SketchExpr`), in two separate modules — gives L4 rule application a clean type signature: `fn apply(&QueryExpr, &Constraints) -> Option`, and the boundary cannot be silently violated. - -```rust -pub enum SketchExpr { - /// Any logical L3 node passes through unchanged when no L4 rule rewrote - /// it — a `Filter` doesn't need a sketch counterpart. - Logical(QueryExpr), - - /// Sketch aggregation. L4 picked the sketch type and parameters from the - /// catalog given the `AggIntent` and `DeploymentConstraints`. - SketchAgg { - child: Box, - sketch: SketchKind, // Kll, Cms, Hll, DDSketch, CmsWithHeap, … - params: SketchParams, // catalog-validated - col: ColumnRef, - by: Vec, - }, - - /// Sketch-aware join (KMV / theta-sketch for join cardinality; - /// join-sample for join sampling). Emitted only when a `Bind*OnJoin` - /// rule fires — L3 always presents the logical `Join` for L4 to choose. - SketchJoin { - outer: Box, - inner: Box, - key: ColumnRef, - sketch: SketchKind, - params: SketchParams, - }, - - /// Subtract one sketch from another. Valid only for sketches with a - /// linear-inverse property (CMS, theta, count-based). Lets the planner - /// compute "all-A minus all-B" cardinality / count without re-scanning. - SketchSubtract { left: Box, right: Box }, - - /// Delete a key from a sketch (CMS update with -1, deletable Bloom - /// filter, …). Valid only for deletion-supporting sketches. - SketchDelete { sketch_input: Box, key: ColumnRef }, - - /// Read out a query result from a built sketch. Inverse of `SketchAgg`. - /// `query` says what to extract — quantile φ, count for key k, cardinality. - SketchEstimate { sketch_input: Box, query: SketchQuery }, - - /// ⊕ — union of sketches across stages / shards. Distinct from L3 - /// `Merge` because sketch union has type constraints (same family, - /// same params). L5 stage allocator emits this when distributing. - SketchMerge { children: Vec }, -} -``` - -The optimizer's job is to selectively replace logical aggregates / joins with their sketch-bound variants when a binding rule fires; everything else stays inside `SketchExpr::Logical(…)`. - -#### Per-node input/output spec for `SketchExpr` - -L4 introduces a new field type into `Schema`: - -```rust -pub enum DataType { - // … the L3 types (Int64, Float64, Utf8, Map<…>, …) … - /// Sketch state. Carries the sketch family + params so the type system - /// rejects merges of incompatible sketches at plan time. - Sketch(SketchKind, SketchParams), -} -``` - -A "sketch-state schema" is a regular `Schema` whose value-bearing field has a `DataType::Sketch(...)` dtype. Reading rules: - -| Node | Input schemas | Output schema | -|---|---|---| -| `Logical(qe)` | whatever the inner L3 node `qe` consumes (per the L3 table above) | whatever `qe` produces — straight pass-through | -| `SketchAgg { child, sketch, params, col, by }` | one input schema; must contain `col` (the column being summarised) and every field referenced in `by` (group keys) | the `by` columns carried over verbatim, followed by one synthetic field of dtype `Sketch(sketch, params)` carrying the partial sketch state per group; `unique_keys = [by]` | -| `SketchJoin { outer, inner, key, sketch, params }` | two input schemas (outer + inner); both must contain `key` with compatible types | one field of dtype `Sketch(sketch, params)` carrying the join-cardinality / join-sample state — read out by a downstream `SketchEstimate` | -| `SketchSubtract { left, right }` | two input schemas, each with exactly one `Sketch(s, p)` field; **`s` and `p` must match** between the two inputs (catalog rejects mismatches at plan time); the `s` family must have `subtractable = true` in the catalog | one `Sketch(s, p)` field carrying the subtracted state (same family + params as inputs) | -| `SketchDelete { sketch_input, key }` | one input schema with a `Sketch(s, p)` field whose catalog entry has `deletable = true`; plus the `key` column to delete | the input schema unchanged in type — sketch state is mutated logically (the key's contribution removed) but the field's `(sketch, params)` signature is preserved | -| `SketchEstimate { sketch_input, query }` | one input schema with a `Sketch(s, p)` field; `query` (e.g. `Quantile(φ)`, `PointCount(k)`, `Cardinality`) must appear in the catalog entry's `supported_intents` for `s` | a regular row-shaped schema carrying the answer — `Float64` for quantile, `Int64` for count / cardinality, an array of `(key, count)` for top-k. The `Sketch(...)` field type does *not* propagate downstream of an Estimate | -| `SketchMerge { children }` | N input schemas, each with a `Sketch(s, p)` field; **all N must agree on `(s, p)`**; the `s` family must have `mergeable = true` in the catalog | one `Sketch(s, p)` field carrying the unioned state (same family + params as inputs) | - -Two type-system invariants make L4 robust: - -1. **Sketch-family mismatch is a plan-time error.** `SketchSubtract` over `Sketch(KLL, …)` and `Sketch(CMS, …)` fails type-checking before L5 ever sees it. -2. **Catalog capability flags gate which nodes can fire.** `SketchSubtract` requires `subtractable`, `SketchDelete` requires `deletable`, `SketchMerge` requires `mergeable`. The catalog (see §6 `core::physical::sketch_catalog`) is the single source of truth for these flags; binding rules consult it before producing the node. - -**Why a `Source` sum instead of two parallel `QueryExpr` trees:** most `QueryExpr` nodes (`Filter`, `Aggregate`) are data-model-agnostic — filter semantics are the same whether the input is a time-series window or a table scan. Only the leaf `Scan` differs. Keeping one tree with a polymorphic leaf means L4 rules like `BindKllOnQuantile` work uniformly across both data models; rules that care about data-model specifics (stage-aware push-down for TS; join-selectivity for tabular) gate on `source.data_model()` + `intent.requires()`. - -**Sketches are data-model-agnostic by construction.** KLL / CMS / HLL / DDSketch ingest a stream of values. That stream can come from a time-series window (`Source::TimeSeries`) or a table column (`Source::Table`); the sketch doesn't know or care. So `BindKllOnQuantile` works identically regardless of `Source`. - -#### Implementation status - -**Phase C** (`feat(controller): Phase C — sketch_algebra L4 IR (SketchExpr + Bind* rules)`) lands the typed `SketchExpr` IR + `Bind*` rules in `controller/src/sketch_algebra/`. Module layout: - -- `sketch_expr.rs` — `SketchExpr` enum (`Logical` / `SketchAgg` / `SketchEstimate` / `SketchMerge` / `LetBinding` / `Ref`). -- `params.rs` — `SketchKind` + per-family `SketchParams` (`KllParams{k}`, `DDSketchParams{alpha}`, `HllParams{precision}`, `CmsParams{w,d}`, `CountSketchParams{w,d,with_heap}`). -- `schema.rs` — `SketchStateSchema` with the `(SketchKind, SketchParams)` field-type and the catalog-capability flags (`mergeable` / `subtractable` / `deletable`). -- `rules/{bind_kll_quantile, bind_ddsketch_quantile, bind_cms_count, bind_cms_topk, bind_hll_cardinality}.rs` — five `Bind*` rules implementing the `Rule` trait. -- `lower.rs` — `bind_query_expr(&QueryExpr, AccuracyTarget) -> Result`. - -**Scope reduction.** The variant set ships the subset DC + PromQL needs. `SketchJoin`, `SketchSubtract`, `SketchDelete` from the spec above are intentionally *not* surfaced yet — they're gated on rules that haven't landed. Adding them is purely additive. - -**Planner-side migration is opt-in.** `controller/src/planner/rules.rs` exposes `bind_workload_typed(&QueryWorkload) -> Option` and an env-var gate `USE_TYPED_SKETCH_ALGEBRA=1`. Existing call sites continue to use the legacy untyped binding path (`algebra::directory::sketch_type_for_agg`); the typed path runs in parallel for callers that opt in. **Phase E** (stage_split refactor) is the natural migration point at which the typed path becomes the only path. - -### `core::lower` — L1 → L2 → L3 passes - -One pass per language, each producing the same `intent_algebra::QueryExpr`: - -```rust -pub fn lower_promql(ast: PromqlAst, schema: &MetricSchema) -> Result; -pub fn lower_sql(ast: SqlAst, schema: &TableSchema) -> Result; -pub fn lower_datafusion(ast: DfAst, ctx: &DfContext) -> Result; -pub fn lower_elasticdsl(ast: EsAst, schema: &IndexSchema) -> Result; -``` - -Once a query hits L3 it's language-agnostic. All deployment models downstream see the same IR. - -### `core::pipeline` — orchestration - -The L1→…→L5 driver. Parameterised on a deployment model's optimizer rules (L4) + emitter (L5): - -```rust -pub struct Pipeline { - deployment_model: S, -} - -impl Pipeline { - pub fn run(&self, workload: &QueryWorkload) -> Result { - let l3: Vec = workload.queries() - .iter() - .map(|q| self.parse_and_lower(q)) // L1→L2→L3 - .collect::>()?; - let l4 = self.deployment_model.optimizer().optimize(l3)?; // L4 - let l5 = self.deployment_model.physical().lower(l4)?; // L5 - self.deployment_model.emitter().emit(&l5) // L5 → bytes/protobuf/DF plan - } -} -``` - -Core owns the driver. Deployment models own what goes into each deployment-model-specific seam. - -### `core::optimizer` — Layer 4 framework - -Core ships the **rule engine + trait surface + a shared rule library**. Deployment models pick which rules to enable. - -```rust -// trait (in core) -pub trait OptimizerRule: Send + Sync { - fn name(&self) -> &'static str; - fn category(&self) -> RuleCategory; // PushDown | Fusion | Elim | Bind | ... - fn priority(&self) -> u16; - fn apply(&self, expr: &QueryExpr, c: &DeploymentConstraints) -> Option; -} - -// engine (in core) — fixed-point iteration, cycle detection, priority ordering -pub struct RuleEngine { /* ... */ } -impl RuleEngine { - pub fn new(rules: Vec>) -> Self { /* ... */ } - pub fn run(&self, exprs: Vec, c: &DeploymentConstraints) - -> Result, OptError>; -} - -// shared rule library (in core::optimizer::rules) — opt-in from deployment models -pub mod rules { - pub struct BindKllOnQuantile; // AggIntent::Quantile → bind KLL(k by accuracy) - pub struct BindCmsOnCount; // AggIntent::Count → bind CMS(w, d) - pub struct BindHllOnCardinality; // AggIntent::Cardinality → bind HLL(p) - pub struct FusionPassthrough; // Aggregate over Filter → push Filter under Aggregate - pub struct ElimNoopFilter; // Filter(true) → child - // ... etc - impl OptimizerRule for BindKllOnQuantile { /* ... */ } -} -``` - -Deployment models compose rule sets by picking from the shared library + adding their own: - -```rust -// in deployment-model-asaplifecycle -use asap_control_core::optimizer::{RuleEngine, rules::*}; -fn rule_set() -> Vec> { - vec![ - Box::new(BindKllOnQuantile), - Box::new(BindCmsOnCount), - Box::new(FusionPassthrough), - // DC-specific additions: - Box::new(StageAwarePushDown), // pushes ops to edge when possible - Box::new(TransmissionCostRewrite), // uses TCO model to defer aggregation - ] -} -``` - -Core also ships `DeploymentConstraints` as a trait object; each deployment model supplies a concrete impl with its deployment's memory budgets, network topology, available sketch backends, and the registered `Executor` list (one entry per concrete runtime instance, each tagged with its `StageId`). The executor list is populated from the deployment model's discovery channel — OpAMP for DC lifecycle, static config for single-backend query, the in-process `SessionContext` itself for fusion. - -### `core::physical` — Layer 5 framework - -```rust -pub trait PhysicalPlanner { - type Topology: TopologyDescriptor; - type Output; - fn lower(&self, l4: Vec, t: &Self::Topology) -> Result; -} - -pub trait TopologyDescriptor { - fn stages(&self) -> &[StageDescriptor]; - fn edges(&self) -> &[StageEdge]; -} - -// pre-baked topologies in core -pub mod topology { - pub struct ThreeStage { /* edge → gateway → backend */ } - pub struct SingleStage { /* backend-only */ } - pub struct ZeroStage; /* in-process */ -} - -/// Identifier for a stage role (categorical tier in the data lifecycle). -pub struct StageId(pub String); // "edge" / "gateway" / "backend" / "in-process" - -/// A **concrete runtime instance** that executes a sub-DAG of the plan. -/// One stage may have many executors — e.g. a 50-host edge fleet is 50 -/// executors all with `stage = StageId("edge")`; a singleton backend is -/// one executor at `stage = StageId("backend")`. The stage allocator -/// works at stage granularity; the deployment-model planner / emitter -/// materialises each stage's sub-DAG into one concrete config per executor. -pub struct Executor { - pub id: ExecutorId, // stable identifier across replans - pub stage: StageId, // which stage this executor occupies - pub capabilities: ExecutorCaps, // memory budget, available sketch backends, network neighbours - pub address: ExecutorAddr, // OpAMP agent / HTTP endpoint / in-process handle -} - -pub struct ExecutorId(pub String); - -pub enum ExecutorAddr { - OpAmpAgent(AgentId), // OTel collector managed via OpAMP (DC lifecycle) - HttpEndpoint(Url), // ASAPQuery-backend, etc. - InProcess, // asap-fusion library-mode SessionContext -} - -// generic stage allocator — given a QueryExpr tree + a topology, decide which -// ops land on which stage subject to constraints. Stage-level only; per-executor -// fan-out happens in the deployment model's PhysicalPlanner using the executor -// list from `DeploymentConstraints::executors()`. -pub struct StageAllocator; -impl StageAllocator { - pub fn allocate( - &self, exprs: &[QueryExpr], topology: &T, c: &DeploymentConstraints, - ) -> Result, PlanError>; -} - -// sketch catalogue — what sketches exist, what they support, what params they -// accept. Built at startup from the registered sketch backends; queried by L4 -// binding rules to map an `AggIntent` → `SketchKind` + `SketchParams`, and by -// L5 to instantiate the chosen sketch. -pub struct SketchCatalog { - pub entries: Vec, -} - -pub struct SketchEntry { - pub kind: SketchKind, // Kll, Cms, Hll, DDSketch, KMV, Theta, … - /// Which `AggIntent` variants this sketch can serve. - pub supported_intents: &'static [IntentTag], // Quantile, Count, Cardinality, JoinCardinality, … - /// Mergeability — sketches built on disjoint inputs combine without re-scan. - /// (CMS / HLL / KLL / theta = mergeable; SpaceSaving = approximately; - /// some heavy-hitter variants = no.) - pub mergeable: Mergeability, - /// Whether the sketch supports point deletion (CMS update with -1, - /// deletable Bloom filter) — gates `SketchExpr::SketchDelete`. - pub deletable: bool, - /// Whether the sketch admits a linear inverse (CMS, theta, count-based) - /// — gates `SketchExpr::SketchSubtract`. - pub subtractable: bool, - /// Accuracy / confidence model: error bounds as a function of params. - /// `(eps, delta)` for randomised sketches; absolute error for KLL; etc. - pub accuracy: AccuracyModel, - /// Aggregation keys this sketch supports natively. Some sketches are - /// keyed (CMS over (label, value)); others are unkeyed (HLL). - pub aggregated_keys: KeyShape, // Unkeyed | KeyedScalar | KeyedTuple - /// Parameter ranges + defaults. Catalog rejects out-of-range params at - /// L4 bind time so L5 never sees an unsupported configuration. - pub param_ranges: ParamRanges, - /// Memory + CPU model used by `CostModel`. Function of params. - pub cost_model: SketchCostModel, -} -``` - -Deployment models use these pieces: - -```rust -// in deployment-model-asaplifecycle -use asap_control_core::physical::{StageAllocator, topology::ThreeStage, Executor}; - -impl PhysicalPlanner for LifecyclePlanner { - type Topology = ThreeStage; - type Output = Vec<(ExecutorId, ExecutorPlan)>; // one entry per executor - fn lower(&self, l4: Vec, t: &ThreeStage) -> Result<_, _> { - // 1. Stage-level: colour the DAG by StageId. - let assignments = StageAllocator.allocate(&l4, t, &self.constraints)?; - // 2. Per-executor fan-out: for each Executor in the deployment, - // pick the sub-DAG assigned to its stage and produce its config. - let executors: &[Executor] = self.constraints.executors(); - // deployment-model-specific post-processing: DC's delta_cost logic, - // backend_client push preparation, etc. - Ok(/* ... */) - } -} -``` - -> **Implementation status (Phase E).** The typed L5 framework lives in -> `controller/src/stage_split/` (this PR): -> - `stage_id.rs` — `StageId` enum (`Edge`/`Gateway`/`Backend`) + -> `Topology` enum. Phase E ships only `Topology::ThreeStage`; the -> `SingleStage` / `ZeroStage` variants surface as future-proofing -> stubs that error cleanly via `AllocateError::UnsupportedTopology`. -> - `colored_dag.rs` — `ColoredDag { topology, nodes, edges }` IR, the -> allocator's output. `cut_edges()` exposes cross-stage edges for -> future wire-format insertion (Phase G+). -> - `allocator.rs` — `StageAllocator::allocate(expr, topology) -> -> Result`. Implements the §6 -> batched-queries colouring rules: `Logical(Scan/Window/Aggregate)` -> + `SketchAgg` → Edge; `SketchMerge` → Gateway; `SketchEstimate` → -> Backend; `LetBinding`/`Ref` colour by their bound expression's -> stage. The L3 `Aggregate{exact}` (e.g. `Max`) reachable through -> `Logical` colours Edge — the design.md "root of q3" backend -> placement is exercised when the same exact aggregation appears -> *above* a SketchMerge sibling structure (a Phase G enhancement -> that adds an explicit `Logical(Merge)` SketchExpr variant for the -> gateway hop). -> - `emitter.rs` — `Emitter` trait + `ThreeStageEmitter` that lowers -> `ColoredDag` → `HashMap`. `StageConfig` -> carries the structural facts each downstream consumer needs: -> - `Edge` → `EdgeStageConfig { source_metric, label_filters, -> window_secs, sketch_processors, exporter_target }`. Sketch -> processor names follow the catalog (`KLL`, -> `ddsketch`, `HLL`, `countmin`, -> `countsketch`). -> - `Gateway` → `GatewayStageConfig { otlp_receiver_port, -> merge_processors, exporter_target }`. -> - `Backend` → `BackendStageConfig { aggregations, readouts }` — -> the aggregation_id ↔ (sketch_kind, params) mapping the backend's -> `OtlpReceiver` + readout catalog need. -> -> The typed path is opt-in via the `USE_TYPED_STAGE_SPLIT` env var -> consulted by `controller/src/planner/stage_split.rs::split_typed_three_stage`; -> existing untyped callers (`split_expr_by_stage`) continue to run -> unchanged. OpAMP push (`crate::opamp::OpampServer::push_to_role`) -> and backend `StreamingConfig` POST (`crate::backend_client`) wiring -> against the typed `StageConfig` is downstream (Phase G+) — Phase E -> ships only the structured per-stage output, not the wire push. - -### `core::plan` — shared traits bridging layers - -```rust -pub trait DeploymentModel { - type Topology: TopologyDescriptor; - type EmitterOutput; - fn rules(&self) -> Vec>; - fn topology(&self) -> &Self::Topology; - fn physical(&self) -> &dyn PhysicalPlanner; - fn emitter(&self) -> &dyn PlanEmitter; - fn constraints(&self) -> &dyn DeploymentConstraints; -} -``` - -### `core::workload` - -> **Implementation status (this PR).** The new types `QueryLanguage`, -> `AccuracyTarget`, `QueryShape`, `DataShape`, `QueryId`, `BindingName`, -> and a `WorkloadPlan` container live in `controller/src/types_v2.rs`. -> `analyzer::QuerySpec` carries the new fields (`id`, `language`, -> `accuracy`, `dollars`, `deployment_model`, `shape`, `data`) as -> `#[serde(default)]` additions; defaults preserve legacy behaviour for -> existing `POST /api/v1/plan` callers and the `workloads.yaml` -> pre-population path. `Analyzer::analyze` enforces the L1 cross-product -> rejections (`Streaming × Batch`, `Streaming × Mutable`) from the table -> below and resolves typed `accuracy` over legacy `accuracy_sla` with -> the typed form taking precedence. The fields are not yet load-bearing -> in `replan.rs` / `planner/` cost or binding decisions — that's a -> separate downstream PR. `WorkloadPlan` is a container only; the CSE -> pass that populates `bindings` is deferred until the L3 algebra grows -> `LetBinding` / `Ref`. - -One public type for every kind of input: - -```rust -pub enum QueryWorkload { - /// A single QuerySpec from an HTTP POST. DC's current entry point. - Single(QuerySpec), - /// A hand-authored YAML workload file. asap-planner-rs's current entry point. - AuthoredSet(Vec), - /// A Prometheus query log replayed. asap-planner-rs's alt entry point. - QueryLog(QueryLogReplay), -} - -pub struct QuerySpec { - /// Stable identifier — preserved across `replan` cycles so the runtime - /// can correlate plan outputs with the originating spec, and L4 reuse - /// rules can name shared producers across consumers in the same workload. - pub id: QueryId, - - /// Source-language query string + which language it's written in. - /// L1 parses `query` against `language`; no schema lookup yet. - pub query: String, - pub language: QueryLanguage, // PromQL | Sql | DataFusion | ElasticDsl - - /// Evaluation time range. Some languages (PromQL) carry their own ranges - /// inside `query`; this field wins when both are set. - pub time_range: TimeRange, - - /// Per-target SLA. Drives L4 binding (which sketch / which params) and - /// L5 stage placement (push down or not). The three are independent — - /// `accuracy: Exact` disables all `Bind*` rules; an unset latency / - /// dollars target lets the cost model pick freely. - pub accuracy: AccuracyTarget, // Exact | Epsilon(f64) | EpsilonDelta { ε, δ } - pub latency: Option, // p99 evaluation latency target - pub dollars: Option, // per-evaluation budget - - /// Routing hint: which deployment model should plan this query. When - /// unset, the runtime defaults to the deployment model bound to the - /// inbound HTTP route (`POST /plan/lifecycle` vs `POST /plan/query`). - pub deployment_model: Option, - - /// How the query is *evaluated*: one-shot, continuous, or scheduled. - /// Distinct from `data` below — a one-shot query against a streaming - /// source is "evaluate now over the latest window"; a streaming query - /// against a batch source doesn't make sense and is rejected at L1. - /// Drives L4 binding (mergeable vs one-shot sketch family) and the - /// L5 wire format (`OneShot` emits config + result; `Streaming` and - /// `Periodic` emit a config that keeps running). - pub shape: QueryShape, - - /// Shape of the *data* feeding the query. Workload-level summary — - /// per-leaf detail rides on `Source::data_shape` (§6 L3). For a join - /// over streaming + batch this is `Mixed`, and the planner reads the - /// per-leaf shape during L4. Drives binding choices: an - /// `AppendOnlyStream` unlocks incremental, mergeable sketches and - /// retraction-free aggregation; `Batch` lets the planner pick a - /// non-mergeable estimator (e.g. exact percentile over a sort) that - /// wouldn't survive a distributed streaming setting; `Mutable` - /// requires retraction-aware operators (out of scope today — the - /// planner refuses sketch binding and falls back to re-scan). - pub data: DataShape, -} - -pub enum QueryLanguage { PromQL, Sql, DataFusion, ElasticDsl } -pub enum AccuracyTarget { Exact, Epsilon(f64), EpsilonDelta { eps: f64, delta: f64 } } - -pub enum QueryShape { - /// Evaluate once. Plan, execute, return result, discard state. - /// SQL ad-hoc queries; one-off PromQL via `POST /plan`. - OneShot, - /// Continuous query — output stream that the executor keeps emitting - /// as new data arrives. No fixed cadence; the runtime emits whenever - /// the underlying state changes. Streaming dashboards, alerting - /// expressions evaluated by the agent rather than by a poller. - Streaming, - /// Re-evaluated at a fixed cadence — Prometheus recording rules, - /// scheduled dashboard panels, alerting evaluation cycles. The - /// planner amortises sketch / aggregate build cost across evaluations - /// within the cadence and reuses state between adjacent windows. - Periodic { every: Duration }, -} - -pub enum DataShape { - /// Bounded relation, fully materialised at plan time. SQL tables, - /// Parquet / CSV files, DataFusion in-process tables. - Batch, - /// Append-only stream — events arrive over time, never updated or - /// deleted. Metrics, logs, event streams. The common case for - /// asaplifecycle and asapquery. - AppendOnlyStream, - /// Mutable relation — inserts + updates + deletes. Operational - /// databases, CRUD-style tables. Sketch binding is currently refused - /// for this shape (no retraction-aware sketches in the catalog yet). - Mutable, - /// Join across sources of differing shape. The planner consults - /// `Source::data_shape` per leaf during L4; this variant exists so - /// callers don't have to flatten a workload-level summary. - Mixed, -} -``` - -All three deployment models take `&QueryWorkload`. Same type, different planners. Fields are deployment-model-agnostic — anything deployment-model-specific (DC's stage-budget overrides, fusion's `SessionContext` handle) rides on `DeploymentConstraints`, not on `QuerySpec`. - -The `shape` × `data` cross-product isn't fully populated. Combinations the planner accepts and rejects: - -| `shape` \ `data` | `Batch` | `AppendOnlyStream` | `Mutable` | `Mixed` | -|---|---|---|---|---| -| `OneShot` | ✓ ad-hoc SQL, fusion | ✓ "evaluate now over latest window" | ✓ via re-scan, no sketch binding | ✓ per-leaf shape decides | -| `Streaming` | ✗ rejected at L1 (no stream over a static dataset) | ✓ canonical streaming case | ✗ rejected (no retraction support) | ✓ if streaming leaves dominate | -| `Periodic { every }` | ✓ scheduled batch report | ✓ recording rules | ✓ via re-scan | ✓ | - -### `core::cost` - -Extracted from DC's `planner/*` (delta, online, pareto, tco). Deployment models pick the models they need. - -```rust -pub trait CostModel { - fn accuracy(&self, plan: &Plan) -> Accuracy; - fn latency(&self, plan: &Plan) -> Duration; - fn dollars(&self, plan: &Plan) -> Dollars; - - /// Workload-level total cost. A straight sum of per-plan costs - /// under-estimates how good a plan is when multiple queries share - /// computation — a sketch or a precomputed aggregate built for - /// one query serves the others for free. Implementations must - /// identify reusable sub-expressions across `plans` and credit - /// their build cost once, so the planner prefers plans that - /// maximise reuse when the total-cost objective allows. - fn workload_cost(&self, plans: &[Plan]) -> WorkloadCost; -} - -pub struct WorkloadCost { - pub total_latency: Duration, - pub total_dollars: Dollars, - /// Per-plan contribution, for EXPLAIN / observability. - pub per_plan: Vec, - /// Sub-expressions built once, consumed by ≥2 plans. Drives - /// decisions like "build a KLL for p99 that q1 and q2 both read" - /// vs "run exact select-n per query". - pub reused: Vec, -} -``` - -The reuse model is not sketch-specific: any primitive that is costly to build once and cheap to query (sketches, materialized aggregates, cached scan results, future wavelet summaries) plugs into the same `ReusedComponent` accounting. - -#### Implementation status - -Phase F (in `controller/src/planner/cost_model.rs`) lands the workload-cost shape that pairs with the `Schema::unique_keys`-based CSE legality gate above: - -- `workload_cost(plan: &WorkloadCostPlan<'_>) -> Result` — walks the L3 IR DAG (`intent_algebra::QueryExpr`) post-order, memoises by `LetBinding` name, credits each shared producer once across consumers. Returns `WorkloadCost { total_dollars, per_root_breakdown, reused_savings }` — the `reused_savings` field exposes the gap between the bundled total and the naive sum-over-roots, so EXPLAIN can show what shared-producer credit was worth. -- `WorkloadCostPlan { bindings, roots }` — the cost-model's view of `types_v2::WorkloadPlan` carrying real `&QueryExpr` references rather than the `QueryExprPlaceholder` JSON-wire string. Collapses into `types_v2::WorkloadPlan` when the placeholder is swapped for live `QueryExpr` downstream. - -Per-node cost primitives at L3 (`node_cost_scan`, `node_cost_window`, `node_cost_aggregate`, `intent_cost`) are coarse-but-monotonic placeholders calibrated against schema width and intent kind. Calibration against real benchmarks is downstream; what Phase F pins is the *shape* — costs are positive, additive over sub-trees, and the savings invariant `bundled_total ≤ naive_sum` holds with `savings = naive_sum − bundled_total`. - -Not yet shipped at L3 cost (deferred): the `total_latency` field, the per-plan `Contribution` split (Phase F's `per_root_breakdown` carries one `f64` per root, not the full latency / dollars / accuracy tuple), and the `ReusedComponent` enumeration (savings are reported as a scalar, not as a per-component vector — the per-component breakdown is downstream when more producer kinds (sketches, materialized aggregates) become candidates for sharing). The L4 `score` / `score_with` path above remains the per-plan dollars / latency / memory cost; the L3 `workload_cost` is the bundled-plan credit on top. - -### `core::emit` - -```rust -pub trait PlanEmitter { - type PlanInput; - type Output; // YAML bytes, OpAMP RemoteConfig, rewritten LogicalPlan - fn emit(&self, plan: &Self::PlanInput) -> Result; -} -``` - -Concrete emitters live in deployment model crates: - -- `deployment-model-asapquery::yaml::StreamingConfigEmitter` → `streaming_config.yaml` bytes -- `deployment-model-asapquery::yaml::InferenceConfigEmitter` → `inference_config.yaml` bytes -- `deployment-model-asaplifecycle::emit::OpAmpRemoteConfigEmitter` → `opamp::RemoteConfig` protobuf -- `deployment-model-asapfusion::emit::DataFusionPlanEmitter` → rewritten `datafusion::LogicalPlan` - -### `core::registry` - -The extension point. The runtime binary does: - -```rust -let mut reg = DeploymentModelRegistry::new(); -reg.register::(); -reg.register::(); -reg.register::(); -// add more... -``` - -Registration is by-type; the runtime looks up by `DeploymentModelId` (either from the `QuerySpec.deployment_model` field or from an HTTP route). A 4th deployment model is added by: - -1. New crate `deployment-model-/` -2. Implement `DeploymentModel` trait (wraps a `Planner` + its `PlanEmitter`s) -3. Register in `bin/asap-controller/main.rs` - -No core change. - -### End-to-end example — one query through all five layers - -A worked trace of a single PromQL query as it flows L1 → L5. The query is intentionally simple (one metric, one window, one aggregate) so the IR shapes stay readable; production workloads have larger DAGs but the per-layer transformation is the same. - -**Input.** A `QuerySpec` arrives at `POST /plan` carrying: - -```text -quantile_over_time(0.99, http_request_duration_seconds{service="api"}[5m]) -``` - -with `accuracy: AccuracyTarget::Epsilon(0.01)` and `language: PromQL`. - -#### L1 — query language (parse to `PromqlAst`) - -`core::query_language::promql::parse` wraps `promql-parser`. Output (sketch — actual fields come from the upstream crate): - -```rust -PromqlAst::Call { - func: BuiltinFn::QuantileOverTime, - args: vec![ - Expr::Number(0.99), - Expr::MatrixSelector { - name: "http_request_duration_seconds", - matchers: vec![LabelMatcher::Eq("service", "api")], - range: Duration::from_secs(300), - }, - ], -} -``` - -Pure language-level parse — no schema lookup, no sketch awareness. Same call returns the same AST regardless of deployment model. - -#### L2 — language logical plan (`PromqlLogicalPlan` tree) - -`core::lower::promql::lower_to_logical` walks the AST against a `MetricSchema` resolved from Prometheus's `/api/v1/labels`. The result preserves PromQL semantics — `quantile_over_time` is a range-vector aggregate, distinct from a generic `Aggregate { Quantile }` over a `Window`: - -```rust -PromqlLogicalPlan::RangeAggregate { - func: PromqlRangeAggFunc::QuantileOverTime { q: 0.99 }, - range: Duration::from_secs(300), - matrix: Box::new(PromqlLogicalPlan::MatrixSelector { - metric: "http_request_duration_seconds", - matchers: vec![("service", LabelOp::Eq, "api")], - }), -} -``` - -Per language: the SQL form `SELECT approx_percentile(latency, 0.99) FROM events WHERE service='api' AND ts > now() - INTERVAL '5 min'` lands in `SqlLogicalPlan` with a different shape. L2 is per-language; the shapes converge at L3. - -#### L3 — intent algebra (`QueryExpr` + `AggIntent`) - -`core::lower::promql::lower_to_intent` strips PromQL-specific shapes. `quantile_over_time` does **not** survive — the canonical L3 form for a windowed quantile is `Window` over `Aggregate{Quantile}` (see §6 design rule 1, "no `WindowedAgg`"; see §6 "Why no `QuantileOverTime` intent?"). The accuracy target threads through from the `QuerySpec`: - -```rust -QueryExpr::Aggregate { - by: vec![], - aggs: vec![AggIntent::Quantile { - q: 0.99, - accuracy: AccuracyTarget::Epsilon(0.01), - }], - having: None, - child: Box::new(QueryExpr::Window { - kind: WindowKind::Sliding, - size: Duration::from_secs(300), - slide: None, - child: Box::new(QueryExpr::Scan { - source: Source::TimeSeries { - metric: MetricRef::from("http_request_duration_seconds"), - time: TimeRange::default(), // supplied by QuerySpec at evaluation - labels: LabelFilter::eq("service", "api"), - }, - predicates: vec![], - }), - }), -} -``` - -Per-edge schemas (derived per the §6 input/output spec): - -| Edge | Schema | -|---|---| -| `Scan` → `Window` | `{value: Float64, ts: Int64@time_index, service: Utf8}` | -| `Window` → `Aggregate` | the above + `{window_id: Int64, window_start: Int64, window_end: Int64}` | -| `Aggregate` → root | `{p99_value: Float64}` (one column per `AggIntent::Quantile`, typed via `output_type`) | - -This IR is identical across deployment models — same DAG, same intent, same accuracy target. Below this point each deployment model picks its own L4 rules. - -#### L4 — sketch algebra (`SketchExpr`) - -The shared rule `core::optimizer::rules::BindKllOnQuantile` matches `Aggregate { aggs: [Quantile{q, accuracy}] }`, consults the sketch catalog (KLL has `supported_intents: [Quantile]`, is mergeable, satisfies `ε=0.01` at `k=200`), and rewrites the matched sub-DAG into a `SketchAgg` wrapped in a `SketchEstimate` (the readout). Everything not rewritten passes through in `SketchExpr::Logical(...)`: - -```rust -SketchExpr::SketchEstimate { - sketch_input: Box::new(SketchExpr::SketchAgg { - child: Box::new(SketchExpr::Logical(/* the L3 Window+Scan subtree */)), - sketch: SketchKind::Kll, - params: SketchParams::Kll(KllParams { k: 200 }), - col: ColumnRef::value(), - by: vec![], - }), - query: SketchQuery::Quantile { q: 0.99 }, -} -``` - -Catalog-derived edge schemas: - -| Edge | Schema | -|---|---| -| inner `Logical(Window)` → `SketchAgg` | as L3 above (logical pass-through) | -| `SketchAgg` → `SketchEstimate` | `{kll: Sketch(Kll, KllParams{k:200})}` | -| `SketchEstimate` → root | `{p99_value: Float64}` (the `Sketch(...)` field type does not propagate past `SketchEstimate`) | - -The `Sketch(Kll, KllParams{k:200})` field type is the L4 type-system invariant — a downstream `SketchMerge` over a mismatched `Sketch(Cms, …)` input would fail at plan time, before L5 ever sees it. `BindKllOnQuantile` lives once in core and fires for all three deployment models; deployment-model-specific rules (lifecycle's `StageAwarePushDown`, fusion's `HashModeRule`) chain before or after. - -#### L5 — physical plan (stage allocation + emission, per deployment model) - -L4 chose the sketch family + params. L5 colors the DAG by `StageId` and emits per-executor configs. Same `SketchExpr` input; topology and emitter differ per deployment model. - -##### deployment-model-asaplifecycle (3-stage, OpAMP + backend POST) - -`StageAllocator` colors against `topology::ThreeStage`: - -| Node | StageId | Why | -|---|---|---| -| `Scan` + `Window` + `SketchAgg` | `"edge"` | scrape happens on the agent host; KLL build is mergeable so it's safe to run early | -| `SketchMerge` (inserted on cut edge) | `"gateway"` | catalog says `KLL.mergeable = true`; reduces N edge streams to 1 | -| `SketchEstimate` | `"backend"` | readout where users query | - -`PhysicalPlanner` then fans out via `DeploymentConstraints::executors()`. Given a deployment with 3 edge agents + 1 gateway + 1 backend: - -```rust -vec![ - (ExecutorId("edge-001"), OpAmpRemoteConfig { /* OTel YAML: scrape http_request_duration_seconds{service=api}, - sliding 5m window, KLL k=200, forward to gateway-001 */ }), - (ExecutorId("edge-002"), OpAmpRemoteConfig { /* identical OTel YAML; differs only by agent_id */ }), - (ExecutorId("edge-003"), OpAmpRemoteConfig { /* … */ }), - (ExecutorId("gateway-001"), OpAmpRemoteConfig { /* receive 3 KLL streams, SketchMerge, forward to backend */ }), - (ExecutorId("backend-001"), AsapqueryBackendConfig { /* read merged KLL, SketchEstimate q=0.99 */ }), -] -``` - -The 3 edge configs are byte-identical except for `agent_id` — that's the §3 "one stage may have N executors all materialised from the same per-stage sub-DAG" property. The `AsapqueryBackendConfig` is produced by calling `deployment-model-asapquery::yaml::StreamingConfigEmitter` (see §8 asymmetric dep). - -##### deployment-model-asapquery (1-stage, YAML) - -`StageAllocator` against `topology::SingleStage` returns everything on `"backend"`. `PhysicalPlanner` produces one config per backend executor (typically one): - -```yaml -# streaming_config.yaml — emitted by deployment-model-asapquery::yaml::StreamingConfigEmitter -operators: - - name: kll_p99_request_duration - kind: DatasketchesKLL - params: { k: 200 } - input: - metric: http_request_duration_seconds - label_filter: { service: api } - window: { kind: sliding, size: 5m } -estimates: - - sketch: kll_p99_request_duration - query: { kind: quantile, q: 0.99 } -``` - -This is the byte-for-byte format ASAPQuery-backend already consumes — `inference_config.yaml` is emitted alongside by `InferenceConfigEmitter` for the readout side. Both are wire invariants (see §10). - -##### deployment-model-asapfusion (0-stage, in-process `LogicalPlan`) - -`StageAllocator` against `topology::ZeroStage` returns the whole DAG in-process. `PhysicalPlanner` rewrites the caller's `datafusion::LogicalPlan`, replacing the original `Aggregate(quantile(0.99, ...))` node with an `Extension` wrapping the KLL sketch op: - -```rust -LogicalPlan::Extension(Extension { - node: Arc::new(SketchAggExt { - kind: SketchKind::Kll, - params: KllParams { k: 200 }, - input: /* original Window+Filter+Scan subtree, unchanged */, - estimate: SketchQuery::Quantile { q: 0.99 }, - }), -}) -``` - -Returned to the caller's `SessionContext` for execution by `asap-fusion`'s in-process operator. No wire format, no external executor; the data plane is the caller's process. Note also that `asap-fusion`'s entry point is `Source::Table` (not `Source::TimeSeries`) — this PromQL example is only illustrative for the fusion deployment model, which in practice consumes a pre-built DataFusion `LogicalPlan` and skips L1 (see §8). - -#### What this example demonstrates - -- **L1 → L3 are deployment-model-independent.** All three deployment models see the same `QueryExpr` for this query. -- **L4 binding is a shared rule.** `BindKllOnQuantile` lives once in `core::optimizer::rules` and fires for all three. -- **L5 is where deployment models diverge.** Same `SketchExpr` → three topologies → three emitter outputs. The topology is a parameter, not an axis of code (§3). -- **The sketch path is selected, not mandated.** If the `QuerySpec` had `accuracy: AccuracyTarget::Exact`, no `Bind*` rule would fire; L4 would pass `SketchExpr::Logical(QueryExpr::Aggregate{…})` through, and L5 would target an exact `HashAgg` (§1 goal 6). - -### End-to-end example — batched queries with shared sub-DAGs - -The single-query trace above shows one `QueryExpr` flowing through all five layers. This example shows what changes when the `QueryWorkload` is an `AuthoredSet` of related queries — the case `CostModel::workload_cost` and §6 design rule 5 (DAG fan-in) are designed for. The point is that reuse is expressed as **shared nodes in a single DAG**, not as side-channel caching. - -**Input.** A `QueryWorkload::AuthoredSet` arrives with three PromQL queries on the same metric: - -```text -q1: quantile_over_time(0.99, http_request_duration_seconds{service="api"}[5m]) -q2: quantile_over_time(0.95, http_request_duration_seconds{service="api"}[5m]) -q3: max_over_time(http_request_duration_seconds{service="api"}[5m]) -``` - -All three carry `accuracy: Epsilon(0.01)`. Planning them independently would scan + window the same metric three times. The DAG-shaped IR collapses the shared work. - -#### L3 — common sub-expression elimination produces fan-in - -After per-query L1→L2→L3 lowering, the three `QueryExpr` trees are identical below the `Aggregate` node. A workload-level CSE pass (`core::lower::workload::dedupe_subtrees`) hoists the shared sub-DAG behind a `LetBinding`, leaving three `Aggregate` nodes that fan in to one producer: - -```text - Scan{http_request_duration_seconds, service="api"} - │ - Window{Sliding, 5m} ◄── shared producer - ╱ │ ╲ - Aggregate Aggregate Aggregate - [Quantile{0.99,ε}] [Quantile{0.95,ε}] [Max] - │ │ │ - q1 q2 q3 -``` - -Multi-root DAGs live one level above `QueryExpr`. `QueryExpr` stays single-root (one query in, one root out) — the workload-level container holds N roots plus the hoisted bindings they share: - -```rust -// core::workload — output of L1→L2→L3 lowering for a QueryWorkload -pub struct WorkloadPlan { - /// Named shared producers, hoisted out of individual queries by the CSE - /// pass. Each binding is referenced by ≥2 roots via `QueryExpr::Ref`. - pub bindings: Vec<(BindingName, QueryExpr)>, - /// One root per QuerySpec in the workload, in input order. - pub roots: Vec<(QueryId, QueryExpr)>, -} -``` - -For the example above: - -```rust -WorkloadPlan { - bindings: vec![ - ("windowed_latency".into(), /* Scan → Window subtree */), - ], - roots: vec![ - (q1_id, QueryExpr::Aggregate { - by: vec![], - aggs: vec![AggIntent::Quantile { q: 0.99, accuracy: Epsilon(0.01) }], - child: Box::new(QueryExpr::Ref("windowed_latency".into())), - having: None, - }), - (q2_id, /* same shape, q=0.95, same Ref */), - (q3_id, /* Aggregate { aggs: [AggIntent::Max] } over the same Ref */), - ], -} -``` - -Three `QueryExpr::Ref` nodes, one binding; the fan-in is explicit. `CostModel::workload_cost` credits the Scan + Window build cost once across {q1, q2, q3} via `WorkloadCost::reused`, which is what makes the bundled plan beat three independent plans on `total_dollars` (§6 `core::cost`). Within-query CTE fan-in (SQL `WITH`, PromQL recording rules) keeps using `QueryExpr::LetBinding`/`Ref` unchanged — `WorkloadPlan` only adds the cross-query layer. - -CSE legality leans on `Schema::unique_keys` (§6 Schema flow): two `QueryExpr::Ref` consumers can share a producer only when its output schema is provably stable across reads — the unique-key metadata is what lets the deduper assert that without re-running the producer's logic. - -#### L4 — sketch reuse across q1 and q2 - -`BindKllOnQuantile` fires three times (once per `Quantile` intent in the workload), but a follow-on rule `MergeKllSketches` recognises that q1 and q2 read from the same input at the same accuracy. KLL state is independent of `q` — one sketch serves any quantile readout — so the two `SketchAgg{KLL}` nodes collapse into one, with two `SketchEstimate` parents reading `q=0.99` and `q=0.95`. q3 (`Max`) takes a separate path: KLL doesn't expose max, so the rule emits an exact `Aggregate{Max}` over the shared `Window`. - -```text - Scan - │ - Window - ╱ │ ╲ - SketchAgg{KLL,k=200} Aggregate{Max} - ╱ ╲ │ - SketchEstimate SketchEstimate q3 - q=0.99 q=0.95 - │ │ - q1 q2 -``` - -`SketchExpr::LetBinding` carries the two-tier fan-in: the outer let names the `Window` output (read by all three branches), an inner let names the `SketchAgg{KLL}` output (read by both `SketchEstimate` parents). The L4 type system rejects mismatched merges before L5 sees them — both `SketchEstimate` parents must declare `Sketch(Kll, KllParams{k:200})` on their input edge, which is checked locally per the §6.4 sketch-state schema rules. - -#### L5 — colored DAG, one config per executor - -`StageAllocator` colors the bound DAG by `StageId` exactly as for the single query, but the shared nodes are colored once. Under `topology::ThreeStage`: - -| Node | StageId | -|---|---| -| `Scan` + `Window` + `SketchAgg{KLL}` + `Aggregate{Max}` | `"edge"` | -| `SketchMerge` (over KLL streams), `Merge` (over Max streams) | `"gateway"` | -| `SketchEstimate{q=0.99}`, `SketchEstimate{q=0.95}`, root of q3 | `"backend"` | - -The OpAMP config emitted to each edge agent describes one scrape, one window operator, one KLL builder, one max accumulator — feeding two output streams (KLL state, Max state) toward gateway. Per-edge memory drops from `3× scan + 3× window + 2× KLL + 1× Max` (independent plans) to `1× scan + 1× window + 1× KLL + 1× Max`; bandwidth across the edge→gateway cut drops correspondingly. - -#### What this example demonstrates - -- **Reuse is a DAG property, not a sketch property.** The Scan + Window collapse is L3 CSE; the KLL collapse is L4 sketch-aware; both expressed as fan-in in the same DAG. No new IR surface, no caching layer. -- **`Schema::unique_keys` becomes load-bearing.** The CSE pass uses it to prove two `Ref`s read from a stable producer. Without it, the deduper has to be conservative and reuse drops on the floor — which is why §6 keeps the field on the `Schema` struct even though single-query plans don't read it (§6 Schema flow). -- **`CostModel::workload_cost` decides whether to share.** Reuse isn't free — when q1 demands a tighter accuracy than q2, the smaller-budget sketch may not satisfy both, and keeping them separate can be cheaper. `MergeKllSketches` is gated by `workload_cost`, not unconditional. -- **L5 is unchanged.** Same allocator, same emitter, same wire format — the only difference is that the DAG has multiple roots and shared interior nodes. The single-query path is the degenerate case where `outputs.len() == 1` and no fan-in fires. - -## 7. Runtime crate details - -Lifted from DC controller with zero semantic change: - -- `http/` — axum server on `:8080`. Routes: `POST /plan`, `POST /replan`, `GET /plans/:id`, `GET /status`, `GET /metrics` -- `opamp/` — WebSocket OpAMP server on `:4320`. Same protocol DC speaks today. -- `monitor/` — `Scraper` polls Prometheus, emits `Violation` -- `replan/` — subscribes to violations + expiry ticks, re-invokes the deployment model registry -- `store/` — in-memory `PlanStore` + `WorkloadStore`. Pluggable backend later. -- `backend_client/` — pushes `StreamingConfig` YAML to ASAPQuery-backend's `/api/v1/streaming-config` endpoint. Factored out so new deployment models can push to other backends. - -Runtime depends on `core` but NOT on any deployment model crate directly. It talks to deployment models via the registry. - -## 8. Deployment model crate details (each owns its L4 + L5) - -Each deployment model crate is a library with: - -- A `DeploymentModel` impl that registers its planner, its emitters, and its HTTP routes (if any). -- Its own config types — no shared config crate. -- Its own cost model — maybe using `core::cost` primitives, maybe not. -- Its own integration tests. - -### `deployment-model-asaplifecycle` (thin) - -- **Data model**: time-series. L2→L3 lowering produces `QueryExpr` with `Source::TimeSeries` leaves. -- **L4 rules**: picks from `core::optimizer::rules::*` (Bind*, Fusion*, Elim*) + adds DC-specific rules that require stage awareness (`StageAwarePushDown`, `TransmissionCostRewrite`). Adding a stage-specific rule = a new file in `deployment-model-asaplifecycle/src/rules.rs`, impl `OptimizerRule`. Unchanged rules come from core. -- **L5 topology**: `core::physical::topology::ThreeStage` (edge → gateway → backend). No deployment-model-specific allocator logic — `StageAllocator` handles the tree walk; lifecycle only declares what the topology looks like. -- **Cost models**: `delta / online-EMA / Pareto / TCO` — these live in `deployment-model-asaplifecycle/src/cost.rs` because they're specific to the DC deployment's network/compute assumptions. Implement `core::optimizer::cost::CostModel`. -- **L5 emitters**: `OpAmpRemoteConfigEmitter` (per-role OTel YAML over OpAMP WebSocket) and `AsapqueryBackendConfigEmitter` (calls `deployment-model-asapquery`'s `StreamingConfigEmitter` for the YAML bytes, then POSTs to backend). -- **HTTP route**: `POST /plan` for full-lifecycle planning, `POST /replan` for SLA-triggered. -- **Size estimate**: ~1500 LOC (was ~5000 pre-refactor). Cost models are the bulk; rule selection + topology + emitter are each a few hundred lines. - -### `deployment-model-asapquery` (thin) - -- **Data model**: time-series. L2→L3 lowering produces `QueryExpr` with `Source::TimeSeries` leaves. -- **Inherited L1**: uses `core::query_language::promql` and `core::query_language::sql`. -- **NEW L2 tree** (Phase 4 work): defines `PromqlLogicalPlan` in `core::logical_plan::promql` that expresses the five pattern shapes asap-planner-rs currently template-matches as first-class L2 nodes. Replaces the pattern-catalogue approach with a proper L1→L2 tree rewrite. SQL side gets a matching `SqlLogicalPlan`. -- **L3 intent**: maps `Statistic` enum (9 variants) onto `core::intent_algebra::AggIntent` subset. Planner's `Topk` maps directly to `AggIntent::TopK` (heavy-hitter intent, served by SpaceSaving / CMS-with-heap at L4). -- **L4 rules**: picks from `core::optimizer::rules::*` (all `Bind*` rules are relevant since this deployment model covers most sketch types) + a deployment-model-specific sketch-binding rule for the precompute engine's flavor (which sketches are available, what params, `DeltaSetAggregator` auto-injection before CMS/HydraKLL). This rule absorbs `map_statistic_to_precompute_operator`'s sketch-binding half. -- **L5 topology**: `core::physical::topology::SingleStage` (backend-only). No stage-split; `StageAllocator` returns everything on one stage trivially. -- **L5 emitters**: `StreamingConfigEmitter` + `InferenceConfigEmitter` (YAML bytes). **Authoritative** for these two formats — `deployment-model-asaplifecycle` calls them when it needs to POST to ASAPQuery-backend. -- **Extra L1 inputs**: `query_log/` for Prometheus-query-log replay (unique to this deployment model); `schema/` for `PromQLSchema` discovery from a live Prometheus URL. -- **HTTP route**: `POST /plan/query` (JSON `QuerySpec` in, YAML stream out). Also backs the `bin/asap-query` one-shot CLI. -- **Size estimate**: ~2500 LOC (was ~6000 pre-refactor — saved by picking from shared rule library; still pays the L2 tree + L3/L4 split refactor cost, which is one-time). - -### `deployment-model-asapfusion` (thin) - -- **Data model**: tabular. L2→L3 lowering produces `QueryExpr` with `Source::Table` leaves (and, in future, `Source::Join` when fusion extends to multi-table). Crucially **not time-indexed** — fusion works on arbitrary DataFusion relations; time is just another column if present. -- **L1 opt-out**: fusion consumes a pre-built DataFusion `LogicalPlan` from its caller. `core::query_language` is not invoked. Library-mode deployment model. -- **L2 inherited from DataFusion**: DataFusion's `LogicalPlan` *is* fusion's L2 tree. `core::logical_plan::datafusion` is a thin re-export of `datafusion::logical_expr::LogicalPlan` so deployment models that want to take a DataFusion plan as input have a canonical name for it. -- **L3 intent**: `SubPopulationAnalyticsType` (3 variants: `Count`, `Sum`, `Quantile`) maps to `core::intent_algebra::AggIntent` subset. -- **L4 rules**: picks `BindCmsOnCount` and `BindKllOnQuantile` from `core::optimizer::rules::*` (which already cover fusion's `SketchConfigRule` semantics) + deployment-model-specific `HashModeRule`. Rules operate on DataFusion `LogicalPlan` (via Extension wrapping), not on `QueryExpr` trees — this lets DataFusion's `context.state().optimize` run *after* fusion's rewrites, keeping free reuse of DF's standard optimizer passes. -- **L5 topology**: `core::physical::topology::ZeroStage` (in-process). -- **L5 emitter**: rewritten `datafusion::LogicalPlan`. Not a wire format — this deployment model is library-mode. -- **Executor**: `ASAPExecutor` wraps a DataFusion `SessionContext`. Users construct `deployment-model-asapfusion` in-process. -- **HTTP route**: none by default. -- **Conformance cost**: near zero. Fusion's `SketchConfigRule` logic is replaced with picks from `core::optimizer::rules::*`; its `HashModeRule` stays deployment-model-specific. -- **Size estimate**: ~1800 LOC (was ~3000 pre-refactor; the `SketchConfigRule` code folds into core's shared rule library). - -The sketch microbenchmarks (KLL/CMS) move with the crate and keep running. The TODO items from `asap-fusion/TODO.md` (batch/multi-query execution, time semantics, distributed model) remain open but are now filed against `deployment-model-asapfusion/TODO.md` in-repo. - -### Data plane communication - -Control plane (ASAPController) and data plane (OTel agents / ASAPQuery-backend / DataFusion runtimes) always talk **over wire**, never via in-process calls. This is unchanged from today: - -| Deployment model | Data plane lives in | Wire protocol | -|---|---|---| -| lifecycle | DataCollector (OTel collectors, agent + backend roles) | OpAMP WebSocket (config push) + HTTP POST (`StreamingConfig` → ASAPQuery-backend) + Prometheus scrape (metrics in) | -| query | ASAPQuery-backend (query engine, SimpleMapStore) | HTTP POST `/api/v1/streaming-config` + `/api/v1/plan` (capability-miss callback in) + YAML file on disk (init-container mode) | -| fusion | Caller's DataFusion `SessionContext` | in-process library call (no wire) | - -**Data plane code stays in its original repo.** ASAPController only owns the control plane. The merger doesn't move OTel collectors out of DataCollector, doesn't move the query engine out of ASAPQuery-backend, and doesn't move DataFusion out of asap-fusion's users. Each data plane keeps its own release cadence. - -### Deployment model placement: in-repo or out-of-repo - -Because core owns the L4/L5 infrastructure (not just L1-3), a deployment model crate is small and largely self-contained. That means deployment models can live either: - -- **Inside ASAPController workspace** — `crates/deployment-model-/`, lockstep release with core, cross-deployment-model changes are one PR. -- **In their own downstream repo** — declares `asap-control-core` as a git-tagged dep, releases independently, owns its own CI. - -Both produce functionally identical artifacts because they pick from the same `core::optimizer::rules` library and use the same traits. The placement is a **deployment / team-ownership decision**, not an architectural fork. - -Default recommendation: -- **deployment-model-asaplifecycle** and **deployment-model-asapquery** in ASAPController (they share `deployment-model-asapquery`'s YAML emitter via a workspace `path = "../deployment-model-asapquery"` dep — trivial in-workspace). -- **deployment-model-asapfusion** out-of-tree in `asap-fusion` repo (research project with independent benchmark cadence; depends on `asap-control-core` + `asap-control-optimizer` as published git tags). - -Future deployment models choose whichever placement fits the team that owns them. - -### Asymmetric dependency - -`deployment-model-asaplifecycle` depends on `deployment-model-asapquery` because the `StreamingConfig` YAML emitter is authoritative there. When both live in ASAPController workspace this is a trivial `path =` dep. If one is ever moved out-of-tree, we lift the emitter into core to avoid a cross-repo Cargo dep. - -## 9. Extension point — a 4th deployment model - -With the L4/L5 framework in core, adding a new deployment model is mostly picking + a bit of glue. A hypothetical "edge caching" deployment model that decides which queries to cache at the edge vs. backend would land as: - -``` -crates/deployment-model-edge-cache/ # or your own repo -├── Cargo.toml # depends on asap-control-core -├── src/ -│ ├── lib.rs # impl DeploymentModel for EdgeCacheDeploymentModel -│ ├── rules.rs # pick core rules + add EdgeCacheBindRule -│ ├── topology.rs # TwoStage { edge, backend } TopologyDescriptor -│ ├── cost.rs # impl CostModel — hit ratio × bandwidth -│ └── emit/ -│ └── edge_agent_config.rs # impl PlanEmitter — emits edge-agent YAML -└── tests/ -``` - -Total touch outside the new crate: -- 1 line in `bin/asap-controller/main.rs` to register (if in-workspace) OR a published binary in your own repo -- 1 line in workspace `Cargo.toml` (if in-workspace) -- optional: add `DeploymentModelId::EdgeCache` to `core::registry` - -Typical size: ~500-2000 LOC depending on how deployment-model-specific the rules / cost model / emitter are. If the deployment model accepts core's defaults everywhere, ~300 LOC is realistic. - -### Standalone binary for one deployment model - -If you want a slim binary that runs only one deployment model: - -``` -bin/asap-edge-cache/ -├── Cargo.toml # depends only on runtime + deployment-model-edge-cache, not other deployment models -└── src/main.rs # registers only EdgeCacheDeploymentModel; no DataFusion, no query YAML -``` - -Cargo builds this with its own minimal dep tree — no `datafusion`, no `promql-parser` unless this deployment model needs it. Useful when a deployment is dedicated to one deployment model (e.g. an edge-caching-only service). - -## 10. Wire protocols — what changes, what doesn't - -**Unchanged** (hard contract with other systems): -- `POST /api/v1/streaming-config` on ASAPQuery-backend — consumed YAML format -- `POST /api/v1/plan` on DC controller (from backend capability-miss) — request JSON -- OpAMP `ServerToAgent.remote_config` payload — OTel collector YAML -- Prometheus scrape format - -**Internal** (controller's own surface, still HTTP+JSON for now): -- `POST /plan` — new unified entry point that takes `QueryWorkload` in, returns a plan ID + list of emitted artifacts (URIs). `/api/v1/plan` proxies to this. -- `GET /plans/:id` — plan inspection -- `GET /status` — runtime + deployment model health - -**New** (internal-only, proto): -- `proto/asap_control.proto` defines `Plan`, `PlanNode`, `Expr` for persistence + store-internal serialization. Not on the wire between services. Optional; initial migration skips this and persists via `serde_json`. - -## 11. Dependencies and Cargo surface - -Workspace `Cargo.toml`: - -```toml -[workspace] -resolver = "2" -members = [ - "crates/core", - "crates/runtime", - "crates/deployment-model-asaplifecycle", - "crates/deployment-model-asapquery", - "crates/deployment-model-asapfusion", - "crates/control-proto", - "crates/testing", - "bin/asap-controller", - "bin/asap-query", -] - -[workspace.dependencies] -tokio = "1" -axum = "0.7" -reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } -prost = "0.13" -tonic = "0.12" -serde = { version = "1", features = ["derive"] } -serde_yaml = "0.9" -tracing = "0.1" -thiserror = "1" -# deployment-model-specific deps kept in deployment model crates -``` - -Core depends only on `serde`, `tracing`, `thiserror`, and small utility crates. No HTTP, no DataFusion, no OpAMP. - -Runtime depends on core + `axum` + `reqwest` + `tokio-tungstenite` + `prost` (OpAMP proto). - -Deployment models depend on core. deployment-model-asapquery also pulls in `promql-parser`, `sqlparser`. deployment-model-asapfusion pulls in `datafusion` + `arrow`. deployment-model-asaplifecycle pulls in `sqlparser` + `promql-parser` + the cost-model math crates. - -This matters: a user who only wants `deployment-model-asapfusion` (e.g., an offline benchmark) gets DataFusion but NOT axum/OpAMP. - -## 12. Open questions - -1. **Do we need a cross-deployment-model cost model?** Today DC's lifecycle planner and asap-planner-rs's query planner have overlapping but not identical cost models. Answer for now: keep them separate in deployment model crates; let them re-converge organically. If a third deployment model needs the same model, lift at that point. - -2. **Where does the backend's `ControllerClient.create_plan` call land?** Initially: HTTP `POST /api/v1/plan` on the controller, handled by `deployment-model-asaplifecycle` (same as today). Long-term: could route by `QuerySpec.deployment_model` to a different planner, but that's a follow-up. - -3. **Which deployment model owns `StreamingConfig` YAML emission?** Both `deployment-model-asaplifecycle` and `deployment-model-asapquery` emit it today (DC's `config/generate_streaming_config_yaml` and `output/generator.rs` in `asap-planner-rs`). Plan: **one emitter in `deployment-model-asapquery`**, called from both. This is why `deployment-model-asaplifecycle` depends on `deployment-model-asapquery` in the dependency graph (asymmetric — query does not depend on lifecycle). - -4. **Do we vendor DataFusion's IR into core?** No. `deployment-model-asapfusion` owns its DataFusion-flavored plan; `core::plan::Plan` stays an enum with a variant that wraps a `datafusion::LogicalPlan` behind a feature flag. Core itself never reaches into DataFusion types. - -5. **OpAMP proto: vendored, or from crates.io?** Today DC vendors. Recommend: keep vendored in `proto/opamp.proto`, generate via `prost-build` in `crates/control-proto`. Same thing DC does today, just moved. - -6. **What happens to ASAPQuery-backend's `asap-planner-rs` directory post-migration?** Deleted. ASAPQuery-backend's docker-compose drops the `asap-planner-rs` init container; the controller's `deployment-model-asapquery` now runs in-process (service mode) or as the `asap-query` CLI (one-shot mode). The ASAPQuery-backend repo shrinks by two directories. - -7. **Versioning?** Start at `0.1.0` on the workspace. Deployment models can rev independently later via per-crate versions, but initially lockstep. - -8. **What if a future deployment model can't fit a tree L2?** L2 is currently a per-language tree, mandatory for every deployment model. asap-planner-rs's Phase 4 conformance cost (reverse-engineering PromQL pattern templates into a `PromqlLogicalPlan` tree) was taken deliberately to keep the architecture uniform. If a future deployment model's source language doesn't map naturally onto a tree (e.g. a constraint-based or dataflow-graph query language), that's the moment to re-examine the L2 contract — the `core::logical_plan` module is a single Rust trait + per-language types, not a deep assumption baked across the codebase. Until then, L2 = tree. - -9. **Deployment model placement — in ASAPController workspace, or in own repo?** Both supported, same architecture either way (see §8 "Deployment model placement"). Default: `deployment-model-asaplifecycle` and `deployment-model-asapquery` in ASAPController (easy cross-deployment-model changes, shared YAML emitter); `deployment-model-asapfusion` in the `asap-fusion` repo (research cadence). A new deployment model picks based on team ownership and release cadence preferences. - -10. **How far do shared L4/L5 rules live in core before they become deployment-model-specific?** The rule of thumb: if ≥2 current deployment models would use it, it lives in `core::optimizer::rules::*`. If only 1 deployment model uses it *and* it depends on deployment-model-specific types (DataFusion's `LogicalPlan`, OTel YAML shape), it lives in the deployment model crate. `SketchConfigRule`-style "bind intent to concrete sketch" rules belong in core (shared). `StageAwarePushDown` (which needs DC's stage graph) belongs in `deployment-model-asaplifecycle`. When a rule straddles — e.g. a fusion rewrite that *could* be generalized to any L4-compatible plan — start it in the deployment model; lift to core once a second deployment model wants it. Don't pre-emptively generalize. - -11. **How does the design support non-time-series data (asap-fusion's tabular queries, future OLAP deployment models)?** Already handled — see §3 "Data-model support" and §6 `core::intent_algebra`. `QueryExpr::Scan` wraps a `Source` sum (`TimeSeries` / `Table` / `Join`); `AggIntent::requires() -> DataModel` tags which intents apply to which data models; sketches themselves are data-model-agnostic. asap-fusion uses `Source::Table` exclusively; ASAPQuery uses `Source::TimeSeries`; a future OLAP deployment model picks whichever fits. The only implementation cost is that L4 rules which genuinely don't apply across data models (e.g. "merge overlapping time windows") must gate on `source.data_model()`; data-model-agnostic rules (the `Bind*` family) need no changes. - -### Resolved during the L3 IR cleanup (see §6 `core::intent_algebra`) - -The following questions came up during review of the previous `QueryExpr` draft and are now settled. Listed here so the trail is visible: - -- **Output of `SketchAgg` vs `WindowedAgg`?** Neither node exists at L3 anymore. `SketchAgg` is an L4 sketch-bound node (`SketchExpr::SketchAgg`) emitted by binding rules; `WindowedAgg` was redundant with `Window` over `Aggregate` and removed. Output of `SketchAgg` is a sketch-typed column carrying the partial state; `SketchEstimate` reads it out into a scalar / vector. -- **Is `TopK` different from `Sort + Limit`?** They overlap on heavy-hitter queries but are different concepts. `TopK` is now an `AggIntent` at L3 (heavy-hitter intent — SpaceSaving / CMS-with-heap / Misra-Gries serve it as a single primitive), while generic `Sort + Limit` survives as `QueryExpr` operators for non-heavy-hitter cases (`ORDER BY name LIMIT 10`). L1→L2→L3 lowering picks the intent form when it recognises a heavy-hitter pattern (`ORDER BY count DESC LIMIT k`, PromQL `topk(k, …)`) and the operator form otherwise. -- **What is `JoinSketch`?** A sketch-aware join (KMV / theta / join-sample). Moved to L4 (`SketchExpr::SketchJoin`) so the choice "exact join vs sketch join" is an L4 cost decision against `Join`, not a competing L3 surface. -- **`HistogramQuantile` and `PromQLSubquery` feel out of place** — they were. Both removed from L3. `histogram_quantile` lowers in PromQL L1→L2 to bucket reads + a `Quantile` intent. `[range:resolution]` is a driver that expands into multiple queries during PromQL L1→L2 lowering, not a DAG node. -- **Sketch subtract / delete / estimate operators?** Added: `SketchExpr::SketchSubtract`, `SketchDelete`, `SketchEstimate`. Catalog flags (`subtractable`, `deletable`) gate which sketch families admit them. -- **What's `Dedup` exactly?** SQL `DISTINCT`. Renamed to `Distinct { cols }` and generalised from one column to N. -- **Why differentiate `Quantile` and `QuantileOverTime`?** No reason — the surrounding `Window` already encodes the temporal axis. `QuantileOverTime` removed from `AggIntent`. -- **`WindowedAgg` vs `Window` + `Agg`?** Equivalent; `WindowedAgg` removed. `WindowKind::{Tumbling, Sliding, Session}` lives on the `Window` node so the streaming-window kind is explicit; SQL analytic `OVER (...)` stays on the separate `WindowFunc` node. -- **Need input/output specs more fine-grained than `QueryExpr`?** Done: every L3 edge carries a typed `Schema` (fields + time index + unique-key sets), and §6 lists per-node input/output schemas. - -## 13. Future work - -### Extending the primitive set beyond sketches - -The L4 rule engine + `OptimizerRule` trait are **primitive-agnostic** — nothing in the framework is sketch-specific above the rule library. Future primitive classes land as additional rules against the same trait + additional entries in `CostModel`, not as new translation phases or parallel IRs. Candidates we explicitly anticipate: - -1. **Wavelets** — a sibling approximation family (Haar / DWT + coefficient thresholding). Strong on smooth, low-entropy signals where thresholded coefficient sets dramatically out-compress randomized sketches. Slots in as a new physical alternative for the same `AggIntent` variants sketches serve today (range-sum, heavy-hitter, quantile). No change to L3. - -2. **Reuse / precomputation as first-class primitives.** Materialized views, cached scan results, shared sub-expressions. The cost-model hook already exists (`CostModel::workload_cost` + `ReusedComponent`); the missing piece is rules that *introduce* a reuse node — e.g. "build this aggregate once for `q1`, rewrite `q2` to read from it". Orthogonal to approximation: you can reuse an exact aggregate or an approximate one. - -3. **Other approximation algorithms** — sampling, coresets, online PCA / linear-regression normal equations, naive-Bayes-with-conjugate-priors. Anything with a monoid-shaped build + bounded error fits the same contract a sketch does. - -The guiding principle: **same framework, different rule.** A new primitive class is never a translation-layer change — only a new rule, a new cost-model entry, and (if it introduces a new runtime op) a new physical operator in L5. - -## 14. Glossary - -Terms that are project-specific or that get conflated. Where the term has a Rust counterpart, the type is shown in backticks. - -### Architecture roles - -- **Deployment model** — A concrete bundle of (L4 rule choices) + (L5 topology) + (emitter), packaged as a `deployment-model-*` crate. Three exist today: `asaplifecycle`, `asapquery`, `asapfusion`. -- **Stage** (`StageId`) — A *categorical tier* in the data lifecycle: edge / gateway / backend / in-process. Roles, not instances. Declared by the deployment model's `TopologyDescriptor`. -- **Executor** (`Executor`) — A *concrete runtime instance* occupying a stage: a specific OTel agent, the ASAPQuery-backend process, a DataFusion `SessionContext`. One stage may have N executors (e.g. 50 edge agents). Carries `id`, `stage: StageId`, `capabilities`, `address`. -- **Topology** (`TopologyDescriptor`) — Declares which stages exist + how they connect (3-stage / 1-stage / 0-stage). Categorical, not instance-level. -- **DeploymentConstraints** — Trait object owned by each deployment model. Carries memory budgets, network topology, available sketch backends, and the registered `Executor` list. Threaded through L4 rules and the stage allocator. - -### Layer drivers - -- **RuleEngine** — Generic L4 driver in core (fixed-point iteration + cycle detection + priority ordering). One implementation; same code for every deployment model. -- **OptimizerRule** — Trait for an L4 rewrite. Categories: `PushDown | Fusion | Elim | Bind | StageRouting`. Deployment models pick which rules from `core::optimizer::rules` + add their own. -- **CostModel** — Trait scoring a plan's accuracy / latency / dollars. Generic implementations in core; deployment-model-specific cost models (DC's delta / online / pareto / TCO) live in their crates. -- **StageAllocator** — Generic L5 algorithm in core. Colors the L4-bound DAG by `StageId`. **Stage-granularity only — no executor knowledge.** One implementation; same code for every deployment model. -- **PhysicalPlanner** — Per-deployment-model L5 *driver* (one impl per deployment model). Calls `StageAllocator`, then fans the per-stage sub-DAGs out to executors via `DeploymentConstraints::executors()`, then produces the deployment-specific output type. Compare to allocator: planner = whole L5 driver, allocator = the stage-coloring step it uses. -- **PlanEmitter** — Per-deployment-model trait that serialises the planner's output to its wire format (OpAMP `RemoteConfig` / `streaming_config.yaml` / rewritten DataFusion `LogicalPlan`). - -### IR by layer - -- **L1 — query language** — Raw query string + parser (PromQL / SQL / DataFusion / ElasticDSL). -- **L2 — logical plan** — Per-language relational algebra tree (`PromqlLogicalPlan`, `SqlLogicalPlan`, etc.). -- **L3 — intent algebra** (`QueryExpr`, `core::intent_algebra`) — Symbolic, intent-only IR. No sketch type, no params. `AggIntent` carries accuracy targets only. The "algebra" is the operator surface (`Scan`, `Filter`, `Aggregate`, `Window`, …); the algebra is *intent-only* because no sketches have been bound yet. -- **L4 — sketch algebra** (`SketchExpr`, `core::sketch_algebra`) — Same DAG shape as L3, but sketches now committed (kind + params). Produced by L4 binding rules; consumed by L5 emitters. Note the naming inversion vs. earlier drafts: "sketch algebra" is L4 (where sketches actually live), not L3. -- **L5 — physical plan** — Stage-assigned, executor-targeted, ready to serialize. Produced by `PhysicalPlanner`, written out by `PlanEmitter`. - -### Metadata sources (see §6 "DAG schema, DB schema, sketch catalog") - -- **DAG schema** (`Schema`) — Columns + types + `unique_keys` carried on every L3/L4/L5 edge. Type-checked locally at each node. -- **DB schema** / **source schema** (`SchemaCatalog`) — Data-plane metadata (Prometheus TSDB / SQL `information_schema` / DataFusion catalog). Read by `core::lower::*` for L1→L2 symbol resolution. -- **Sketch catalog** (`SketchCatalog`) — Runtime registry of available primitives: what sketches exist, mergeability, deletability, parameter ranges, accuracy model. Consulted by L4 binding rules. **Not a schema** — a catalog of available primitives, not a description of a stream. - -### Workload + identity - -- **QueryWorkload** — Top-level controller input: one or more `QuerySpec`s plus workload features (batch vs streaming, reuse opportunity, data source). Arrives via HTTP POST, OpAMP capability-miss callback, YAML file, or query-log replay. -- **QuerySpec** — A single query: source-language string + accuracy / latency / cost target. - -## 15. Success criteria - -The migration is done when: - -1. `asap-controller` binary runs and passes DC controller's existing integration tests (OpAMP push, backend config POST, SLA replan). -2. `asap-query` binary takes the same YAML input asap-planner-rs does today and produces byte-identical `streaming_config.yaml` + `inference_config.yaml` (fuzz-test against a corpus of fixtures). -3. ASAPQuery-backend's docker-compose no longer starts `asap-planner-rs`; the controller handles both shapes. -4. `asap-fusion`'s microbenchmarks still run under `deployment-model-asapfusion` with identical numbers. -5. The `DataCollector/controller/`, `ASAPQuery/asap-planner-rs/`, `ASAPQuery-backend/asap-planner-rs/`, and `asap-fusion/` directories are deletable (or already deleted) without breaking any currently-running deployment. -6. A new hypothetical deployment model can be added with zero changes outside its crate + one line in `bin/asap-controller/main.rs`. diff --git a/controller/docs/query-to-sketch-translation.md b/controller/docs/query-to-sketch-translation.md deleted file mode 100644 index 295d2887..00000000 --- a/controller/docs/query-to-sketch-translation.md +++ /dev/null @@ -1,745 +0,0 @@ -# Query-to-Sketch Translation: How QL Maps to Sketch Execution - -This document explains how a PromQL or SQL query is translated through the -controller's five-layer architecture and ultimately mapped to sketch-based -distributed execution. - -## 1. Five-Layer Architecture - -The controller is structured as a five-layer pipeline. Each layer has a -clear input, output, and responsibility: - -``` -Query workloads - │ - │ Layer 1 — Query Language - │ (PromQL, SQL, DataFusion, ElasticDSL, ...) - ▼ -Language-specific AST - │ - │ Layer 2 — Language Logical Plan - │ (each language's own relational/query algebra) - ▼ -Language Logical Plan - │ - │ Layer 3 — Sketch Logical Plan (Sketch Algebra) - │ (language-independent, implementation-independent) - ▼ -Sketch Logical Plan - │ - │ Layer 4 — Sketch Optimizer - │ (rewrite rules on the sketch logical plan) - ▼ -Optimised Sketch Logical Plan - │ - │ Layer 5 — Physical Execution Plan - │ (concrete implementations for a specific deployment) - ▼ -Physical Plan (edge processors, backend sketchDB, backend original DB, object store) -``` - -### What each layer owns - -| Layer | Input | Output | Responsibility | -|---|---|---|---| -| **1. Query Language** | query string | language AST | grammar, parsing | -| **2. Language Logical Plan** | AST | language-specific relational plan | language semantics (PromQL instant/range vectors, SQL frames, Elastic buckets) | -| **3. Sketch Logical Plan** | language plan | sketch algebra tree (`QueryExpr`) | **what** to compute: aggregation intent + accuracy requirement + window semantics — no sketch names, no implementation details | -| **4. Sketch Optimizer** | sketch plan + deployment constraints | optimised sketch plan | cost-aware rewrites: push-down, fusion, elimination, budget-driven deferral — considers physical deployment constraints (memory budgets, network topology, available backends) | -| **5. Physical Plan** | optimised plan + deployment config | executable plan | **how** to execute: edge processors (sketch build), backend sketchDB (merge + query), backend original DB (exact), object store (raw backup) | - -### Key design principle - -**Layers 1–3 are query-language-independent and workload-independent.** -They define *what* to compute without reference to any specific query language, -sketch implementation, or deployment topology. A `Quantile { φ=0.99, accuracy=0.01 }` -intent is the same whether it came from PromQL, SQL, DataFusion, or ElasticDSL, -and whether the deployment is a single node or a 1000-agent fleet. - -**Layer 4 is deployment-constraint-aware.** -The optimizer considers physical deployment constraints — memory budgets per stage, -network bandwidth, available backends — when applying cost-based rewrite rules -(e.g., deferring a sketch from Agent to Backend when the agent memory budget is -exceeded, or fusing TopK when the downstream merge is expensive). - -**Layer 5 is deployment-specific.** -The physical planner commits to concrete implementations based on the specific -setup: edge processors, backend sketchDB, backend original DB, or object store. - -### `AggIntent` — the Layer 3 aggregation vocabulary - -| `AggIntent` variant | Meaning | Physical candidates (Layer 5) | -|---|---|---| -| `Quantile { quantiles, accuracy }` | "I need quantile estimates at these φ values within this error" | DDSketch, KLL, t-digest, PromSketch EHKLL | -| `Cardinality { accuracy }` | "I need a distinct-count estimate within this error" | HLL, UnivMon, PromSketch EHUniv | -| `Frequency { accuracy }` | "I need frequency estimates within this error" | CountSketch, CountMinSketch | -| `Extrema { min, max }` | "I need exact min/max" | ExactMinMax, DDSketch at φ=0/1 | -| `PerPartition { inner, keys }` | "Run inner once per distinct key tuple" | Hydra, per-key sketch instances | -| `Exact(Sum\|Count\|Avg\|Min\|Max)` | "No sketch benefit — exact computation" | Raw passthrough, DB-side | - -The flow: -- **Layers 1–2** (parsers): "this query needs a quantile at φ=0.99" → `Aggregate { Quantile(0.99) }` -- **Layer 3** (lowering): `Aggregate` → `SketchAgg { AggIntent::Quantile }` (shared by all languages) -- **Layer 4** (optimizer): rewrites the plan considering deployment constraints -- **Layer 5** (physical planner): "for this deployment, DDSketch at the edge is cheapest" or "KLL at the backend sketchDB is better for this workload" - -## 2. Sketch Logical Plan: `QueryExpr` (Layer 3) - -`QueryExpr` (`algebra/expr.rs`) is the sketch algebra IR — a **logical plan** that -normalises all query languages into a common algebraic form. - -| | AST (syntax tree) | Logical Plan (QueryExpr) | -|---|---|---| -| **Structure** | Mirrors the grammar | Mirrors relational algebra operators | -| **Semantics** | Preserves syntactic details | Preserves only operator semantics | -| **Sketch types** | N/A | Implementation-independent intents (`AggIntent`) | -| **Language** | Language-specific | Language-independent (shared by SQL, PromQL, etc.) | - -QueryExpr has **25 operator variants** organized into categories: - -**Relational core** — standard relational algebra: -- `Source` — base metric / table (leaf node) -- `Filter { pred, input }` — selection (σ) -- `Project { cols, input }` — projection (π) -- `Aggregate { keys, aggs, having, input }` — grouping + aggregation (γ) -- `Join { kind, pred, left, right }` — relational join (⋈) -- `SetOp { kind, all, left, right }` — UNION / INTERSECT / EXCEPT -- `Sort { keys, input }` — ORDER BY -- `Limit { n, offset, input }` — LIMIT / OFFSET - -**Sketch-specific** — operators that express sketch computation intent: -- `SketchAgg { op: AggIntent, col, input }` — sketch aggregation intent (what, not how) -- `WindowedAgg { agg: AggIntent, window: WindowSpec, col, input }` — bundled window + sketch agg (window defines sketch lifecycle) -- `Partition { keys, input }` — GROUP BY distribution for distributed sketches -- `Dedup { col, input }` — deduplication (absorbed by cardinality sketches) -- `TopK { k, by, input }` — top-K heavy-hitter query -- `Merge { inputs }` — sketch merge (linearity: sketch(A∪B) = merge(sketch(A), sketch(B))) -- `JoinSketch { join_key, outer, inner }` — sketch-aware join push-down - -**Time / streaming** — window operators: -- `Window { duration, slide, input }` — standalone time window (batching) - -**PromQL-specific** — operators that preserve PromQL semantics: -- `HistogramQuantile { phi, input }` — `histogram_quantile(φ, …)` -- `PromQLSubquery { range, resolution, input }` — `expr[range:resolution]` -- `BinaryOp { op, lhs, rhs, vector_match }` — vector binary arithmetic with matching - -**Structural** — subqueries and bindings: -- `Subquery`, `LetBinding`, `Ref`, `WindowFunc` - -### Window operators: `Window` vs `WindowedAgg` - -| Operator | Use | Why separate | -|---|---|---| -| `Window { duration, slide }` | Standalone time batching (no sketch) | Used when the sketch op is a separate `SketchAgg` child node | -| `WindowedAgg { agg, window, col }` | Bundled window + sketch aggregation | In sketch systems the window defines the sketch lifecycle (when to flush/reset). Bundling lets the physical planner choose the best implementation (edge tumbling flush vs backend sketchDB EH vs original DB time_bucket). | - -`WindowSpec` supports five window kinds: - -| `WindowKind` | Semantics | Example | -|---|---|---| -| `Tumbling { size }` | Fixed-size, non-overlapping | PromQL implicit, SQL `TUMBLE(ts, '5m')`, Elastic `fixed_interval` | -| `Sliding { size, slide }` | Fixed-size, overlapping | PromQL `[5m]` range vector, SQL `HOP(ts, '1m', '5m')` | -| `Unbounded` | All samples, no time dimension | SQL `GROUP BY key` without time | -| `Landmark` | From epoch to now (cumulative) | Running aggregates | -| `Session { gap }` | Gap-based, closes after inactivity | Elastic session windows | - -## 3. Layers 1–3: Query Language → Language Plan → Sketch Algebra - -### 3.1 Layer 1→2: Language AST → Language Logical Plan - -Each parser takes a language-specific AST (from an external crate) and produces -a **language logical plan** using relational operators (`Aggregate`, `Window`, -`Filter`, `Sort`, `Limit`, etc.) with generic `AggFunc` variants — no sketch -names at this layer. - -**PromQL** (`query_parser/promql.rs`): - -``` -PromQL AST (promql-parser crate) - ↓ walk_qe(ast_node, ctx) -Language Logical Plan (Aggregate + AggFunc + Window) -``` - -The walker carries context downward: `partition` (GROUP BY keys), `topk` (K value), -`outer_count` (whether wrapped in `count(…)`). - -| PromQL construct | Layer 2 output | -|---|---| -| `quantile_over_time(φ, m[5m])` | `Aggregate { Quantile(φ), input: Window { 5m, Filter(Source) } }` | -| `histogram_quantile(φ, rate(…))` | `HistogramQuantile { φ, Aggregate { Quantile(φ), Window(...) } }` | -| `count_over_time(m[5m])` | `Aggregate { Count, input: Window { 5m, Source } }` | -| `avg_over_time(m[5m])` | `Aggregate { Avg, input: Window { 5m, Source } }` | -| `topk(k, …) by (dims)` | `TopK { k, Partition { dims, inner } }` | -| `a + b` | `BinaryOp { Add, lhs, rhs, VectorMatch }` | -| `m[5m:1m]` | `PromQLSubquery { range: 5m, step: 1m, inner }` | - -**SQL** (`query_parser/sql.rs`): - -``` -SQL AST (sqlparser crate) - ↓ extract_select_qe(select, order_by, limit, offset) -Language Logical Plan (Aggregate + AggFunc + Sort + Limit) -``` - -The SQL parser builds the plan bottom-up from SELECT clauses: - -| SQL construct | Layer 2 output | -|---|---| -| `FROM table` | `Source(table)` | -| `WHERE pred` | `Filter(ScalarExpr, Source)` | -| `JOIN … ON` | `Join(kind, pred, left, right)` | -| `GROUP BY keys` + agg functions | `Aggregate { keys, aggs: [AggItem { func }] }` | -| `TUMBLE(ts, INTERVAL '5m')` | `Aggregate { input: Window { 5m, Source } }` | -| `ORDER BY … DESC` | `Sort(keys, input)` | -| `LIMIT n` | `Limit(n, input)` | -| `UNION ALL` | `SetOp(Union, all, left, right)` | - -### 3.2 Layer 2→3: Language Logical Plan → Sketch Algebra (lowering) - -The shared `lower_to_sketch_algebra()` pass (`algebra/lower.rs`) converts -language-independent `Aggregate { AggFunc }` nodes into sketch algebra -`SketchAgg { AggIntent }` nodes. This is the same pass for both PromQL and SQL. - -**Algorithm**: - -``` -lower_to_sketch_algebra(expr): - Recursively walk the QueryExpr tree. - For each single-agg Aggregate node: - - 1. Map AggFunc → AggIntent (implementation-independent): - Quantile(φ) → AggIntent::Quantile { [φ], accuracy } - CountDistinct → AggIntent::Cardinality { accuracy } - Count (w/ GROUP BY) → AggIntent::Frequency { accuracy } - Avg → AggIntent::Quantile { [0.5], accuracy } (median proxy) - Min → AggIntent::Extrema { min: true } - Max → AggIntent::Extrema { max: true } - StdDev → AggIntent::Quantile { [0.25, 0.75], accuracy } (IQR proxy) - Sum/Rate/Delta → AggIntent::Exact(Sum) - Count (no GROUP BY) → stays as Aggregate (no sketch benefit) - - 2. If the Aggregate's input is a Window, fuse into WindowedAgg: - Aggregate { AggFunc, input: Window { duration } } - → WindowedAgg { AggIntent, WindowSpec { Tumbling(duration) }, input } - - 3. If the Aggregate had GROUP BY keys, wrap with Partition: - → Partition { keys, input: SketchAgg/WindowedAgg } - - Multi-agg Aggregates and HAVING clauses pass through unchanged. -``` - -| Layer 2 input | Layer 3 output | -|---|---| -| `Aggregate { Quantile(0.99), Window { 5m, Source } }` | `WindowedAgg { Quantile([0.99]), Tumbling(5m), Source }` | -| `Aggregate { CountDistinct, Source }` | `SketchAgg { Cardinality, Source }` | -| `Aggregate { Count, keys: [region], Source }` | `Partition { [region], SketchAgg { Frequency, Source } }` | -| `Aggregate { Avg, keys: [symbol], Source }` | `Partition { [symbol], SketchAgg { Quantile([0.5]), Source } }` | -| `Aggregate { Sum, Source }` | `SketchAgg { Exact(Sum), Source }` | -| `Aggregate { Count (no GROUP BY), Source }` | unchanged (no sketch benefit) | - -**SQL function → AggFunc mapping**: - -| SQL function | AggFunc | Sketch candidate | -|---|---|---| -| `COUNT(*)` with GROUP BY | `Count` | CountSketch / CountMinSketch | -| `COUNT(*)` without GROUP BY | `Count` | Exact (no sketch benefit) | -| `COUNT(DISTINCT col)` | `CountDistinct` | HLL | -| `SUM(col)` | `Sum` | Exact (not sketchable) | -| `AVG(col)` | `Avg` | DDSketch (p50 proxy) or Exact(Avg) | -| `MIN(col)` | `Min` | DDSketch (φ=0.0) or ExactMinMax | -| `MAX(col)` | `Max` | DDSketch (φ=1.0) or ExactMinMax | - -Note: the parser emits `Aggregate { func: Avg }` — it does **not** emit sketch ops. -Sketch assignment happens later in the optimizer (R9 HydraConversion) and allocator. -The SQL parser only produces relational operators; the PromQL parser is more aggressive -and emits `SketchAgg` nodes directly because PromQL functions like `quantile_over_time` -have a 1-to-1 mapping to sketch types. - -## 4. Concrete Example: PromQL (all 5 layers) - -### Query -```promql -quantile_over_time(0.99, http_request_duration{env="prod"}[5m]) -``` - -### Layer 1 — Language AST - -The `promql-parser` crate parses the string into a PromQL AST: -`Call("quantile_over_time", [NumberLiteral(0.99), MatrixSelector("http_request_duration", {env="prod"}, 5m)])` - -### Layer 2 — Language Logical Plan (parser output) - -The PromQL parser emits **relational operators only** — `Aggregate { AggFunc }` + `Window`, -no sketch names: - -``` -Aggregate { - keys: [], - aggs: [AggItem { func: Quantile(0.99), col: SampleValue }], - input: Window { - duration: 5m, - input: Filter { - pred: Column("env") = Literal("prod"), - input: Source("http_request_duration") - } - } -} -``` - -### Layer 3 — Sketch Logical Plan (after lowering) - -The shared `lower_to_sketch_algebra()` pass converts `Aggregate { Quantile }` to -`AggIntent::Quantile` and fuses with `Window` into `WindowedAgg`: - -``` -WindowedAgg { - agg: Quantile { quantiles: [0.99], accuracy: 0.01 }, - window: WindowSpec { kind: Tumbling { size: 5m } }, - col: SampleValue, - input: Filter { - pred: Column("env") = Literal("prod"), - input: Source("http_request_duration") - } -} -``` - -Note: no sketch implementation names — just "I need a quantile at φ=0.99 with ≤1% error." - -### Layer 4 — Optimizer - -R1 (PredicatePushDown): filter is already below the window — no change. Tree is returned as-is. - -### Layer 5 — Physical Plan - -`physical::plan(expr, config)` produces a `PhysicalNode` tree. For this simple -query, all nodes are at the Agent — no Exchange boundaries: - -``` -OtelSketchBuild { DDSketch, OtelTumblingFlush(5m) } [AgentCollector] - └── Filter { env="prod" } [AgentCollector] - └── OtlpScan [AgentCollector] -``` - -Resolution: `Quantile([0.99], 0.01)` → `DDSketch { relative_accuracy: 0.01, quantiles: [0.99] }`, -`Tumbling(5m)` at AgentCollector → `OtelTumblingFlush { 5m }`. - -### Execution - -1. **Agent** receives raw samples → filters `env="prod"` → batches 5m windows → DDSketch → emit -2. **Backend** merges DDSketches from N agents -3. **Query time**: extract 0.99 quantile from merged DDSketch - ---- - -## 5. Concrete Example: PromQL with Top-K (all 5 layers) - -### Query -```promql -topk by (service) (10, count_over_time(requests{env="prod"}[1m])) -``` - -### Layer 1 — Language AST - -The `promql-parser` crate parses this as: -`Aggregate(op="topk", param=10, modifier=By(["service"]), expr=Call("count_over_time", MatrixSelector("requests", {env="prod"}, 1m)))` - -### Layer 2 — Language Logical Plan (parser output) - -The PromQL parser emits relational operators. The `by (service)` partition keys -are propagated into the inner `Aggregate`'s GROUP BY keys, so the lowering pass -can see `Count WITH GROUP BY` → `Frequency`: - -``` -TopK { - k: 10, - by: ["service"], - input: Aggregate { - keys: ["service"], - aggs: [AggItem { func: Count, col: SampleValue }], - input: Window { - duration: 1m, - input: Filter { - pred: Column("env") = Literal("prod"), - input: Source("requests") - } - } - } -} -``` - -### Layer 3 — Sketch Logical Plan (after lowering) - -`lower_to_sketch_algebra()` converts `Aggregate { Count, keys: ["service"] }` → -`Partition { ["service"], WindowedAgg { Frequency } }`. The `Window + Aggregate` -fuses into `WindowedAgg`: - -``` -TopK { - k: 10, - by: ["service"], - input: Partition { - keys: By(["service"]), - input: WindowedAgg { - agg: Frequency { accuracy: 0.001 }, - window: WindowSpec { kind: Tumbling { size: 1m } }, - col: SampleValue, - input: Filter { - pred: env = "prod", - input: Source("requests") - } - } - } -} -``` - -Note: `Frequency`, not `CountSketch` — implementation-independent. Both CountSketch -and CountMinSketch are valid candidates; the physical planner decides. - -### Layer 4 — Optimizer - -R1 (PredicatePushDown): filter already below window — no change. - -### Layer 5 — Physical Plan - -`physical::plan(expr, config)` produces a multi-stage `PhysicalNode` tree with -Exchange nodes at stage boundaries: - -``` -TopK { k: 10 } [QueryEngine] - └── Exchange { SketchBinary } [QueryEngine] - └── HashAggregate { keys: ["service"] } [BackendCollector] - └── Exchange { Otlp } [BackendCollector] - └── OtelSketchBuild { CountSketch, [AgentCollector] - OtelTumblingFlush(1m) } - └── Filter { env="prod" } [AgentCollector] - └── OtlpScan [AgentCollector] -``` - -Three stages, two Exchange boundaries: -- **Agent → Backend** (Otlp): sketch data flows from agent collectors to merge tier -- **Backend → QueryEngine** (SketchBinary): merged sketches flow to query engine for top-K - -### Execution - -1. **Agent** → filters → builds CountSketch per 1m window → emits via OTLP -2. **Backend** → merges CountSketches per service -3. **QueryEngine** → extracts top-10 services by frequency - ---- - -## 6. Concrete Example: SQL (all 5 layers) - -### Query -```sql -SELECT symbol, AVG(price) FROM trades GROUP BY symbol -``` - -### Layer 1 — Language AST - -`sqlparser` produces: `Select { projection: [Identifier("symbol"), Function(AVG, "price")], from: [Table("trades")], group_by: [Identifier("symbol")] }` - -### Layer 2 — Language Logical Plan - -Both parsers emit the same kind of output — relational `Aggregate { AggFunc }`: - -``` -Aggregate { - keys: ["symbol"], - aggs: [AggItem { func: Avg, col: Named("price"), alias: "avg" }], - having: None, - input: Source("trades") -} -``` - -### Layer 3 — Sketch Logical Plan (after lowering) - -`lower_to_sketch_algebra()` converts `Avg` → `Quantile { [0.5], 0.01 }` (median proxy): - -``` -Partition { - keys: By(["symbol"]), - input: SketchAgg { - op: Quantile { quantiles: [0.5], accuracy: 0.01 }, - col: Named("price"), - input: Source("trades") - } -} -``` - -However, `Avg` is **non-mergeable** (`avg(A∪B) ≠ merge(avg(A), avg(B))`). -The stage-split will route this to DB for exact computation. - -### Layer 4 — Optimizer - -No rewrites applicable. - -### Layer 5 — Physical Plan - -`physical::plan()` assigns the non-mergeable Aggregate to the Database: - -``` -DbQuery { GROUP BY ["symbol"] } [Database] - └── Exchange { RawSamples } [Database] - └── OtlpScan [AgentCollector] -``` - -The Agent passes raw samples through to the Database, which computes exact AVG. - ---- - -## 7. Concrete Example: SQL with TUMBLE window (all 5 layers) - -### Query -```sql -SELECT region, COUNT(DISTINCT user_id) AS cnt -FROM sessions -GROUP BY region, TUMBLE(ts, INTERVAL '5' MINUTE) -ORDER BY cnt DESC LIMIT 10 -``` - -### Layer 1–2 — Parse to relational operators - -The SQL parser detects `TUMBLE(ts, INTERVAL '5' MINUTE)` in GROUP BY and emits -a `Window` node. `COUNT(DISTINCT user_id)` becomes `AggFunc::CountDistinct`: - -``` -Limit { - n: 10, - input: Sort { - keys: [{ col: "cnt", desc: true }], - input: Aggregate { - keys: ["region"], - aggs: [AggItem { func: CountDistinct, col: Named("user_id"), alias: "cnt" }], - input: Window { - duration: 5m, - input: Source("sessions") - } - } - } -} -``` - -### Layer 3 — Sketch Logical Plan (after lowering) - -`lower_to_sketch_algebra()` converts `CountDistinct` → `Cardinality { 0.01 }` and -fuses `Window + Aggregate` → `WindowedAgg`: - -``` -Limit { - n: 10, - input: Sort { - input: Partition { - keys: By(["region"]), - input: WindowedAgg { - agg: Cardinality { accuracy: 0.01 }, - window: WindowSpec { kind: Tumbling { size: 5m } }, - col: Named("user_id"), - input: Source("sessions") - } - } - } -} -``` - -### Layer 4 — Optimizer - -**R5 (TopKFusion)**: `Limit(10, Sort(desc, ...))` → fused into `TopK { k: 10 }` - -### Layer 5 — Physical Plan - -`physical::plan()` produces a multi-stage tree: - -``` -TopK { k: 10 } [QueryEngine] - └── Exchange { SketchBinary } [QueryEngine] - └── HashAggregate { keys: ["region"] } [BackendCollector] - └── Exchange { Otlp } [BackendCollector] - └── OtelSketchBuild { HLL, [AgentCollector] - OtelTumblingFlush(5m) } - └── OtlpScan [AgentCollector] -``` - -Resolution: `Cardinality(0.01)` → `HLL { precision: 14 }`, `Tumbling(5m)` → `OtelTumblingFlush`. - -### Execution - -1. **Agent**: builds one HLL per region per 5m window → emits via OTLP -2. **Backend**: merges HLLs from N agents (HLL merge = set union) -3. **QueryEngine**: extracts cardinality per region → top 10 - ---- - -## 8. Optimizer: Formulation of the Sketch Placement Problem - -### Optimization Goal - -Given a set of query workloads Q = {q₁, q₂, …, qₙ} and a deployment with -pipeline stages S = {Agent, BackendCollector, BackendDB, OriginalDB, ObjectStore}, -the optimizer solves: - -``` -minimize TotalCost(P) -subject to Accuracy(qᵢ, P) ≤ accuracy_sla(qᵢ) ∀ qᵢ ∈ Q - Latency(qᵢ, P) ≤ latency_sla(qᵢ) ∀ qᵢ ∈ Q - Throughput(qᵢ, P) ≥ throughput_sla(qᵢ) ∀ qᵢ ∈ Q - ResourceUsage(s, P) ≤ Budget(s) ∀ s ∈ S -``` - -where P is the physical plan (sketch type assignment + stage placement + window -configuration for each query operator). - -### Cost Model - -The total cost decomposes into per-stage costs: - -``` -TotalCost(P) = Σ_s [ BandwidthCost(s) + MemoryCost(s) + CPUCost(s) + StorageCost(s) ] -``` - -Each term is the aggregate resource consumption across all queries assigned to -that stage: - -| Cost component | Formula | -|---|---| -| `BandwidthCost(s)` | Σ_q transmission_bytes(sketch(q)) × flush_rate(q) | -| `MemoryCost(s)` | Σ_q memory_per_series(sketch(q)) × series_count(q) | -| `CPUCost(s)` | Σ_q cpu_per_insert(sketch(q)) × samples_per_sec(q) | -| `StorageCost(s)` | Σ_q transmission_bytes(sketch(q)) × retention(q) | - -### Constraints - -**Per-stage resource budgets** — each stage has memory, CPU, disk, and bandwidth limits: - -``` -∀ s ∈ S: - Σ_q memory_per_series(sketch(q, s)) × series_count(q) ≤ s.memory_bytes - Σ_q cpu_per_insert(sketch(q, s)) × samples_per_sec(q) ≤ s.cpu_budget - Σ_q transmission_bytes(sketch(q, s)) × flush_rate(q) ≤ s.bandwidth_budget -``` - -**Accuracy constraint** — sketch error must be within the query's SLA: - -``` -∀ qᵢ: - error(sketch_type(qᵢ), sketch_params(qᵢ)) ≤ accuracy_sla(qᵢ) -``` - -For example: DDSketch with `relative_accuracy = 0.01` guarantees ≤1% relative error -on quantile queries. HLL with `precision = 14` guarantees ≤0.8% relative error -on cardinality. - -**Functional constraint** — the sketch must support the query's aggregation intent: - -``` -∀ qᵢ: - intent(qᵢ) ∈ sketch_capability(sketch_type(qᵢ)).supported_intents -``` - -For example: a `Cardinality` intent can only be served by a sketch with -`SupportedIntent::Cardinality` (HLL, UnivMon), not by DDSketch. - -### Decision Variables - -For each query operator `op` in the plan: - -1. **Sketch type selection**: `sketch_type(op) ∈ candidates(intent(op))` - - Quantile → {DDSketch, KLL} - - Cardinality → {HLL} - - Frequency → {CountSketch, CountMinSketch} - -2. **Stage placement**: `stage(op) ∈ S` - - Subject to `stage_budget(stage(op)).fits(sketch_capability(sketch_type(op)))` - - Deferral chain: Agent → BackendCollector → BackendDB - -3. **Window configuration**: `window(op) ∈ {Tumbling(d), Sliding(d, s), Unbounded}` - - Subject to sketch capability: `sketch_capability(type).supports_sliding_window` - -4. **Delta encoding**: `delta(op) ∈ {true, false}` - - Subject to: `sketch_capability(type).supports_delta` - - Reduces bandwidth at the cost of reconstruction at the receiver - -### Cross-Query Optimization: What to Precompute - -When multiple queries share overlapping time series or aggregation patterns, the -optimizer can amortise costs: - -**Shared sketch reuse**: if q₁ = `quantile_over_time(0.99, m[5m])` and -q₂ = `quantile_over_time(0.5, m[5m])`, a single DDSketch serves both -(DDSketch can answer any quantile from one structure). - -**Precomputation decision**: a query should be precomputed (sketch maintained -continuously) rather than computed on-demand when: - -``` -precompute(q) = true iff repeat_interval(q) < query_latency_sla(q) -``` - -i.e., the query fires more often than the system can recompute it from raw data. -Precomputed sketches are maintained at the Agent and merged at the Backend, -with the Precompute Engine answering queries against the merged state. - -**Multi-query sketch sharing matrix**: for N queries over the same metric, the -optimizer builds a sharing matrix: - -| | DDSketch | HLL | CountSketch | -|---|---|---|---| -| q₁: quantile(0.99) | ✓ serves | ✗ | ✗ | -| q₂: quantile(0.5) | ✓ **shared with q₁** | ✗ | ✗ | -| q₃: count_distinct | ✗ | ✓ serves | ✗ | -| q₄: topk(10) | ✗ | ✗ | ✓ serves | - -One DDSketch instance serves both q₁ and q₂ → memory cost counted once, not twice. - -### Current Implementation - -The optimizer currently solves a simplified version: - -1. **Per-query greedy**: each query is optimised independently (no cross-query sharing yet) -2. **Sketch selection**: `CostModelPlanner` scores all candidates per query, picks cheapest meeting accuracy SLA -3. **Stage placement**: `physical::decide_sketch_placement()` checks `StageBudget::fits(SketchCapability)` per stage in order: Agent → Backend → QueryEngine -4. **Precomputation**: `should_precompute(q)` checks `repeat_interval < latency_sla` - -Future work: -- Global optimisation across queries (shared sketch instances) -- Joint sketch+stage+window optimisation (currently done greedily per dimension) -- Workload-adaptive re-optimisation (replan when query patterns change) - ---- - -## 9. Sketch Directory: Which Sketch for Which Operation? - -The sketch directory (`algebra/directory.rs`) maps aggregation types to candidate -sketch families. The `CostModelPlanner` scores all candidates and picks the -cheapest that meets the accuracy SLA. - -### Candidates per Aggregation Type - -| Aggregation | Candidates (default first) | When non-default is chosen | -|---|---|---| -| Quantile | **DDSketch**, KLL | KLL when memory-constrained | -| Cardinality | **HLL** | Single candidate | -| Frequency | **CountSketch**, CountMinSketch | Based on cost model scoring | - -### Sketch Type → OTel Collector Processor - -| SketchType | Go processor | Key parameters | -|---|---|---| -| DDSketch | `ddsketch` | relative_accuracy, quantiles | -| KLL | `KLL` | k, quantiles | -| HLL | `HLL` | (fixed precision in Go code) | -| CountSketch | `countsketch` | epsilon, delta | -| CountMinSketch | `countmin` | rows, cols, metric_name | - -### Stage Assignment Rules - -| QueryExpr node | Default stage | Deferral trigger | -|---|---|---| -| Source, Filter, Window, SketchAgg | **Agent** | Memory budget exceeded → Backend | -| Partition, Merge, Dedup, Exact(Sum/Count/Min/Max) | **Backend** | Memory exceeded → Precompute | -| TopK, HistogramQuantile, BinaryOp, PromQLSubquery | **Precompute** | — | -| Exact(Avg) | **DB** | Non-mergeable — cannot distribute | - -When an Agent sketch exceeds the memory budget, it is deferred to Backend. -If it also exceeds the Backend budget, it moves to Precompute. Every deferral -is logged in `StagedPlan.deferral_log` for observability. diff --git a/controller/proto/anyvalue.proto b/controller/proto/anyvalue.proto deleted file mode 100644 index 8ed97315..00000000 --- a/controller/proto/anyvalue.proto +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2019, OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// This file is copied and modified from https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/common/v1/common.proto -// Modifications: -// - Removal of unneeded InstrumentationLibrary and StringKeyValue messages. -// - Change of go_package to reference a package in this repo. -// - Removal of gogoproto usage. - -syntax = "proto3"; - -package opamp.proto; - -option go_package = "github.com/open-telemetry/opamp-go/protobufs"; - -// AnyValue is used to represent any type of attribute value. AnyValue may contain a -// primitive value such as a string or integer or it may contain an arbitrary nested -// object containing arrays, key-value lists and primitives. -message AnyValue { - // The value is one of the listed fields. It is valid for all values to be unspecified - // in which case this AnyValue is considered to be "null". - oneof value { - string string_value = 1; - bool bool_value = 2; - int64 int_value = 3; - double double_value = 4; - ArrayValue array_value = 5; - KeyValueList kvlist_value = 6; - bytes bytes_value = 7; - } -} - -// ArrayValue is a list of AnyValue messages. We need ArrayValue as a message -// since oneof in AnyValue does not allow repeated fields. -message ArrayValue { - // Array of values. The array may be empty (contain 0 elements). - repeated AnyValue values = 1; -} - -// KeyValueList is a list of KeyValue messages. We need KeyValueList as a message -// since `oneof` in AnyValue does not allow repeated fields. Everywhere else where we need -// a list of KeyValue messages (e.g. in Span) we use `repeated KeyValue` directly to -// avoid unnecessary extra wrapping (which slows down the protocol). The 2 approaches -// are semantically equivalent. -message KeyValueList { - // A collection of key/value pairs of key-value pairs. The list may be empty (may - // contain 0 elements). - repeated KeyValue values = 1; -} - -// KeyValue is a key-value pair that is used to store Span attributes, Link -// attributes, etc. -message KeyValue { - string key = 1; - AnyValue value = 2; -} diff --git a/controller/proto/controller.proto b/controller/proto/controller.proto deleted file mode 100644 index 5eb87cde..00000000 --- a/controller/proto/controller.proto +++ /dev/null @@ -1,121 +0,0 @@ -syntax = "proto3"; - -package controller.v1; - -option go_package = "github.com/ProjectASAP/controller/proto/controllerv1"; - -import "google/protobuf/duration.proto"; -import "google/protobuf/timestamp.proto"; - -// ── Enumerations ────────────────────────────────────────────────────────────── - -enum AggType { - AGG_TYPE_UNSPECIFIED = 0; - AGG_TYPE_QUANTILE = 1; - AGG_TYPE_CARDINALITY = 2; - AGG_TYPE_FREQUENCY = 3; -} - -enum SketchType { - SKETCH_TYPE_UNSPECIFIED = 0; - SKETCH_TYPE_DDSKETCH = 1; - SKETCH_TYPE_KLL = 2; - SKETCH_TYPE_HLL = 3; - SKETCH_TYPE_COUNT_SKETCH = 4; - SKETCH_TYPE_COUNT_MIN_SKETCH = 5; -} - -enum OutputMode { - OUTPUT_MODE_UNSPECIFIED = 0; - OUTPUT_MODE_RAW = 1; - OUTPUT_MODE_SKETCH = 2; -} - -enum ProcessorMode { - PROCESSOR_MODE_UNSPECIFIED = 0; - PROCESSOR_MODE_BATCH = 1; - PROCESSOR_MODE_WINDOW = 2; -} - -// ── Core types ──────────────────────────────────────────────────────────────── - -message QueryWorkload { - string metric_name = 1; - map label_filters = 2; - repeated string group_by_labels = 3; - repeated AggType aggregations = 4; - google.protobuf.Duration time_window = 5; - google.protobuf.Duration repeat_every = 6; - double accuracy_sla = 7; // relative error, e.g. 0.01 - google.protobuf.Duration latency_sla = 8; -} - -message SketchParams { - double relative_accuracy = 1; // DDSketch - int32 k = 2; // KLL - int32 precision = 3; // HLL - int32 rows = 4; // CountSketch / CountMinSketch - int32 cols = 5; - repeated double quantiles = 6; -} - -message AgentCollectorConfig { - OutputMode output_mode = 1; - SketchType sketch_type = 2; - SketchParams sketch_params = 3; - repeated string aggregate_by = 4; - repeated string label_matchers = 5; - google.protobuf.Duration window_duration = 6; - ProcessorMode mode = 7; - bool transmit_sketch = 8; - bool drop_original = 9; -} - -message GatewayCollectorConfig { - bool passthrough = 1; -} - -message BackendCollectorConfig { - SketchType merge_sketch_type = 1; - repeated string group_by = 2; -} - -message PrecomputeJob { - string query_expr = 1; - google.protobuf.Duration granularity = 2; - string sketch_source = 3; - string store_path = 4; -} - -message CollectionPlan { - AgentCollectorConfig agent_config = 1; - GatewayCollectorConfig gateway_config = 2; - BackendCollectorConfig backend_config = 3; - repeated PrecomputeJob precompute = 4; - google.protobuf.Timestamp valid_until = 5; -} - -// ── RPCs ────────────────────────────────────────────────────────────────────── - -message PlanRequest { - QueryWorkload workload = 1; -} - -message PlanResponse { - CollectionPlan plan = 1; - string status = 2; -} - -message PlanStatusRequest { - string metric_name = 1; -} - -message PlanStatusResponse { - CollectionPlan current_plan = 1; - string status = 2; -} - -service ControllerService { - rpc Plan(PlanRequest) returns (PlanResponse); - rpc GetPlanStatus(PlanStatusRequest) returns (PlanStatusResponse); -} diff --git a/controller/proto/feedback.proto b/controller/proto/feedback.proto deleted file mode 100644 index 301c6825..00000000 --- a/controller/proto/feedback.proto +++ /dev/null @@ -1,52 +0,0 @@ -// ASAP runtime-samples wire schema. -// -// Carries batches of `sketch-core::report::Record` from an -// agent's `sketch-runtime::GrpcExporter` into the controller's -// real-time decision loop. The Record payload is forwarded as -// JSON so the controller stays forward-compatible with new -// v1 schema fields without a proto rebump. -// -// Schema version is `sketch-core::report::SCHEMA_VERSION` (1 -// today). A bump implies both sides get recompiled. - -syntax = "proto3"; - -package asap.runtime.v1; - -// One record per emitted bench window. Mirrors -// `sketch-core::report::Record`: the `source` / `sketch` / -// `impl_name` tuple is the per-key routing handle on the -// controller side (→ `RuntimeSamplesStore`'s `SampleKey`); -// the rest of the record rides in `payload_json` so new -// fields on `Record` flow through without a proto change. -message RuntimeRecord { - string source = 1; - string sketch = 2; - string impl_name = 3; - uint32 schema_version = 4; - string payload_json = 5; -} - -// A batched push. `GrpcExporter`'s default batch is 64 records -// / 100 ms (whichever triggers first) — small enough to keep -// decision latency bounded, large enough to amortize gRPC -// per-call overhead. -message PushBatch { - repeated RuntimeRecord records = 1; -} - -message PushAck { - // Number of records the server accepted (could be less than - // records.size() if a few failed validation; log + continue - // on the client side in that case). - uint64 accepted = 1; -} - -service RuntimeSamples { - // Unary RPC per batch — simpler than bidi streaming, and the - // client already batches before pushing so HTTP/2 mux handles - // throughput without a per-message acks-on-stream dance. - // tonic negotiates gzip if `Accept-Encoding: gzip` is set on - // the server. - rpc Push(PushBatch) returns (PushAck); -} diff --git a/controller/proto/opamp.proto b/controller/proto/opamp.proto deleted file mode 100644 index 219118c8..00000000 --- a/controller/proto/opamp.proto +++ /dev/null @@ -1,1079 +0,0 @@ -// Copyright 2021, OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// OpAMP: Open Agent Management Protocol (https://github.com/open-telemetry/opamp-spec) - -syntax = "proto3"; - -package opamp.proto; - -import "anyvalue.proto"; - -option go_package = "github.com/open-telemetry/opamp-go/protobufs"; - -message AgentToServer { - // Globally unique identifier of the running instance of the Agent. SHOULD remain - // unchanged for the lifetime of the Agent process. - // MUST be 16 bytes long and SHOULD be generated using the UUID v7 spec. - bytes instance_uid = 1; - - // The sequence number is incremented by 1 for every AgentToServer sent - // by the Agent. This allows the Server to detect that it missed a message when - // it notices that the sequence_num is not exactly by 1 greater than the previously - // received one. - uint64 sequence_num = 2; - - // Data that describes the Agent, its type, where it runs, etc. - // May be omitted if nothing changed since last AgentToServer message. - AgentDescription agent_description = 3; - - // Bitmask of flags defined by AgentCapabilities enum. - // All bits that are not defined in AgentCapabilities enum MUST be set to 0 by - // the Agent. This allows extending the protocol and the AgentCapabilities enum - // in the future such that old Agents automatically report that they don't - // support the new capability. - // This field MUST be always set. - uint64 capabilities = 4; - - // The current health of the Agent and sub-components. The top-level ComponentHealth represents - // the health of the Agent overall. May be omitted if nothing changed since last AgentToServer - // message. - // Status: [Beta] - ComponentHealth health = 5; - - // The current effective configuration of the Agent. The effective configuration is - // the one that is currently used by the Agent. The effective configuration may be - // different from the remote configuration received from the Server earlier, e.g. - // because the Agent uses a local configuration instead (or in addition). - // - // This field SHOULD be unset if the effective config is unchanged since the last - // AgentToServer message. - EffectiveConfig effective_config = 6; - - // The status of the remote config that was previously received from the Server. - // This field SHOULD be unset if the remote config status is unchanged since the - // last AgentToServer message. - RemoteConfigStatus remote_config_status = 7; - - // The list of the Agent packages, including package statuses. This field SHOULD be - // unset if this information is unchanged since the last AgentToServer message for - // this Agent was sent in the stream. - // Status: [Beta] - PackageStatuses package_statuses = 8; - - // AgentDisconnect MUST be set in the last AgentToServer message sent from the - // Agent to the Server. - AgentDisconnect agent_disconnect = 9; - - // Bit flags as defined by AgentToServerFlags bit masks. - uint64 flags = 10; - - // A request to create connection settings. This field is set for flows where - // the Agent initiates the creation of connection settings. - // Status: [Development] - ConnectionSettingsRequest connection_settings_request = 11; - - // A message indicating custom capabilities supported by the Agent. - // Status: [Development] - CustomCapabilities custom_capabilities = 12; - - // A custom message sent from an Agent to the Server. - // Status: [Development] - CustomMessage custom_message = 13; - - // A message indicating the components that are available for configuration on the agent. - // Status: [Development] - AvailableComponents available_components = 14; - - // The status of the OfferedConnectionSettings that was previously received - // from the Server. This field SHOULD be unset if the offered connection - // settings status is unchanged since the last AgentToServer message. - // Status: [Development] - ConnectionSettingsStatus connection_settings_status = 15; -} - -enum AgentToServerFlags { - AgentToServerFlags_Unspecified = 0; - - // AgentToServerFlags is a bit mask. Values below define individual bits. - - // The Agent requests Server go generate a new instance_uid, which will - // be sent back in ServerToAgent message - AgentToServerFlags_RequestInstanceUid = 0x00000001; -} - -// AgentDisconnect is the last message sent from the Agent to the Server. The Server -// SHOULD forget the association of the Agent instance with the message stream. -// -// If the message stream is closed in the transport layer then the Server SHOULD -// forget association of all Agent instances that were previously established for -// this message stream using AgentConnect message, even if the corresponding -// AgentDisconnect message were not explicitly received from the Agent. -message AgentDisconnect { -} - -// ConnectionSettingsRequest is a request from the Agent to the Server to create -// and respond with an offer of connection settings for the Agent. -// Status: [Development] -message ConnectionSettingsRequest { - // Request for OpAMP connection settings. If this field is unset - // then the ConnectionSettingsRequest message is empty and is not actionable - // for the Server. - OpAMPConnectionSettingsRequest opamp = 1; - - // In the future we can add request fields for non-OpAMP connection types - // (own telemetry, other connections). -} - -// OpAMPConnectionSettingsRequest is a request for the Server to produce -// a OpAMPConnectionSettings in its response. -// Status: [Development] -message OpAMPConnectionSettingsRequest { - // A request to create a client certificate. This is used to initiate a - // Client Signing Request (CSR) flow. - // Required. - CertificateRequest certificate_request = 1; -} - -// Status: [Development] -message CertificateRequest { - // PEM-encoded Client Certificate Signing Request (CSR), signed by client's private key. - // The Server SHOULD validate the request and SHOULD respond with a - // OpAMPConnectionSettings where the certificate.cert contains the issued - // certificate. - bytes csr = 1; -} - -// AvailableComponents contains metadata relating to the components included -// within the agent. -// status: [Development] -message AvailableComponents { - // A map of a unique component ID to details about the component. - // This may be omitted from the message if the server has not - // explicitly requested it be sent by setting the ReportAvailableComponents - // flag in the previous ServerToAgent message. - map components = 1; - - // Agent-calculated hash of the components. - // This hash should be included in every AvailableComponents message. - bytes hash = 2; - } - -message ComponentDetails { - // Extra key/value pairs that may be used to describe the component. - // The key/value pairs are according to semantic conventions, see: - // https://opentelemetry.io/docs/specs/semconv/ - // - // For example, you may use the "code" semantic conventions to - // report the location of the code for a specific component: - // https://opentelemetry.io/docs/specs/semconv/attributes-registry/code/ - // - // Or you may use the "vcs" semantic conventions to report the - // repository the component may be a part of: - // https://opentelemetry.io/docs/specs/semconv/attributes-registry/vcs/ - repeated KeyValue metadata = 1; - - // A map of component ID to sub components details. It can nest as deeply as needed to - // describe the underlying system. - map sub_component_map = 2; -} - - -message ServerToAgent { - // Agent instance uid. MUST match the instance_uid field in AgentToServer message. - // Used for multiplexing messages from/to multiple agents using one message stream. - bytes instance_uid = 1; - - // error_response is set if the Server wants to indicate that something went wrong - // during processing of an AgentToServer message. If error_response is set then - // all other fields below must be unset and vice versa, if any of the fields below is - // set then error_response must be unset. - ServerErrorResponse error_response = 2; - - // remote_config field is set when the Server has a remote config offer for the Agent. - AgentRemoteConfig remote_config = 3; - - // This field is set when the Server wants the Agent to change one or more - // of its client connection settings (destination, headers, certificate, etc). - // Status: [Beta] - ConnectionSettingsOffers connection_settings = 4; - - // This field is set when the Server has packages to offer to the Agent. - // Status: [Beta] - PackagesAvailable packages_available = 5; - - // Bit flags as defined by ServerToAgentFlags bit masks. - uint64 flags = 6; - - // Bitmask of flags defined by ServerCapabilities enum. - // All bits that are not defined in ServerCapabilities enum MUST be set to 0 - // by the Server. This allows extending the protocol and the ServerCapabilities - // enum in the future such that old Servers automatically report that they - // don't support the new capability. - // This field MUST be set in the first ServerToAgent sent by the Server and MAY - // be omitted in subsequent ServerToAgent messages by setting it to - // UnspecifiedServerCapability value. - uint64 capabilities = 7; - - // Properties related to identification of the Agent, which can be overridden - // by the Server if needed. - AgentIdentification agent_identification = 8; - - // Allows the Server to instruct the Agent to perform a command, e.g. RESTART. This field should not be specified - // with fields other than instance_uid and capabilities. If specified, other fields will be ignored and the command - // will be performed. - // Status: [Beta] - ServerToAgentCommand command = 9; - - // A message indicating custom capabilities supported by the Server. - // Status: [Development] - CustomCapabilities custom_capabilities = 10; - - // A custom message sent from the Server to an Agent. - // Status: [Development] - CustomMessage custom_message = 11; -} - -enum ServerToAgentFlags { - ServerToAgentFlags_Unspecified = 0; - - // Flags is a bit mask. Values below define individual bits. - - // ReportFullState flag can be used by the Server if the Agent did not include the - // particular bit of information in the last status report (which is an allowed - // optimization) but the Server detects that it does not have it (e.g. was - // restarted and lost state). The detection happens using - // AgentToServer.sequence_num values. - // The Server asks the Agent to report full status. - ServerToAgentFlags_ReportFullState = 0x00000001; - - // ReportAvailableComponents flag can be used by the server if the Agent did - // not include the full AvailableComponents message, but only the hash. - // If this flag is specified, the agent will populate available_components.components - // with a full description of the agent's components. - // Status: [Development] - ServerToAgentFlags_ReportAvailableComponents = 0x00000002; -} - -enum ServerCapabilities { - // The capabilities field is unspecified. - ServerCapabilities_Unspecified = 0; - - // The Server can accept status reports. This bit MUST be set, since all Server - // MUST be able to accept status reports. - ServerCapabilities_AcceptsStatus = 0x00000001; - // The Server can offer remote configuration to the Agent. - ServerCapabilities_OffersRemoteConfig = 0x00000002; - // The Server can accept EffectiveConfig in AgentToServer. - ServerCapabilities_AcceptsEffectiveConfig = 0x00000004; - // The Server can offer Packages. - // Status: [Beta] - ServerCapabilities_OffersPackages = 0x00000008; - // The Server can accept Packages status. - // Status: [Beta] - ServerCapabilities_AcceptsPackagesStatus = 0x00000010; - // The Server can offer connection settings. - // Status: [Beta] - ServerCapabilities_OffersConnectionSettings = 0x00000020; - // The Server can accept ConnectionSettingsRequest and respond with an offer. - // Status: [Development] - ServerCapabilities_AcceptsConnectionSettingsRequest = 0x00000040; - - // Add new capabilities here, continuing with the least significant unused bit. -} - -// The OpAMPConnectionSettings message is a collection of fields which comprise an -// offer from the Server to the Agent to use the specified settings for OpAMP -// connection. -// Status: [Beta] -message OpAMPConnectionSettings { - // OpAMP Server URL This MUST be a WebSocket or HTTP URL and MUST be non-empty, for - // example: "wss://example.com:4318/v1/opamp" - string destination_endpoint = 1; - - // Optional headers to use when connecting. Typically used to set access tokens or - // other authorization headers. For HTTP-based protocols the Agent should - // set these in the request headers. - // For example: - // key="Authorization", Value="Basic YWxhZGRpbjpvcGVuc2VzYW1l". - Headers headers = 2; - - // The Agent should use the offered certificate to connect to the destination - // from now on. If the Agent is able to validate and connect using the offered - // certificate the Agent SHOULD forget any previous client certificates - // for this connection. - // This field is optional: if omitted the client SHOULD NOT use a client-side certificate. - // This field can be used to perform a client certificate revocation/rotation. - TLSCertificate certificate = 3; - - // The Agent MUST periodically send an AgentToServer message if the - // AgentCapabilities_ReportsHeartbeat capability is true. At a minimum the instance_uid - // field MUST be set. - // - // An HTTP Client MUST use the value as polling interval, if heartbeat_interval_seconds is non-zero. - // - // A heartbeat is used to keep the connection active and inform the server that the Agent - // is still alive and active. - // - // If this field has no value or is set to 0, the Agent should not send any heartbeats. - // Status: [Development] - uint64 heartbeat_interval_seconds = 4; - - // Optional connection specific TLS settings. - // Status: [Development] - TLSConnectionSettings tls = 5; - - // Optional connection specific proxy settings. - // Status: [Development] - ProxyConnectionSettings proxy = 6; -} - -// The TelemetryConnectionSettings message is a collection of fields which comprise an -// offer from the Server to the Agent to use the specified settings for a network -// connection to report own telemetry. -// Status: [Beta] -message TelemetryConnectionSettings { - // The value MUST be a full URL an OTLP/HTTP/Protobuf receiver with path. Schema - // SHOULD begin with "https://", for example "https://example.com:4318/v1/metrics" - // The Agent MAY refuse to send the telemetry if the URL begins with "http://". - string destination_endpoint = 1; - - // Optional headers to use when connecting. Typically used to set access tokens or - // other authorization headers. For HTTP-based protocols the Agent should - // set these in the request headers. - // For example: - // key="Authorization", Value="Basic YWxhZGRpbjpvcGVuc2VzYW1l". - Headers headers = 2; - - // The Agent should use the offered certificate to connect to the destination - // from now on. If the Agent is able to validate and connect using the offered - // certificate the Agent SHOULD forget any previous client certificates - // for this connection. - // This field is optional: if omitted the client SHOULD NOT use a client-side certificate. - // This field can be used to perform a client certificate revocation/rotation. - TLSCertificate certificate = 3; - - // Optional connection specific TLS settings. - // Status: [Development] - TLSConnectionSettings tls = 4; - - // Optional connection specific proxy settings. - // Status: [Development] - ProxyConnectionSettings proxy = 5; -} - -// The OtherConnectionSettings message is a collection of fields which comprise an -// offer from the Server to the Agent to use the specified settings for a network -// connection. It is not required that all fields in this message are specified. -// The Server may specify only some of the fields, in which case it means that -// the Server offers the Agent to change only those fields, while keeping the -// rest of the fields unchanged. -// -// For example the Server may send a ConnectionSettings message with only the -// certificate field set, while all other fields are unset. This means that -// the Server wants the Agent to use a new certificate and continue sending to -// the destination it is currently sending using the current header and other -// settings. -// -// For fields which reference other messages the field is considered unset -// when the reference is unset. -// -// For primitive field (string) we rely on the "flags" to describe that the -// field is not set (this is done to overcome the limitation of old protoc -// compilers don't generate methods that allow to check for the presence of -// the field. -// Status: [Beta] -message OtherConnectionSettings { - // A URL, host:port or some other destination specifier. - string destination_endpoint = 1; - - // Optional headers to use when connecting. Typically used to set access tokens or - // other authorization headers. For HTTP-based protocols the Agent should - // set these in the request headers. - // For example: - // key="Authorization", Value="Basic YWxhZGRpbjpvcGVuc2VzYW1l". - Headers headers = 2; - - // The Agent should use the offered certificate to connect to the destination - // from now on. If the Agent is able to validate and connect using the offered - // certificate the Agent SHOULD forget any previous client certificates - // for this connection. - // This field is optional: if omitted the client SHOULD NOT use a client-side certificate. - // This field can be used to perform a client certificate revocation/rotation. - TLSCertificate certificate = 3; - - // Other connection settings. These are Agent-specific and are up to the Agent - // interpret. - map other_settings = 4; - - // Optional connection specific TLS settings. - // Status: [Development] - TLSConnectionSettings tls = 5; - - // Optional connection specific proxy settings. - // Status: [Development] - ProxyConnectionSettings proxy = 6; -} - - -// TLSConnectionSettings are optional connection settings that can be passed to -// the client in order to specify TLS configuration. -// Status: [Development] -message TLSConnectionSettings { - // Provides CA cert contents as a string. - string ca_pem_contents = 1; - - // Load system CA pool alongside any passed CAs. - bool include_system_ca_certs_pool = 2; - - // skip certificate verification. - bool insecure_skip_verify = 3; - - // Miniumum accepted TLS version; default "1.2". - string min_version = 4; - - // Maxiumum accepted TLS version; default "". - string max_version = 5; - - // Explicit list of cipher suites. - repeated string cipher_suites = 6; -} - -// Status: [Development] -message ProxyConnectionSettings { - // A URL, host:port or some other destination specifier. - string url = 1; - - // Optional headers to send to proxies during CONNECT requests. - // These headers can be ignored for non-HTTP based proxies. - // For example: - // key="Authorization", Value="Basic YWxhZGRpbjpvcGVuc2VzYW1l". - Headers connect_headers = 2; -} - -// Status: [Beta] -message Headers { - repeated Header headers = 1; -} - -// Status: [Beta] -message Header { - string key = 1; - string value = 2; -} - -// Status: [Beta] -message TLSCertificate { - // The (cert,private_key) pair should be issued and signed by a Certificate - // Authority (CA) that the destination Server recognizes. - // - // It is highly recommended that the private key of the CA certificate is NOT - // stored on the destination Server otherwise compromising the Server will allow - // a malicious actor to issue valid Server certificates which will be automatically - // trusted by all agents and will allow the actor to trivially MITM Agent-to-Server - // traffic of all servers that use this CA certificate for their Server-side - // certificates. - // - // Alternatively the certificate may be self-signed, assuming the Server can - // verify the certificate. - - // PEM-encoded certificate. Required. - bytes cert = 1; - - // PEM-encoded private key of the certificate. Required. - bytes private_key = 2; - - // PEM-encoded certificate of the signing CA. - // Optional. MUST be specified if the certificate is CA-signed. - // Can be stored by TLS-terminating intermediary proxies in order to verify - // the connecting client's certificate in the future. - // It is not recommended that the Agent accepts this CA as an authority for - // any purposes. - bytes ca_cert = 3; -} - -// Status: [Beta] -message ConnectionSettingsOffers { - // Hash of all settings, including settings that may be omitted from this message - // because they are unchanged. - bytes hash = 1; - - // Settings to connect to the OpAMP Server. - // If this field is not set then the Agent should assume that the settings are - // unchanged and should continue using existing settings. - // The Agent MUST verify the offered connection settings by actually connecting - // before accepting the setting to ensure it does not loose access to the OpAMP - // Server due to invalid settings. - OpAMPConnectionSettings opamp = 2; - - // Settings to connect to an OTLP metrics backend to send Agent's own metrics to. - // If this field is not set then the Agent should assume that the settings - // are unchanged. - // - // Once accepted the Agent should periodically send to the specified destination - // its own metrics, i.e. metrics of the Agent process and any custom metrics that - // describe the Agent state. - // - // All attributes specified in the identifying_attributes field in AgentDescription - // message SHOULD be also specified in the Resource of the reported OTLP metrics. - // - // Attributes specified in the non_identifying_attributes field in - // AgentDescription message may be also specified in the Resource of the reported - // OTLP metrics, in which case they SHOULD have exactly the same values. - // - // Process metrics MUST follow the conventions for processes: - // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/semantic_conventions/process-metrics.md - TelemetryConnectionSettings own_metrics = 3; - - // Similar to own_metrics, but for traces. - TelemetryConnectionSettings own_traces = 4; - - // Similar to own_metrics, but for logs. - TelemetryConnectionSettings own_logs = 5; - - // Another set of connection settings, with a string name associated with each. - // How the Agent uses these is Agent-specific. Typically the name represents - // the name of the destination to connect to (as it is known to the Agent). - // If this field is not set then the Agent should assume that the other_connections - // settings are unchanged. - map other_connections = 6; -} - -// List of packages that the Server offers to the Agent. -// Status: [Beta] -message PackagesAvailable { - // Map of packages. Keys are package names, values are the packages available for download. - map packages = 1; - - // Aggregate hash of all remotely installed packages. The Agent SHOULD include this - // value in subsequent PackageStatuses messages. This in turn allows the management - // Server to identify that a different set of packages is available for the Agent - // and specify the available packages in the next ServerToAgent message. - // - // This field MUST be always set if the management Server supports packages - // of agents. - // - // The hash is calculated as an aggregate of all packages names and content. - bytes all_packages_hash = 2; -} - -// Each Agent is composed of one or more packages. A package has a name and -// content stored in a file. The content of the files, functionality -// provided by the packages, how they are stored and used by the Agent side is Agent -// type-specific and is outside the concerns of the OpAMP protocol. -// -// If the Agent does not have an installed package with the specified name then -// it SHOULD download it from the specified URL and install it. -// -// If the Agent already has an installed package with the specified name -// but with a different hash then the Agent SHOULD download and -// install the package again, since it is a different version of the same package. -// -// If the Agent has an installed package with the specified name and the same -// hash then the Agent does not need to do anything, it already -// has the right version of the package. -// Status: [Beta] -message PackageAvailable { - PackageType type = 1; - - // The package version that is available on the Server side. The Agent may for - // example use this information to avoid downloading a package that was previously - // already downloaded and failed to install. - string version = 2; - - // The downloadable file of the package. - DownloadableFile file = 3; - - // The hash of the package. SHOULD be calculated based on all other fields of the - // PackageAvailable message and content of the file of the package. The hash is - // used by the Agent to determine if the package it has is different from the - // package the Server is offering. - bytes hash = 4; -} - -// The type of the package, either an addon or a top-level package. -// Status: [Beta] -enum PackageType { - PackageType_TopLevel = 0; - PackageType_Addon = 1; -} - -// Status: [Beta] -message DownloadableFile { - // The URL from which the file can be downloaded using HTTP GET request. - // The Server at the specified URL SHOULD support range requests - // to allow for resuming downloads. - string download_url = 1; - - // The hash of the file content. Can be used by the Agent to verify that the file - // was downloaded correctly. - bytes content_hash = 2; - - // Optional signature of the file content. Can be used by the Agent to verify the - // authenticity of the downloaded file, for example can be the - // [detached GPG signature](https://www.gnupg.org/gph/en/manual/x135.html#AEN160). - // The exact signing and verification method is Agent specific. See - // https://github.com/open-telemetry/opamp-spec/blob/main/specification.md#code-signing - // for recommendations. - bytes signature = 3; - - // Optional headers to use when downloading a file. Typically used to set - // access tokens or other authorization headers. For HTTP-based protocols - // the Agent should set these in the request headers. - // For example: - // key="Authorization", Value="Basic YWxhZGRpbjpvcGVuc2VzYW1l". - // Status: [Development] - Headers headers = 4; -} - -message ServerErrorResponse { - ServerErrorResponseType type = 1; - - // Error message in the string form, typically human readable. - string error_message = 2; - - oneof Details { - // Additional information about retrying if type==UNAVAILABLE. - RetryInfo retry_info = 3; - } -} - -enum ServerErrorResponseType { - // Unknown error. Something went wrong, but it is not known what exactly. - // The Agent SHOULD NOT retry the message. - // The error_message field may contain a description of the problem. - ServerErrorResponseType_Unknown = 0; - - // The AgentToServer message was malformed. The Agent SHOULD NOT retry - // the message. - ServerErrorResponseType_BadRequest = 1; - - // The Server is overloaded and unable to process the request. The Agent - // should retry the message later. retry_info field may be optionally - // set with additional information about retrying. - ServerErrorResponseType_Unavailable = 2; -} - -message RetryInfo { - uint64 retry_after_nanoseconds = 1; -} - -// ServerToAgentCommand is sent from the Server to the Agent to request that the Agent -// perform a command. -// Status: [Beta] -message ServerToAgentCommand { - CommandType type = 1; -} - -// Status: [Beta] -enum CommandType { - // The Agent should restart. This request will be ignored if the Agent does not - // support restart. - CommandType_Restart = 0; -} - -//////////////////////////////////////////////////////////////////////////////////// -// Status reporting - -message AgentDescription { - // Attributes that identify the Agent. - // Keys/values are according to OpenTelemetry semantic conventions, see: - // https://github.com/open-telemetry/opentelemetry-specification/tree/main/specification/resource/semantic_conventions - // - // For standalone running Agents (such as OpenTelemetry Collector) the following - // attributes SHOULD be specified: - // - service.name should be set to a reverse FQDN that uniquely identifies the - // Agent type, e.g. "io.opentelemetry.collector" - // - service.namespace if it is used in the environment where the Agent runs. - // - service.version should be set to version number of the Agent build. - // - service.instance.id should be set. It may be set equal to the Agent's - // instance uid (equal to ServerToAgent.instance_uid field) or any other value - // that uniquely identifies the Agent in combination with other attributes. - // - any other attributes that are necessary for uniquely identifying the Agent's - // own telemetry. - // - // The Agent SHOULD also include these attributes in the Resource of its own - // telemetry. The combination of identifying attributes SHOULD be sufficient to - // uniquely identify the Agent's own telemetry in the destination system to which - // the Agent sends its own telemetry. - repeated KeyValue identifying_attributes = 1; - - // Attributes that do not necessarily identify the Agent but help describe - // where it runs. - // The following attributes SHOULD be included: - // - os.type, os.version - to describe where the Agent runs. - // - host.* to describe the host the Agent runs on. - // - cloud.* to describe the cloud where the host is located. - // - any other relevant Resource attributes that describe this Agent and the - // environment it runs in. - // - any user-defined attributes that the end user would like to associate - // with this Agent. - repeated KeyValue non_identifying_attributes = 2; - - // TODO: add ability to specify related entities (such as the Service the Agent is - // is responsible/associated with). -} - -enum AgentCapabilities { - // The capabilities field is unspecified. - AgentCapabilities_Unspecified = 0; - // The Agent can report status. This bit MUST be set, since all Agents MUST - // report status. - AgentCapabilities_ReportsStatus = 0x00000001; - // The Agent can accept remote configuration from the Server. - AgentCapabilities_AcceptsRemoteConfig = 0x00000002; - // The Agent will report EffectiveConfig in AgentToServer. - AgentCapabilities_ReportsEffectiveConfig = 0x00000004; - // The Agent can accept package offers. - // Status: [Beta] - AgentCapabilities_AcceptsPackages = 0x00000008; - // The Agent can report package status. - // Status: [Beta] - AgentCapabilities_ReportsPackageStatuses = 0x00000010; - // The Agent can report own trace to the destination specified by - // the Server via ConnectionSettingsOffers.own_traces field. - // Status: [Beta] - AgentCapabilities_ReportsOwnTraces = 0x00000020; - // The Agent can report own metrics to the destination specified by - // the Server via ConnectionSettingsOffers.own_metrics field. - // Status: [Beta] - AgentCapabilities_ReportsOwnMetrics = 0x00000040; - // The Agent can report own logs to the destination specified by - // the Server via ConnectionSettingsOffers.own_logs field. - // Status: [Beta] - AgentCapabilities_ReportsOwnLogs = 0x00000080; - // The can accept connections settings for OpAMP via - // ConnectionSettingsOffers.opamp field. - // Status: [Beta] - AgentCapabilities_AcceptsOpAMPConnectionSettings = 0x00000100; - // The can accept connections settings for other destinations via - // ConnectionSettingsOffers.other_connections field. - // Status: [Beta] - AgentCapabilities_AcceptsOtherConnectionSettings = 0x00000200; - // The Agent can accept restart requests. - // Status: [Beta] - AgentCapabilities_AcceptsRestartCommand = 0x00000400; - // The Agent will report Health via AgentToServer.health field. - AgentCapabilities_ReportsHealth = 0x00000800; - // The Agent will report RemoteConfig status via AgentToServer.remote_config_status field. - AgentCapabilities_ReportsRemoteConfig = 0x00001000; - // The Agent can report heartbeats. - // This is specified by the ServerToAgent.OpAMPConnectionSettings.heartbeat_interval_seconds field. - // If this capability is true, but the Server does not set a heartbeat_interval_seconds field, the - // Agent should use its own configured interval, which by default will be 30s. The Server may not - // know the configured interval and should not make assumptions about it. - // Status: [Development] - AgentCapabilities_ReportsHeartbeat = 0x00002000; - // The agent will report AvailableComponents via the AgentToServer.available_components field. - // Status: [Development] - AgentCapabilities_ReportsAvailableComponents = 0x00004000; - // The agent will report ConnectionSettingsOffers status via AgentToServer.connection_settings_status field. - // Status: [Development] - AgentCapabilities_ReportsConnectionSettingsStatus = 0x00008000; - // Add new capabilities here, continuing with the least significant unused bit. -} - -// The health of the Agent and sub-components -// Status: [Beta] -message ComponentHealth { - // Set to true if the component is up and healthy. - bool healthy = 1; - - // Timestamp since the component is up, i.e. when the component was started. - // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. - // If the component is not running MUST be set to 0. - fixed64 start_time_unix_nano = 2; - - // Human-readable error message if the component is in erroneous state. SHOULD be set - // when healthy==false. - string last_error = 3; - - // Component status represented as a string. The status values are defined by agent-specific - // semantics and not at the protocol level. - string status = 4; - - // The time when the component status was observed. Value is UNIX Epoch time in - // nanoseconds since 00:00:00 UTC on 1 January 1970. - fixed64 status_time_unix_nano = 5; - - // A map to store more granular, sub-component health. It can nest as deeply as needed to - // describe the underlying system. - map component_health_map = 6; -} - -message EffectiveConfig { - // The effective config of the Agent. - AgentConfigMap config_map = 1; -} - -message RemoteConfigStatus { - // The hash of the remote config that was last received by this Agent in the - // AgentRemoteConfig.config_hash field. - // The Server SHOULD compare this hash with the config hash - // it has for the Agent and if the hashes are different the Server MUST include - // the remote_config field in the response in the ServerToAgent message. - bytes last_remote_config_hash = 1; - - RemoteConfigStatuses status = 2; - - // Optional error message if status==FAILED. - string error_message = 3; -} - -// Status: [Development] -message ConnectionSettingsStatus { - // The hash of the connection settings that was last recieved by this Agent - // in the connection_settings.hash field. The Server SHOULD compare this - // hash with the OfferedConnectionSettings hash it has for the Agent and if - // the hashes are different the Server MUST include the connection_settings - // field in the response in the ServerToAgent message. - bytes last_connection_settings_hash = 1; - - ConnectionSettingsStatuses status = 2; - - // Optional error message if status==FAILED. - string error_message = 3; -} - -// Status: [Development] -enum ConnectionSettingsStatuses { - // The value of status field is not set. - ConnectionSettingsStatuses_UNSET = 0; - - // ConnectionSettings were successfully applied by the Agent. - ConnectionSettingsStatuses_APPLIED = 1; - - // Agent is currently applying the ConnectionSettings that it received. - ConnectionSettingsStatuses_APPLYING = 2; - - // Agent tried to apply the ConnectionSettings it received earlier, but failed. - // See error_message for more details. - ConnectionSettingsStatuses_FAILED = 3; -} - -enum RemoteConfigStatuses { - // The value of status field is not set. - RemoteConfigStatuses_UNSET = 0; - - // Remote config was successfully applied by the Agent. - RemoteConfigStatuses_APPLIED = 1; - - // Agent is currently applying the remote config that it received earlier. - RemoteConfigStatuses_APPLYING = 2; - - // Agent tried to apply the config received earlier, but it failed. - // See error_message for more details. - RemoteConfigStatuses_FAILED = 3; -} - -// The PackageStatuses message describes the status of all packages that the Agent -// has or was offered. -// Status: [Beta] -message PackageStatuses { - // A map of PackageStatus messages, where the keys are package names. - // The key MUST match the name field of PackageStatus message. - map packages = 1; - - // The aggregate hash of all packages that this Agent previously received from the - // Server via PackagesAvailable message. - // - // The Server SHOULD compare this hash to the aggregate hash of all packages that - // it has for this Agent and if the hashes are different the Server SHOULD send - // an PackagesAvailable message to the Agent. - bytes server_provided_all_packages_hash = 2; - - // This field is set if the Agent encountered an error when processing the - // PackagesAvailable message and that error is not related to any particular single - // package. - // The field must be unset is there were no processing errors. - string error_message = 3; -} - -// The status of a single package. -// Status: [Beta] -message PackageStatus { - // Package name. MUST be always set and MUST match the key in the packages field - // of PackageStatuses message. - string name = 1; - - // The version of the package that the Agent has. - // MUST be set if the Agent has this package. - // MUST be empty if the Agent does not have this package. This may be the case - // for example if the package was offered by the Server but failed to install - // and the Agent did not have this package previously. - string agent_has_version = 2; - - // The hash of the package that the Agent has. - // MUST be set if the Agent has this package. - // MUST be empty if the Agent does not have this package. This may be the case for - // example if the package was offered by the Server but failed to install and the - // Agent did not have this package previously. - bytes agent_has_hash = 3; - - // The version of the package that the Server offered to the Agent. - // MUST be set if the installation of the package is initiated by an earlier offer - // from the Server to install this package. - // - // MUST be empty if the Agent has this package but it was installed locally and - // was not offered by the Server. - // - // Note that it is possible for both agent_has_version and server_offered_version - // fields to be set and to have different values. This is for example possible if - // the Agent already has a version of the package successfully installed, the Server - // offers a different version, but the Agent fails to install that version. - string server_offered_version = 4; - - // The hash of the package that the Server offered to the Agent. - // MUST be set if the installation of the package is initiated by an earlier - // offer from the Server to install this package. - // - // MUST be empty if the Agent has this package but it was installed locally and - // was not offered by the Server. - // - // Note that it is possible for both agent_has_hash and server_offered_hash - // fields to be set and to have different values. This is for example possible if - // the Agent already has a version of the package successfully installed, the - // Server offers a different version, but the Agent fails to install that version. - bytes server_offered_hash = 5; - - PackageStatusEnum status = 6; - - // Error message if the status is erroneous. - string error_message = 7; - - // Optional details that may be of interest to a user. - // Should only be set if status is Downloading. - // Status: [Development] - PackageDownloadDetails download_details = 8; -} - - -// Additional details that an agent can use to describe an in-progress package download. -// Status: [Development] -message PackageDownloadDetails { - // The package download progress as a percentage. - double download_percent = 1; - - // The current package download rate in bytes per second. - double download_bytes_per_second = 2; -} - -// The status of this package. -// Status: [Beta] -enum PackageStatusEnum { - // Package is successfully installed by the Agent. - // The error_message field MUST NOT be set. - PackageStatusEnum_Installed = 0; - - // Installation of this package has not yet started. - PackageStatusEnum_InstallPending = 1; - - // Agent is currently installing the package. - // server_offered_hash field MUST be set to indicate the version that the - // Agent is installing. The error_message field MUST NOT be set. - PackageStatusEnum_Installing = 2; - - // Agent tried to install the package but installation failed. - // server_offered_hash field MUST be set to indicate the version that the Agent - // tried to install. The error_message may also contain more details about - // the failure. - PackageStatusEnum_InstallFailed = 3; - - // Agent is currently downloading the package. - // server_offered_hash field MUST be set to indicate the version that the - // Agent is installing. The error_message field MUST NOT be set. - // Status: [Development] - PackageStatusEnum_Downloading = 4; -} - -// Properties related to identification of the Agent, which can be overridden -// by the Server if needed -message AgentIdentification { - // When new_instance_uid is set, Agent MUST update instance_uid - // to the value provided and use it for all further communication. - // MUST be 16 bytes long and SHOULD be generated using the UUID v7 spec. - bytes new_instance_uid = 1; -} - -///////////////////////////////////////////////////////////////////////////////////// -// Config messages -///////////////////////////////////////////////////////////////////////////////////// - -message AgentRemoteConfig { - // Agent config offered by the management Server to the Agent instance. SHOULD NOT be - // set if the config for this Agent has not changed since it was last requested (i.e. - // AgentConfigRequest.last_remote_config_hash field is equal to - // AgentConfigResponse.config_hash field). - AgentConfigMap config = 1; - - // Hash of "config". The Agent SHOULD include this value in subsequent - // RemoteConfigStatus messages in the last_remote_config_hash field. This in turn - // allows the management Server to identify that a new config is available for the Agent. - // - // This field MUST be always set if the management Server supports remote configuration - // of agents. - // - // Management Server must choose a hashing function that guarantees lack of hash - // collisions in practice. - bytes config_hash = 2; -} - -message AgentConfigMap { - // Map of configs. Keys are config file names or config section names. - // The configuration is assumed to be a collection of one or more named config files - // or sections. - // For agents that use a single config file or section the map SHOULD contain a single - // entry and the key may be an empty string. - map config_map = 1; -} - -message AgentConfigFile { - // Config file or section body. The content, format and encoding depends on the Agent - // type. The content_type field may optionally describe the MIME type of the body. - bytes body = 1; - - // Optional MIME Content-Type that describes what's in the body field, for - // example "text/yaml". - string content_type = 2; -} - -///////////////////////////////////////////////////////////////////////////////////// -// Custom messages -///////////////////////////////////////////////////////////////////////////////////// - -message CustomCapabilities { - // A list of custom capabilities that are supported. Each capability is a reverse FQDN - // with optional version information that uniquely identifies the custom capability - // and should match a capability specified in a supported CustomMessage. - // Status: [Development] - repeated string capabilities = 1; -} - -message CustomMessage { - // A reverse FQDN that uniquely identifies the capability and matches one of the - // capabilities in the CustomCapabilities message. - // Status: [Development] - string capability = 1; - - // Type of message within the capability. The capability defines the types of custom - // messages that are used to implement the capability. The type must only be unique - // within the capability. - // Status: [Development] - string type = 2; - - // Binary data of the message. The capability must specify the format of the contents - // of the data for each custom message type it defines. - // Status: [Development] - bytes data = 3; -} diff --git a/controller/sketch_capabilities.yml b/controller/sketch_capabilities.yml deleted file mode 100644 index c3daa3cb..00000000 --- a/controller/sketch_capabilities.yml +++ /dev/null @@ -1,66 +0,0 @@ -# Sketch capability profiles. -# -# These numbers describe the performance characteristics of each sketch -# implementation. The optimizer and physical planner use them to decide -# which sketch fits within a stage's budget. -# -# Source: initial values are estimates. Run benchmarks to calibrate: -# ./e2esdkbench --sketch-type=ddsketch --series=1000 --duration=60s -# Then update this file with measured values. -# -# Env var: CONTROLLER_SKETCH_CAPABILITIES=path/to/this/file.yml - -ddsketch: - insert_throughput: 10000000.0 # samples/sec (single core) - query_throughput: 50000000.0 # queries/sec (single core) - memory_bytes_per_series: 4096 # bytes - cpu_micros_per_insert: 0.1 # µs/sample - transmission_bytes: 4096 # bytes per flush - supported_intents: [quantile, extrema] - mergeable: true - supports_delta: true - supports_sliding_window: false - -kll: - insert_throughput: 5000000.0 - query_throughput: 20000000.0 - memory_bytes_per_series: 8192 - cpu_micros_per_insert: 0.2 - transmission_bytes: 8192 - supported_intents: [quantile, extrema] - mergeable: true - supports_delta: false # KLL uses random compaction, not additive - supports_sliding_window: false - -hll: - insert_throughput: 20000000.0 - query_throughput: 100000000.0 - memory_bytes_per_series: 16384 # 2^14 registers - cpu_micros_per_insert: 0.05 - transmission_bytes: 16384 - supported_intents: [cardinality] - mergeable: true - supports_delta: true - supports_sliding_window: false - -count_sketch: - insert_throughput: 8000000.0 - query_throughput: 10000000.0 - memory_bytes_per_series: 80000 # 2000 width × 5 depth × 8 bytes - cpu_micros_per_insert: 0.5 - transmission_bytes: 80000 - supported_intents: [frequency] - mergeable: true - supports_delta: true - supports_sliding_window: false - -count_min_sketch: - insert_throughput: 8000000.0 - query_throughput: 10000000.0 - memory_bytes_per_series: 80000 - cpu_micros_per_insert: 0.5 - transmission_bytes: 80000 - supported_intents: [frequency] - mergeable: true - supports_delta: true - supports_sliding_window: false diff --git a/controller/sketch_params_default.yml b/controller/sketch_params_default.yml deleted file mode 100644 index 9e16336f..00000000 --- a/controller/sketch_params_default.yml +++ /dev/null @@ -1,36 +0,0 @@ -# Sketch parameter defaults for the controller planner. -# -# All values here are the compiled-in defaults. Override any field to -# customise sketch sizing without recompiling. -# -# Env var: CONTROLLER_SKETCH_DEFAULTS=path/to/this/file.yaml - -# Quantile grid used when the query does not specify explicit φ values. -quantile_grid: [0.0, 0.25, 0.5, 0.75, 0.9, 0.99, 1.0] - -ddsketch: - # DDSketch relative accuracy (α). Lower = more buckets = higher memory. - relative_accuracy: 0.01 - -kll: - # Minimum k parameter (clamped from ⌈1/accuracy_sla⌉). - min_k: 32 - -hll: - # HLL precision for coarse SLA (accuracy > precision_threshold). - precision_coarse: 10 - # HLL precision for fine SLA (accuracy ≤ precision_threshold). - precision_fine: 14 - # SLA boundary between coarse and fine precision bands. - precision_threshold: 0.02 - -count_sketch: - # Relative error bound (ε ≈ 1/√cols for equivalent width). - epsilon: 0.022 - # Error probability (δ ≈ e^(−rows) for equivalent depth). - delta: 0.007 - -count_min_sketch: - rows: 5 - cols: 2048 - metric_name: "countsketch_partition" diff --git a/controller/src/accuracy.rs b/controller/src/accuracy.rs deleted file mode 100644 index bd7aaa53..00000000 --- a/controller/src/accuracy.rs +++ /dev/null @@ -1,257 +0,0 @@ -//! Theoretical accuracy profile of a chosen `SketchParams`. -//! -//! Mirrors `ASAPQuery-backend/src/stores/sketch_db/accuracy.rs` — -//! both sides compute the same (ε, δ) from the same sketch type -//! + parameters, so a plan the controller validates here meets -//! the same bound the backend will later surface on query -//! responses. The two files must be kept in lockstep; a golden -//! test at the bottom of this module pins the numeric parity. -//! -//! ## Why the controller needs this -//! -//! The planner picks `SketchParams` (width/depth/K/α/precision) -//! to meet a user-supplied `accuracy_sla`. Today the cost model -//! approximates the bound inline in a few places -//! (`cost_model.rs:156-158`). Centralising the derivation here: -//! -//! * lets `RulesPlanner` / `CostModelPlanner` / any future -//! planner compute the post-hoc ε of the chosen plan and -//! verify it actually meets the SLA. -//! * surfaces the bound to downstream systems (backend via -//! `/api/v1/plan` response; dashboards via -//! `asap_otel_processor_accuracy_epsilon` gauge) without the -//! caller re-deriving from scratch. -//! -//! ## Bounds we encode -//! -//! | Sketch | `kind` | ε formula | δ formula | -//! |-------------------|-----------------------|----------------------|--------------------| -//! | CountMinSketch | `AdditiveFrequency` | e / w | 1 / 2^d | -//! | CountSketch(ε, δ) | `AdditiveFrequency` | ε (user-supplied) | δ (user-supplied) | -//! | HLL(p) | `RelativeCardinality` | 1.04 / √(2^p) | — (Gaussian σ) | -//! | KLL(k) | `RankQuantile` | 2.296 / √k | 0.01 (fixed) | -//! | DDSketch(α) | `RelativeQuantile` | α | 0 (deterministic) | -//! -//! Citations (verbatim from the backend): -//! * CMS — Cormode & Muthukrishnan, *J. Algorithms* 55(1) 2005 -//! * CountSketch — Charikar, Chen, Farach-Colton, ICALP 2002 -//! * HLL — Flajolet et al., DMTCS 2007 -//! * KLL — Karnin, Lang, Liberty, FOCS 2016 -//! * DDSketch — Masson, Rim, Lee, VLDB 2019 - -use serde::{Deserialize, Serialize}; - -use crate::types::SketchParams; - -/// How to interpret [`AccuracyProfile::epsilon`]. Same snake_case -/// serialisation as the backend's `AccuracyKind` so JSON records -/// produced here can be cross-compared with backend output. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum AccuracyKind { - Exact, - AdditiveFrequency, - RelativeCardinality, - RankQuantile, - RelativeQuantile, - TopK, -} - -impl AccuracyKind { - pub fn as_str(&self) -> &'static str { - match self { - AccuracyKind::Exact => "exact", - AccuracyKind::AdditiveFrequency => "additive_frequency", - AccuracyKind::RelativeCardinality => "relative_cardinality", - AccuracyKind::RankQuantile => "rank_quantile", - AccuracyKind::RelativeQuantile => "relative_quantile", - AccuracyKind::TopK => "top_k", - } - } -} - -/// Theoretical accuracy bound for one `SketchParams`. -/// Serialises identically to the backend's `AccuracyProfile` so a -/// controller-produced JSON record can be diffed against a -/// backend-produced one byte-for-byte. -#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] -pub struct AccuracyProfile { - pub epsilon: f64, - pub delta: f64, - pub kind: AccuracyKind, -} - -impl AccuracyProfile { - /// Exact (ε = δ = 0). - pub fn exact() -> Self { - Self { - epsilon: 0.0, - delta: 0.0, - kind: AccuracyKind::Exact, - } - } - - /// One-line summary for ops dashboards / logs. Byte-equal to - /// the backend's `AccuracyProfile::summary`. - pub fn summary(&self) -> String { - format!( - "accuracy: ε={}, δ={}, kind={}", - self.epsilon, - self.delta, - self.kind.as_str() - ) - } - - /// Derive the theoretical bound for a given `SketchParams`. - pub fn derive(params: &SketchParams) -> Self { - match params { - SketchParams::CountMinSketch { rows, cols, .. } => { - let cols = (*cols).max(1) as f64; - let rows = (*rows).max(1) as i32; - Self { - epsilon: std::f64::consts::E / cols, - delta: 0.5_f64.powi(rows), - kind: AccuracyKind::AdditiveFrequency, - } - } - SketchParams::CountSketch { epsilon, delta } => Self { - epsilon: *epsilon, - delta: *delta, - kind: AccuracyKind::AdditiveFrequency, - }, - SketchParams::HLL { precision } => { - let m = (1u64 << precision) as f64; - Self { - epsilon: 1.04 / m.sqrt(), - delta: 0.0, - kind: AccuracyKind::RelativeCardinality, - } - } - SketchParams::KLL { k, .. } => { - let k = (*k).max(1) as f64; - Self { - epsilon: 2.296 / k.sqrt(), - delta: 0.01, - kind: AccuracyKind::RankQuantile, - } - } - SketchParams::DDSketch { - relative_accuracy, .. - } => Self { - epsilon: *relative_accuracy, - delta: 0.0, - kind: AccuracyKind::RelativeQuantile, - }, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn cms_bound_is_e_over_w() { - let p = AccuracyProfile::derive(&SketchParams::CountMinSketch { - rows: 3, - cols: 1000, - metric_name: "m".into(), - }); - assert_eq!(p.kind, AccuracyKind::AdditiveFrequency); - assert!((p.epsilon - std::f64::consts::E / 1000.0).abs() < 1e-12); - assert!((p.delta - 0.125).abs() < 1e-12); - } - - #[test] - fn countsketch_passes_through_user_supplied_bounds() { - let p = AccuracyProfile::derive(&SketchParams::CountSketch { - epsilon: 0.01, - delta: 0.001, - }); - assert_eq!(p.kind, AccuracyKind::AdditiveFrequency); - assert_eq!(p.epsilon, 0.01); - assert_eq!(p.delta, 0.001); - } - - #[test] - fn hll_p14_matches_flajolet_bound() { - let p = AccuracyProfile::derive(&SketchParams::HLL { precision: 14 }); - assert_eq!(p.kind, AccuracyKind::RelativeCardinality); - // 1.04 / √16384 = 0.008125 - assert!((p.epsilon - 0.008125).abs() < 1e-9); - } - - #[test] - fn kll_k200_matches_karnin_lang_liberty_bound() { - let p = AccuracyProfile::derive(&SketchParams::KLL { - k: 200, - quantiles: vec![0.5, 0.95, 0.99], - }); - assert_eq!(p.kind, AccuracyKind::RankQuantile); - assert!((p.epsilon - 2.296 / 200.0_f64.sqrt()).abs() < 1e-12); - assert!((p.delta - 0.01).abs() < 1e-12); - } - - #[test] - fn ddsketch_alpha_passes_through_verbatim() { - for alpha in [0.005, 0.01, 0.02, 0.05] { - let p = AccuracyProfile::derive(&SketchParams::DDSketch { - relative_accuracy: alpha, - quantiles: vec![0.5, 0.99], - }); - assert_eq!(p.kind, AccuracyKind::RelativeQuantile); - assert_eq!(p.epsilon, alpha); - assert_eq!(p.delta, 0.0); - } - } - - /// **Parity contract** with the backend. Pinned numeric - /// values are identical to what `ASAPQuery-backend`'s - /// `AccuracyProfile::derive` produces for the matching - /// `AggregationConfig`. Any change that drifts these tests - /// likely needs a matching change on the backend side — - /// and vice versa. - #[test] - fn golden_parity_with_backend() { - // HLL precision=14: ε = 0.008125, kind = relative_cardinality - let p = AccuracyProfile::derive(&SketchParams::HLL { precision: 14 }); - assert_eq!(p.summary(), "accuracy: ε=0.008125, δ=0, kind=relative_cardinality"); - - // KLL k=200: ε = 2.296 / √200, kind = rank_quantile, δ = 0.01 - let p = AccuracyProfile::derive(&SketchParams::KLL { - k: 200, - quantiles: vec![], - }); - let expected_eps = 2.296_f64 / 200.0_f64.sqrt(); - assert_eq!( - p.summary(), - format!("accuracy: ε={}, δ=0.01, kind=rank_quantile", expected_eps) - ); - - // DDSketch α=0.02: passes verbatim, δ=0, kind=relative_quantile - let p = AccuracyProfile::derive(&SketchParams::DDSketch { - relative_accuracy: 0.02, - quantiles: vec![], - }); - assert_eq!(p.summary(), "accuracy: ε=0.02, δ=0, kind=relative_quantile"); - } - - #[test] - fn kind_serialises_to_snake_case() { - let p = AccuracyProfile::derive(&SketchParams::HLL { precision: 14 }); - let json = serde_json::to_string(&p).unwrap(); - assert!(json.contains("\"kind\":\"relative_cardinality\"")); - } - - #[test] - fn round_trip_through_serde() { - let src = AccuracyProfile { - epsilon: 0.008125, - delta: 0.0, - kind: AccuracyKind::RelativeCardinality, - }; - let json = serde_json::to_string(&src).unwrap(); - let back: AccuracyProfile = serde_json::from_str(&json).unwrap(); - assert_eq!(back, src); - } -} diff --git a/controller/src/algebra/allocator.rs b/controller/src/algebra/allocator.rs deleted file mode 100644 index fe57bead..00000000 --- a/controller/src/algebra/allocator.rs +++ /dev/null @@ -1,946 +0,0 @@ -//! Sketch allocator — converts a [`QueryExpr`] tree into an annotated -//! [`PlanNode`] tree, assigning every operator to a [`PipelineStage`] and -//! choosing between sketch and exact execution. -//! -//! # Algorithm -//! -//! 1. Walk the tree bottom-up (children before parents). -//! 2. For each node, determine the *preferred* stage using the rules below. -//! 3. If the preferred stage exceeds its resource budget, demote to the next -//! stage in the chain: `Agent → Backend → Precompute → Db`. -//! 4. Annotate the node with the chosen sketch type, delta-encoding flag, -//! and a human-readable rationale. -//! -//! ## Stage assignment rules -//! -//! | Node type | Default stage | Condition | -//! |-----------|---------------|-----------| -//! | Source, Filter, Window, Partition, Dedup | Agent | Always | -//! | SketchAgg (sketachable op, mergeable) | Agent | budget OK | -//! | SketchAgg (sketchable, mergeable) | Backend | agent budget exceeded | -//! | SketchAgg (sketchable, not mergeable: Avg) | Db | always | -//! | SketchAgg (exact: Sum/Count/Min/Max) | Backend | mergeable | -//! | TopK | Precompute | always | -//! | Merge, JoinSketch | Backend | always | -//! | Aggregate, Project, Sort, Limit | Db | always | -//! | WindowFunc, HistogramQuantile | Db | always | -//! | PromQLSubquery, BinaryOp | Precompute | has sketch children | -//! | LetBinding, Subquery | same as body/inner | propagated | - -use super::expr::QueryExpr; -use super::plan::{ - CostEstimate, ExecutionMode, NodeAnnotation, PipelineStage, PlanNode, -}; -use super::expr::{AggIntent, ExactAgg}; -use crate::types::{SketchType, StageResourceBudgets}; - -// ── Resource budget tracker ─────────────────────────────────────────────────── - -/// Mutable budget state, consumed during allocation. -#[derive(Debug, Clone)] -struct BudgetState { - agent_memory_remaining_bytes: f64, - backend_memory_remaining_bytes: f64, -} - -impl BudgetState { - fn from_budgets(b: &StageResourceBudgets) -> Self { - Self { - agent_memory_remaining_bytes: b.agent_memory_bytes - .map(|v| v as f64) - .unwrap_or(f64::INFINITY), - backend_memory_remaining_bytes: b.backend_memory_bytes - .map(|v| v as f64) - .unwrap_or(f64::INFINITY), - } - } - - fn fits_agent(&self, bytes: f64) -> bool { - bytes <= self.agent_memory_remaining_bytes - } - - fn fits_backend(&self, bytes: f64) -> bool { - bytes <= self.backend_memory_remaining_bytes - } - - fn consume_agent(&mut self, bytes: f64) { - self.agent_memory_remaining_bytes = - (self.agent_memory_remaining_bytes - bytes).max(0.0); - } - - fn consume_backend(&mut self, bytes: f64) { - self.backend_memory_remaining_bytes = - (self.backend_memory_remaining_bytes - bytes).max(0.0); - } -} - -// ── Public allocator ────────────────────────────────────────────────────────── - -/// Converts a (pre-optimised) [`QueryExpr`] tree into an annotated -/// [`PlanNode`] tree. -pub struct SketchAllocator { - budgets: StageResourceBudgets, - raw_bytes_per_sec: f64, -} - -impl SketchAllocator { - /// Create an allocator. - /// - /// * `budgets` — per-stage memory caps (from [`StageResourceBudgets`]). - /// * `raw_bytes_per_sec` — baseline bandwidth of the raw OTLP stream, - /// used to estimate compression ratios. - pub fn new(budgets: StageResourceBudgets, raw_bytes_per_sec: f64) -> Self { - Self { budgets, raw_bytes_per_sec } - } - - /// Allocate stages for the entire expression tree. - pub fn allocate(&self, expr: QueryExpr) -> PlanNode { - let mut budget = BudgetState::from_budgets(&self.budgets); - self.alloc_node(expr, &mut budget) - } - - // ── Recursive allocation ────────────────────────────────────────────────── - - fn alloc_node(&self, expr: QueryExpr, budget: &mut BudgetState) -> PlanNode { - match expr { - // ── Leaves ─────────────────────────────────────────────────────── - QueryExpr::Source(_) | QueryExpr::Ref(_) => PlanNode::leaf( - expr, - PipelineStage::Agent, - ExecutionMode::Passthrough, - ), - - // ── Structural / filter nodes — always Agent ────────────────── - QueryExpr::Filter { pred, input } => { - let child = self.alloc_node(*input, budget); - let stage = PipelineStage::Agent; - PlanNode { - expr: QueryExpr::Filter { pred, input: Box::new(child.expr.clone()) }, - stage, - mode: ExecutionMode::Passthrough, - cost: CostEstimate { - bytes_per_sec: self.raw_bytes_per_sec * 0.5, - ..Default::default() - }, - annotation: NodeAnnotation { - rationale: "Filter pushed to Agent to reduce data volume early".into(), - ..Default::default() - }, - children: vec![child], - } - } - - QueryExpr::Window { duration, slide, input } => { - let child = self.alloc_node(*input, budget); - PlanNode { - expr: QueryExpr::Window { - duration, slide, - input: Box::new(child.expr.clone()), - }, - stage: PipelineStage::Agent, - mode: ExecutionMode::Passthrough, - cost: CostEstimate::default(), - annotation: NodeAnnotation { - rationale: "Time window computed at Agent".into(), - ..Default::default() - }, - children: vec![child], - } - } - - QueryExpr::Partition { keys, input } => { - let child = self.alloc_node(*input, budget); - PlanNode { - expr: QueryExpr::Partition { - keys, - input: Box::new(child.expr.clone()), - }, - stage: PipelineStage::Agent, - mode: ExecutionMode::Passthrough, - cost: CostEstimate::default(), - annotation: NodeAnnotation { - rationale: "Partition for GROUP BY at Agent".into(), - ..Default::default() - }, - children: vec![child], - } - } - - QueryExpr::Dedup { col, input } => { - let child = self.alloc_node(*input, budget); - PlanNode { - expr: QueryExpr::Dedup { - col, - input: Box::new(child.expr.clone()), - }, - stage: PipelineStage::Agent, - mode: ExecutionMode::Passthrough, - cost: CostEstimate::default(), - annotation: NodeAnnotation { - rationale: "Dedup at Agent before sketch build".into(), - ..Default::default() - }, - children: vec![child], - } - } - - // ── Sketch aggregation — core allocation logic ──────────────── - QueryExpr::SketchAgg { op, col, input } => { - let child = self.alloc_node(*input, budget); - self.alloc_sketch_agg(op, col, child, budget) - } - - // ── WindowedAgg — treat as SketchAgg (window is informational) ── - QueryExpr::WindowedAgg { agg, window: _, col, input } => { - let child = self.alloc_node(*input, budget); - self.alloc_sketch_agg(agg, col, child, budget) - } - - // ── TopK — Precompute engine ────────────────────────────────── - QueryExpr::TopK { k, by, input } => { - let child = self.alloc_node(*input, budget); - PlanNode { - expr: QueryExpr::TopK { - k, by, - input: Box::new(child.expr.clone()), - }, - stage: PipelineStage::Precompute, - mode: ExecutionMode::Sketch, - cost: CostEstimate { - bytes_per_sec: self.raw_bytes_per_sec * 0.05, - memory_bytes: (k as f64) * 64.0, - ..Default::default() - }, - annotation: NodeAnnotation { - sketch_type: Some(SketchType::CountSketch), - rationale: "TopK assigned to Precompute engine (CountSketch)".into(), - ..Default::default() - }, - children: vec![child], - } - } - - // ── Merge — Backend ─────────────────────────────────────────── - QueryExpr::Merge { inputs } => { - let children: Vec = inputs - .into_iter() - .map(|inp| self.alloc_node(inp, budget)) - .collect(); - let mem: f64 = children.iter().map(|c| c.cost.memory_bytes).sum(); - PlanNode { - expr: QueryExpr::Merge { - inputs: children.iter().map(|c| c.expr.clone()).collect(), - }, - stage: PipelineStage::Backend, - mode: ExecutionMode::Passthrough, - cost: CostEstimate { - bytes_per_sec: self.raw_bytes_per_sec * 0.1, - memory_bytes: mem, - ..Default::default() - }, - annotation: NodeAnnotation { - rationale: "Sketch merge at Backend".into(), - ..Default::default() - }, - children, - } - } - - QueryExpr::JoinSketch { join_key, outer, inner } => { - let outer_node = self.alloc_node(*outer, budget); - let inner_node = self.alloc_node(*inner, budget); - PlanNode { - expr: QueryExpr::JoinSketch { - join_key, - outer: Box::new(outer_node.expr.clone()), - inner: Box::new(inner_node.expr.clone()), - }, - stage: PipelineStage::Backend, - mode: ExecutionMode::Passthrough, - cost: CostEstimate { - bytes_per_sec: self.raw_bytes_per_sec * 0.2, - ..Default::default() - }, - annotation: NodeAnnotation { - rationale: "JoinSketch at Backend: pre-agg inner then merge".into(), - ..Default::default() - }, - children: vec![outer_node, inner_node], - } - } - - // ── Exact / relational — Db ─────────────────────────────────── - QueryExpr::Aggregate { keys, aggs, having, input } => { - let child = self.alloc_node(*input, budget); - PlanNode { - expr: QueryExpr::Aggregate { - keys, aggs, having, - input: Box::new(child.expr.clone()), - }, - stage: PipelineStage::Db, - mode: ExecutionMode::Exact, - cost: CostEstimate { - bytes_per_sec: self.raw_bytes_per_sec, - ..Default::default() - }, - annotation: NodeAnnotation { - rationale: "General Aggregate at Db (exact)".into(), - ..Default::default() - }, - children: vec![child], - } - } - - QueryExpr::Project { cols, input } => { - let child = self.alloc_node(*input, budget); - PlanNode { - expr: QueryExpr::Project { - cols, - input: Box::new(child.expr.clone()), - }, - stage: PipelineStage::Db, - mode: ExecutionMode::Exact, - cost: CostEstimate::default(), - annotation: NodeAnnotation { - rationale: "Project at Db".into(), - ..Default::default() - }, - children: vec![child], - } - } - - QueryExpr::Sort { keys, input } => { - let child = self.alloc_node(*input, budget); - PlanNode { - expr: QueryExpr::Sort { - keys, - input: Box::new(child.expr.clone()), - }, - stage: PipelineStage::Db, - mode: ExecutionMode::Exact, - cost: CostEstimate::default(), - annotation: NodeAnnotation { - rationale: "Sort at Db".into(), - ..Default::default() - }, - children: vec![child], - } - } - - QueryExpr::Limit { n, offset, input } => { - let child = self.alloc_node(*input, budget); - PlanNode { - expr: QueryExpr::Limit { - n, offset, - input: Box::new(child.expr.clone()), - }, - stage: PipelineStage::Db, - mode: ExecutionMode::Exact, - cost: CostEstimate::default(), - annotation: NodeAnnotation { - rationale: "Limit at Db".into(), - ..Default::default() - }, - children: vec![child], - } - } - - QueryExpr::Join { kind, pred, left, right } => { - let left_node = self.alloc_node(*left, budget); - let right_node = self.alloc_node(*right, budget); - PlanNode { - expr: QueryExpr::Join { - kind, pred, - left: Box::new(left_node.expr.clone()), - right: Box::new(right_node.expr.clone()), - }, - stage: PipelineStage::Db, - mode: ExecutionMode::Exact, - cost: CostEstimate { - bytes_per_sec: self.raw_bytes_per_sec, - ..Default::default() - }, - annotation: NodeAnnotation { - rationale: "Join at Db (exact)".into(), - ..Default::default() - }, - children: vec![left_node, right_node], - } - } - - QueryExpr::SetOp { kind, all, left, right } => { - let left_node = self.alloc_node(*left, budget); - let right_node = self.alloc_node(*right, budget); - PlanNode { - expr: QueryExpr::SetOp { - kind, all, - left: Box::new(left_node.expr.clone()), - right: Box::new(right_node.expr.clone()), - }, - stage: PipelineStage::Db, - mode: ExecutionMode::Exact, - cost: CostEstimate::default(), - annotation: NodeAnnotation { - rationale: "SetOp at Db".into(), - ..Default::default() - }, - children: vec![left_node, right_node], - } - } - - QueryExpr::WindowFunc { func, partition_by, order_by, frame, input } => { - let child = self.alloc_node(*input, budget); - PlanNode { - expr: QueryExpr::WindowFunc { - func, partition_by, order_by, frame, - input: Box::new(child.expr.clone()), - }, - stage: PipelineStage::Db, - mode: ExecutionMode::Exact, - cost: CostEstimate::default(), - annotation: NodeAnnotation { - rationale: "Analytic window function at Db".into(), - ..Default::default() - }, - children: vec![child], - } - } - - // ── PromQL-specific ─────────────────────────────────────────── - QueryExpr::HistogramQuantile { phi, input } => { - let child = self.alloc_node(*input, budget); - // If the child is a sketch, elevate to Precompute; - // otherwise fall through to Db. - let stage = if child.mode == ExecutionMode::Sketch { - PipelineStage::Precompute - } else { - PipelineStage::Db - }; - let rationale = format!("histogram_quantile(φ={phi}) at {stage}"); - PlanNode { - expr: QueryExpr::HistogramQuantile { - phi, - input: Box::new(child.expr.clone()), - }, - stage, - mode: ExecutionMode::Sketch, - cost: CostEstimate { - bytes_per_sec: self.raw_bytes_per_sec * 0.02, - ..Default::default() - }, - annotation: NodeAnnotation { - sketch_type: Some(SketchType::DDSketch), - rationale, - ..Default::default() - }, - children: vec![child], - } - } - - QueryExpr::PromQLSubquery { range, resolution, input } => { - let child = self.alloc_node(*input, budget); - let stage = if child.mode == ExecutionMode::Sketch { - PipelineStage::Precompute - } else { - PipelineStage::Db - }; - let rationale = format!("PromQL subquery at {stage}"); - PlanNode { - expr: QueryExpr::PromQLSubquery { - range, resolution, - input: Box::new(child.expr.clone()), - }, - stage, - mode: child.mode.clone(), - cost: CostEstimate::default(), - annotation: NodeAnnotation { - rationale, - ..Default::default() - }, - children: vec![child], - } - } - - QueryExpr::BinaryOp { op, lhs, rhs, vector_match } => { - let left_node = self.alloc_node(*lhs, budget); - let right_node = self.alloc_node(*rhs, budget); - let has_sketch = left_node.mode == ExecutionMode::Sketch - || right_node.mode == ExecutionMode::Sketch; - let stage = if has_sketch { - PipelineStage::Precompute - } else { - PipelineStage::Db - }; - let rationale = format!("BinaryOp at {stage}"); - PlanNode { - expr: QueryExpr::BinaryOp { - op, vector_match, - lhs: Box::new(left_node.expr.clone()), - rhs: Box::new(right_node.expr.clone()), - }, - stage, - mode: if has_sketch { ExecutionMode::Sketch } else { ExecutionMode::Exact }, - cost: CostEstimate::default(), - annotation: NodeAnnotation { - rationale, - ..Default::default() - }, - children: vec![left_node, right_node], - } - } - - // ── Scoping constructs — propagate body's stage ─────────────── - QueryExpr::Subquery { alias, expr } => { - let child = self.alloc_node(*expr, budget); - let stage = child.stage.clone(); - let mode = child.mode.clone(); - PlanNode { - expr: QueryExpr::Subquery { - alias, - expr: Box::new(child.expr.clone()), - }, - stage, - mode, - cost: CostEstimate::default(), - annotation: NodeAnnotation { - rationale: "Subquery inherits inner stage".into(), - ..Default::default() - }, - children: vec![child], - } - } - - QueryExpr::LetBinding { name, expr, body } => { - let expr_node = self.alloc_node(*expr, budget); - let body_node = self.alloc_node(*body, budget); - let stage = body_node.stage.clone(); - let mode = body_node.mode.clone(); - PlanNode { - expr: QueryExpr::LetBinding { - name, - expr: Box::new(expr_node.expr.clone()), - body: Box::new(body_node.expr.clone()), - }, - stage, - mode, - cost: CostEstimate::default(), - annotation: NodeAnnotation { - rationale: "LetBinding: stage = body stage".into(), - ..Default::default() - }, - children: vec![expr_node, body_node], - } - } - } - } - - // ── SketchAgg allocation (budget-driven demotion) ───────────────────────── - - fn alloc_sketch_agg( - &self, - op: AggIntent, - col: super::expr::ColumnRef, - child: PlanNode, - budget: &mut BudgetState, - ) -> PlanNode { - // Exact non-mergeable (Avg) → always Db. - if let AggIntent::Exact(ExactAgg::Avg) = &op { - return PlanNode { - expr: QueryExpr::SketchAgg { - op, - col, - input: Box::new(child.expr.clone()), - }, - stage: PipelineStage::Db, - mode: ExecutionMode::Exact, - cost: CostEstimate { - bytes_per_sec: self.raw_bytes_per_sec, - ..Default::default() - }, - annotation: NodeAnnotation { - rationale: "Exact(Avg) is not mergeable — must run at Db".into(), - ..Default::default() - }, - children: vec![child], - }; - } - - // Exact mergeable (Sum, Count, Min, Max) → Backend. - if let AggIntent::Exact(_) = &op { - return PlanNode { - expr: QueryExpr::SketchAgg { - op, - col, - input: Box::new(child.expr.clone()), - }, - stage: PipelineStage::Backend, - mode: ExecutionMode::Exact, - cost: CostEstimate { - bytes_per_sec: self.raw_bytes_per_sec * 0.8, - ..Default::default() - }, - annotation: NodeAnnotation { - rationale: "Exact(Sum/Count/Min/Max) merged at Backend".into(), - ..Default::default() - }, - children: vec![child], - }; - } - - // Sketch operators: resolve to physical, then try Agent → Backend → Precompute. - let physical = super::physical::resolve(&op); - let mem = estimated_sketch_memory(&op); - let (sketch_type, params) = (physical.sketch_type, physical.sketch_params); - - if budget.fits_agent(mem) { - budget.consume_agent(mem); - return PlanNode { - expr: QueryExpr::SketchAgg { - op, - col, - input: Box::new(child.expr.clone()), - }, - stage: PipelineStage::Agent, - mode: ExecutionMode::Sketch, - cost: CostEstimate { - bytes_per_sec: self.raw_bytes_per_sec * 0.05, - memory_bytes: mem, - compression_ratio: 20.0, - ..Default::default() - }, - annotation: NodeAnnotation { - sketch_type: Some(sketch_type), - sketch_params: Some(params), - rationale: "Sketch at Agent (within budget)".into(), - ..Default::default() - }, - children: vec![child], - }; - } - - if budget.fits_backend(mem) { - budget.consume_backend(mem); - return PlanNode { - expr: QueryExpr::SketchAgg { - op, - col, - input: Box::new(child.expr.clone()), - }, - stage: PipelineStage::Backend, - mode: ExecutionMode::Sketch, - cost: CostEstimate { - bytes_per_sec: self.raw_bytes_per_sec * 0.1, - memory_bytes: mem, - compression_ratio: 10.0, - ..Default::default() - }, - annotation: NodeAnnotation { - sketch_type: Some(sketch_type), - sketch_params: Some(params), - rationale: "Sketch demoted to Backend (Agent budget exceeded)".into(), - budget_demotion: true, - ..Default::default() - }, - children: vec![child], - }; - } - - // Both Agent and Backend budgets exceeded → Precompute. - PlanNode { - expr: QueryExpr::SketchAgg { - op, - col, - input: Box::new(child.expr.clone()), - }, - stage: PipelineStage::Precompute, - mode: ExecutionMode::Sketch, - cost: CostEstimate { - bytes_per_sec: self.raw_bytes_per_sec * 0.2, - memory_bytes: mem, - compression_ratio: 5.0, - ..Default::default() - }, - annotation: NodeAnnotation { - sketch_type: Some(sketch_type), - sketch_params: Some(params), - rationale: "Sketch demoted to Precompute (Agent+Backend budgets exceeded)".into(), - budget_demotion: true, - ..Default::default() - }, - children: vec![child], - } - } -} - -// ── Helpers ─────────────────────────────────────────────────────────────────── - -/// Estimate the memory footprint of a sketch in bytes. -fn estimated_sketch_memory(op: &AggIntent) -> f64 { - super::directory::estimated_sketch_memory_bytes(op) as f64 -} - -// sketch_type_for_op delegated to algebra::directory::sketch_type_and_params. - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use crate::algebra::expr::QueryExpr; - use crate::algebra::plan::{ExecutionMode, PipelineStage}; - use crate::algebra::expr::{AggIntent, ColumnRef, PartitionKeys, SourceSpec}; - use crate::types::{SketchType, StageResourceBudgets}; - use std::time::Duration; - - fn src(name: &str) -> QueryExpr { - QueryExpr::Source(SourceSpec { name: name.into() }) - } - - fn alloc(budgets: StageResourceBudgets, expr: QueryExpr) -> PlanNode { - SketchAllocator::new(budgets, 100_000.0).allocate(expr) - } - - fn unlimited() -> StageResourceBudgets { - StageResourceBudgets::default() - } - - fn tight_agent() -> StageResourceBudgets { - StageResourceBudgets { - agent_memory_bytes: Some(1), // 1 byte — too small for any sketch - ..Default::default() - } - } - - fn tight_all() -> StageResourceBudgets { - StageResourceBudgets { - agent_memory_bytes: Some(1), - backend_memory_bytes: Some(1), - ..Default::default() - } - } - - // ── Source / leaf ───────────────────────────────────────────────────────── - - #[test] - fn source_goes_to_agent() { - let node = alloc(unlimited(), src("cpu")); - assert_eq!(node.stage, PipelineStage::Agent); - assert_eq!(node.mode, ExecutionMode::Passthrough); - } - - // ── Filter ──────────────────────────────────────────────────────────────── - - #[test] - fn filter_at_agent() { - use crate::algebra::expr::{LiteralValue, ScalarExpr}; - let expr = QueryExpr::Filter { - pred: ScalarExpr::Literal(LiteralValue::Bool(true)), - input: Box::new(src("m")), - }; - let node = alloc(unlimited(), expr); - assert_eq!(node.stage, PipelineStage::Agent); - } - - // ── DDSketch within budget → Agent ──────────────────────────────────────── - - #[test] - fn ddsketch_within_budget_goes_to_agent() { - let expr = QueryExpr::SketchAgg { - op: AggIntent::default_quantile(vec![0.99]), - col: ColumnRef::SampleValue, - input: Box::new(src("latency")), - }; - let node = alloc(unlimited(), expr); - assert_eq!(node.stage, PipelineStage::Agent); - assert_eq!(node.mode, ExecutionMode::Sketch); - assert_eq!(node.annotation.sketch_type, Some(SketchType::DDSketch)); - } - - // ── DDSketch tight agent budget → Backend ───────────────────────────────── - - #[test] - fn ddsketch_agent_budget_exceeded_goes_to_backend() { - let expr = QueryExpr::SketchAgg { - op: AggIntent::default_quantile(vec![0.99]), - col: ColumnRef::SampleValue, - input: Box::new(src("latency")), - }; - let node = alloc(tight_agent(), expr); - assert_eq!(node.stage, PipelineStage::Backend); - assert!(node.annotation.budget_demotion); - } - - // ── DDSketch tight agent+backend → Precompute ───────────────────────────── - - #[test] - fn ddsketch_all_budgets_exceeded_goes_to_precompute() { - let expr = QueryExpr::SketchAgg { - op: AggIntent::default_quantile(vec![0.99]), - col: ColumnRef::SampleValue, - input: Box::new(src("latency")), - }; - let node = alloc(tight_all(), expr); - assert_eq!(node.stage, PipelineStage::Precompute); - assert!(node.annotation.budget_demotion); - } - - // ── Exact(Avg) → Db ─────────────────────────────────────────────────────── - - #[test] - fn exact_avg_goes_to_db() { - let expr = QueryExpr::SketchAgg { - op: AggIntent::Exact(ExactAgg::Avg), - col: ColumnRef::Named("price".into()), - input: Box::new(src("trades")), - }; - let node = alloc(unlimited(), expr); - assert_eq!(node.stage, PipelineStage::Db); - assert_eq!(node.mode, ExecutionMode::Exact); - } - - // ── Exact(Sum) → Backend ────────────────────────────────────────────────── - - #[test] - fn exact_sum_goes_to_backend() { - let expr = QueryExpr::SketchAgg { - op: AggIntent::Exact(ExactAgg::Sum), - col: ColumnRef::Named("bytes".into()), - input: Box::new(src("network")), - }; - let node = alloc(unlimited(), expr); - assert_eq!(node.stage, PipelineStage::Backend); - assert_eq!(node.mode, ExecutionMode::Exact); - } - - // ── TopK → Precompute ───────────────────────────────────────────────────── - - #[test] - fn topk_goes_to_precompute() { - let expr = QueryExpr::TopK { - k: 10, - by: vec!["symbol".into()], - input: Box::new(src("trades")), - }; - let node = alloc(unlimited(), expr); - assert_eq!(node.stage, PipelineStage::Precompute); - assert_eq!(node.annotation.sketch_type, Some(SketchType::CountSketch)); - } - - // ── Merge → Backend ─────────────────────────────────────────────────────── - - #[test] - fn merge_goes_to_backend() { - let expr = QueryExpr::Merge { - inputs: vec![src("a"), src("b")], - }; - let node = alloc(unlimited(), expr); - assert_eq!(node.stage, PipelineStage::Backend); - } - - // ── HLL → Agent ─────────────────────────────────────────────────────────── - - #[test] - fn hll_within_budget_at_agent() { - let expr = QueryExpr::SketchAgg { - op: AggIntent::default_cardinality(), - col: ColumnRef::Named("uid".into()), - input: Box::new(src("events")), - }; - let node = alloc(unlimited(), expr); - assert_eq!(node.stage, PipelineStage::Agent); - assert_eq!(node.annotation.sketch_type, Some(SketchType::HLL)); - } - - // ── Frequency → Agent ───────────────────────────────────────────────────── - - #[test] - fn frequency_within_budget_at_agent() { - let expr = QueryExpr::SketchAgg { - op: AggIntent::default_frequency(), - col: ColumnRef::Wildcard, - input: Box::new(src("requests")), - }; - let node = alloc(unlimited(), expr); - assert_eq!(node.stage, PipelineStage::Agent); - assert_eq!(node.annotation.sketch_type, Some(SketchType::CountSketch)); - } - - // ── Join → Db ───────────────────────────────────────────────────────────── - - #[test] - fn join_goes_to_db() { - use crate::algebra::expr::JoinKind; - let expr = QueryExpr::Join { - kind: JoinKind::Inner, - pred: None, - left: Box::new(src("orders")), - right: Box::new(src("items")), - }; - let node = alloc(unlimited(), expr); - assert_eq!(node.stage, PipelineStage::Db); - } - - // ── HistogramQuantile + DDSketch → Precompute ───────────────────────────── - - #[test] - fn histogram_quantile_over_sketch_at_precompute() { - let expr = QueryExpr::HistogramQuantile { - phi: 0.95, - input: Box::new(QueryExpr::SketchAgg { - op: AggIntent::default_quantile(vec![0.95]), - col: ColumnRef::SampleValue, - input: Box::new(src("hist")), - }), - }; - let node = alloc(unlimited(), expr); - assert_eq!(node.stage, PipelineStage::Precompute); - } - - // ── LetBinding inherits body stage ──────────────────────────────────────── - - #[test] - fn let_binding_inherits_body_stage() { - let expr = QueryExpr::LetBinding { - name: "base".into(), - expr: Box::new(src("cpu")), - body: Box::new(QueryExpr::TopK { - k: 5, - by: vec![], - input: Box::new(src("cpu")), - }), - }; - let node = alloc(unlimited(), expr); - assert_eq!(node.stage, PipelineStage::Precompute); - } - - // ── Memory estimate helpers ─────────────────────────────────────────────── - - #[test] - fn cardinality_memory_estimate() { - let mem = estimated_sketch_memory(&AggIntent::default_cardinality()); - assert!(mem > 0.0); - } - - #[test] - fn frequency_memory_estimate() { - let op = AggIntent::default_frequency(); - let mem = estimated_sketch_memory(&op); - assert!(mem > 0.0); - } - - // ── PlanSummary from allocated tree ────────────────────────────────────── - - #[test] - fn plan_summary_shows_bandwidth_saved() { - let expr = QueryExpr::SketchAgg { - op: AggIntent::default_quantile(vec![0.99]), - col: ColumnRef::SampleValue, - input: Box::new(src("latency")), - }; - let node = alloc(unlimited(), expr); - let summary = node.summarise(100_000.0); - // sketch reduces to ~5% → saved ~95 000 B/s - assert!(summary.bandwidth_saved_bytes_per_sec > 50_000.0); - assert!(summary.agent_memory_bytes > 0.0); - } -} diff --git a/controller/src/algebra/directory.rs b/controller/src/algebra/directory.rs deleted file mode 100644 index 925ab15d..00000000 --- a/controller/src/algebra/directory.rs +++ /dev/null @@ -1,304 +0,0 @@ -//! Sketch directory — single source of truth for mapping aggregation -//! operations to sketch types, parameters, and memory estimates. -//! -//! Previously this logic was duplicated across: -//! - `planner/rules.rs` (`select_sketch_type`) -//! - `planner/stage_split.rs` (`agg_op_to_sketch_type`, `agg_op_to_sketch_params`, -//! `estimated_sketch_memory_bytes`) -//! - `algebra/allocator.rs` (`sketch_type_for_op`) -//! -//! All callers now go through this module. - -use crate::algebra::expr::{AggIntent, ExactAgg}; -use crate::types::{ - AggType, SketchDefaults, - SketchParams, SketchType, -}; - -// ── AggType → candidate SketchTypes ────────────────────────────────────────── - -/// Candidate sketch families per aggregation type. -/// -/// Each aggregation type has multiple viable sketch implementations. -/// The first entry is the default; the cost-model planner scores all -/// candidates and picks the cheapest that meets the accuracy SLA. -/// -/// | AggType | Candidates (default first) | -/// |-------------|---------------------------- | -/// | Quantile | DDSketch, KLL | -/// | Cardinality | HLL | -/// | Frequency | CountSketch, CountMinSketch | -pub fn candidates_for_agg(agg: &AggType) -> &'static [SketchType] { - match agg { - AggType::Quantile => &[SketchType::DDSketch, SketchType::KLL], - AggType::Cardinality => &[SketchType::HLL], - AggType::Frequency => &[SketchType::CountSketch, SketchType::CountMinSketch], - } -} - -/// All candidate sketch types for a workload (deduped, stable order). -pub fn candidates_for_workload(aggs: &[AggType]) -> Vec { - let mut out = Vec::new(); - for agg in aggs { - for st in candidates_for_agg(agg) { - if !out.contains(st) { - out.push(st.clone()); - } - } - } - out -} - -/// Pick the default sketch family from a list of aggregation types. -/// -/// Returns the first candidate for the highest-priority aggregation. -/// Priority: Quantile → Cardinality → Frequency. Falls back to DDSketch. -/// -/// For cost-optimised selection, use [`candidates_for_workload`] and score -/// each candidate via the cost model. -pub fn sketch_type_for_agg(aggs: &[AggType]) -> SketchType { - for agg in aggs { - let candidates = candidates_for_agg(agg); - if !candidates.is_empty() { - return candidates[0].clone(); - } - } - SketchType::DDSketch -} - -// ── AggIntent → SketchType ─────────────────────────────────────────────────── - -/// Resolve the concrete [`SketchType`] for an [`AggIntent`] IR node. -pub fn sketch_type_for_op(op: &AggIntent) -> SketchType { - match op { - AggIntent::Quantile { .. } | AggIntent::Extrema { .. } => SketchType::DDSketch, - AggIntent::Cardinality { .. } => SketchType::HLL, - AggIntent::Frequency { .. } => SketchType::CountSketch, - AggIntent::PerPartition { inner, .. } => sketch_type_for_op(inner), - AggIntent::Exact(_) => SketchType::DDSketch, - } -} - -// ── AggIntent → SketchParams ──────────────────────────────────────────────── - -/// Derive [`SketchParams`] from an [`AggIntent`] IR node. -pub fn sketch_params_for_op(op: &AggIntent) -> SketchParams { - match op { - AggIntent::Quantile { quantiles, accuracy } => SketchParams::DDSketch { - relative_accuracy: *accuracy, - quantiles: quantiles.clone(), - }, - AggIntent::Cardinality { accuracy } => { - // registers ≈ (1.04/accuracy)^2, precision = log2(registers) - let registers = ((1.04 / accuracy).powi(2) as u32).next_power_of_two(); - let precision = (registers as f64).log2() as u32; - SketchParams::HLL { precision } - }, - AggIntent::Frequency { accuracy } => { - let width = (std::f64::consts::E / accuracy) as u32; - SketchParams::CountSketch { - epsilon: *accuracy, - delta: 0.01, - } - }, - AggIntent::PerPartition { inner, .. } => sketch_params_for_op(inner), - AggIntent::Extrema { .. } => SketchParams::DDSketch { - relative_accuracy: 0.01, - quantiles: vec![0.0, 1.0], - }, - AggIntent::Exact(_) => SketchParams::default(), - } -} - -/// Combined (type, params) lookup — convenience for callers that need both. -pub fn sketch_type_and_params(op: &AggIntent) -> (SketchType, SketchParams) { - (sketch_type_for_op(op), sketch_params_for_op(op)) -} - -// ── AggIntent → memory estimate ───────────────────────────────────────────── - -/// Estimated sketch memory footprint per series (bytes). -/// -/// Used by `split_expr_by_stage` to decide whether to defer an operation -/// to a later pipeline stage when the budget is exceeded. -pub fn estimated_sketch_memory_bytes(op: &AggIntent) -> u64 { - match op { - AggIntent::Quantile { .. } => 4_096, - AggIntent::Cardinality { accuracy } => { - // HLL: registers ≈ (1.04/accuracy)^2, memory = registers - let registers = ((1.04 / accuracy).powi(2) as u64).next_power_of_two(); - registers.max(16) - }, - AggIntent::Frequency { accuracy } => { - // CMS: width ≈ e/accuracy, depth ≈ 5, memory = width*depth*8 - let width = (std::f64::consts::E / accuracy) as u64; - width * 5 * 8 - }, - AggIntent::Extrema { .. } => 16, - AggIntent::PerPartition { inner, keys } => { - let factor = 1u64 << keys.len().min(10); - estimated_sketch_memory_bytes(inner).saturating_mul(factor) - }, - AggIntent::Exact(_) => 8, - } -} - -// ── SketchType + accuracy SLA → SketchParams (configurable defaults) ───────── - -/// Build default [`SketchParams`] from a [`SketchDefaults`] config and accuracy SLA. -/// -/// Query-specific quantiles override the configured grid when non-empty. -pub fn build_sketch_params( - defaults: &SketchDefaults, - st: &SketchType, - accuracy_sla: f64, - query_quantiles: &[f64], -) -> SketchParams { - let acc = if accuracy_sla <= 0.0 { - defaults.ddsketch.relative_accuracy - } else { - accuracy_sla - }; - let quantiles: Vec = if !query_quantiles.is_empty() { - query_quantiles.to_vec() - } else { - defaults.quantile_grid.clone() - }; - match st { - SketchType::DDSketch => SketchParams::DDSketch { - relative_accuracy: acc, - quantiles, - }, - SketchType::KLL => { - let k = ((1.0 / acc) as u32).max(defaults.kll.min_k); - SketchParams::KLL { k, quantiles } - } - SketchType::HLL => { - let d = &defaults.hll; - let precision = if acc > d.precision_threshold { d.precision_coarse } else { d.precision_fine }; - SketchParams::HLL { precision } - } - SketchType::CountSketch => SketchParams::CountSketch { - epsilon: defaults.count_sketch.epsilon, - delta: defaults.count_sketch.delta, - }, - SketchType::CountMinSketch => SketchParams::CountMinSketch { - rows: defaults.count_min_sketch.rows, - cols: defaults.count_min_sketch.cols, - metric_name: defaults.count_min_sketch.metric_name.clone(), - }, - } -} - -/// Convenience: build default params using compiled-in defaults. -pub fn default_sketch_params(st: &SketchType, accuracy_sla: f64) -> SketchParams { - build_sketch_params(&SketchDefaults::default(), st, accuracy_sla, &[]) -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn agg_type_quantile_maps_to_ddsketch() { - assert_eq!(sketch_type_for_agg(&[AggType::Quantile]), SketchType::DDSketch); - } - - #[test] - fn agg_type_cardinality_maps_to_hll() { - assert_eq!(sketch_type_for_agg(&[AggType::Cardinality]), SketchType::HLL); - } - - #[test] - fn agg_type_frequency_maps_to_countsketch() { - assert_eq!(sketch_type_for_agg(&[AggType::Frequency]), SketchType::CountSketch); - } - - #[test] - fn empty_aggs_default_to_ddsketch() { - assert_eq!(sketch_type_for_agg(&[]), SketchType::DDSketch); - } - - #[test] - fn op_quantile_yields_ddsketch_type_and_params() { - let op = AggIntent::Quantile { quantiles: vec![0.5], accuracy: 0.01 }; - let (st, p) = sketch_type_and_params(&op); - assert_eq!(st, SketchType::DDSketch); - assert!(matches!(p, SketchParams::DDSketch { .. })); - } - - #[test] - fn op_cardinality_yields_hll_type() { - let op = AggIntent::default_cardinality(); - assert_eq!(sketch_type_for_op(&op), SketchType::HLL); - } - - #[test] - fn op_frequency_yields_countsketch() { - let op = AggIntent::default_frequency(); - assert_eq!(sketch_type_for_op(&op), SketchType::CountSketch); - } - - #[test] - fn per_partition_delegates_to_inner() { - let op = AggIntent::PerPartition { - inner: Box::new(AggIntent::default_cardinality()), - keys: vec!["k".into()], - }; - assert_eq!(sketch_type_for_op(&op), SketchType::HLL); - } - - #[test] - fn memory_quantile() { - let op = AggIntent::Quantile { quantiles: vec![0.5], accuracy: 0.01 }; - assert_eq!(estimated_sketch_memory_bytes(&op), 4096); - } - - #[test] - fn memory_per_partition_scales_by_keys() { - let inner = AggIntent::default_cardinality(); - let base_mem = estimated_sketch_memory_bytes(&inner); - let op = AggIntent::PerPartition { - inner: Box::new(inner), - keys: vec!["a".into(), "b".into()], - }; - assert_eq!(estimated_sketch_memory_bytes(&op), base_mem * 4); - } - - #[test] - fn configurable_defaults_override_quantile_grid() { - let mut d = SketchDefaults::default(); - d.quantile_grid = vec![0.5, 0.99]; - let p = build_sketch_params(&d, &SketchType::DDSketch, 0.01, &[]); - assert_eq!(p.quantiles(), &[0.5, 0.99]); - } - - #[test] - fn query_quantiles_override_grid() { - let d = SketchDefaults::default(); - let p = build_sketch_params(&d, &SketchType::DDSketch, 0.01, &[0.1, 0.9]); - assert_eq!(p.quantiles(), &[0.1, 0.9]); - } - - #[test] - fn quantile_candidates_include_ddsketch_and_kll() { - let c = candidates_for_agg(&AggType::Quantile); - assert!(c.contains(&SketchType::DDSketch)); - assert!(c.contains(&SketchType::KLL)); - } - - #[test] - fn frequency_candidates_include_cs_and_cms() { - let c = candidates_for_agg(&AggType::Frequency); - assert!(c.contains(&SketchType::CountSketch)); - assert!(c.contains(&SketchType::CountMinSketch)); - } - - #[test] - fn candidates_for_workload_dedupes() { - let c = candidates_for_workload(&[AggType::Quantile, AggType::Quantile]); - assert_eq!(c.iter().filter(|s| **s == SketchType::DDSketch).count(), 1); - } -} diff --git a/controller/src/algebra/expr.rs b/controller/src/algebra/expr.rs deleted file mode 100644 index 26b2144a..00000000 --- a/controller/src/algebra/expr.rs +++ /dev/null @@ -1,1230 +0,0 @@ -//! General query algebra — the full IR for SQL and PromQL queries. -//! -//! This module defines two mutually recursive expression types: -//! -//! * [`QueryExpr`] — *relational* operators. Each node takes zero or more -//! relations as input and produces a relation. Maps to SQL's FROM / GROUP BY -//! / JOIN / UNION layer and PromQL's binary / sub-query layer. -//! -//! * [`ScalarExpr`] — *scalar* operators. Each node computes a single value -//! from a row. Used for WHERE predicates, SELECT projections, HAVING -//! conditions, and JOIN conditions. -//! -//! # Stage vocabulary -//! -//! Once the [`crate::algebra::allocator::SketchAllocator`] annotates the tree, -//! every node carries a [`PipelineStage`](crate::algebra::plan::PipelineStage) -//! tag that says where the work executes: -//! -//! | Stage | Component | -//! |-----------|----------------------------| -//! | Agent | OTel Collector at the SDK | -//! | Backend | Central merge collector | -//! | Precompute| ASAPQuery engine | -//! | Db | Backend OLAP / exact store | - -use std::time::Duration; - -use crate::types::AggType; - -// ── Shared sketch / predicate types ─────────────────────────────────────────── - -/// Base relation / metric stream source. -#[derive(Debug, Clone)] -pub struct SourceSpec { - /// Table name (SQL) or metric name (PromQL). - pub name: String, -} - -/// How the stream is partitioned. -#[derive(Debug, Clone)] -pub enum PartitionKeys { - /// `by (k1, k2, ...)` — explicit key list. - By(Vec), - /// `without (k1, k2, ...)` — complement; resolved against schema at plan time. - Without(Vec), -} - -impl PartitionKeys { - pub fn keys(&self) -> &[String] { - match self { - PartitionKeys::By(k) | PartitionKeys::Without(k) => k, - } - } - - pub fn is_empty(&self) -> bool { - self.keys().is_empty() - } - - pub fn into_by_keys(self) -> Vec { - match self { - PartitionKeys::By(k) => k, - // For Without, return empty — caller resolves complement. - PartitionKeys::Without(k) => k, - } - } -} - -/// Which column / field the sketch aggregation targets. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum ColumnRef { - /// Explicit column name (SQL: `AVG(price)` → `Named("price")`). - Named(String), - /// The implicit metric sample value (PromQL — always the series value). - SampleValue, - /// All rows / COUNT(*). - Wildcard, -} - -/// Layer 3 — Sketch logical plan aggregation intent. -/// Describes WHAT to compute, not HOW (no sketch implementation names). -#[derive(Debug, Clone, PartialEq)] -pub enum AggIntent { - /// Quantile estimation (DDSketch, KLL, t-digest, etc. at physical layer). - Quantile { quantiles: Vec, accuracy: f64 }, - - /// Cardinality / distinct count (HLL, UnivMon, etc. at physical layer). - Cardinality { accuracy: f64 }, - - /// Frequency estimation / heavy-hitters (CountSketch, CountMinSketch, etc.). - Frequency { accuracy: f64 }, - - /// Min/max extrema. - Extrema { min: bool, max: bool }, - - /// Per-partition wrapper: "run inner intent once per distinct key tuple". - PerPartition { - inner: Box, - keys: Vec, - }, - - /// Exact passthrough — no sketch benefit (SUM, global COUNT, AVG, etc.). - Exact(ExactAgg), -} - -/// Exact (non-sketch) aggregation kinds. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum ExactAgg { - Count, - Sum, - /// **Not mergeable** — carries `(sum, count)` in distributed contexts. - Avg, - Min, - Max, -} - -impl AggIntent { - /// Returns `true` when two instances of this sketch can be merged - /// (i.e., `sketch(A ∪ B) = merge(sketch(A), sketch(B))`). - pub fn is_mergeable(&self) -> bool { - match self { - AggIntent::Exact(ExactAgg::Avg) => false, - AggIntent::PerPartition { inner, .. } => inner.is_mergeable(), - _ => true, - } - } - - /// Map to the coarse [`AggType`] used by the legacy planner. - pub fn to_agg_type(&self) -> AggType { - match self { - AggIntent::Cardinality { .. } => AggType::Cardinality, - AggIntent::Frequency { .. } => AggType::Frequency, - AggIntent::Quantile { .. } | AggIntent::Extrema { .. } => AggType::Quantile, - AggIntent::PerPartition { inner, .. } => inner.to_agg_type(), - AggIntent::Exact(_) => AggType::Quantile, - } - } - - /// Extract quantile φ values for Quantile operators. - pub fn quantiles(&self) -> Vec { - match self { - AggIntent::Quantile { quantiles, .. } => quantiles.clone(), - AggIntent::PerPartition { inner, .. } => inner.quantiles(), - _ => vec![], - } - } - - /// Whether this op implies `exact_required` (no sketch benefit). - pub fn is_exact(&self) -> bool { - matches!(self, AggIntent::Exact(_) | AggIntent::Extrema { .. }) - } - - /// Accuracy parameter (0.0 for exact ops). - pub fn accuracy(&self) -> f64 { - match self { - AggIntent::Quantile { accuracy, .. } - | AggIntent::Cardinality { accuracy, .. } - | AggIntent::Frequency { accuracy, .. } => *accuracy, - AggIntent::PerPartition { inner, .. } => inner.accuracy(), - _ => 0.0, - } - } - - // ── Default constructors (backward compat) ────────────────────────────── - - pub fn default_frequency() -> Self { - AggIntent::Frequency { accuracy: std::f64::consts::E / 2000.0 } - } - pub fn default_cardinality() -> Self { - AggIntent::Cardinality { accuracy: hll_accuracy(14) } - } - pub fn default_quantile(quantiles: Vec) -> Self { - AggIntent::Quantile { quantiles, accuracy: 0.01 } - } - -} - -// ── Accuracy helpers ───────────────────────────────────────────────────────── - -/// HLL accuracy from register count: `1.04 / sqrt(2^registers)`. -pub fn hll_accuracy(registers: u8) -> f64 { - 1.04 / (2.0f64.powi(registers as i32)).sqrt() -} - -/// CountMin accuracy from width: `e / width`. -pub fn countmin_accuracy(width: u32) -> f64 { - std::f64::consts::E / width as f64 -} - -/// Unified window specification — captures all language-level window semantics. -#[derive(Debug, Clone, PartialEq)] -pub struct WindowSpec { - pub kind: WindowKind, - pub time_col: Option, -} - -#[derive(Debug, Clone, PartialEq)] -pub enum WindowKind { - /// Fixed-size, non-overlapping. - Tumbling { size: Duration }, - /// Fixed-size, overlapping (each sample belongs to ceil(size/slide) windows). - Sliding { size: Duration, slide: Duration }, - /// Aggregate all samples (no time dimension). - Unbounded, - /// From epoch to current time. - Landmark, - /// Gap-based: window closes after inactivity. - Session { gap: Duration }, -} - -/// A single filter predicate pushed down to the collector. -#[derive(Debug, Clone)] -pub struct Predicate { - pub col: String, - pub op: FilterOp, - pub val: FilterVal, -} - -#[derive(Debug, Clone, PartialEq)] -pub enum FilterOp { - Eq, - Ne, - Lt, - Le, - Gt, - Ge, - Like, - NotLike, - IsNull, - IsNotNull, - /// PromQL `=~` label matcher (RE2 syntax). - Regex(String), - /// PromQL `!~` label matcher. - NotRegex(String), -} - -#[derive(Debug, Clone)] -pub enum FilterVal { - Str(String), - Num(f64), - Int(i64), - Null, -} - -/// How completely a query can be served by sketches. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum SketchCoverage { - /// All aggregation columns are sketch-mapped. - Full, - /// Some columns are sketch-mapped; others require exact passthrough. - Partial, - /// No sketch applicable; query requires exact execution. - None, -} - -// ── Relational algebra ──────────────────────────────────────────────────────── - -/// Full relational + sketch algebra — the sole query IR. -/// -/// Every variant is a *node* in the logical query plan tree. Leaves are -/// [`QueryExpr::Source`] or [`QueryExpr::Ref`]. Interior nodes combine their -/// `input` child(ren) through the operator they implement. -#[derive(Debug, Clone)] -pub enum QueryExpr { - // ── Base relations ──────────────────────────────────────────────────── - - /// A named metric stream or table. The outermost leaf. - Source(SourceSpec), - - /// Reference to a CTE / let-binding by name. Resolved at plan time. - Ref(String), - - // ── Filtering & projection ──────────────────────────────────────────── - - /// σ — row-level filter (WHERE / PromQL label matchers). - Filter { - pred: ScalarExpr, - input: Box, - }, - - /// π — column projection (SELECT list). - Project { - cols: Vec, - input: Box, - }, - - // ── Aggregation ─────────────────────────────────────────────────────── - - /// γ + α — GROUP BY followed by aggregate functions. - /// - /// `keys` is the GROUP BY column list (empty → global aggregate). - /// `aggs` is the list of aggregate expressions to compute. - /// `having` is an optional post-aggregation predicate. - Aggregate { - keys: Vec, - aggs: Vec, - having: Option, - input: Box, - }, - - // ── Time / streaming operators ──────────────────────────────────────── - - /// ψ — time window (PromQL `[5m]`; SQL tumbling/sliding window). - Window { - duration: Duration, - slide: Option, - input: Box, - }, - - /// γ+α specialisation for sketch aggregations (single sketch per node). - /// - /// Kept separate from [`Self::Aggregate`] so the allocator can reason - /// about which sketch type to use without parsing `AggFunc` variants. - SketchAgg { - op: AggIntent, - col: ColumnRef, - input: Box, - }, - - /// Core sketch algebra operator: windowed aggregation intent. - /// Bundles the window and the aggregation because in sketch systems - /// the window defines the sketch lifecycle (when to flush/reset). - WindowedAgg { - agg: AggIntent, - window: WindowSpec, - col: ColumnRef, - input: Box, - }, - - // ── Distributed / multi-stage operators ────────────────────────────── - - /// Partition the stream by key-tuple (GROUP BY / `by (dims)`). - Partition { - keys: PartitionKeys, - input: Box, - }, - - /// δ — deduplicate on `col` before sketch ingestion. - Dedup { - col: String, - input: Box, - }, - - /// τ — retain only the top-K entries (heavy hitters). - TopK { - k: u64, - by: Vec, - input: Box, - }, - - /// ⊕ — merge sketches from independent branches (distributed union). - Merge { - inputs: Vec, - }, - - // ── Join operators ──────────────────────────────────────────────────── - - /// Relational join. - Join { - kind: JoinKind, - pred: Option, - left: Box, - right: Box, - }, - - /// Sketch-aware join push-down: pre-aggregate on inner side then merge. - JoinSketch { - join_key: String, - outer: Box, - inner: Box, - }, - - // ── Set operators ───────────────────────────────────────────────────── - - /// UNION / INTERSECT / EXCEPT (with or without ALL). - SetOp { - kind: SetOpKind, - all: bool, - left: Box, - right: Box, - }, - - // ── Ordering & limiting ─────────────────────────────────────────────── - - /// ORDER BY. - Sort { - keys: Vec, - input: Box, - }, - - /// LIMIT [OFFSET]. - Limit { - n: u64, - offset: u64, - input: Box, - }, - - // ── Subquery / CTE ──────────────────────────────────────────────────── - - /// Inline subquery with an alias (SQL `(SELECT ...) AS alias`). - Subquery { - alias: String, - expr: Box, - }, - - /// SQL `WITH name AS (expr) IN body` or PromQL recording rule binding. - LetBinding { - name: String, - expr: Box, - body: Box, - }, - - // ── Window functions (analytic functions) ───────────────────────────── - - /// OVER (PARTITION BY … ORDER BY … frame) analytic functions. - WindowFunc { - func: WindowFuncKind, - partition_by: Vec, - order_by: Vec, - frame: Option, - input: Box, - }, - - // ── PromQL-specific operators ───────────────────────────────────────── - - /// `histogram_quantile(φ, )` — converts an HLL / histogram - /// sketch into a quantile estimate. - HistogramQuantile { - phi: f64, - input: Box, - }, - - /// PromQL sub-query syntax: `[range:resolution]`. - PromQLSubquery { - range: Duration, - resolution: Option, - input: Box, - }, - - /// Binary operation between two instant-vector expressions (PromQL `+`, `/`, …). - /// Also used for SQL arithmetic between sub-relations. - BinaryOp { - op: BinaryOpKind, - lhs: Box, - rhs: Box, - vector_match: Option, - }, -} - -// ── Scalar algebra ──────────────────────────────────────────────────────────── - -/// Scalar expression — computes a single value from a row. -/// -/// Used in [`QueryExpr::Filter`] predicates, [`ProjectItem`] expressions, -/// [`QueryExpr::Aggregate`] HAVING clauses, and JOIN conditions. -#[derive(Debug, Clone)] -pub enum ScalarExpr { - /// Column reference: `t.col` or just `col`. - Column(String), - - /// Literal value. - Literal(LiteralValue), - - /// Arithmetic / comparison / logical / regex binary operator. - BinaryOp { - op: BinaryOpKind, - lhs: Box, - rhs: Box, - }, - - /// Unary prefix operator (`NOT`, `-`, `+`). - UnaryOp { - op: UnaryOpKind, - input: Box, - }, - - /// Named function call (e.g. `ABS(x)`, `DATE_TRUNC('hour', ts)`). - FunctionCall { - name: String, - args: Vec, - }, - - /// Scalar sub-query (`SELECT MAX(price) FROM orders`). - ScalarSubquery(Box), - - /// `expr IN (v1, v2, …)` or `NOT IN (…)`. - InList { - expr: Box, - list: Vec, - negated: bool, - }, - - /// `expr IN (SELECT …)` / `NOT IN (SELECT …)`. - InSubquery { - expr: Box, - subquery: Box, - negated: bool, - }, - - /// `expr BETWEEN low AND high` or `NOT BETWEEN …`. - Between { - expr: Box, - low: Box, - high: Box, - negated: bool, - }, - - /// `expr IS NULL` / `IS NOT NULL`. - IsNull { - expr: Box, - negated: bool, - }, - - /// CASE WHEN … THEN … [ELSE …] END. - Case { - operand: Option>, - when_then: Vec<(ScalarExpr, ScalarExpr)>, - else_: Option>, - }, - - /// CAST(expr AS type). - Cast { - expr: Box, - to: DataType, - }, - - /// PromQL vector binary op between two instant-vector expressions where one - /// or both sides produce a scalar in the final result (e.g. `rate(…) > 0.5`). - VectorBinaryOp { - op: BinaryOpKind, - lhs: Box, - rhs: Box, - vector_match: Option, - }, -} - -// ── Supporting enumerations ─────────────────────────────────────────────────── - -/// A single item in a SELECT projection list. -#[derive(Debug, Clone)] -pub struct ProjectItem { - /// Output column name (SQL `AS alias`; None → use expression name). - pub alias: Option, - pub expr: ScalarExpr, -} - -/// One aggregate function in a GROUP BY / AGGREGATE node. -#[derive(Debug, Clone)] -pub struct AggItem { - /// Output column name. - pub alias: String, - /// The aggregate function. - pub func: AggFunc, - /// Column(s) the function operates on. - pub col: ColumnRef, - /// Whether DISTINCT is applied before aggregation. - pub distinct: bool, -} - -/// All aggregate functions that the algebra supports. -/// -/// "Sketchable" variants (Quantile, CountDistinct, HeavyHitters) can be -/// approximated by a sketch in early pipeline stages; the rest require -/// exact computation. -#[derive(Debug, Clone, PartialEq)] -pub enum AggFunc { - Count, - Sum, - Avg, - Min, - Max, - /// Sample / population standard deviation. - StdDev { population: bool }, - /// Sample / population variance. - Variance { population: bool }, - /// Approximate quantile at φ ∈ (0, 1]. Maps to DDSketch. - Quantile(f64), - /// COUNT DISTINCT — maps to HLL. - CountDistinct, - /// Top-K heavy hitters — maps to CountSketch. - HeavyHitters { k: u64 }, - /// PromQL `rate()` — per-second increase over a window. - Rate, - /// PromQL `increase()` — total increase over a window. - Increase, - /// PromQL `delta()` — change over a window (may be negative). - Delta, - /// Arbitrary named aggregate (UDA or extension). - Custom(String), -} - -impl AggFunc { - /// Returns true when this function can be computed from merged partial - /// results: `f(A ∪ B) = combine(f(A), f(B))`. - pub fn is_mergeable(&self) -> bool { - match self { - AggFunc::Avg | AggFunc::StdDev { .. } | AggFunc::Variance { .. } => false, - _ => true, - } - } - - /// Returns true when this function requires sketch approximation to be - /// bandwidth-efficient (i.e. the raw data would be too large to ship). - pub fn is_sketchable(&self) -> bool { - matches!( - self, - AggFunc::Quantile(_) | AggFunc::CountDistinct | AggFunc::HeavyHitters { .. } - ) - } - - /// Suggest the appropriate [`AggIntent`] for this function, if any. - pub fn to_sketch_op(&self) -> Option { - match self { - AggFunc::Quantile(phi) => Some(AggIntent::default_quantile(vec![*phi])), - AggFunc::CountDistinct => Some(AggIntent::default_cardinality()), - AggFunc::HeavyHitters { .. } => Some(AggIntent::default_frequency()), - AggFunc::Count => Some(AggIntent::Exact(ExactAgg::Count)), - AggFunc::Sum => Some(AggIntent::Exact(ExactAgg::Sum)), - AggFunc::Avg => Some(AggIntent::Exact(ExactAgg::Avg)), - AggFunc::Min => Some(AggIntent::Extrema { min: true, max: false }), - AggFunc::Max => Some(AggIntent::Extrema { min: false, max: true }), - _ => None, - } - } -} - -/// Binary operator kinds — used in both [`ScalarExpr::BinaryOp`] and -/// [`QueryExpr::BinaryOp`] (PromQL instant-vector arithmetic). -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum BinaryOpKind { - // Arithmetic - Add, - Sub, - Mul, - Div, - Mod, - Pow, - // Comparison - Eq, - Ne, - Lt, - Le, - Gt, - Ge, - // Logical - And, - Or, - // Bitwise - BitAnd, - BitOr, - BitXor, - // String / pattern - Concat, - Like, - NotLike, - Regex, - NotRegex, - // PromQL-specific - Unless, - Atan2, -} - -/// Unary prefix operators. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum UnaryOpKind { - Negate, - Not, - BitwiseNot, -} - -/// JOIN variant. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum JoinKind { - Inner, - LeftOuter, - RightOuter, - FullOuter, - Cross, - /// Semi-join: return only left rows that have a match (WHERE EXISTS). - Semi, - /// Anti-join: return only left rows that have no match (WHERE NOT EXISTS). - AntiSemi, -} - -/// Set-operation variant. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum SetOpKind { - Union, - Intersect, - Except, -} - -/// PromQL vector matching semantics (`on (…)` / `ignoring (…)` plus -/// `group_left` / `group_right`). -#[derive(Debug, Clone)] -pub struct VectorMatch { - pub kind: VectorMatchKind, - pub labels: Vec, - pub grouping: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum VectorMatchKind { - On, - Ignoring, -} - -#[derive(Debug, Clone)] -pub struct VectorGrouping { - pub side: GroupSide, - pub labels: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum GroupSide { - Left, - Right, -} - -/// ORDER BY sort key. -#[derive(Debug, Clone)] -pub struct SortKey { - pub col: String, - pub desc: bool, - /// NULLS FIRST / NULLS LAST (None → database default). - pub nulls_first: Option, -} - -/// Analytic window function kinds. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum WindowFuncKind { - RowNumber, - Rank, - DenseRank, - PercentRank, - CumeDist, - NTile { n: u64 }, - Lag { offset: u64 }, - Lead { offset: u64 }, - FirstValue, - LastValue, - NthValue { n: u64 }, - /// User-defined analytic function. - Custom(String), -} - -/// ROWS / RANGE frame clause for analytic functions. -#[derive(Debug, Clone)] -pub struct WindowFrame { - pub unit: FrameUnit, - pub start: FrameBound, - pub end: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum FrameUnit { - Rows, - Range, - Groups, -} - -#[derive(Debug, Clone)] -pub enum FrameBound { - UnboundedPreceding, - Preceding(u64), - CurrentRow, - Following(u64), - UnboundedFollowing, -} - -/// Scalar literal. -#[derive(Debug, Clone, PartialEq)] -pub enum LiteralValue { - Null, - Bool(bool), - Int(i64), - Float(f64), - Str(String), - Duration(Duration), -} - -/// SQL / Arrow data types used in CAST expressions. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum DataType { - Boolean, - Int8, - Int16, - Int32, - Int64, - UInt8, - UInt16, - UInt32, - UInt64, - Float32, - Float64, - Utf8, - Binary, - Timestamp, - Date, - Interval, - List(Box), - Struct(Vec<(String, DataType)>), - Custom(String), -} - -impl QueryExpr { - /// Walk the expression tree depth-first and call `f` on every node. - pub fn walk(&self, f: &mut F) { - f(self); - match self { - QueryExpr::Source(_) | QueryExpr::Ref(_) => {} - QueryExpr::Filter { input, .. } - | QueryExpr::Project { input, .. } - | QueryExpr::Window { input, .. } - | QueryExpr::SketchAgg { input, .. } - | QueryExpr::WindowedAgg { input, .. } - | QueryExpr::Partition { input, .. } - | QueryExpr::Dedup { input, .. } - | QueryExpr::TopK { input, .. } - | QueryExpr::Sort { input, .. } - | QueryExpr::Limit { input, .. } - | QueryExpr::WindowFunc { input, .. } - | QueryExpr::HistogramQuantile { input, .. } - | QueryExpr::PromQLSubquery { input, .. } => input.walk(f), - - QueryExpr::Aggregate { input, .. } => input.walk(f), - - QueryExpr::Merge { inputs } => { - for i in inputs { i.walk(f); } - } - QueryExpr::Join { left, right, .. } - | QueryExpr::JoinSketch { outer: left, inner: right, .. } - | QueryExpr::SetOp { left, right, .. } - | QueryExpr::BinaryOp { lhs: left, rhs: right, .. } => { - left.walk(f); - right.walk(f); - } - QueryExpr::Subquery { expr, .. } => expr.walk(f), - QueryExpr::LetBinding { expr, body, .. } => { - expr.walk(f); - body.walk(f); - } - } - } - - /// Returns `true` when the sub-tree contains at least one [`QueryExpr::SketchAgg`] - /// or [`QueryExpr::TopK`] node (i.e. sketch work is present). - pub fn has_sketch_work(&self) -> bool { - let mut found = false; - self.walk(&mut |n| { - if matches!(n, QueryExpr::SketchAgg { .. } | QueryExpr::WindowedAgg { .. } | QueryExpr::TopK { .. }) { - found = true; - } - }); - found - } - - /// Returns the outermost metric/table name from the first `Source` leaf. - pub fn source_name(&self) -> Option<&str> { - match self { - QueryExpr::Source(s) => Some(&s.name), - QueryExpr::Filter { input, .. } - | QueryExpr::Project { input, .. } - | QueryExpr::Window { input, .. } - | QueryExpr::SketchAgg { input, .. } - | QueryExpr::WindowedAgg { input, .. } - | QueryExpr::Partition { input, .. } - | QueryExpr::Dedup { input, .. } - | QueryExpr::TopK { input, .. } - | QueryExpr::Sort { input, .. } - | QueryExpr::Limit { input, .. } - | QueryExpr::Aggregate { input, .. } - | QueryExpr::WindowFunc { input, .. } - | QueryExpr::HistogramQuantile { input, .. } - | QueryExpr::PromQLSubquery { input, .. } => input.source_name(), - QueryExpr::Merge { inputs } => inputs.first()?.source_name(), - QueryExpr::Join { left, .. } - | QueryExpr::JoinSketch { outer: left, .. } - | QueryExpr::SetOp { left, .. } - | QueryExpr::BinaryOp { lhs: left, .. } => left.source_name(), - QueryExpr::Subquery { expr, .. } => expr.source_name(), - QueryExpr::LetBinding { body, .. } => body.source_name(), - QueryExpr::Ref(_) => None, - } - } -} - -// ── Predicate → ScalarExpr conversion ──────────────────────────────────────── - -/// Convert a slice of legacy [`Predicate`]s (AND-list) into a single -/// [`ScalarExpr`] tree. An empty slice becomes `Literal(true)`. -fn scalar_from_predicates(preds: &[Predicate]) -> ScalarExpr { - if preds.is_empty() { - return ScalarExpr::Literal(LiteralValue::Bool(true)); - } - let mut iter = preds.iter().map(scalar_from_predicate); - let first = iter.next().unwrap(); - iter.fold(first, |acc, p| ScalarExpr::BinaryOp { - op: BinaryOpKind::And, - lhs: Box::new(acc), - rhs: Box::new(p), - }) -} - -fn scalar_from_predicate(p: &Predicate) -> ScalarExpr { - let col = ScalarExpr::Column(p.col.clone()); - let val = match &p.val { - FilterVal::Str(s) => ScalarExpr::Literal(LiteralValue::Str(s.clone())), - FilterVal::Num(n) => ScalarExpr::Literal(LiteralValue::Float(*n)), - FilterVal::Int(i) => ScalarExpr::Literal(LiteralValue::Int(*i)), - FilterVal::Null => ScalarExpr::Literal(LiteralValue::Null), - }; - match &p.op { - FilterOp::Eq => bin(BinaryOpKind::Eq, col, val), - FilterOp::Ne => bin(BinaryOpKind::Ne, col, val), - FilterOp::Lt => bin(BinaryOpKind::Lt, col, val), - FilterOp::Le => bin(BinaryOpKind::Le, col, val), - FilterOp::Gt => bin(BinaryOpKind::Gt, col, val), - FilterOp::Ge => bin(BinaryOpKind::Ge, col, val), - FilterOp::Like => bin(BinaryOpKind::Like, col, val), - FilterOp::NotLike => bin(BinaryOpKind::NotLike, col, val), - FilterOp::IsNull => ScalarExpr::IsNull { expr: Box::new(col), negated: false }, - FilterOp::IsNotNull => ScalarExpr::IsNull { expr: Box::new(col), negated: true }, - FilterOp::Regex(r) => bin( - BinaryOpKind::Regex, - col, - ScalarExpr::Literal(LiteralValue::Str(r.clone())), - ), - FilterOp::NotRegex(r) => bin( - BinaryOpKind::NotRegex, - col, - ScalarExpr::Literal(LiteralValue::Str(r.clone())), - ), - } -} - -fn bin(op: BinaryOpKind, lhs: ScalarExpr, rhs: ScalarExpr) -> ScalarExpr { - ScalarExpr::BinaryOp { op, lhs: Box::new(lhs), rhs: Box::new(rhs) } -} - -// ── Display helpers ─────────────────────────────────────────────────────────── - -impl std::fmt::Display for BinaryOpKind { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let s = match self { - BinaryOpKind::Add => "+", BinaryOpKind::Sub => "-", - BinaryOpKind::Mul => "*", BinaryOpKind::Div => "/", - BinaryOpKind::Mod => "%", BinaryOpKind::Pow => "^", - BinaryOpKind::Eq => "=", BinaryOpKind::Ne => "!=", - BinaryOpKind::Lt => "<", BinaryOpKind::Le => "<=", - BinaryOpKind::Gt => ">", BinaryOpKind::Ge => ">=", - BinaryOpKind::And => "AND", BinaryOpKind::Or => "OR", - BinaryOpKind::BitAnd => "&", BinaryOpKind::BitOr => "|", - BinaryOpKind::BitXor => "XOR", - BinaryOpKind::Concat => "||", - BinaryOpKind::Like => "LIKE", BinaryOpKind::NotLike => "NOT LIKE", - BinaryOpKind::Regex => "=~", BinaryOpKind::NotRegex => "!~", - BinaryOpKind::Unless => "unless", BinaryOpKind::Atan2 => "atan2", - }; - write!(f, "{s}") - } -} - -impl std::fmt::Display for AggFunc { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - AggFunc::Count => write!(f, "COUNT"), - AggFunc::Sum => write!(f, "SUM"), - AggFunc::Avg => write!(f, "AVG"), - AggFunc::Min => write!(f, "MIN"), - AggFunc::Max => write!(f, "MAX"), - AggFunc::StdDev { .. } => write!(f, "STDDEV"), - AggFunc::Variance { .. } => write!(f, "VARIANCE"), - AggFunc::Quantile(p) => write!(f, "QUANTILE({p})"), - AggFunc::CountDistinct => write!(f, "COUNT_DISTINCT"), - AggFunc::HeavyHitters { k } => write!(f, "HEAVY_HITTERS({k})"), - AggFunc::Rate => write!(f, "rate"), - AggFunc::Increase => write!(f, "increase"), - AggFunc::Delta => write!(f, "delta"), - AggFunc::Custom(s) => write!(f, "{s}"), - } - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use std::time::Duration; - - fn src(name: &str) -> QueryExpr { - QueryExpr::Source(SourceSpec { name: name.into() }) - } - - // ── has_sketch_work ─────────────────────────────────────────────────────── - - #[test] - fn has_sketch_work_true_when_ddsketch_present() { - let qe = QueryExpr::SketchAgg { - op: AggIntent::default_quantile(vec![0.5]), - col: ColumnRef::SampleValue, - input: Box::new(src("m")), - }; - assert!(qe.has_sketch_work()); - } - - #[test] - fn has_sketch_work_false_for_plain_source() { - let qe = QueryExpr::Source(SourceSpec { name: "x".into() }); - assert!(!qe.has_sketch_work()); - } - - // ── source_name ─────────────────────────────────────────────────────────── - - #[test] - fn source_name_extracted_through_chain() { - let qe = QueryExpr::Window { - duration: Duration::from_secs(60), - slide: None, - input: Box::new(QueryExpr::Filter { - pred: ScalarExpr::Literal(LiteralValue::Bool(true)), - input: Box::new(src("my_metric")), - }), - }; - assert_eq!(qe.source_name(), Some("my_metric")); - } - - // ── AggFunc helpers ─────────────────────────────────────────────────────── - - #[test] - fn agg_func_mergeability() { - assert!(AggFunc::Sum.is_mergeable()); - assert!(AggFunc::Count.is_mergeable()); - assert!(AggFunc::Min.is_mergeable()); - assert!(AggFunc::Max.is_mergeable()); - assert!(!AggFunc::Avg.is_mergeable()); - assert!(!AggFunc::StdDev { population: false }.is_mergeable()); - assert!(!AggFunc::Variance { population: true }.is_mergeable()); - } - - #[test] - fn agg_func_sketchability() { - assert!(AggFunc::Quantile(0.99).is_sketchable()); - assert!(AggFunc::CountDistinct.is_sketchable()); - assert!(AggFunc::HeavyHitters { k: 10 }.is_sketchable()); - assert!(!AggFunc::Avg.is_sketchable()); - assert!(!AggFunc::Sum.is_sketchable()); - } - - #[test] - fn agg_func_to_sketch_op_quantile() { - let op = AggFunc::Quantile(0.99).to_sketch_op(); - assert!(matches!(op, Some(AggIntent::Quantile { .. }))); - } - - #[test] - fn agg_func_to_sketch_op_count_distinct() { - let op = AggFunc::CountDistinct.to_sketch_op(); - assert!(matches!(op, Some(AggIntent::Cardinality { .. }))); - } - - #[test] - fn agg_func_to_sketch_op_heavy_hitters() { - let op = AggFunc::HeavyHitters { k: 50 }.to_sketch_op(); - assert!(matches!(op, Some(AggIntent::Frequency { .. }))); - } - - // ── ScalarExpr predicate list conversion ────────────────────────────────── - - #[test] - fn empty_pred_list_becomes_literal_true() { - let s = scalar_from_predicates(&[]); - assert!(matches!(s, ScalarExpr::Literal(LiteralValue::Bool(true)))); - } - - #[test] - fn two_preds_become_and_tree() { - let preds = vec![ - Predicate { col: "a".into(), op: FilterOp::Eq, val: FilterVal::Int(1) }, - Predicate { col: "b".into(), op: FilterOp::Gt, val: FilterVal::Num(2.0) }, - ]; - let s = scalar_from_predicates(&preds); - assert!(matches!(s, ScalarExpr::BinaryOp { op: BinaryOpKind::And, .. })); - } - - // ── BinaryOpKind display ────────────────────────────────────────────────── - - #[test] - fn binary_op_kind_display() { - assert_eq!(BinaryOpKind::Add.to_string(), "+"); - assert_eq!(BinaryOpKind::And.to_string(), "AND"); - assert_eq!(BinaryOpKind::Regex.to_string(), "=~"); - assert_eq!(BinaryOpKind::NotRegex.to_string(), "!~"); - assert_eq!(BinaryOpKind::Unless.to_string(), "unless"); - } - - // ── Complex nested tree ─────────────────────────────────────────────────── - - #[test] - fn complex_nested_tree() { - // TopK(10, Partition(symbol, Window(5m, SketchAgg(CountSketch, Source(price))))) - let qe = QueryExpr::TopK { - k: 10, - by: vec![], - input: Box::new(QueryExpr::Partition { - keys: PartitionKeys::By(vec!["symbol".into()]), - input: Box::new(QueryExpr::Window { - duration: Duration::from_secs(300), - slide: None, - input: Box::new(QueryExpr::SketchAgg { - op: AggIntent::default_frequency(), - col: ColumnRef::Wildcard, - input: Box::new(src("price")), - }), - }), - }), - }; - assert!(qe.has_sketch_work()); - assert_eq!(qe.source_name(), Some("price")); - } - - // ── LetBinding and Subquery ─────────────────────────────────────────────── - - #[test] - fn let_binding_construction() { - let expr = QueryExpr::LetBinding { - name: "base".into(), - expr: Box::new(QueryExpr::Source(SourceSpec { name: "cpu".into() })), - body: Box::new(QueryExpr::Ref("base".into())), - }; - match expr { - QueryExpr::LetBinding { name, .. } => assert_eq!(name, "base"), - _ => panic!(), - } - } - - #[test] - fn histogram_quantile_node() { - let expr = QueryExpr::HistogramQuantile { - phi: 0.95, - input: Box::new(QueryExpr::Source(SourceSpec { name: "hist".into() })), - }; - match expr { - QueryExpr::HistogramQuantile { phi, .. } => { - assert!((phi - 0.95).abs() < 1e-9); - } - _ => panic!(), - } - } - - #[test] - fn promql_subquery_node() { - let expr = QueryExpr::PromQLSubquery { - range: Duration::from_secs(3600), - resolution: Some(Duration::from_secs(60)), - input: Box::new(QueryExpr::Source(SourceSpec { name: "m".into() })), - }; - match expr { - QueryExpr::PromQLSubquery { range, resolution, .. } => { - assert_eq!(range, Duration::from_secs(3600)); - assert_eq!(resolution, Some(Duration::from_secs(60))); - } - _ => panic!(), - } - } - - // ── AggIntent and related types ────────────────────────────────────────── - - #[test] - fn agg_intent_cardinality_is_mergeable() { - assert!(AggIntent::default_cardinality().is_mergeable()); - } - - #[test] - fn agg_intent_exact_avg_not_mergeable() { - assert!(!AggIntent::Exact(ExactAgg::Avg).is_mergeable()); - } - - #[test] - fn agg_intent_per_partition_mergeability_from_inner() { - let pp_card = AggIntent::PerPartition { - inner: Box::new(AggIntent::default_cardinality()), - keys: vec!["region".into()], - }; - assert!(pp_card.is_mergeable()); - let pp_avg = AggIntent::PerPartition { - inner: Box::new(AggIntent::Exact(ExactAgg::Avg)), - keys: vec!["region".into()], - }; - assert!(!pp_avg.is_mergeable()); - } - - #[test] - fn partition_keys_without_variant() { - let keys = PartitionKeys::Without(vec!["instance".into()]); - assert_eq!(keys.keys(), &["instance".to_string()]); - assert!(!keys.is_empty()); - } - - #[test] - fn agg_intent_to_agg_type() { - use crate::types::AggType; - assert_eq!(AggIntent::default_cardinality().to_agg_type(), AggType::Cardinality); - assert_eq!(AggIntent::default_frequency().to_agg_type(), AggType::Frequency); - assert_eq!(AggIntent::default_quantile(vec![0.5]).to_agg_type(), AggType::Quantile); - } - - #[test] - fn agg_intent_is_exact() { - assert!(AggIntent::Exact(ExactAgg::Sum).is_exact()); - assert!(AggIntent::Extrema { min: true, max: false }.is_exact()); - assert!(!AggIntent::default_cardinality().is_exact()); - } - - #[test] - fn sketch_coverage_classification() { - let ops: Vec = vec![ - AggIntent::default_cardinality(), - AggIntent::Exact(ExactAgg::Sum), - ]; - let has_sketch = ops.iter().any(|o| !o.is_exact()); - let has_exact = ops.iter().any(|o| o.is_exact()); - let cov = match (has_sketch, has_exact) { - (true, false) => SketchCoverage::Full, - (true, true) => SketchCoverage::Partial, - _ => SketchCoverage::None, - }; - assert_eq!(cov, SketchCoverage::Partial); - } -} diff --git a/controller/src/algebra/lower.rs b/controller/src/algebra/lower.rs deleted file mode 100644 index 760c220d..00000000 --- a/controller/src/algebra/lower.rs +++ /dev/null @@ -1,522 +0,0 @@ -//! Layer 2→3 lowering: convert relational Aggregate nodes to sketch AggIntent. -//! -//! This pass walks a [`QueryExpr`] tree and converts `Aggregate { AggFunc }` -//! nodes to `SketchAgg { AggIntent }` where the aggregation can benefit from -//! sketch-based execution. It is shared by both the PromQL and SQL parsers. -//! -//! # What gets lowered -//! -//! A single-agg `Aggregate` node whose `AggFunc` maps to a sketch intent is -//! replaced by `SketchAgg { AggIntent }`. If the `Aggregate` had GROUP BY -//! keys, they become a wrapping `Partition` node. -//! -//! Multi-agg `Aggregate` nodes or those with HAVING clauses pass through -//! unchanged — the physical planner handles them. - -use crate::algebra::expr::*; - -/// Lower a Layer 2 `QueryExpr` (relational operators only) to Layer 3 -/// (sketch algebra with `AggIntent`). -/// -/// This pass walks the tree and converts `Aggregate { AggFunc }` nodes -/// to `SketchAgg { AggIntent }` where the aggregation can benefit from -/// sketch-based execution. -pub fn lower_to_sketch_algebra(expr: QueryExpr) -> QueryExpr { - match expr { - QueryExpr::Aggregate { keys, aggs, having, input } => { - let input = lower_to_sketch_algebra(*input); - lower_aggregate(keys, aggs, having, Box::new(input)) - } - - // ── Single-input nodes: recurse ────────────────────────────────── - QueryExpr::Filter { pred, input } => QueryExpr::Filter { - pred, - input: Box::new(lower_to_sketch_algebra(*input)), - }, - QueryExpr::Project { cols, input } => QueryExpr::Project { - cols, - input: Box::new(lower_to_sketch_algebra(*input)), - }, - QueryExpr::Window { duration, slide, input } => { - let lowered_input = lower_to_sketch_algebra(*input); - // Fuse Window + SketchAgg → WindowedAgg (the window defines sketch lifecycle). - if let QueryExpr::SketchAgg { op, col, input: sketch_input } = lowered_input { - let window = WindowSpec { - kind: match slide { - Some(s) => WindowKind::Sliding { size: duration, slide: s }, - None => WindowKind::Tumbling { size: duration }, - }, - time_col: None, - }; - QueryExpr::WindowedAgg { agg: op, window, col, input: sketch_input } - } else { - QueryExpr::Window { duration, slide, input: Box::new(lowered_input) } - } - } - QueryExpr::SketchAgg { op, col, input } => QueryExpr::SketchAgg { - op, - col, - input: Box::new(lower_to_sketch_algebra(*input)), - }, - QueryExpr::WindowedAgg { agg, window, col, input } => QueryExpr::WindowedAgg { - agg, - window, - col, - input: Box::new(lower_to_sketch_algebra(*input)), - }, - QueryExpr::Partition { keys, input } => QueryExpr::Partition { - keys, - input: Box::new(lower_to_sketch_algebra(*input)), - }, - QueryExpr::Dedup { col, input } => QueryExpr::Dedup { - col, - input: Box::new(lower_to_sketch_algebra(*input)), - }, - QueryExpr::TopK { k, by, input } => QueryExpr::TopK { - k, - by, - input: Box::new(lower_to_sketch_algebra(*input)), - }, - QueryExpr::Sort { keys, input } => QueryExpr::Sort { - keys, - input: Box::new(lower_to_sketch_algebra(*input)), - }, - QueryExpr::Limit { n, offset, input } => QueryExpr::Limit { - n, - offset, - input: Box::new(lower_to_sketch_algebra(*input)), - }, - QueryExpr::WindowFunc { func, partition_by, order_by, frame, input } => QueryExpr::WindowFunc { - func, - partition_by, - order_by, - frame, - input: Box::new(lower_to_sketch_algebra(*input)), - }, - QueryExpr::HistogramQuantile { phi, input } => QueryExpr::HistogramQuantile { - phi, - input: Box::new(lower_to_sketch_algebra(*input)), - }, - QueryExpr::PromQLSubquery { range, resolution, input } => QueryExpr::PromQLSubquery { - range, - resolution, - input: Box::new(lower_to_sketch_algebra(*input)), - }, - - // ── Two-input nodes: recurse into both ───���────────────────────── - QueryExpr::BinaryOp { op, lhs, rhs, vector_match } => QueryExpr::BinaryOp { - op, - lhs: Box::new(lower_to_sketch_algebra(*lhs)), - rhs: Box::new(lower_to_sketch_algebra(*rhs)), - vector_match, - }, - QueryExpr::Join { kind, pred, left, right } => QueryExpr::Join { - kind, - pred, - left: Box::new(lower_to_sketch_algebra(*left)), - right: Box::new(lower_to_sketch_algebra(*right)), - }, - QueryExpr::JoinSketch { join_key, outer, inner } => QueryExpr::JoinSketch { - join_key, - outer: Box::new(lower_to_sketch_algebra(*outer)), - inner: Box::new(lower_to_sketch_algebra(*inner)), - }, - QueryExpr::SetOp { kind, all, left, right } => QueryExpr::SetOp { - kind, - all, - left: Box::new(lower_to_sketch_algebra(*left)), - right: Box::new(lower_to_sketch_algebra(*right)), - }, - - // ── Multi-input / container nodes ─────��────────────────────────── - QueryExpr::Merge { inputs } => QueryExpr::Merge { - inputs: inputs.into_iter().map(lower_to_sketch_algebra).collect(), - }, - QueryExpr::Subquery { alias, expr } => QueryExpr::Subquery { - alias, - expr: Box::new(lower_to_sketch_algebra(*expr)), - }, - QueryExpr::LetBinding { name, expr, body } => QueryExpr::LetBinding { - name, - expr: Box::new(lower_to_sketch_algebra(*expr)), - body: Box::new(lower_to_sketch_algebra(*body)), - }, - - // ── Leaf nodes: pass through ────────���──────────────────────────── - QueryExpr::Source(_) | QueryExpr::Ref(_) => expr, - } -} - -/// Try to lower a single `Aggregate` node to `SketchAgg`. -fn lower_aggregate( - keys: Vec, - aggs: Vec, - having: Option, - input: Box, -) -> QueryExpr { - // Only lower single-agg Aggregates without HAVING. - if aggs.len() == 1 && having.is_none() { - let agg = &aggs[0]; - - // COUNT(*) without GROUP BY is a simple row count — no sketch benefit. - if matches!(agg.func, AggFunc::Count) && keys.is_empty() { - return QueryExpr::Aggregate { keys, aggs, having, input }; - } - - if let Some(intent) = agg_func_to_intent(&agg.func) { - // If the input is a Window, fuse into WindowedAgg (the window - // defines the sketch lifecycle — flush/reset/merge semantics). - let sketch = if let QueryExpr::Window { duration, slide, input: win_input } = *input { - let window = WindowSpec { - kind: match slide { - Some(s) => WindowKind::Sliding { size: duration, slide: s }, - None => WindowKind::Tumbling { size: duration }, - }, - time_col: None, - }; - QueryExpr::WindowedAgg { - agg: intent, - window, - col: agg.col.clone(), - input: win_input, - } - } else { - QueryExpr::SketchAgg { - op: intent, - col: agg.col.clone(), - input, - } - }; - if keys.is_empty() { - return sketch; - } else { - return QueryExpr::Partition { - keys: PartitionKeys::By(keys), - input: Box::new(sketch), - }; - } - } - } - - // Multi-agg or non-sketchable: keep as relational Aggregate. - QueryExpr::Aggregate { keys, aggs, having, input } -} - -/// Map an [`AggFunc`] to an [`AggIntent`] for sketch execution. -fn agg_func_to_intent(func: &AggFunc) -> Option { - match func { - AggFunc::Quantile(phi) => Some(AggIntent::default_quantile(vec![*phi])), - AggFunc::CountDistinct => Some(AggIntent::default_cardinality()), - AggFunc::HeavyHitters { .. } => Some(AggIntent::default_frequency()), - AggFunc::Count => Some(AggIntent::default_frequency()), - AggFunc::Avg => Some(AggIntent::Quantile { - quantiles: vec![0.5], - accuracy: 0.01, - }), - AggFunc::Min => Some(AggIntent::Extrema { min: true, max: false }), - AggFunc::Max => Some(AggIntent::Extrema { min: false, max: true }), - AggFunc::StdDev { .. } => Some(AggIntent::Quantile { - quantiles: vec![0.25, 0.75], - accuracy: 0.01, - }), - AggFunc::Variance { .. } => Some(AggIntent::Quantile { - quantiles: vec![0.25, 0.75], - accuracy: 0.01, - }), - AggFunc::Sum | AggFunc::Rate | AggFunc::Increase | AggFunc::Delta => { - Some(AggIntent::Exact(ExactAgg::Sum)) - } - AggFunc::Custom(_) => None, - } -} - -// ── Tests ───────────���───────────────────────────��───────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use std::time::Duration; - - fn src(name: &str) -> QueryExpr { - QueryExpr::Source(SourceSpec { name: name.into() }) - } - - fn make_agg(func: AggFunc, input: QueryExpr) -> QueryExpr { - QueryExpr::Aggregate { - keys: vec![], - aggs: vec![AggItem { - alias: "v".into(), - func, - col: ColumnRef::SampleValue, - distinct: false, - }], - having: None, - input: Box::new(input), - } - } - - fn make_agg_with_keys(func: AggFunc, keys: Vec, input: QueryExpr) -> QueryExpr { - QueryExpr::Aggregate { - keys, - aggs: vec![AggItem { - alias: "v".into(), - func, - col: ColumnRef::SampleValue, - distinct: false, - }], - having: None, - input: Box::new(input), - } - } - - // ── Single-agg lowering ────────────────────────────────────────────────── - - #[test] - fn quantile_lowered_to_sketch_agg() { - let expr = make_agg(AggFunc::Quantile(0.99), src("m")); - let lowered = lower_to_sketch_algebra(expr); - assert!(matches!(lowered, QueryExpr::SketchAgg { op: AggIntent::Quantile { .. }, .. })); - } - - #[test] - fn count_distinct_lowered_to_cardinality() { - let expr = make_agg(AggFunc::CountDistinct, src("m")); - let lowered = lower_to_sketch_algebra(expr); - assert!(matches!(lowered, QueryExpr::SketchAgg { op: AggIntent::Cardinality { .. }, .. })); - } - - #[test] - fn count_without_group_by_stays_aggregate() { - // COUNT(*) without GROUP BY is a simple row count — no sketch benefit. - let expr = make_agg(AggFunc::Count, src("m")); - let lowered = lower_to_sketch_algebra(expr); - assert!(matches!(lowered, QueryExpr::Aggregate { .. })); - } - - #[test] - fn count_with_group_by_lowered_to_frequency() { - let expr = make_agg_with_keys(AggFunc::Count, vec!["region".into()], src("m")); - let lowered = lower_to_sketch_algebra(expr); - match &lowered { - QueryExpr::Partition { input, .. } => { - assert!(matches!(input.as_ref(), QueryExpr::SketchAgg { op: AggIntent::Frequency { .. }, .. })); - } - other => panic!("expected Partition(SketchAgg), got {other:?}"), - } - } - - #[test] - fn sum_lowered_to_exact() { - let expr = make_agg(AggFunc::Sum, src("m")); - let lowered = lower_to_sketch_algebra(expr); - assert!(matches!(lowered, QueryExpr::SketchAgg { op: AggIntent::Exact(ExactAgg::Sum), .. })); - } - - #[test] - fn avg_lowered_to_quantile_p50() { - let expr = make_agg(AggFunc::Avg, src("m")); - let lowered = lower_to_sketch_algebra(expr); - match &lowered { - QueryExpr::SketchAgg { op: AggIntent::Quantile { quantiles, .. }, .. } => { - assert_eq!(quantiles, &[0.5]); - } - other => panic!("expected SketchAgg(Quantile), got {other:?}"), - } - } - - #[test] - fn min_lowered_to_extrema() { - let expr = make_agg(AggFunc::Min, src("m")); - let lowered = lower_to_sketch_algebra(expr); - assert!(matches!(lowered, QueryExpr::SketchAgg { op: AggIntent::Extrema { min: true, max: false }, .. })); - } - - #[test] - fn max_lowered_to_extrema() { - let expr = make_agg(AggFunc::Max, src("m")); - let lowered = lower_to_sketch_algebra(expr); - assert!(matches!(lowered, QueryExpr::SketchAgg { op: AggIntent::Extrema { min: false, max: true }, .. })); - } - - #[test] - fn stddev_lowered_to_iqr_quantile() { - let expr = make_agg(AggFunc::StdDev { population: false }, src("m")); - let lowered = lower_to_sketch_algebra(expr); - match &lowered { - QueryExpr::SketchAgg { op: AggIntent::Quantile { quantiles, .. }, .. } => { - assert!(quantiles.contains(&0.25) && quantiles.contains(&0.75)); - } - other => panic!("expected SketchAgg(Quantile), got {other:?}"), - } - } - - #[test] - fn custom_func_not_lowered() { - let expr = make_agg(AggFunc::Custom("my_udf".into()), src("m")); - let lowered = lower_to_sketch_algebra(expr); - assert!(matches!(lowered, QueryExpr::Aggregate { .. })); - } - - // ── GROUP BY keys become Partition ────��─────────────────────────────────── - - #[test] - fn group_by_wraps_with_partition() { - let expr = make_agg_with_keys( - AggFunc::Quantile(0.5), - vec!["host".into()], - src("m"), - ); - let lowered = lower_to_sketch_algebra(expr); - match &lowered { - QueryExpr::Partition { keys, input } => { - assert_eq!(keys.keys(), &["host".to_string()]); - assert!(matches!(input.as_ref(), QueryExpr::SketchAgg { .. })); - } - other => panic!("expected Partition, got {other:?}"), - } - } - - // ── Multi-agg not lowered ──────────────────────────────────────────────── - - #[test] - fn multi_agg_not_lowered() { - let expr = QueryExpr::Aggregate { - keys: vec![], - aggs: vec![ - AggItem { - alias: "c".into(), - func: AggFunc::Count, - col: ColumnRef::Wildcard, - distinct: false, - }, - AggItem { - alias: "s".into(), - func: AggFunc::Sum, - col: ColumnRef::Named("x".into()), - distinct: false, - }, - ], - having: None, - input: Box::new(src("m")), - }; - let lowered = lower_to_sketch_algebra(expr); - assert!(matches!(lowered, QueryExpr::Aggregate { .. })); - } - - // ── Recursive lowering ───────────────────��────────────────────���────────── - - #[test] - fn window_wrapping_aggregate_fuses_to_windowed_agg() { - // Aggregate inside a Window → fused WindowedAgg. - let expr = QueryExpr::Window { - duration: Duration::from_secs(300), - slide: None, - input: Box::new(make_agg(AggFunc::Quantile(0.99), src("m"))), - }; - let lowered = lower_to_sketch_algebra(expr); - match &lowered { - QueryExpr::WindowedAgg { agg, window, .. } => { - assert!(matches!(agg, AggIntent::Quantile { .. })); - assert!(matches!(window.kind, WindowKind::Tumbling { .. })); - } - other => panic!("expected WindowedAgg, got {other:?}"), - } - } - - #[test] - fn sliding_window_fuses_to_windowed_agg_sliding() { - let expr = QueryExpr::Window { - duration: Duration::from_secs(300), - slide: Some(Duration::from_secs(60)), - input: Box::new(make_agg(AggFunc::Quantile(0.5), src("m"))), - }; - let lowered = lower_to_sketch_algebra(expr); - match &lowered { - QueryExpr::WindowedAgg { window, .. } => { - assert!(matches!(window.kind, WindowKind::Sliding { .. })); - } - other => panic!("expected WindowedAgg(Sliding), got {other:?}"), - } - } - - #[test] - fn window_wrapping_non_sketchable_stays_separate() { - // Custom func is not sketchable → Window stays, Aggregate stays. - let expr = QueryExpr::Window { - duration: Duration::from_secs(300), - slide: None, - input: Box::new(make_agg(AggFunc::Custom("my_udf".into()), src("m"))), - }; - let lowered = lower_to_sketch_algebra(expr); - assert!(matches!(lowered, QueryExpr::Window { .. })); - } - - #[test] - fn lowering_recurses_into_binary_op() { - let expr = QueryExpr::BinaryOp { - op: BinaryOpKind::Add, - lhs: Box::new(make_agg(AggFunc::Sum, src("a"))), - rhs: Box::new(make_agg(AggFunc::CountDistinct, src("b"))), - vector_match: None, - }; - let lowered = lower_to_sketch_algebra(expr); - match &lowered { - QueryExpr::BinaryOp { lhs, rhs, .. } => { - assert!(matches!(lhs.as_ref(), QueryExpr::SketchAgg { op: AggIntent::Exact(_), .. })); - assert!(matches!(rhs.as_ref(), QueryExpr::SketchAgg { op: AggIntent::Cardinality { .. }, .. })); - } - other => panic!("expected BinaryOp, got {other:?}"), - } - } - - #[test] - fn lowering_recurses_into_topk() { - let expr = QueryExpr::TopK { - k: 10, - by: vec!["symbol".into()], - input: Box::new(make_agg_with_keys( - AggFunc::Count, - vec!["symbol".into()], - src("m"), - )), - }; - let lowered = lower_to_sketch_algebra(expr); - match &lowered { - QueryExpr::TopK { input, .. } => { - // Count with GROUP BY is lowered to Partition(SketchAgg(Frequency)) - assert!(matches!(input.as_ref(), QueryExpr::Partition { .. })); - } - other => panic!("expected TopK, got {other:?}"), - } - } - - #[test] - fn histogram_quantile_passthrough() { - let expr = QueryExpr::HistogramQuantile { - phi: 0.95, - input: Box::new(make_agg(AggFunc::Quantile(0.95), src("m"))), - }; - let lowered = lower_to_sketch_algebra(expr); - match &lowered { - QueryExpr::HistogramQuantile { phi, input } => { - assert!((phi - 0.95).abs() < 1e-9); - assert!(matches!(input.as_ref(), QueryExpr::SketchAgg { .. })); - } - other => panic!("expected HistogramQuantile, got {other:?}"), - } - } - - #[test] - fn rate_lowered_to_exact_sum() { - let expr = make_agg(AggFunc::Rate, src("m")); - let lowered = lower_to_sketch_algebra(expr); - assert!(matches!(lowered, QueryExpr::SketchAgg { op: AggIntent::Exact(ExactAgg::Sum), .. })); - } - - #[test] - fn delta_lowered_to_exact_sum() { - let expr = make_agg(AggFunc::Delta, src("m")); - let lowered = lower_to_sketch_algebra(expr); - assert!(matches!(lowered, QueryExpr::SketchAgg { op: AggIntent::Exact(ExactAgg::Sum), .. })); - } -} diff --git a/controller/src/algebra/mod.rs b/controller/src/algebra/mod.rs deleted file mode 100644 index 6cbfc080..00000000 --- a/controller/src/algebra/mod.rs +++ /dev/null @@ -1,44 +0,0 @@ -//! Sketch algebra — the 5-layer query translation pipeline. -//! -//! # Layer architecture -//! -//! | Layer | Module | Role | -//! |-------|--------|------| -//! | **3. Sketch Logical Plan** | [`expr`] | `QueryExpr` + `AggIntent` — implementation-independent algebra | -//! | **3. Sketch Logical Plan** | [`directory`] | Candidate sketch types per `AggIntent`, memory estimation | -//! | **4. Sketch Optimizer** | [`optimizer`] | 12 algebraic rewrite rules (fixed-point iteration) | -//! | **5. Physical Plan** | [`physical`] | `PhysicalAggOp` — resolves `AggIntent` → concrete `SketchType` + `SketchParams` | -//! | **5. Physical Plan** | [`allocator`] | `SketchAllocator` — assigns physical ops to pipeline stages | -//! | **5. Physical Plan** | [`plan`] | `PlanNode` tree with cost estimates and stage annotations | -//! -//! Layers 1–2 (language parsing) live in `query_parser/`. -//! -//! # Typical usage -//! -//! ```rust,ignore -//! use controller::query_parser; -//! use controller::algebra::{expr::QueryExpr, optimizer::QueryOptimizer, physical}; -//! -//! // Layers 1–3: parse query string → sketch logical plan. -//! let query_expr = query_parser::parse_query_expr("quantile_over_time(0.99, latency[5m])")?; -//! -//! // Layer 4: optimise (algebraic rewrite rules). -//! let (opt_expr, _iters) = QueryOptimizer::new(raw_bps).optimize(query_expr); -//! -//! // Layer 5: resolve logical AggIntent → physical SketchType + SketchParams. -//! // (done automatically by stage_split / allocator via physical::resolve) -//! ``` - -pub mod allocator; -pub mod directory; -pub mod expr; -pub mod lower; -pub mod optimizer; -pub mod physical; -pub mod plan; - -// Convenience re-exports. -pub use allocator::SketchAllocator; -pub use expr::{AggFunc, AggIntent, BinaryOpKind, QueryExpr, ScalarExpr, WindowKind, WindowSpec}; -pub use optimizer::QueryOptimizer; -pub use plan::{CostEstimate, ExecutionMode, PipelineStage, PlanNode, PlanSummary}; diff --git a/controller/src/algebra/optimizer.rs b/controller/src/algebra/optimizer.rs deleted file mode 100644 index 89135acf..00000000 --- a/controller/src/algebra/optimizer.rs +++ /dev/null @@ -1,1418 +0,0 @@ -//! Cost-based fixed-point query optimizer. -//! -//! The optimizer applies a set of algebraic rewrite rules to a -//! [`QueryExpr`](super::expr::QueryExpr) tree until no rule fires (fixed -//! point). Each rule is a pure function `QueryExpr → Option`: -//! returning `None` means "this rule does not apply here". -//! -//! # Rules implemented -//! -//! | Rule | Name | Description | -//! |------|------|-------------| -//! | R1 | `PredicatePushDown` | Push `Filter` below `Window`, `Partition`, `Aggregate` | -//! | R2 | `MergeLifting` | Lift mergeable sketch aggs above `Partition` nodes | -//! | R3 | `HLLDedupElim` | Eliminate `Dedup` before HLL (HLL is inherently distinct) | -//! | R4 | `FilterWindowSwap` | Swap `Filter` below `Window` to reduce window input size | -//! | R5 | `TopKFusion` | Absorb `Limit` / `TopK` into a `CountSketch` agg | -//! | R6 | `HistogramQuantileFusion` | Recognise `HistogramQuantile(φ, Agg(DDSketch))` and mark | -//! | R7 | `SubqueryDecorrelation` | Hoist correlated `ScalarSubquery` to a `LetBinding` | -//! | R8 | `CommonSubexprElim` | Extract identical sub-trees into `LetBinding`s | -//! | R9 | `HydraConversion` | Convert multi-key `Partition + Agg` into `Hydra` sketch | -//! | R10| `WindowMerge` | Merge adjacent `Window` nodes with the same duration | -//! | R11| `PartitionElim` | Remove `Partition` with empty key list (becomes global agg) | -//! | R12| `SetOpFusion` | Fuse `SetOp(Union, Merge, Merge)` into a single `Merge` | -//! -//! ## Cost model integration -//! -//! Rules R1–R4 are cost-free (always beneficial). Rules R5–R12 consult a -//! [`CostModel`] that estimates bandwidth, memory, and CPU overhead. A -//! rewrite is only applied when the estimated cost improves. - -use std::collections::HashMap; - -use super::expr::{QueryExpr, ScalarExpr, SetOpKind, SortKey}; -use super::expr::{AggIntent, PartitionKeys, SourceSpec}; - -// ── Cost model interface ────────────────────────────────────────────────────── - -/// Estimated cost of evaluating an expression at a given bandwidth. -#[derive(Debug, Clone, Default)] -pub struct NodeCost { - pub bytes_per_sec: f64, - pub memory_bytes: f64, - pub cpu_per_sample: f64, -} - -/// Pluggable cost oracle. The default implementation uses simple heuristics. -pub trait CostModel: Send + Sync { - /// Estimate the cost of the expression tree rooted at `expr`. - fn estimate(&self, expr: &QueryExpr) -> NodeCost; - - /// Deployment constraints (memory budgets, available backends, etc.). - /// Returns `None` if no constraints are configured (unconstrained mode). - fn constraints(&self) -> Option<&DeploymentConstraints> { None } -} - -// ── Sketch capabilities ───────────────────────────────────────────────────── - -/// Performance and capability profile for a single sketch implementation. -/// -/// Used by the optimizer to compare candidates and by the physical planner -/// to check whether a sketch fits within a stage's budget. -#[derive(Debug, Clone)] -pub struct SketchCapability { - /// Insertion throughput (samples/sec at 1 core). - pub insert_throughput: f64, - /// Query throughput (queries/sec at 1 core). - pub query_throughput: f64, - /// Memory footprint per series (bytes). - pub memory_bytes_per_series: u64, - /// CPU cost per insert (µs/sample). - pub cpu_micros_per_insert: f64, - /// Transmission size per flush (bytes). - pub transmission_bytes: u64, - /// Which logical aggregation intents this sketch supports. - pub supported_intents: Vec, - /// Whether the sketch supports merge (sketch(A∪B) = merge(sketch(A), sketch(B))). - pub mergeable: bool, - /// Whether the sketch supports delta encoding. - pub supports_delta: bool, - /// Whether the sketch supports sliding windows natively. - pub supports_sliding_window: bool, -} - -/// A logical aggregation intent that a sketch can serve. -#[derive(Debug, Clone, PartialEq)] -pub enum SupportedIntent { - Quantile, - Cardinality, - Frequency, - Extrema, -} - -/// YAML-serializable capability profile (for loading from config). -#[derive(Debug, Clone, serde::Deserialize)] -struct SketchCapabilityYaml { - insert_throughput: f64, - query_throughput: f64, - memory_bytes_per_series: u64, - cpu_micros_per_insert: f64, - transmission_bytes: u64, - supported_intents: Vec, - mergeable: bool, - supports_delta: bool, - supports_sliding_window: bool, -} - -impl SketchCapabilityYaml { - fn to_capability(&self) -> SketchCapability { - let intents = self.supported_intents.iter().filter_map(|s| match s.as_str() { - "quantile" => Some(SupportedIntent::Quantile), - "cardinality" => Some(SupportedIntent::Cardinality), - "frequency" => Some(SupportedIntent::Frequency), - "extrema" => Some(SupportedIntent::Extrema), - _ => None, - }).collect(); - SketchCapability { - insert_throughput: self.insert_throughput, - query_throughput: self.query_throughput, - memory_bytes_per_series: self.memory_bytes_per_series, - cpu_micros_per_insert: self.cpu_micros_per_insert, - transmission_bytes: self.transmission_bytes, - supported_intents: intents, - mergeable: self.mergeable, - supports_delta: self.supports_delta, - supports_sliding_window: self.supports_sliding_window, - } - } -} - -/// YAML file structure for all sketch capabilities. -#[derive(Debug, Clone, serde::Deserialize)] -struct SketchCapabilitiesFile { - ddsketch: SketchCapabilityYaml, - kll: SketchCapabilityYaml, - hll: SketchCapabilityYaml, - count_sketch: SketchCapabilityYaml, - count_min_sketch: SketchCapabilityYaml, -} - -/// Load sketch capabilities from a YAML file. -/// -/// Falls back to built-in defaults if the file is missing or malformed. -pub fn load_sketch_capabilities(path: &str) -> std::collections::HashMap { - use crate::types::SketchType; - if let Ok(contents) = std::fs::read_to_string(path) { - if let Ok(file) = serde_yaml::from_str::(&contents) { - let mut map = std::collections::HashMap::new(); - map.insert(SketchType::DDSketch, file.ddsketch.to_capability()); - map.insert(SketchType::KLL, file.kll.to_capability()); - map.insert(SketchType::HLL, file.hll.to_capability()); - map.insert(SketchType::CountSketch, file.count_sketch.to_capability()); - map.insert(SketchType::CountMinSketch, file.count_min_sketch.to_capability()); - return map; - } - } - // Fallback: built-in defaults. - let mut map = std::collections::HashMap::new(); - for st in &[SketchType::DDSketch, SketchType::KLL, SketchType::HLL, SketchType::CountSketch, SketchType::CountMinSketch] { - map.insert(st.clone(), sketch_capability(st)); - } - map -} - -/// Built-in capability profiles for known sketch types. -/// -/// These are compiled-in defaults. For deployment-specific values, load from -/// `sketch_capabilities.yml` via [`load_sketch_capabilities`], or run benchmarks -/// with `e2esdkbench` and update the YAML. -pub fn sketch_capability(st: &crate::types::SketchType) -> SketchCapability { - use crate::types::SketchType; - match st { - SketchType::DDSketch => SketchCapability { - insert_throughput: 10_000_000.0, - query_throughput: 50_000_000.0, - memory_bytes_per_series: 4_096, - cpu_micros_per_insert: 0.1, - transmission_bytes: 4_096, - supported_intents: vec![SupportedIntent::Quantile, SupportedIntent::Extrema], - mergeable: true, - supports_delta: true, - supports_sliding_window: false, - }, - SketchType::KLL => SketchCapability { - insert_throughput: 5_000_000.0, - query_throughput: 20_000_000.0, - memory_bytes_per_series: 8_192, - cpu_micros_per_insert: 0.2, - transmission_bytes: 8_192, - supported_intents: vec![SupportedIntent::Quantile, SupportedIntent::Extrema], - mergeable: true, - supports_delta: false, - supports_sliding_window: false, - }, - SketchType::HLL => SketchCapability { - insert_throughput: 20_000_000.0, - query_throughput: 100_000_000.0, - memory_bytes_per_series: 16_384, - cpu_micros_per_insert: 0.05, - transmission_bytes: 16_384, - supported_intents: vec![SupportedIntent::Cardinality], - mergeable: true, - supports_delta: true, - supports_sliding_window: false, - }, - SketchType::CountSketch => SketchCapability { - insert_throughput: 8_000_000.0, - query_throughput: 10_000_000.0, - memory_bytes_per_series: 80_000, - cpu_micros_per_insert: 0.5, - transmission_bytes: 80_000, - supported_intents: vec![SupportedIntent::Frequency], - mergeable: true, - supports_delta: true, - supports_sliding_window: false, - }, - SketchType::CountMinSketch => SketchCapability { - insert_throughput: 8_000_000.0, - query_throughput: 10_000_000.0, - memory_bytes_per_series: 80_000, - cpu_micros_per_insert: 0.5, - transmission_bytes: 80_000, - supported_intents: vec![SupportedIntent::Frequency], - mergeable: true, - supports_delta: true, - supports_sliding_window: false, - }, - } -} - -// ── Stage budgets ─────────────────────────────────────────────────────────── - -/// Resource budget for a single pipeline stage. -/// -/// All fields are `Option` — `None` means unbounded / unconstrained. -#[derive(Debug, Clone, Default)] -pub struct StageBudget { - /// Memory budget (bytes). - pub memory_bytes: Option, - /// CPU budget (µs per sample). - pub cpu_micros_per_sample: Option, - /// Disk budget (bytes). - pub disk_bytes: Option, - /// Egress bandwidth budget (bytes/sec). - pub bandwidth_bytes_per_sec: Option, -} - -impl StageBudget { - /// Check whether a sketch fits within this stage's budget. - pub fn fits(&self, cap: &SketchCapability) -> bool { - if let Some(mem) = self.memory_bytes { - if cap.memory_bytes_per_series > mem { return false; } - } - if let Some(cpu) = self.cpu_micros_per_sample { - if cap.cpu_micros_per_insert > cpu { return false; } - } - if let Some(bw) = self.bandwidth_bytes_per_sec { - // Rough: transmission bytes per flush ÷ 1 second - if cap.transmission_bytes as f64 > bw { return false; } - } - true - } -} - -// ── Deployment constraints ────────────────────────────────────────────────── - -/// Full deployment specification: per-stage budgets. -/// -/// The optimizer uses stage budgets to penalise plans that exceed capacity. -/// The physical planner uses `StageBudget::fits(SketchCapability)` to decide -/// concrete placement. -#[derive(Debug, Clone, Default)] -pub struct DeploymentConstraints { - /// Edge / agent collector (sketch build). - pub agent: StageBudget, - /// Backend collector (sketch merge). - pub backend_collector: StageBudget, - /// Backend sketchDB (precompute engine / query engine). - pub backend_db: StageBudget, - /// Backend original DB (exact computation). - pub original_db: StageBudget, - /// Object store (S3 raw backup). - pub object_store: StageBudget, -} - -impl DeploymentConstraints { - /// Build from [`StageResourceBudgets`] (the workload-derived budgets). - pub fn from_budgets(budgets: &crate::types::StageResourceBudgets) -> Self { - Self { - agent: StageBudget { - memory_bytes: budgets.agent_memory_bytes, - cpu_micros_per_sample: budgets.agent_cpu_micros_per_sample, - ..Default::default() - }, - backend_collector: StageBudget { - memory_bytes: budgets.backend_memory_bytes, - ..Default::default() - }, - backend_db: StageBudget { - memory_bytes: budgets.precompute_memory_bytes, - ..Default::default() - }, - ..Default::default() - } - } -} - -/// Default cost model — simple heuristics, no schema statistics. -pub struct DefaultCostModel { - pub raw_bytes_per_sec: f64, - pub deployment: Option, -} - -impl CostModel for DefaultCostModel { - fn constraints(&self) -> Option<&DeploymentConstraints> { - self.deployment.as_ref() - } - - fn estimate(&self, expr: &QueryExpr) -> NodeCost { - // Sketch nodes reduce bandwidth; exact nodes pass through. - let factor = match expr { - QueryExpr::SketchAgg { op, .. } | QueryExpr::WindowedAgg { agg: op, .. } => match op { - AggIntent::Quantile { .. } => 0.05, - AggIntent::Cardinality { .. } => 0.02, - AggIntent::Frequency { .. } => 0.03, - AggIntent::Exact(_) => 1.0, - _ => 0.1, - }, - QueryExpr::Merge { inputs } => 1.0 / (inputs.len().max(1) as f64), - QueryExpr::Filter { .. } => 0.5, - QueryExpr::TopK { k, .. } => (*k as f64).recip().min(0.1), - QueryExpr::Partition { .. } => 0.8, // partition adds overhead - QueryExpr::Dedup { .. } => 0.9, - _ => 1.0, - }; - - let memory = match expr { - QueryExpr::SketchAgg { op, .. } | QueryExpr::WindowedAgg { agg: op, .. } => - crate::algebra::directory::estimated_sketch_memory_bytes(op) as f64, - _ => self.raw_bytes_per_sec * factor * 0.01, - }; - - let base = NodeCost { - bytes_per_sec: self.raw_bytes_per_sec * factor, - memory_bytes: memory, - cpu_per_sample: factor * 10.0, - }; - - // Apply deployment constraint penalties. - if let Some(dc) = &self.deployment { - // Map each expression to its default stage budget. - let stage_budget = match expr { - QueryExpr::SketchAgg { .. } | QueryExpr::WindowedAgg { .. } - | QueryExpr::Source(_) | QueryExpr::Filter { .. } - | QueryExpr::Window { .. } => &dc.agent, - QueryExpr::Partition { .. } | QueryExpr::Merge { .. } - | QueryExpr::Dedup { .. } => &dc.backend_collector, - QueryExpr::TopK { .. } | QueryExpr::HistogramQuantile { .. } - | QueryExpr::BinaryOp { .. } | QueryExpr::PromQLSubquery { .. } => &dc.backend_db, - QueryExpr::Aggregate { .. } => &dc.original_db, - _ => &dc.agent, - }; - - // For sketch nodes, check sketch capability against stage budget. - if let QueryExpr::SketchAgg { op, .. } | QueryExpr::WindowedAgg { agg: op, .. } = expr { - let sketch_type = crate::algebra::directory::sketch_type_for_op(op); - let cap = sketch_capability(&sketch_type); - if !stage_budget.fits(&cap) { - // Sketch doesn't fit — apply 10× penalty across all dimensions. - return NodeCost { - bytes_per_sec: base.bytes_per_sec * 10.0, - memory_bytes: base.memory_bytes * 10.0, - cpu_per_sample: base.cpu_per_sample * 10.0, - }; - } - // Sliding window: penalise if sketch doesn't support it natively. - if let QueryExpr::WindowedAgg { window, .. } = expr { - if matches!(window.kind, crate::algebra::expr::WindowKind::Sliding { .. }) - && !cap.supports_sliding_window - { - return NodeCost { - cpu_per_sample: base.cpu_per_sample * 5.0, - ..base - }; - } - } - } else { - // Non-sketch nodes: check basic budget constraints. - if let Some(budget) = stage_budget.memory_bytes { - if base.memory_bytes > budget as f64 { - return NodeCost { memory_bytes: base.memory_bytes * 10.0, ..base }; - } - } - if let Some(bw) = stage_budget.bandwidth_bytes_per_sec { - if base.bytes_per_sec > bw { - return NodeCost { bytes_per_sec: base.bytes_per_sec * 10.0, ..base }; - } - } - } - } - base - } -} - -// ── Rewrite rule trait ──────────────────────────────────────────────────────── - -/// A single algebraic rewrite rule. -pub trait RewriteRule: Send + Sync { - /// Human-readable name for logging. - fn name(&self) -> &'static str; - - /// Try to rewrite `expr`. Returns `Some(new_expr)` if the rule fired, - /// `None` otherwise. The rule is applied top-down: the optimizer will - /// also recurse into the children of `new_expr`. - fn try_rewrite(&self, expr: QueryExpr, model: &dyn CostModel) -> Option; -} - -// ── R1: PredicatePushDown ───────────────────────────────────────────────────── - -/// Push `Filter` nodes as deep as possible — reduces data volume early. -/// -/// Transformations applied (all strictly beneficial, no cost model check): -/// * `Filter(p, Window(d, e))` → `Window(d, Filter(p, e))` -/// * `Filter(p, Partition(k, e))` → `Partition(k, Filter(p, e))` -/// * `Filter(p, Sort(k, e))` → `Sort(k, Filter(p, e))` -/// * `Filter(p, Limit(n, o, e))` → `Limit(n, o, Filter(p, e))` (NB: only safe when p is on input cols) -pub struct PredicatePushDown; - -impl RewriteRule for PredicatePushDown { - fn name(&self) -> &'static str { "PredicatePushDown" } - - fn try_rewrite(&self, expr: QueryExpr, _model: &dyn CostModel) -> Option { - match expr { - QueryExpr::Filter { pred, input } => { - match *input { - // Filter below Window - QueryExpr::Window { duration, slide, input: inner } => Some( - QueryExpr::Window { - duration, - slide, - input: Box::new(QueryExpr::Filter { - pred, - input: inner, - }), - } - ), - // Filter below Partition - QueryExpr::Partition { keys, input: inner } => Some( - QueryExpr::Partition { - keys, - input: Box::new(QueryExpr::Filter { - pred, - input: inner, - }), - } - ), - // Filter below Sort (safe when pred references input columns only) - QueryExpr::Sort { keys, input: inner } => Some( - QueryExpr::Sort { - keys, - input: Box::new(QueryExpr::Filter { - pred, - input: inner, - }), - } - ), - // Not applicable — reconstruct - other => Some(QueryExpr::Filter { pred, input: Box::new(other) }), - } - } - _ => None, - } - } -} - -// ── R2: MergeLifting ────────────────────────────────────────────────────────── - -/// Lift mergeable `SketchAgg` ops above `Merge` nodes. -/// -/// `SketchAgg(op, Merge([a, b]))` → `Merge([SketchAgg(op, a), SketchAgg(op, b)])` -/// -/// Only applied for `is_mergeable()` ops so we don't incorrectly distribute -/// `Avg` or `StdDev`. -pub struct MergeLifting; - -impl RewriteRule for MergeLifting { - fn name(&self) -> &'static str { "MergeLifting" } - - fn try_rewrite(&self, expr: QueryExpr, _model: &dyn CostModel) -> Option { - match expr { - QueryExpr::SketchAgg { ref op, ref col, ref input } - if op.is_mergeable() => - { - if let QueryExpr::Merge { inputs } = input.as_ref() { - let new_inputs: Vec = inputs.iter().map(|branch| { - QueryExpr::SketchAgg { - op: op.clone(), - col: col.clone(), - input: Box::new(branch.clone()), - } - }).collect(); - return Some(QueryExpr::Merge { inputs: new_inputs }); - } - None - } - _ => None, - } - } -} - -// ── R3: HLLDedupElim ───────────────────────────────────────────────────────── - -/// Eliminate `Dedup` nodes that immediately precede an HLL aggregation. -/// -/// HLL counts distinct values intrinsically; an explicit dedup step is -/// redundant and wastes CPU / memory. -/// -/// `SketchAgg(HLL, Dedup(col, e))` → `SketchAgg(HLL, e)` -pub struct HLLDedupElim; - -impl RewriteRule for HLLDedupElim { - fn name(&self) -> &'static str { "HLLDedupElim" } - - fn try_rewrite(&self, expr: QueryExpr, _model: &dyn CostModel) -> Option { - match expr { - QueryExpr::SketchAgg { op: AggIntent::Cardinality { accuracy }, col, input } => { - if let QueryExpr::Dedup { input: inner, .. } = *input { - return Some(QueryExpr::SketchAgg { - op: AggIntent::Cardinality { accuracy }, - col, - input: inner, - }); - } - None - } - _ => None, - } - } -} - -// ── R4: FilterWindowSwap ────────────────────────────────────────────────────── - -/// Push `Filter` below `Window` when the predicate references only source -/// columns (not windowed aggregates). -/// -/// Identical to the push-down in R1 for the Window case, but checked -/// separately so the optimizer can attribute the transformation correctly -/// in logs. -pub struct FilterWindowSwap; - -impl RewriteRule for FilterWindowSwap { - fn name(&self) -> &'static str { "FilterWindowSwap" } - - fn try_rewrite(&self, expr: QueryExpr, _model: &dyn CostModel) -> Option { - // Handled by PredicatePushDown — mark as no-op here to avoid double-fire. - match expr { - QueryExpr::Filter { pred, input } => { - if let QueryExpr::Window { duration, slide, input: inner } = *input { - return Some(QueryExpr::Window { - duration, - slide, - input: Box::new(QueryExpr::Filter { pred, input: inner }), - }); - } - None - } - _ => None, - } - } -} - -// ── R5: TopKFusion ──────────────────────────────────────────────────────────── - -/// Fuse a `Limit(k, TopK(_, e))` or `Limit(k, Sort(_, e))` into a single -/// `TopK(k, e)` node that the allocator maps to a `CountSketch`. -/// -/// `Limit(n, Sort([col DESC], e))` → `TopK(n, [col], e)` -pub struct TopKFusion; - -impl RewriteRule for TopKFusion { - fn name(&self) -> &'static str { "TopKFusion" } - - fn try_rewrite(&self, expr: QueryExpr, _model: &dyn CostModel) -> Option { - match expr { - QueryExpr::Limit { n, offset: 0, input } => { - if let QueryExpr::Sort { keys, input: inner } = *input { - // Only fuse when all keys are DESC (top-k semantics). - if keys.iter().all(|k| k.desc) { - let by: Vec = keys.into_iter().map(|k| k.col).collect(); - return Some(QueryExpr::TopK { k: n, by, input: inner }); - } - } - None - } - _ => None, - } - } -} - -// ── R6: HistogramQuantileFusion ─────────────────────────────────────────────── - -/// Recognise `HistogramQuantile(φ, SketchAgg(DDSketch([φ]), …))` and -/// simplify to a single annotated node that the allocator handles as one -/// DDSketch query. -/// -/// `HistogramQuantile(φ, SketchAgg(DDSketch(qs), col, e))` -/// where `qs` contains `φ` -/// → `HistogramQuantile(φ, SketchAgg(DDSketch(qs), col, e))` [marked fused] -/// -/// In practice we just ensure the quantile is in the DDSketch's quantile -/// list so the allocator emits a single sketch with the right φ. -pub struct HistogramQuantileFusion; - -impl RewriteRule for HistogramQuantileFusion { - fn name(&self) -> &'static str { "HistogramQuantileFusion" } - - fn try_rewrite(&self, expr: QueryExpr, _model: &dyn CostModel) -> Option { - match expr { - QueryExpr::HistogramQuantile { phi, input } => { - match *input { - QueryExpr::SketchAgg { - op: AggIntent::Quantile { quantiles, accuracy }, - col, - input: inner, - } => { - if !quantiles.contains(&phi) { - let mut new_qs = quantiles; - new_qs.push(phi); - new_qs.sort_by(|a, b| a.partial_cmp(b).unwrap()); - Some(QueryExpr::HistogramQuantile { - phi, - input: Box::new(QueryExpr::SketchAgg { - op: AggIntent::Quantile { quantiles: new_qs, accuracy }, - col, - input: inner, - }), - }) - } else { - Some(QueryExpr::HistogramQuantile { - phi, - input: Box::new(QueryExpr::SketchAgg { - op: AggIntent::Quantile { quantiles, accuracy }, - col, - input: inner, - }), - }) - } - } - other => Some(QueryExpr::HistogramQuantile { phi, input: Box::new(other) }), - } - } - _ => None, - } - } -} - -// ── R7: SubqueryDecorrelation ───────────────────────────────────────────────── - -/// Hoist correlated `ScalarSubquery` references into `LetBinding`s so that -/// the subquery is evaluated once rather than once per row. -/// -/// This rule is a structural marker — full correlated-subquery detection -/// requires a binder pass that is out of scope here. We handle the simple -/// case: a `Filter` whose predicate contains a `ScalarSubquery` that does -/// not reference the filter's own input. -pub struct SubqueryDecorrelation; - -impl RewriteRule for SubqueryDecorrelation { - fn name(&self) -> &'static str { "SubqueryDecorrelation" } - - fn try_rewrite(&self, expr: QueryExpr, _model: &dyn CostModel) -> Option { - match expr { - QueryExpr::Filter { pred, input } => { - if let Some((name, sq_expr, new_pred)) = extract_scalar_subquery(pred) { - return Some(QueryExpr::LetBinding { - name: name.clone(), - expr: Box::new(sq_expr), - body: Box::new(QueryExpr::Filter { - pred: new_pred, - input, - }), - }); - } - None - } - _ => None, - } - } -} - -/// If `pred` contains a `ScalarSubquery`, extract it as -/// `(binding_name, subquery_expr, pred_with_ref)`. -fn extract_scalar_subquery( - pred: ScalarExpr, -) -> Option<(String, QueryExpr, ScalarExpr)> { - match pred { - ScalarExpr::BinaryOp { op, lhs, rhs } => { - // Check lhs - if let ScalarExpr::ScalarSubquery(sq) = *lhs { - let name = "__subq_0".to_string(); - let new_pred = ScalarExpr::BinaryOp { - op, - lhs: Box::new(ScalarExpr::Column(name.clone())), - rhs, - }; - return Some((name, *sq, new_pred)); - } - // Check rhs - if let ScalarExpr::ScalarSubquery(sq) = *rhs { - let name = "__subq_0".to_string(); - let new_pred = ScalarExpr::BinaryOp { - op, - lhs, - rhs: Box::new(ScalarExpr::Column(name.clone())), - }; - return Some((name, *sq, new_pred)); - } - None - } - _ => None, - } -} - -// ── R8: CommonSubexprElim ───────────────────────────────────────────────────── - -/// Identify identical sub-trees that appear in multiple branches of a `Merge` -/// node and hoist them into a `LetBinding`. -/// -/// This is a conservative implementation: only `Source` nodes with the same -/// name are deduplicated (the common case where the same metric appears in -/// multiple union branches). -pub struct CommonSubexprElim; - -impl RewriteRule for CommonSubexprElim { - fn name(&self) -> &'static str { "CommonSubexprElim" } - - fn try_rewrite(&self, expr: QueryExpr, _model: &dyn CostModel) -> Option { - match expr { - QueryExpr::Merge { ref inputs } => { - // Count occurrences of each source name. - let mut counts: HashMap = HashMap::new(); - for inp in inputs { - if let Some(name) = inp.source_name() { - *counts.entry(name.to_string()).or_insert(0) += 1; - } - } - let repeated: Vec = counts.into_iter() - .filter(|(_, c)| *c > 1) - .map(|(n, _)| n) - .collect(); - if repeated.is_empty() { - return None; - } - // Hoist the first repeated source into a LetBinding. - let name = repeated.into_iter().next()?; - let binding_name = format!("__cse_{name}"); - let new_inputs: Vec = inputs.iter().cloned().map(|inp| { - if inp.source_name() == Some(name.as_str()) { - QueryExpr::Ref(binding_name.clone()) - } else { - inp - } - }).collect(); - Some(QueryExpr::LetBinding { - name: binding_name, - expr: Box::new(QueryExpr::Source(SourceSpec { name })), - body: Box::new(QueryExpr::Merge { inputs: new_inputs }), - }) - } - _ => None, - } - } -} - -// ── R9: HydraConversion ─────────────────────────────────────────────────────── - -/// Convert `Partition(keys, SketchAgg(op, col, e))` where `keys` has ≥ 2 -/// dimensions into `SketchAgg(Hydra{inner: op, keys}, col, e)`. -/// -/// Hydra is a sketch-of-sketches that handles multi-dimensional GROUP BY -/// more efficiently than one sketch per group tuple. -pub struct HydraConversion; - -impl RewriteRule for HydraConversion { - fn name(&self) -> &'static str { "HydraConversion" } - - fn try_rewrite(&self, expr: QueryExpr, model: &dyn CostModel) -> Option { - match expr { - QueryExpr::Partition { keys: PartitionKeys::By(ref key_list), ref input } - if key_list.len() >= 2 => - { - if let QueryExpr::SketchAgg { op: ref inner_op, ref col, input: ref inner_input } = - **input - { - if matches!( - inner_op, - AggIntent::Quantile { .. } - | AggIntent::Cardinality { .. } - | AggIntent::Frequency { .. } - ) { - let hydra_op = AggIntent::PerPartition { - inner: Box::new(inner_op.clone()), - keys: key_list.clone(), - }; - let candidate = QueryExpr::SketchAgg { - op: hydra_op, - col: col.clone(), - input: inner_input.clone(), - }; - let old_cost = model.estimate(&expr); - let new_cost = model.estimate(&candidate); - if new_cost.memory_bytes < old_cost.memory_bytes - || new_cost.bytes_per_sec < old_cost.bytes_per_sec - { - return Some(candidate); - } - } - } - None - } - _ => None, - } - } -} - -// ── R10: WindowMerge ───────────────────────────────────────────────────────── - -/// Merge two adjacent `Window` nodes with the same `duration` into one. -/// -/// `Window(d, Window(d, e))` → `Window(d, e)` -pub struct WindowMerge; - -impl RewriteRule for WindowMerge { - fn name(&self) -> &'static str { "WindowMerge" } - - fn try_rewrite(&self, expr: QueryExpr, _model: &dyn CostModel) -> Option { - match expr { - QueryExpr::Window { duration, slide, input } => { - if let QueryExpr::Window { duration: inner_d, slide: inner_s, input: inner_e } = - *input - { - if duration == inner_d && slide == inner_s { - return Some(QueryExpr::Window { - duration, - slide, - input: inner_e, - }); - } - } - None - } - _ => None, - } - } -} - -// ── R11: PartitionElim ──────────────────────────────────────────────────────── - -/// Remove `Partition` with an empty key list — equivalent to a global -/// aggregation with no GROUP BY. -/// -/// `Partition(By([]), e)` → `e` -pub struct PartitionElim; - -impl RewriteRule for PartitionElim { - fn name(&self) -> &'static str { "PartitionElim" } - - fn try_rewrite(&self, expr: QueryExpr, _model: &dyn CostModel) -> Option { - match expr { - QueryExpr::Partition { keys, input } if keys.is_empty() => Some(*input), - _ => None, - } - } -} - -// ── R12: SetOpFusion ────────────────────────────────────────────────────────── - -/// Fuse `SetOp(Union, Merge([…]), Merge([…]))` into a single `Merge([…, …])`. -pub struct SetOpFusion; - -impl RewriteRule for SetOpFusion { - fn name(&self) -> &'static str { "SetOpFusion" } - - fn try_rewrite(&self, expr: QueryExpr, _model: &dyn CostModel) -> Option { - match expr { - QueryExpr::SetOp { - kind: SetOpKind::Union, - all: true, - left, - right, - } => { - match (*left, *right) { - (QueryExpr::Merge { inputs: mut li }, QueryExpr::Merge { inputs: mut ri }) => { - li.append(&mut ri); - Some(QueryExpr::Merge { inputs: li }) - } - (l, r) => Some(QueryExpr::SetOp { - kind: SetOpKind::Union, - all: true, - left: Box::new(l), - right: Box::new(r), - }), - } - } - _ => None, - } - } -} - -// ── Optimizer ───────────────────────────────────────────────────────────────── - -/// Fixed-point query optimizer. -/// -/// Call [`QueryOptimizer::optimize`] to rewrite a [`QueryExpr`] tree. -/// The optimizer iterates over all registered rules until no rule fires. -pub struct QueryOptimizer { - rules: Vec>, - cost_model: Box, - /// Maximum number of fixed-point iterations (prevents infinite loops). - max_iters: usize, -} - -impl QueryOptimizer { - /// Create an optimizer with the default rule set and cost model. - pub fn new(raw_bytes_per_sec: f64) -> Self { - Self { - rules: default_rules(), - cost_model: Box::new(DefaultCostModel { raw_bytes_per_sec, deployment: None }), - max_iters: 32, - } - } - - /// Create an optimizer with deployment constraints. - pub fn with_constraints(raw_bytes_per_sec: f64, constraints: DeploymentConstraints) -> Self { - Self { - rules: default_rules(), - cost_model: Box::new(DefaultCostModel { - raw_bytes_per_sec, - deployment: Some(constraints), - }), - max_iters: 32, - } - } - - /// Create an optimizer with a custom cost model. - pub fn with_cost_model(cost_model: Box) -> Self { - Self { - rules: default_rules(), - cost_model, - max_iters: 32, - } - } - - /// Set maximum fixed-point iterations (default: 32). - pub fn max_iters(mut self, n: usize) -> Self { - self.max_iters = n; - self - } - - /// Optimize `expr` until fixed point or `max_iters` iterations. - /// - /// Returns the rewritten tree and the number of iterations actually run. - pub fn optimize(&self, expr: QueryExpr) -> (QueryExpr, usize) { - let mut current = expr; - for iter in 0..self.max_iters { - let (next, changed) = self.apply_all(current); - current = next; - if !changed { - return (current, iter + 1); - } - } - (current, self.max_iters) - } - - /// Apply all rules once to every node in the tree (single pass). - /// Returns `(new_tree, did_anything_change)`. - fn apply_all(&self, expr: QueryExpr) -> (QueryExpr, bool) { - // First recurse into children, then try rules at this node. - let (expr_with_new_children, child_changed) = self.recurse_children(expr); - let (final_expr, this_changed) = self.apply_rules_at(expr_with_new_children); - (final_expr, child_changed || this_changed) - } - - /// Apply all rules at the current node (no recursion). - fn apply_rules_at(&self, mut expr: QueryExpr) -> (QueryExpr, bool) { - let mut changed = false; - for rule in &self.rules { - if let Some(new_expr) = rule.try_rewrite(expr.clone(), self.cost_model.as_ref()) { - expr = new_expr; - changed = true; - // After firing, restart from the first rule (fixed-point per node). - break; - } - } - (expr, changed) - } - - /// Recurse into children, rebuilding the node with rewritten children. - fn recurse_children(&self, expr: QueryExpr) -> (QueryExpr, bool) { - macro_rules! recurse { - ($child:expr) => {{ - let (e, c) = self.apply_all(*$child); - (Box::new(e), c) - }}; - } - match expr { - QueryExpr::Source(_) | QueryExpr::Ref(_) => (expr, false), - - QueryExpr::Filter { pred, input } => { - let (new_input, c) = recurse!(input); - (QueryExpr::Filter { pred, input: new_input }, c) - } - QueryExpr::Project { cols, input } => { - let (new_input, c) = recurse!(input); - (QueryExpr::Project { cols, input: new_input }, c) - } - QueryExpr::Aggregate { keys, aggs, having, input } => { - let (new_input, c) = recurse!(input); - (QueryExpr::Aggregate { keys, aggs, having, input: new_input }, c) - } - QueryExpr::Window { duration, slide, input } => { - let (new_input, c) = recurse!(input); - (QueryExpr::Window { duration, slide, input: new_input }, c) - } - QueryExpr::SketchAgg { op, col, input } => { - let (new_input, c) = recurse!(input); - (QueryExpr::SketchAgg { op, col, input: new_input }, c) - } - QueryExpr::WindowedAgg { agg, window, col, input } => { - let (new_input, c) = recurse!(input); - (QueryExpr::WindowedAgg { agg, window, col, input: new_input }, c) - } - QueryExpr::Partition { keys, input } => { - let (new_input, c) = recurse!(input); - (QueryExpr::Partition { keys, input: new_input }, c) - } - QueryExpr::Dedup { col, input } => { - let (new_input, c) = recurse!(input); - (QueryExpr::Dedup { col, input: new_input }, c) - } - QueryExpr::TopK { k, by, input } => { - let (new_input, c) = recurse!(input); - (QueryExpr::TopK { k, by, input: new_input }, c) - } - QueryExpr::Sort { keys, input } => { - let (new_input, c) = recurse!(input); - (QueryExpr::Sort { keys, input: new_input }, c) - } - QueryExpr::Limit { n, offset, input } => { - let (new_input, c) = recurse!(input); - (QueryExpr::Limit { n, offset, input: new_input }, c) - } - QueryExpr::WindowFunc { func, partition_by, order_by, frame, input } => { - let (new_input, c) = recurse!(input); - (QueryExpr::WindowFunc { func, partition_by, order_by, frame, input: new_input }, c) - } - QueryExpr::HistogramQuantile { phi, input } => { - let (new_input, c) = recurse!(input); - (QueryExpr::HistogramQuantile { phi, input: new_input }, c) - } - QueryExpr::PromQLSubquery { range, resolution, input } => { - let (new_input, c) = recurse!(input); - (QueryExpr::PromQLSubquery { range, resolution, input: new_input }, c) - } - QueryExpr::Merge { inputs } => { - let (new_inputs, changed): (Vec<_>, Vec<_>) = inputs - .into_iter() - .map(|inp| self.apply_all(inp)) - .unzip(); - (QueryExpr::Merge { inputs: new_inputs }, changed.into_iter().any(|c| c)) - } - QueryExpr::Join { kind, pred, left, right } => { - let (new_left, cl) = recurse!(left); - let (new_right, cr) = recurse!(right); - (QueryExpr::Join { kind, pred, left: new_left, right: new_right }, cl || cr) - } - QueryExpr::JoinSketch { join_key, outer, inner } => { - let (new_outer, co) = recurse!(outer); - let (new_inner, ci) = recurse!(inner); - (QueryExpr::JoinSketch { join_key, outer: new_outer, inner: new_inner }, co || ci) - } - QueryExpr::SetOp { kind, all, left, right } => { - let (new_left, cl) = recurse!(left); - let (new_right, cr) = recurse!(right); - (QueryExpr::SetOp { kind, all, left: new_left, right: new_right }, cl || cr) - } - QueryExpr::BinaryOp { op, lhs, rhs, vector_match } => { - let (new_lhs, cl) = recurse!(lhs); - let (new_rhs, cr) = recurse!(rhs); - (QueryExpr::BinaryOp { op, lhs: new_lhs, rhs: new_rhs, vector_match }, cl || cr) - } - QueryExpr::Subquery { alias, expr } => { - let (new_expr, c) = recurse!(expr); - (QueryExpr::Subquery { alias, expr: new_expr }, c) - } - QueryExpr::LetBinding { name, expr, body } => { - let (new_expr, ce) = recurse!(expr); - let (new_body, cb) = recurse!(body); - (QueryExpr::LetBinding { name, expr: new_expr, body: new_body }, ce || cb) - } - } - } -} - -/// Construct the default ordered rule set. -fn default_rules() -> Vec> { - vec![ - Box::new(PredicatePushDown), - Box::new(FilterWindowSwap), - Box::new(HLLDedupElim), - Box::new(WindowMerge), - Box::new(PartitionElim), - Box::new(TopKFusion), - Box::new(HistogramQuantileFusion), - Box::new(MergeLifting), - Box::new(SetOpFusion), - Box::new(HydraConversion), - Box::new(SubqueryDecorrelation), - Box::new(CommonSubexprElim), - ] -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use crate::algebra::expr::{LiteralValue, ScalarExpr}; - use crate::algebra::expr::{AggIntent, ColumnRef, PartitionKeys, SourceSpec}; - use std::time::Duration; - - fn src(name: &str) -> QueryExpr { - QueryExpr::Source(SourceSpec { name: name.into() }) - } - - fn opt() -> QueryOptimizer { - QueryOptimizer::new(100_000.0) - } - - // ── R1: PredicatePushDown ───────────────────────────────────────────────── - - #[test] - fn r1_pushes_filter_below_window() { - let expr = QueryExpr::Filter { - pred: ScalarExpr::Literal(LiteralValue::Bool(true)), - input: Box::new(QueryExpr::Window { - duration: Duration::from_secs(60), - slide: None, - input: Box::new(src("m")), - }), - }; - let (result, _) = opt().optimize(expr); - assert!( - matches!(&result, QueryExpr::Window { input, .. } - if matches!(input.as_ref(), QueryExpr::Filter { .. })), - "filter should be inside window: {result:?}" - ); - } - - #[test] - fn r1_pushes_filter_below_partition() { - let expr = QueryExpr::Filter { - pred: ScalarExpr::Literal(LiteralValue::Bool(true)), - input: Box::new(QueryExpr::Partition { - keys: PartitionKeys::By(vec!["host".into()]), - input: Box::new(src("cpu")), - }), - }; - let (result, _) = opt().optimize(expr); - assert!( - matches!(&result, QueryExpr::Partition { input, .. } - if matches!(input.as_ref(), QueryExpr::Filter { .. })) - ); - } - - // ── R3: HLLDedupElim ───────────────────────────────────────────────────── - - #[test] - fn r3_removes_dedup_before_hll() { - let expr = QueryExpr::SketchAgg { - op: AggIntent::default_cardinality(), - col: ColumnRef::Named("user_id".into()), - input: Box::new(QueryExpr::Dedup { - col: "user_id".into(), - input: Box::new(src("events")), - }), - }; - let (result, _) = opt().optimize(expr); - assert!( - !matches!(&result, QueryExpr::SketchAgg { input, .. } - if matches!(input.as_ref(), QueryExpr::Dedup { .. })), - "Dedup should be eliminated before HLL" - ); - } - - // ── R5: TopKFusion ──────────────────────────────────────────────────────── - - #[test] - fn r5_fuses_limit_sort_to_topk() { - let expr = QueryExpr::Limit { - n: 10, - offset: 0, - input: Box::new(QueryExpr::Sort { - keys: vec![SortKey { col: "count".into(), desc: true, nulls_first: None }], - input: Box::new(src("events")), - }), - }; - let (result, _) = opt().optimize(expr); - assert!( - matches!(&result, QueryExpr::TopK { k: 10, .. }), - "expected TopK(10), got {result:?}" - ); - } - - #[test] - fn r5_does_not_fuse_ascending_sort() { - // ASC sort → not a top-k query. - let expr = QueryExpr::Limit { - n: 10, - offset: 0, - input: Box::new(QueryExpr::Sort { - keys: vec![SortKey { col: "ts".into(), desc: false, nulls_first: None }], - input: Box::new(src("events")), - }), - }; - let (result, _) = opt().optimize(expr); - assert!( - !matches!(&result, QueryExpr::TopK { .. }), - "ascending sort should not become TopK" - ); - } - - // ── R6: HistogramQuantileFusion ─────────────────────────────────────────── - - #[test] - fn r6_adds_phi_to_ddsketch_quantiles() { - let expr = QueryExpr::HistogramQuantile { - phi: 0.95, - input: Box::new(QueryExpr::SketchAgg { - op: AggIntent::Quantile { quantiles: vec![0.5], accuracy: 0.01 }, - col: ColumnRef::SampleValue, - input: Box::new(src("latency")), - }), - }; - let (result, _) = opt().optimize(expr); - match &result { - QueryExpr::HistogramQuantile { input, .. } => { - if let QueryExpr::SketchAgg { op: AggIntent::Quantile { quantiles, .. }, .. } = - input.as_ref() - { - assert!(quantiles.contains(&0.95), "0.95 should be in DDSketch quantiles"); - assert!(quantiles.contains(&0.5), "0.5 should still be present"); - } else { - panic!("expected DDSketch under HistogramQuantile"); - } - } - other => panic!("unexpected {other:?}"), - } - } - - // ── R10: WindowMerge ────────────────────────────────────────────────────── - - #[test] - fn r10_merges_duplicate_windows() { - let expr = QueryExpr::Window { - duration: Duration::from_secs(300), - slide: None, - input: Box::new(QueryExpr::Window { - duration: Duration::from_secs(300), - slide: None, - input: Box::new(src("m")), - }), - }; - let (result, _) = opt().optimize(expr); - assert!( - !matches!(&result, QueryExpr::Window { input, .. } - if matches!(input.as_ref(), QueryExpr::Window { .. })), - "duplicate window should be merged" - ); - } - - // ── R11: PartitionElim ──────────────────────────────────────────────────── - - #[test] - fn r11_removes_empty_partition() { - let expr = QueryExpr::Partition { - keys: PartitionKeys::By(vec![]), - input: Box::new(src("m")), - }; - let (result, _) = opt().optimize(expr); - assert!( - matches!(&result, QueryExpr::Source(_)), - "empty Partition should be eliminated" - ); - } - - // ── Fixed-point convergence ─────────────────────────────────────────────── - - #[test] - fn optimizer_reaches_fixed_point_on_simple_tree() { - let expr = src("m"); - let (result, iters) = opt().optimize(expr); - assert!(iters < 5, "should converge quickly on source-only tree"); - assert!(matches!(result, QueryExpr::Source(_))); - } - - #[test] - fn optimizer_chain_of_rewrites() { - // Filter(Window(Dedup(HLL(Source)))) → - // R1: Window(Filter(Dedup(HLL(Source)))) - // R3: Window(Filter(HLL(Source))) (HLL absorbs Dedup) - let expr = QueryExpr::Filter { - pred: ScalarExpr::Literal(LiteralValue::Bool(true)), - input: Box::new(QueryExpr::Window { - duration: Duration::from_secs(60), - slide: None, - input: Box::new(QueryExpr::SketchAgg { - op: AggIntent::default_cardinality(), - col: ColumnRef::Named("uid".into()), - input: Box::new(QueryExpr::Dedup { - col: "uid".into(), - input: Box::new(src("events")), - }), - }), - }), - }; - let (result, _iters) = opt().optimize(expr); - // The Dedup should be gone. - let mut dedup_found = false; - result.walk(&mut |n| { - if matches!(n, QueryExpr::Dedup { .. }) { - dedup_found = true; - } - }); - assert!(!dedup_found, "Dedup should have been eliminated"); - } - - // ── R2: MergeLifting ───────────────────────────────────────────────────── - - #[test] - fn r2_lifts_mergeable_sketch_above_merge() { - let expr = QueryExpr::SketchAgg { - op: AggIntent::default_cardinality(), - col: ColumnRef::Named("uid".into()), - input: Box::new(QueryExpr::Merge { - inputs: vec![src("shard_a"), src("shard_b")], - }), - }; - let (result, _) = opt().optimize(expr); - assert!( - matches!(&result, QueryExpr::Merge { inputs } - if inputs.iter().all(|i| matches!(i, QueryExpr::SketchAgg { .. }))), - "HLL should be pushed into each Merge branch" - ); - } - - // ── R12: SetOpFusion ───────────────────────────────────────────────────── - - #[test] - fn r12_fuses_union_of_merges() { - let expr = QueryExpr::SetOp { - kind: SetOpKind::Union, - all: true, - left: Box::new(QueryExpr::Merge { inputs: vec![src("a"), src("b")] }), - right: Box::new(QueryExpr::Merge { inputs: vec![src("c")] }), - }; - let (result, _) = opt().optimize(expr); - match result { - QueryExpr::Merge { inputs } => assert_eq!(inputs.len(), 3), - other => panic!("expected Merge(3), got {other:?}"), - } - } - - // ── DeploymentConstraints tests ───────────────────────────────────── - - #[test] - fn constraints_from_budgets() { - let budgets = crate::types::StageResourceBudgets { - agent_memory_bytes: Some(4096), - backend_memory_bytes: Some(1_000_000), - ..Default::default() - }; - let dc = DeploymentConstraints::from_budgets(&budgets); - assert_eq!(dc.agent.memory_bytes, Some(4096)); - assert_eq!(dc.backend_collector.memory_bytes, Some(1_000_000)); - } - - #[test] - fn constrained_optimizer_penalises_large_sketch() { - let dc = DeploymentConstraints { - agent: StageBudget { memory_bytes: Some(1), ..Default::default() }, - ..Default::default() - }; - let opt = QueryOptimizer::with_constraints(1000.0, dc); - let expr = QueryExpr::SketchAgg { - op: AggIntent::default_quantile(vec![0.99]), - col: ColumnRef::SampleValue, - input: Box::new(QueryExpr::Source(SourceSpec { name: "m".into() })), - }; - let cost = opt.cost_model.estimate(&expr); - // Memory should be heavily penalised (10× multiplier) - assert!(cost.memory_bytes > 10_000.0, - "expected penalised memory, got {}", cost.memory_bytes); - } - - #[test] - fn unconstrained_optimizer_normal_cost() { - let opt = QueryOptimizer::new(1000.0); - let expr = QueryExpr::SketchAgg { - op: AggIntent::default_quantile(vec![0.99]), - col: ColumnRef::SampleValue, - input: Box::new(QueryExpr::Source(SourceSpec { name: "m".into() })), - }; - let cost = opt.cost_model.estimate(&expr); - // Normal cost, no penalty - assert!(cost.memory_bytes < 10_000.0, - "expected normal memory, got {}", cost.memory_bytes); - } -} diff --git a/controller/src/algebra/physical.rs b/controller/src/algebra/physical.rs deleted file mode 100644 index 01246dc9..00000000 --- a/controller/src/algebra/physical.rs +++ /dev/null @@ -1,938 +0,0 @@ -//! Layer 5 — Physical plan IR. -//! -//! Maps the implementation-independent [`AggIntent`] (Layer 3) to concrete -//! sketch implementations and pipeline stages. -//! -//! # Key types -//! -//! - [`PhysicalAggOp`] — resolved AggIntent → concrete SketchType + SketchParams -//! - [`PhysicalOp`] — a physical operator (sketch build, merge, exchange, eval, etc.) -//! - [`PhysicalNode`] — a node in the physical plan tree (operator + placement + cost) -//! - [`Placement`] — where a physical operator runs (Agent, Backend, PromSketch, DB, etc.) - -use std::time::Duration; - -use crate::algebra::directory; -use crate::algebra::expr::{AggIntent, WindowKind, WindowSpec}; -use crate::types::{SketchParams, SketchType}; - -// ── PhysicalAggOp (resolved sketch intent) ────────────────────────────────── - -/// A resolved physical aggregation operation. -/// -/// This is the output of `resolve()`: concrete sketch implementation -/// chosen for a logical [`AggIntent`]. -#[derive(Debug, Clone)] -pub struct PhysicalAggOp { - /// The logical intent this was derived from. - pub intent: AggIntent, - /// Concrete sketch type. - pub sketch_type: SketchType, - /// Concrete sketch parameters. - pub sketch_params: SketchParams, - /// Estimated memory footprint per series (bytes). - pub estimated_memory_bytes: u64, -} - -/// Resolve an [`AggIntent`] into a [`PhysicalAggOp`] using default mapping. -/// -/// This is the Layer 3 → Layer 5 boundary. -pub fn resolve(intent: &AggIntent) -> PhysicalAggOp { - PhysicalAggOp { - intent: intent.clone(), - sketch_type: directory::sketch_type_for_op(intent), - sketch_params: directory::sketch_params_for_op(intent), - estimated_memory_bytes: directory::estimated_sketch_memory_bytes(intent), - } -} - -// ── Physical operators ────────────────────────────────────────────────────── - -/// A physical operator — concrete implementation of a logical operator. -#[derive(Debug, Clone)] -pub enum PhysicalOp { - // ── Scan / ingest ───────────────────────────────────────────── - /// Read raw OTLP metrics from an SDK or scrape target. - OtlpScan { - endpoint: String, - label_matchers: Vec, - }, - - /// Read from an existing PromSketch store. - PromSketchScan { - store_addr: String, - series_selector: String, - }, - - // ── Sketch build ────────────────────────────────────────────── - /// Build sketch via OTel Collector processor (tumbling window flush). - OtelSketchBuild { - sketch_type: SketchType, - sketch_params: SketchParams, - window: PhysicalWindow, - delta_encoding: bool, - }, - - /// Build sketch via PromSketch's ExponentialHistogram layer. - PromSketchBuild { - sketch_type: SketchType, - eh_k: usize, - time_window: Duration, - }, - - // ── Sketch merge ────────────────────────────────────────────── - /// Merge sketches from N upstream nodes. - SketchMerge { - sketch_type: SketchType, - group_by: Vec, - }, - - // ── Sketch query ────────────────────────────────────────────── - /// Extract result from a sketch (quantile, cardinality, frequency). - SketchEval { - sketch_type: SketchType, - func: EvalFunc, - }, - - // ── Data exchange ───────────────────────────────────────────── - /// Data transfer between pipeline stages. - Exchange { - format: ExchangeFormat, - }, - - // ── Relational / passthrough ────────────────────────────────── - /// Filter rows. - Filter { pred: String }, - /// Top-K ranking. - TopK { k: u64 }, - /// Hash-partitioned aggregation. - HashAggregate { keys: Vec }, - /// SQL query to database. - DbQuery { sql: String }, - /// Passthrough — no transformation. - Passthrough, -} - -/// Physical window implementation. -#[derive(Debug, Clone)] -pub enum PhysicalWindow { - /// OTel Collector: `time.NewTicker` flush + sketch reset. - OtelTumblingFlush { duration: Duration }, - /// PromSketch ExponentialHistogram: time-decaying buckets. - PromSketchEH { eh_k: usize, time_window: Duration }, - /// Database-side: `GROUP BY time_bucket(interval, ts)`. - SqlTimeBucket { interval: Duration, time_col: String }, - /// No windowing (unbounded / landmark). - None, -} - -/// What to extract from a sketch at query time. -#[derive(Debug, Clone)] -pub enum EvalFunc { - Quantile(Vec), - Cardinality, - Frequency { key: String }, - TopK { k: u64 }, - Extrema { min: bool, max: bool }, -} - -/// Data format for Exchange operators. -#[derive(Debug, Clone)] -pub enum ExchangeFormat { - /// OTLP gRPC / HTTP. - Otlp, - /// Sketch-specific binary (merged sketch bytes). - SketchBinary, - /// Raw samples (for non-sketch path). - RawSamples, -} - -/// Where a physical operator runs. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum Placement { - /// Agent OTel Collector (co-located with SDK). - AgentCollector, - /// Backend OTel Collector (merge tier). - BackendCollector, - /// PromSketch store (ASAPQuery). - PromSketchStore, - /// General query engine (ASAPQuery / DataFusion). - QueryEngine, - /// Database (ClickHouse, TimescaleDB, etc.). - Database, -} - -// ── Physical plan tree ────────────────────────────────────────────────────── - -/// A node in the physical plan tree. -#[derive(Debug, Clone)] -pub struct PhysicalNode { - /// The physical operator at this node. - pub op: PhysicalOp, - /// Where this operator runs. - pub placement: Placement, - /// Estimated cost. - pub cost: PhysicalCost, - /// Child nodes (ordered: left, right, or input list). - pub children: Vec, -} - -/// Cost estimate for a physical operator. -#[derive(Debug, Clone, Default)] -pub struct PhysicalCost { - /// Estimated output bandwidth (bytes/sec). - pub bytes_per_sec: f64, - /// Estimated memory usage (bytes). - pub memory_bytes: f64, - /// Estimated CPU cost (microseconds per sample). - pub cpu_per_sample: f64, -} - -// ── Window resolution ─────────────────────────────────────────────────────── - -/// Resolve a logical [`WindowSpec`] to a [`PhysicalWindow`] for a given placement. -pub fn resolve_window(window: &WindowSpec, placement: &Placement) -> PhysicalWindow { - match (&window.kind, placement) { - (WindowKind::Tumbling { size }, Placement::AgentCollector) => - PhysicalWindow::OtelTumblingFlush { duration: *size }, - (WindowKind::Tumbling { size }, Placement::PromSketchStore) => - PhysicalWindow::PromSketchEH { eh_k: 50, time_window: *size }, - (WindowKind::Sliding { size, .. }, Placement::PromSketchStore) => - PhysicalWindow::PromSketchEH { eh_k: 50, time_window: *size }, - (WindowKind::Tumbling { size }, Placement::Database) => - PhysicalWindow::SqlTimeBucket { - interval: *size, - time_col: window.time_col.clone().unwrap_or_else(|| "ts".into()), - }, - (WindowKind::Unbounded | WindowKind::Landmark, _) => - PhysicalWindow::None, - // Fallback: tumbling at the given size for any other combo. - (WindowKind::Tumbling { size } | WindowKind::Sliding { size, .. } | WindowKind::Session { gap: size }, _) => - PhysicalWindow::OtelTumblingFlush { duration: *size }, - } -} - -// ── Physical planner ──────────────────────────────────────────────────────── - -use crate::algebra::expr::*; -use crate::algebra::optimizer::DeploymentConstraints; -use crate::types::StageResourceBudgets; - -/// Physical planner configuration. -#[derive(Debug, Clone)] -pub struct PhysicalPlannerConfig { - pub budgets: StageResourceBudgets, - pub constraints: DeploymentConstraints, -} - -/// Build a physical plan from an optimized `QueryExpr`. -/// -/// Walks the logical tree bottom-up, assigning each node to a pipeline stage -/// (`Placement`), resolving sketch intents to concrete implementations, and -/// inserting `Exchange` nodes at stage boundaries. -pub fn plan(expr: &QueryExpr, config: &PhysicalPlannerConfig) -> PhysicalNode { - plan_node(expr, config) -} - -fn plan_node(expr: &QueryExpr, config: &PhysicalPlannerConfig) -> PhysicalNode { - match expr { - // ── Leaf: scan at Agent ───────────────────────────────────── - QueryExpr::Source(s) => PhysicalNode { - op: PhysicalOp::OtlpScan { - endpoint: String::new(), - label_matchers: vec![], - }, - placement: Placement::AgentCollector, - cost: PhysicalCost::default(), - children: vec![], - }, - - // ── Filter: same placement as child ───────────────────────── - QueryExpr::Filter { pred, input } => { - let child = plan_node(input, config); - PhysicalNode { - placement: child.placement.clone(), - op: PhysicalOp::Filter { pred: format!("{pred:?}") }, - cost: PhysicalCost::default(), - children: vec![child], - } - } - - // ── SketchAgg: resolve intent → physical, place at Agent or defer ── - QueryExpr::SketchAgg { op, col, input } => { - let child = plan_node(input, config); - let resolved = resolve(op); - let placement = decide_sketch_placement(&resolved, config); - - let physical_op = PhysicalOp::OtelSketchBuild { - sketch_type: resolved.sketch_type.clone(), - sketch_params: resolved.sketch_params.clone(), - window: PhysicalWindow::None, - delta_encoding: false, - }; - - let mut node = PhysicalNode { - op: physical_op, - placement: placement.clone(), - cost: PhysicalCost { - memory_bytes: resolved.estimated_memory_bytes as f64, - ..Default::default() - }, - children: vec![child], - }; - // Insert exchange if child is at a different stage - insert_exchange_if_needed(&mut node); - node - } - - // ── WindowedAgg: resolve + place with window ──────────────── - QueryExpr::WindowedAgg { agg, window, col, input } => { - let child = plan_node(input, config); - let resolved = resolve(agg); - let placement = decide_sketch_placement(&resolved, config); - let phys_window = resolve_window(window, &placement); - - let physical_op = PhysicalOp::OtelSketchBuild { - sketch_type: resolved.sketch_type.clone(), - sketch_params: resolved.sketch_params.clone(), - window: phys_window, - delta_encoding: false, - }; - - let mut node = PhysicalNode { - op: physical_op, - placement: placement.clone(), - cost: PhysicalCost { - memory_bytes: resolved.estimated_memory_bytes as f64, - ..Default::default() - }, - children: vec![child], - }; - insert_exchange_if_needed(&mut node); - node - } - - // ── Partition / Merge: Backend stage ──────────────────────── - QueryExpr::Partition { keys, input } => { - let child = plan_node(input, config); - let mut node = PhysicalNode { - op: PhysicalOp::HashAggregate { keys: keys.keys().to_vec() }, - placement: Placement::BackendCollector, - cost: PhysicalCost::default(), - children: vec![child], - }; - insert_exchange_if_needed(&mut node); - node - } - - QueryExpr::Merge { inputs } => { - let children: Vec = inputs.iter() - .map(|i| plan_node(i, config)) - .collect(); - let sketch_type = children.first() - .and_then(|c| match &c.op { - PhysicalOp::OtelSketchBuild { sketch_type, .. } => Some(sketch_type.clone()), - _ => None, - }) - .unwrap_or(SketchType::DDSketch); - PhysicalNode { - op: PhysicalOp::SketchMerge { sketch_type, group_by: vec![] }, - placement: Placement::BackendCollector, - cost: PhysicalCost::default(), - children, - } - } - - QueryExpr::Dedup { col, input } => { - let child = plan_node(input, config); - let mut node = PhysicalNode { - op: PhysicalOp::Filter { pred: format!("dedup({col})") }, - placement: Placement::BackendCollector, - cost: PhysicalCost::default(), - children: vec![child], - }; - insert_exchange_if_needed(&mut node); - node - } - - // ── TopK / HistogramQuantile / BinaryOp: QueryEngine stage ── - QueryExpr::TopK { k, input, .. } => { - let child = plan_node(input, config); - let mut node = PhysicalNode { - op: PhysicalOp::TopK { k: *k }, - placement: Placement::QueryEngine, - cost: PhysicalCost::default(), - children: vec![child], - }; - insert_exchange_if_needed(&mut node); - node - } - - QueryExpr::HistogramQuantile { phi, input } => { - let child = plan_node(input, config); - let mut node = PhysicalNode { - op: PhysicalOp::SketchEval { - sketch_type: SketchType::DDSketch, - func: EvalFunc::Quantile(vec![*phi]), - }, - placement: Placement::QueryEngine, - cost: PhysicalCost::default(), - children: vec![child], - }; - insert_exchange_if_needed(&mut node); - node - } - - QueryExpr::BinaryOp { op, lhs, rhs, .. } => { - let left = plan_node(lhs, config); - let right = plan_node(rhs, config); - PhysicalNode { - op: PhysicalOp::Passthrough, - placement: Placement::QueryEngine, - cost: PhysicalCost::default(), - children: vec![left, right], - } - } - - QueryExpr::PromQLSubquery { input, .. } => { - let child = plan_node(input, config); - let mut node = PhysicalNode { - op: PhysicalOp::Passthrough, - placement: Placement::QueryEngine, - cost: PhysicalCost::default(), - children: vec![child], - }; - insert_exchange_if_needed(&mut node); - node - } - - // ── Aggregate (non-sketch, exact): Database stage ─────────── - QueryExpr::Aggregate { keys, input, .. } => { - let child = plan_node(input, config); - let mut node = PhysicalNode { - op: PhysicalOp::DbQuery { sql: format!("GROUP BY {:?}", keys) }, - placement: Placement::Database, - cost: PhysicalCost::default(), - children: vec![child], - }; - insert_exchange_if_needed(&mut node); - node - } - - // ── Sort / Limit / Project: inherit child placement ───────── - QueryExpr::Sort { input, .. } - | QueryExpr::Limit { input, .. } - | QueryExpr::Project { input, .. } - | QueryExpr::Window { input, .. } - | QueryExpr::WindowFunc { input, .. } => { - let child = plan_node(input, config); - PhysicalNode { - op: PhysicalOp::Passthrough, - placement: child.placement.clone(), - cost: PhysicalCost::default(), - children: vec![child], - } - } - - // ── Join: both children, QueryEngine placement ────────────── - QueryExpr::Join { left, right, .. } - | QueryExpr::JoinSketch { outer: left, inner: right, .. } - | QueryExpr::SetOp { left, right, .. } => { - let l = plan_node(left, config); - let r = plan_node(right, config); - PhysicalNode { - op: PhysicalOp::Passthrough, - placement: Placement::QueryEngine, - cost: PhysicalCost::default(), - children: vec![l, r], - } - } - - // ── Subquery / LetBinding ─────────────────────────────────── - QueryExpr::Subquery { expr, .. } => plan_node(expr, config), - QueryExpr::LetBinding { body, .. } => plan_node(body, config), - QueryExpr::Ref(_) => PhysicalNode { - op: PhysicalOp::Passthrough, - placement: Placement::QueryEngine, - cost: PhysicalCost::default(), - children: vec![], - }, - } -} - -/// Decide where a sketch operation runs based on memory budget. -/// Decide where a sketch runs based on deployment constraints and sketch capability. -/// -/// Uses `StageBudget::fits(SketchCapability)` to check each stage in order: -/// Agent → BackendCollector → QueryEngine. -fn decide_sketch_placement(resolved: &PhysicalAggOp, config: &PhysicalPlannerConfig) -> Placement { - use crate::algebra::optimizer::sketch_capability; - - let cap = sketch_capability(&resolved.sketch_type); - - // Try Agent first. - if config.constraints.agent.fits(&cap) { - return Placement::AgentCollector; - } - - // Agent budget exceeded — try Backend. - if config.constraints.backend_collector.fits(&cap) { - return Placement::BackendCollector; - } - - // Both exceeded — defer to QueryEngine. - Placement::QueryEngine -} - -/// If a node's child is at a different stage, insert an Exchange node between them. -fn insert_exchange_if_needed(node: &mut PhysicalNode) { - let parent_placement = node.placement.clone(); - for child in &mut node.children { - if child.placement != parent_placement { - let format = match (&child.placement, &parent_placement) { - (Placement::AgentCollector, Placement::BackendCollector) => ExchangeFormat::Otlp, - (Placement::AgentCollector, Placement::QueryEngine) => ExchangeFormat::Otlp, - (Placement::BackendCollector, Placement::QueryEngine) => ExchangeFormat::SketchBinary, - (Placement::AgentCollector, Placement::Database) => ExchangeFormat::RawSamples, - _ => ExchangeFormat::Otlp, - }; - // Wrap the child in an Exchange node - let original_child = std::mem::replace(child, PhysicalNode { - op: PhysicalOp::Passthrough, - placement: parent_placement.clone(), - cost: PhysicalCost::default(), - children: vec![], - }); - *child = PhysicalNode { - op: PhysicalOp::Exchange { format }, - placement: parent_placement.clone(), - cost: PhysicalCost::default(), - children: vec![original_child], - }; - } - } -} - -impl PhysicalNode { - /// Count total nodes in the tree. - pub fn node_count(&self) -> usize { - 1 + self.children.iter().map(|c| c.node_count()).sum::() - } - - /// Collect all distinct placements in the tree. - pub fn placements(&self) -> Vec { - let mut out = vec![self.placement.clone()]; - for child in &self.children { - for p in child.placements() { - if !out.contains(&p) { - out.push(p); - } - } - } - out - } - - /// Count Exchange nodes (= stage boundary crossings). - pub fn exchange_count(&self) -> usize { - let self_count = if matches!(self.op, PhysicalOp::Exchange { .. }) { 1 } else { 0 }; - self_count + self.children.iter().map(|c| c.exchange_count()).sum::() - } - - /// Extract a flat [`StagedPlan`] from this physical plan tree. - /// - /// Walks the tree and populates each sub-plan based on node placement - /// and operator type. This bridges the physical planner to the existing - /// config generators that consume `StagedPlan`. - pub fn to_staged_plan(&self) -> crate::types::StagedPlan { - use crate::types::{ - AgentSubPlan, BackendSubPlan, DbSubPlan, PrecomputeSubPlan, StagedPlan, - }; - - let mut staged = StagedPlan::default(); - self.collect_into_staged(&mut staged); - staged - } - - fn collect_into_staged(&self, staged: &mut crate::types::StagedPlan) { - use crate::types::StagedPlan; - - match (&self.placement, &self.op) { - // Agent: sketch build → populate agent sub-plan - (Placement::AgentCollector, PhysicalOp::OtelSketchBuild { - sketch_type, sketch_params, window, .. - }) => { - staged.agent.sketch_type = Some(sketch_type.clone()); - staged.agent.sketch_params = sketch_params.clone(); - if let PhysicalWindow::OtelTumblingFlush { duration } = window { - staged.agent.window_secs = Some(duration.as_secs()); - } - } - - // Agent: filter → label filters - (Placement::AgentCollector, PhysicalOp::Filter { pred }) => { - staged.agent.label_filters.push(pred.clone()); - } - - // Backend: merge/aggregate - (Placement::BackendCollector, PhysicalOp::SketchMerge { group_by, .. }) => { - staged.backend.has_merge = true; - staged.backend.group_by = group_by.clone(); - } - (Placement::BackendCollector, PhysicalOp::HashAggregate { keys }) => { - staged.backend.has_merge = true; - staged.backend.group_by = keys.clone(); - } - (Placement::BackendCollector, PhysicalOp::Filter { .. }) => { - staged.backend.has_dedup = true; - } - - // QueryEngine: TopK, SketchEval - (Placement::QueryEngine, PhysicalOp::TopK { k }) => { - staged.precompute.active = true; - staged.precompute.topk = Some(*k); - } - (Placement::QueryEngine, PhysicalOp::SketchEval { .. }) => { - staged.precompute.active = true; - } - (Placement::QueryEngine, PhysicalOp::Passthrough) => { - staged.precompute.active = true; - } - - // Database: exact computation - (Placement::Database, PhysicalOp::DbQuery { sql }) => { - staged.db.active = true; - staged.db.query_expr = sql.clone(); - } - - // Exchange: record deferral - (_, PhysicalOp::Exchange { format }) => { - staged.deferral_log.push(format!("Exchange({:?})", format)); - } - - _ => {} - } - - // Recurse into children - for child in &self.children { - child.collect_into_staged(staged); - } - } -} - -// ── Public entry point for main.rs ────────────────────────────────────────── - -/// Run the full physical planning pipeline: optimize → plan → staged plan. -/// -/// This is the single function `main.rs` calls to get a `StagedPlan` -/// from a parsed `QueryExpr`. -pub fn physical_plan_to_staged( - expr: &QueryExpr, - budgets: &StageResourceBudgets, -) -> (crate::types::StagedPlan, PhysicalNode) { - let constraints = DeploymentConstraints::from_budgets(budgets); - let config = PhysicalPlannerConfig { - budgets: budgets.clone(), - constraints, - }; - let tree = plan(expr, &config); - let staged = tree.to_staged_plan(); - (staged, tree) -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn resolve_quantile() { - let p = resolve(&AggIntent::default_quantile(vec![0.99])); - assert_eq!(p.sketch_type, SketchType::DDSketch); - assert!(matches!(p.sketch_params, SketchParams::DDSketch { .. })); - assert!(p.estimated_memory_bytes > 0); - } - - #[test] - fn resolve_cardinality() { - let p = resolve(&AggIntent::default_cardinality()); - assert_eq!(p.sketch_type, SketchType::HLL); - assert!(matches!(p.sketch_params, SketchParams::HLL { .. })); - } - - #[test] - fn resolve_frequency() { - let p = resolve(&AggIntent::default_frequency()); - assert_eq!(p.sketch_type, SketchType::CountSketch); - assert!(matches!(p.sketch_params, SketchParams::CountSketch { .. })); - } - - #[test] - fn resolve_preserves_intent() { - let intent = AggIntent::Quantile { quantiles: vec![0.5, 0.99], accuracy: 0.005 }; - let p = resolve(&intent); - assert_eq!(p.intent, intent); - } - - #[test] - fn tumbling_window_at_agent() { - let ws = WindowSpec { - kind: WindowKind::Tumbling { size: Duration::from_secs(300) }, - time_col: None, - }; - let pw = resolve_window(&ws, &Placement::AgentCollector); - assert!(matches!(pw, PhysicalWindow::OtelTumblingFlush { .. })); - } - - #[test] - fn sliding_window_at_promsketch() { - let ws = WindowSpec { - kind: WindowKind::Sliding { size: Duration::from_secs(300), slide: Duration::from_secs(60) }, - time_col: None, - }; - let pw = resolve_window(&ws, &Placement::PromSketchStore); - assert!(matches!(pw, PhysicalWindow::PromSketchEH { .. })); - } - - #[test] - fn tumbling_window_at_database() { - let ws = WindowSpec { - kind: WindowKind::Tumbling { size: Duration::from_secs(60) }, - time_col: Some("event_time".into()), - }; - let pw = resolve_window(&ws, &Placement::Database); - match pw { - PhysicalWindow::SqlTimeBucket { interval, time_col } => { - assert_eq!(interval, Duration::from_secs(60)); - assert_eq!(time_col, "event_time"); - } - other => panic!("expected SqlTimeBucket, got {other:?}"), - } - } - - #[test] - fn unbounded_window_is_none() { - let ws = WindowSpec { kind: WindowKind::Unbounded, time_col: None }; - let pw = resolve_window(&ws, &Placement::AgentCollector); - assert!(matches!(pw, PhysicalWindow::None)); - } - - // ── Physical planner tests ────────────────────────────────────────── - - fn default_config() -> PhysicalPlannerConfig { - PhysicalPlannerConfig { - budgets: StageResourceBudgets::default(), - constraints: DeploymentConstraints::default(), - } - } - - fn src(name: &str) -> QueryExpr { - QueryExpr::Source(SourceSpec { name: name.into() }) - } - - #[test] - fn plan_simple_sketch_at_agent() { - // SketchAgg { Quantile, Source } → Agent placement - let expr = QueryExpr::SketchAgg { - op: AggIntent::default_quantile(vec![0.99]), - col: ColumnRef::SampleValue, - input: Box::new(src("m")), - }; - let node = plan(&expr, &default_config()); - assert_eq!(node.placement, Placement::AgentCollector); - assert!(matches!(node.op, PhysicalOp::OtelSketchBuild { .. })); - assert_eq!(node.children.len(), 1); // Source child - } - - #[test] - fn plan_windowed_agg_has_window() { - let expr = QueryExpr::WindowedAgg { - agg: AggIntent::default_quantile(vec![0.5]), - window: WindowSpec { kind: WindowKind::Tumbling { size: Duration::from_secs(300) }, time_col: None }, - col: ColumnRef::SampleValue, - input: Box::new(src("m")), - }; - let node = plan(&expr, &default_config()); - assert_eq!(node.placement, Placement::AgentCollector); - match &node.op { - PhysicalOp::OtelSketchBuild { window, .. } => { - assert!(matches!(window, PhysicalWindow::OtelTumblingFlush { .. })); - } - other => panic!("expected OtelSketchBuild, got {other:?}"), - } - } - - #[test] - fn plan_topk_at_query_engine() { - let expr = QueryExpr::TopK { - k: 10, - by: vec!["svc".into()], - input: Box::new(QueryExpr::SketchAgg { - op: AggIntent::default_frequency(), - col: ColumnRef::SampleValue, - input: Box::new(src("m")), - }), - }; - let node = plan(&expr, &default_config()); - assert_eq!(node.placement, Placement::QueryEngine); - assert!(matches!(node.op, PhysicalOp::TopK { k: 10 })); - } - - #[test] - fn plan_topk_inserts_exchange() { - // TopK(QueryEngine) wrapping SketchAgg(Agent) → Exchange between them - let expr = QueryExpr::TopK { - k: 5, - by: vec![], - input: Box::new(QueryExpr::SketchAgg { - op: AggIntent::default_frequency(), - col: ColumnRef::SampleValue, - input: Box::new(src("m")), - }), - }; - let node = plan(&expr, &default_config()); - assert!(node.exchange_count() > 0, "expected Exchange between Agent and QueryEngine"); - } - - #[test] - fn plan_partition_at_backend() { - let expr = QueryExpr::Partition { - keys: PartitionKeys::By(vec!["region".into()]), - input: Box::new(QueryExpr::SketchAgg { - op: AggIntent::default_cardinality(), - col: ColumnRef::SampleValue, - input: Box::new(src("m")), - }), - }; - let node = plan(&expr, &default_config()); - assert_eq!(node.placement, Placement::BackendCollector); - } - - #[test] - fn plan_aggregate_at_database() { - let expr = QueryExpr::Aggregate { - keys: vec!["symbol".into()], - aggs: vec![AggItem { - alias: "avg".into(), - func: AggFunc::Avg, - col: ColumnRef::Named("price".into()), - distinct: false, - }], - having: None, - input: Box::new(src("trades")), - }; - let node = plan(&expr, &default_config()); - assert_eq!(node.placement, Placement::Database); - } - - #[test] - fn plan_full_pipeline_has_multiple_stages() { - // TopK(Partition(WindowedAgg(Filter(Source)))) - // Should span: Agent → Backend → QueryEngine - let expr = QueryExpr::TopK { - k: 10, - by: vec!["svc".into()], - input: Box::new(QueryExpr::Partition { - keys: PartitionKeys::By(vec!["svc".into()]), - input: Box::new(QueryExpr::WindowedAgg { - agg: AggIntent::default_frequency(), - window: WindowSpec { kind: WindowKind::Tumbling { size: Duration::from_secs(60) }, time_col: None }, - col: ColumnRef::SampleValue, - input: Box::new(QueryExpr::Filter { - pred: ScalarExpr::Literal(LiteralValue::Bool(true)), - input: Box::new(src("requests")), - }), - }), - }), - }; - let node = plan(&expr, &default_config()); - let placements = node.placements(); - assert!(placements.contains(&Placement::AgentCollector), "should have Agent: {placements:?}"); - assert!(placements.contains(&Placement::BackendCollector), "should have Backend: {placements:?}"); - assert!(placements.contains(&Placement::QueryEngine), "should have QueryEngine: {placements:?}"); - assert!(node.exchange_count() >= 2, "should have ≥2 exchanges: {}", node.exchange_count()); - } - - #[test] - fn plan_budget_deferral() { - // With tiny agent budget, sketch should defer to Backend - let budgets = StageResourceBudgets { - agent_memory_bytes: Some(1), // 1 byte = too small - ..Default::default() - }; - let config = PhysicalPlannerConfig { - constraints: DeploymentConstraints::from_budgets(&budgets), - budgets, - }; - let expr = QueryExpr::SketchAgg { - op: AggIntent::default_quantile(vec![0.99]), - col: ColumnRef::SampleValue, - input: Box::new(src("m")), - }; - let node = plan(&expr, &config); - assert_eq!(node.placement, Placement::BackendCollector, - "sketch should be deferred to Backend when agent budget is tiny"); - } - - // ── to_staged_plan tests ──────────────────────────────────────────── - - #[test] - fn staged_plan_simple_sketch() { - let expr = QueryExpr::WindowedAgg { - agg: AggIntent::Quantile { quantiles: vec![0.99], accuracy: 0.01 }, - window: WindowSpec { kind: WindowKind::Tumbling { size: Duration::from_secs(300) }, time_col: None }, - col: ColumnRef::SampleValue, - input: Box::new(src("m")), - }; - let (staged, _) = physical_plan_to_staged(&expr, &StageResourceBudgets::default()); - assert_eq!(staged.agent.sketch_type, Some(SketchType::DDSketch)); - assert_eq!(staged.agent.window_secs, Some(300)); - assert!(!staged.precompute.active); - assert!(!staged.db.active); - } - - #[test] - fn staged_plan_topk_multi_stage() { - let expr = QueryExpr::TopK { - k: 10, - by: vec!["svc".into()], - input: Box::new(QueryExpr::Partition { - keys: PartitionKeys::By(vec!["svc".into()]), - input: Box::new(QueryExpr::WindowedAgg { - agg: AggIntent::default_frequency(), - window: WindowSpec { kind: WindowKind::Tumbling { size: Duration::from_secs(60) }, time_col: None }, - col: ColumnRef::SampleValue, - input: Box::new(src("m")), - }), - }), - }; - let (staged, tree) = physical_plan_to_staged(&expr, &StageResourceBudgets::default()); - // Agent has sketch - assert!(staged.agent.sketch_type.is_some()); - // Backend has merge - assert!(staged.backend.has_merge); - // Precompute has topk - assert!(staged.precompute.active); - assert_eq!(staged.precompute.topk, Some(10)); - // Exchanges recorded in deferral log - assert!(tree.exchange_count() >= 2); - } - - #[test] - fn staged_plan_exact_agg_at_db() { - let expr = QueryExpr::Aggregate { - keys: vec!["symbol".into()], - aggs: vec![AggItem { - alias: "avg".into(), - func: AggFunc::Avg, - col: ColumnRef::Named("price".into()), - distinct: false, - }], - having: None, - input: Box::new(src("trades")), - }; - let (staged, _) = physical_plan_to_staged(&expr, &StageResourceBudgets::default()); - assert!(staged.db.active); - assert!(staged.agent.sketch_type.is_none()); - } -} diff --git a/controller/src/algebra/plan.rs b/controller/src/algebra/plan.rs deleted file mode 100644 index 4cf237ae..00000000 --- a/controller/src/algebra/plan.rs +++ /dev/null @@ -1,428 +0,0 @@ -//! Annotated plan nodes — the output of the [`super::allocator::SketchAllocator`]. -//! -//! After the optimizer rewrites a [`QueryExpr`](super::expr::QueryExpr) tree, -//! the allocator wraps every node in a [`PlanNode`] that carries: -//! -//! * **`stage`** — which pipeline component executes this operator. -//! * **`mode`** — whether the operator uses sketch approximation or exact -//! computation. -//! * **`cost`** — estimated memory and bandwidth cost at this node. -//! * **`annotation`** — additional hints for the code-generator (e.g. which -//! sketch type to use, whether delta encoding is enabled). -//! -//! The annotated plan tree is serialisable to JSON so it can be included in -//! the `/api/v1/plan` response for observability. - -use serde::{Deserialize, Serialize}; - -use super::expr::QueryExpr; - -// ── Pipeline stages ─────────────────────────────────────────────────────────── - -/// Which component in the data pipeline executes an operator. -/// -/// The ordering `Agent < Backend < Precompute < Db` mirrors the data-flow -/// direction: data originates at the Agent and flows toward the Db. -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum PipelineStage { - /// SDK-side OTel Collector — highest bandwidth savings, lowest latency. - Agent, - /// Central merge collector — aggregates partial sketches from many agents. - Backend, - /// ASAPQuery pre-computation engine — materialises recurring queries. - Precompute, - /// Exact OLAP / time-series database — last resort for non-sketchable ops. - Db, -} - -impl std::fmt::Display for PipelineStage { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let s = match self { - PipelineStage::Agent => "agent", - PipelineStage::Backend => "backend", - PipelineStage::Precompute => "precompute", - PipelineStage::Db => "db", - }; - write!(f, "{s}") - } -} - -// ── Execution mode ──────────────────────────────────────────────────────────── - -/// Whether an operator uses sketch approximation or runs exactly. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ExecutionMode { - /// The operator produces an approximate result via a data sketch. - Sketch, - /// The operator computes an exact result (no error bounds). - Exact, - /// The operator is a structural / routing node (merge, partition, …) - /// that does not itself aggregate — its mode is determined by its children. - Passthrough, -} - -// ── Cost estimate ───────────────────────────────────────────────────────────── - -/// Estimated resource cost of a single plan node. -/// -/// The allocator fills this in using the same cost model as the legacy -/// [`crate::planner::cost_model`]. All fields default to `0.0` for nodes -/// whose cost is negligible or unknown. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct CostEstimate { - /// Outbound bytes per second produced by this node. - pub bytes_per_sec: f64, - /// Memory footprint of the sketch or intermediate state (bytes). - pub memory_bytes: f64, - /// CPU overhead per input sample (µs). - pub cpu_micros_per_sample: f64, - /// Compression ratio relative to the raw OTLP baseline (≥ 1.0 is better). - pub compression_ratio: f64, -} - -// ── Node annotation ─────────────────────────────────────────────────────────── - -/// Extra hints attached to a plan node by the allocator. -/// -/// Not all fields are relevant to all node types; unused fields are `None`. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct NodeAnnotation { - /// Sketch type selected by the allocator (only for sketch nodes). - pub sketch_type: Option, - /// Sketch parameters (width/depth/registers/epsilon). - pub sketch_params: Option, - /// Whether delta encoding should be used at this node. - pub delta_enabled: bool, - /// Minimum cell-change threshold for delta encoding (T). - pub delta_threshold: f64, - /// Human-readable explanation of why this stage/mode was chosen. - pub rationale: String, - /// Whether this node was demoted to a later stage due to budget overflow. - pub budget_demotion: bool, -} - -// ── Plan node ───────────────────────────────────────────────────────────────── - -/// An annotated node in the physical execution plan. -/// -/// The `expr` field holds the logical operator; the surrounding fields -/// describe where and how it runs. -#[derive(Debug, Clone)] -pub struct PlanNode { - /// The logical operator at this node. - pub expr: QueryExpr, - /// Which pipeline stage executes this operator. - pub stage: PipelineStage, - /// Sketch vs. exact vs. passthrough. - pub mode: ExecutionMode, - /// Estimated resource cost. - pub cost: CostEstimate, - /// Allocator hints for code-generation. - pub annotation: NodeAnnotation, - /// Child plan nodes (mirrors `expr`'s children after annotation). - pub children: Vec, -} - -impl PlanNode { - /// Create a leaf `PlanNode` (no children) with default cost/annotation. - pub fn leaf(expr: QueryExpr, stage: PipelineStage, mode: ExecutionMode) -> Self { - Self { - expr, - stage, - mode, - cost: CostEstimate::default(), - annotation: NodeAnnotation::default(), - children: vec![], - } - } - - /// Recursively collect all nodes at a given stage, depth-first. - pub fn nodes_at_stage(&self, target: &PipelineStage) -> Vec<&PlanNode> { - let mut out = vec![]; - if &self.stage == target { - out.push(self); - } - for c in &self.children { - out.extend(c.nodes_at_stage(target)); - } - out - } - - /// Recursively collect all sketch nodes (mode == Sketch). - pub fn sketch_nodes(&self) -> Vec<&PlanNode> { - let mut out = vec![]; - if self.mode == ExecutionMode::Sketch { - out.push(self); - } - for c in &self.children { - out.extend(c.sketch_nodes()); - } - out - } - - /// Total estimated bandwidth of all nodes at `stage` (bytes/sec). - pub fn stage_bandwidth(&self, stage: &PipelineStage) -> f64 { - self.nodes_at_stage(stage) - .iter() - .map(|n| n.cost.bytes_per_sec) - .sum() - } - - /// Total estimated memory of all nodes at `stage` (bytes). - pub fn stage_memory(&self, stage: &PipelineStage) -> f64 { - self.nodes_at_stage(stage) - .iter() - .map(|n| n.cost.memory_bytes) - .sum() - } - - /// Returns a flat, depth-first list of `(depth, node)` pairs for display. - pub fn flatten(&self) -> Vec<(usize, &PlanNode)> { - let mut out = vec![]; - self.flatten_inner(0, &mut out); - out - } - - fn flatten_inner<'a>(&'a self, depth: usize, out: &mut Vec<(usize, &'a PlanNode)>) { - out.push((depth, self)); - for c in &self.children { - c.flatten_inner(depth + 1, out); - } - } -} - -// ── Plan summary (serialisable) ─────────────────────────────────────────────── - -/// A serialisable summary of the full annotated plan, suitable for inclusion -/// in the `/api/v1/plan` JSON response. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct PlanSummary { - /// Total estimated bandwidth saved vs raw OTLP (bytes/sec). - pub bandwidth_saved_bytes_per_sec: f64, - /// Total estimated agent memory for all sketch nodes (bytes). - pub agent_memory_bytes: f64, - /// Total estimated backend memory for all sketch nodes (bytes). - pub backend_memory_bytes: f64, - /// Whether any node was demoted due to budget overflow. - pub has_budget_demotion: bool, - /// List of per-node stage + mode + rationale entries. - pub node_annotations: Vec, -} - -/// One row in the [`PlanSummary::node_annotations`] table. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct NodeSummaryEntry { - pub node_kind: String, - pub stage: PipelineStage, - pub mode: ExecutionMode, - pub rationale: String, - pub memory_bytes: f64, - pub bytes_per_sec: f64, -} - -impl PlanNode { - /// Build a [`PlanSummary`] from this root node. - pub fn summarise(&self, raw_bytes_per_sec: f64) -> PlanSummary { - let flat = self.flatten(); - let agent_mem: f64 = flat.iter() - .filter(|(_, n)| n.stage == PipelineStage::Agent) - .map(|(_, n)| n.cost.memory_bytes) - .sum(); - let backend_mem: f64 = flat.iter() - .filter(|(_, n)| n.stage == PipelineStage::Backend) - .map(|(_, n)| n.cost.memory_bytes) - .sum(); - let plan_bw: f64 = flat.iter() - .filter(|(_, n)| matches!( - n.stage, PipelineStage::Agent | PipelineStage::Backend - )) - .map(|(_, n)| n.cost.bytes_per_sec) - .fold(f64::INFINITY, f64::min); // min of outbound paths - let saved = if raw_bytes_per_sec > plan_bw { - raw_bytes_per_sec - plan_bw - } else { - 0.0 - }; - let has_demotion = flat.iter().any(|(_, n)| n.annotation.budget_demotion); - let entries = flat.iter().map(|(_, n)| { - NodeSummaryEntry { - node_kind: format!("{:?}", n.expr).split_whitespace().next().unwrap_or("?").to_string(), - stage: n.stage.clone(), - mode: n.mode.clone(), - rationale: n.annotation.rationale.clone(), - memory_bytes: n.cost.memory_bytes, - bytes_per_sec: n.cost.bytes_per_sec, - } - }).collect(); - PlanSummary { - bandwidth_saved_bytes_per_sec: saved, - agent_memory_bytes: agent_mem, - backend_memory_bytes: backend_mem, - has_budget_demotion: has_demotion, - node_annotations: entries, - } - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use crate::algebra::expr::QueryExpr; - use crate::algebra::expr::SourceSpec; - - fn source_node(name: &str, stage: PipelineStage) -> PlanNode { - PlanNode::leaf( - QueryExpr::Source(SourceSpec { name: name.into() }), - stage, - ExecutionMode::Passthrough, - ) - } - - // ── PipelineStage ordering ──────────────────────────────────────────────── - - #[test] - fn stage_ordering_agent_lt_db() { - assert!(PipelineStage::Agent < PipelineStage::Db); - assert!(PipelineStage::Agent < PipelineStage::Backend); - assert!(PipelineStage::Backend < PipelineStage::Precompute); - assert!(PipelineStage::Precompute < PipelineStage::Db); - } - - // ── nodes_at_stage ──────────────────────────────────────────────────────── - - #[test] - fn nodes_at_stage_collects_correctly() { - let root = PlanNode { - expr: QueryExpr::Source(SourceSpec { name: "root".into() }), - stage: PipelineStage::Agent, - mode: ExecutionMode::Sketch, - cost: CostEstimate { memory_bytes: 100.0, ..Default::default() }, - annotation: NodeAnnotation::default(), - children: vec![ - source_node("child_agent", PipelineStage::Agent), - source_node("child_backend", PipelineStage::Backend), - ], - }; - let agent_nodes = root.nodes_at_stage(&PipelineStage::Agent); - assert_eq!(agent_nodes.len(), 2); // root + child_agent - let backend_nodes = root.nodes_at_stage(&PipelineStage::Backend); - assert_eq!(backend_nodes.len(), 1); - } - - // ── sketch_nodes ───────────────────────────────────────────────────────── - - #[test] - fn sketch_nodes_only_returns_sketch_mode() { - let root = PlanNode { - expr: QueryExpr::Source(SourceSpec { name: "r".into() }), - stage: PipelineStage::Agent, - mode: ExecutionMode::Sketch, - cost: CostEstimate::default(), - annotation: NodeAnnotation::default(), - children: vec![ - PlanNode::leaf( - QueryExpr::Source(SourceSpec { name: "exact_child".into() }), - PipelineStage::Db, - ExecutionMode::Exact, - ), - PlanNode::leaf( - QueryExpr::Source(SourceSpec { name: "sketch_child".into() }), - PipelineStage::Backend, - ExecutionMode::Sketch, - ), - ], - }; - let sn = root.sketch_nodes(); - assert_eq!(sn.len(), 2); // root (Sketch) + sketch_child - } - - // ── stage_bandwidth / stage_memory ──────────────────────────────────────── - - #[test] - fn stage_bandwidth_sums_nodes_at_stage() { - let root = PlanNode { - expr: QueryExpr::Source(SourceSpec { name: "r".into() }), - stage: PipelineStage::Agent, - mode: ExecutionMode::Sketch, - cost: CostEstimate { bytes_per_sec: 500.0, ..Default::default() }, - annotation: NodeAnnotation::default(), - children: vec![ - PlanNode { - expr: QueryExpr::Source(SourceSpec { name: "c".into() }), - stage: PipelineStage::Agent, - mode: ExecutionMode::Passthrough, - cost: CostEstimate { bytes_per_sec: 200.0, ..Default::default() }, - annotation: NodeAnnotation::default(), - children: vec![], - }, - ], - }; - assert!((root.stage_bandwidth(&PipelineStage::Agent) - 700.0).abs() < 1e-6); - } - - // ── flatten ─────────────────────────────────────────────────────────────── - - #[test] - fn flatten_returns_depth_zero_for_root() { - let root = source_node("r", PipelineStage::Agent); - let flat = root.flatten(); - assert_eq!(flat.len(), 1); - assert_eq!(flat[0].0, 0); // depth = 0 - } - - #[test] - fn flatten_depth_increments_per_level() { - let root = PlanNode { - expr: QueryExpr::Source(SourceSpec { name: "r".into() }), - stage: PipelineStage::Agent, - mode: ExecutionMode::Passthrough, - cost: CostEstimate::default(), - annotation: NodeAnnotation::default(), - children: vec![source_node("c1", PipelineStage::Backend)], - }; - let flat = root.flatten(); - assert_eq!(flat[0].0, 0); - assert_eq!(flat[1].0, 1); - } - - // ── PlanSummary ─────────────────────────────────────────────────────────── - - #[test] - fn summarise_reports_bandwidth_saved() { - let root = PlanNode { - expr: QueryExpr::Source(SourceSpec { name: "r".into() }), - stage: PipelineStage::Agent, - mode: ExecutionMode::Sketch, - cost: CostEstimate { - bytes_per_sec: 1_000.0, - memory_bytes: 256.0, - ..Default::default() - }, - annotation: NodeAnnotation::default(), - children: vec![], - }; - // Raw baseline is 10 000 B/s; plan reduces to 1 000 B/s → saved = 9 000. - let summary = root.summarise(10_000.0); - assert!((summary.bandwidth_saved_bytes_per_sec - 9_000.0).abs() < 1.0); - assert!((summary.agent_memory_bytes - 256.0).abs() < 1.0); - assert!(!summary.has_budget_demotion); - } - - #[test] - fn summarise_detects_budget_demotion() { - let root = PlanNode { - expr: QueryExpr::Source(SourceSpec { name: "r".into() }), - stage: PipelineStage::Backend, - mode: ExecutionMode::Sketch, - cost: CostEstimate::default(), - annotation: NodeAnnotation { budget_demotion: true, ..Default::default() }, - children: vec![], - }; - let summary = root.summarise(0.0); - assert!(summary.has_budget_demotion); - } -} diff --git a/controller/src/analyzer.rs b/controller/src/analyzer.rs deleted file mode 100644 index 46aed67c..00000000 --- a/controller/src/analyzer.rs +++ /dev/null @@ -1,699 +0,0 @@ -use std::collections::{HashMap, HashSet}; -use std::time::Duration; -use anyhow::{anyhow, Context}; -use serde::{Deserialize, Serialize}; - -use crate::query_parser; -use crate::types::{AggType, QueryWorkload, SketchType, WorkloadCharacteristics}; -use crate::types_v2::{AccuracyTarget, DataShape, QueryId, QueryLanguage, QueryShape}; - -// ── Public API ──────────────────────────────────────────────────────────────── - -/// JSON-friendly representation of a query workload submitted by callers. -/// -/// There are two ways to populate a `QuerySpec`: -/// -/// 1. **Explicit fields** — supply `metric_name`, `aggregations`, -/// `time_window`, etc. directly. This is the original API. -/// -/// 2. **Query string** — supply a raw PromQL or SQL string in -/// `query_string`. The analyzer parses it and fills in `metric_name`, -/// `aggregations`, `group_by_labels`, `label_filters`, and `time_window` -/// automatically. Any explicit fields that are non-empty / non-default -/// **override** the parsed values, so the two approaches compose. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct QuerySpec { - /// Raw PromQL or SQL query string to parse (SP-1 automatic extraction). - /// When provided, metric_name / aggregations / time_window may be omitted - /// and will be derived from the query. - #[serde(default)] - pub query_string: Option, - - /// Metric name override. Required when `query_string` is absent. - #[serde(default)] - pub metric_name: String, - #[serde(default)] - pub label_filters: HashMap, - #[serde(default)] - pub group_by_labels: Vec, - /// Aggregation type overrides ("quantile", "cardinality", "frequency"). - /// Required when `query_string` is absent. - #[serde(default)] - pub aggregations: Vec, - /// Time window override (e.g. "5m"). Required when `query_string` is absent. - #[serde(default)] - pub time_window: String, - #[serde(default)] - pub repeat_every: Option, - pub accuracy_sla: f64, - pub latency_sla: Option, - /// Optional: pin a specific sketch type, bypassing the cost-model planner. - pub sketch_type: Option, - /// Observable data-stream characteristics used for delta / raw-vs-sketch - /// bandwidth comparison. Omit to use conservative defaults. - #[serde(default)] - pub workload: WorkloadCharacteristics, - - // ── design.md alignment: new fields, defaulted for back-compat ──────── - // - // These fields converge `QuerySpec` toward the typed schema in - // `controller/docs/design.md` §6 `core::workload`. Each is defaulted - // so the existing JSON API surface (POST /api/v1/plan handlers, - // pre-population from `workloads.yaml`, the test fixtures elsewhere - // in the controller) keeps working without supplying them. The - // planner does not yet consume these — see `Analyzer::analyze` for - // the L1 cross-product validation that does fire today. - - /// Stable identifier preserved across replan cycles. Optional; - /// auto-derived from `metric_name + accuracy_sla` if omitted - /// (existing API callers don't supply this). - #[serde(default)] - pub id: Option, - - /// Source language. Inferred from `query_string` syntax / parser - /// dispatch when omitted (existing API callers default to PromQL - /// behavior, which matches today's `query_parser::parse_query`). - #[serde(default)] - pub language: Option, - - /// Typed accuracy target. When present, takes precedence over the - /// legacy `accuracy_sla: f64` field. When absent, the legacy field - /// is converted to `Epsilon(1.0 - accuracy_sla)` (or `Exact` when - /// `accuracy_sla == 1.0`). - #[serde(default)] - pub accuracy: Option, - - /// Per-evaluation $ budget. Optional; the cost model picks freely - /// when unset. - #[serde(default)] - pub dollars: Option, - - /// Deployment-model routing hint. Optional; defaults to the model - /// bound to the inbound HTTP route. - #[serde(default)] - pub deployment_model: Option, - - /// Evaluation cadence shape. Defaults to `OneShot`. - #[serde(default = "default_query_shape")] - pub shape: QueryShape, - - /// Source data shape. Defaults to `AppendOnlyStream` (the - /// asap-collector / asap-query default). - #[serde(default = "default_data_shape")] - pub data: DataShape, -} - -fn default_query_shape() -> QueryShape { QueryShape::default() } -fn default_data_shape() -> DataShape { DataShape::default() } - -pub struct Analyzer; - -impl Analyzer { - pub fn new() -> Self { Self } - - pub fn analyze(&self, spec: QuerySpec) -> anyhow::Result { - if !(0.0..=1.0).contains(&spec.accuracy_sla) { - return Err(anyhow!("accuracy_sla must be in [0,1], got {}", spec.accuracy_sla)); - } - - // ── design.md L1: shape × data cross-product check ───────────────── - // The cross-product table in design.md §6 enumerates which - // (shape, data) combinations the planner accepts. The two - // hard rejections are at L1 because they have no semantically - // valid plan: a streaming query over a static dataset, and a - // streaming query over a mutable relation (no retraction-aware - // sketches in the catalog yet). Everything else is accepted - // here — downstream rule firing can still narrow further. - match (&spec.shape, &spec.data) { - (QueryShape::Streaming, DataShape::Batch) => { - return Err(anyhow!( - "QueryShape::Streaming over DataShape::Batch is rejected at L1: \ - no semantically valid plan (no stream over a static dataset). \ - See controller/docs/design.md §6 cross-product table." - )); - } - (QueryShape::Streaming, DataShape::Mutable) => { - return Err(anyhow!( - "QueryShape::Streaming over DataShape::Mutable is rejected at L1: \ - no retraction-aware sketches in the catalog yet. \ - See controller/docs/design.md §6 cross-product table." - )); - } - _ => {} - } - - // ── design.md accuracy precedence: typed `accuracy` > legacy ─────── - // When the caller supplies `accuracy: Some(AccuracyTarget)` it - // takes precedence. Otherwise the legacy `accuracy_sla: f64` - // field is translated into the typed form. The downstream - // planner currently consumes the legacy `f64` field; we keep - // it populated either way so cost-model behaviour does not - // regress for callers that supply the new field. Once the - // planner switches to consuming `AccuracyTarget` directly - // (separate downstream PR), this back-translation stops being - // needed. - let accuracy_sla = match &spec.accuracy { - Some(AccuracyTarget::Exact) => 1.0, - Some(AccuracyTarget::Epsilon(eps)) => (1.0 - eps).clamp(0.0, 1.0), - Some(AccuracyTarget::EpsilonDelta { eps, .. }) => (1.0 - eps).clamp(0.0, 1.0), - None => spec.accuracy_sla, - }; - - // ── Step 1: parse query_string if provided ───────────────────────── - let parsed = spec.query_string.as_deref() - .map(|q| query_parser::parse_query(q)) - .transpose() - .with_context(|| "failed to parse query_string")?; - - // ── Step 2: resolve metric_name ──────────────────────────────────── - let metric_name = if !spec.metric_name.trim().is_empty() { - spec.metric_name.clone() - } else if let Some(ref p) = parsed { - p.metric_name.clone() - } else { - return Err(anyhow!( - "metric_name is required (or provide query_string)" - )); - }; - - // ── Step 3: resolve aggregations ─────────────────────────────────── - let aggregations = if !spec.aggregations.is_empty() { - parse_agg_types(&spec.aggregations)? - } else if let Some(ref p) = parsed { - if p.aggregations.is_empty() && !p.exact_required { - return Err(anyhow!( - "could not infer aggregation type from query_string; \ - provide explicit aggregations" - )); - } - p.aggregations.clone() - } else { - return Err(anyhow!("at least one aggregation is required")); - }; - - // ── Step 4: resolve time_window ──────────────────────────────────── - let time_window = if !spec.time_window.trim().is_empty() { - let d = parse_duration(&spec.time_window) - .with_context(|| format!("invalid time_window {:?}", spec.time_window))?; - if d.is_zero() { - return Err(anyhow!("time_window must be positive")); - } - d - } else if let Some(ref p) = parsed { - p.time_window - } else { - return Err(anyhow!("time_window is required (or provide query_string)")); - }; - - // ── Step 5: resolve dimensions (group_by + label_filter keys) ────── - // Parsed values are the base; explicit spec fields override / extend. - let parsed_group_by = parsed.as_ref().map(|p| p.group_by_labels.as_slice()).unwrap_or(&[]); - let parsed_filters: HashMap = - parsed.as_ref().map(|p| p.label_filters.clone()).unwrap_or_default(); - - let merged_filters: HashMap = { - let mut m = parsed_filters; - m.extend(spec.label_filters.clone()); // explicit overrides parsed - m - }; - - let filter_keys: Vec = merged_filters.keys().cloned().collect(); - let all_group_by: Vec = dedup_dims( - &dedup_dims(parsed_group_by, &spec.group_by_labels), - &filter_keys, - ); - - // ── Step 6: scalar fields ────────────────────────────────────────── - let repeat_every = spec.repeat_every.as_deref() - .map(parse_duration) - .transpose() - .with_context(|| "invalid repeat_every")?; - - let latency_sla = spec.latency_sla.as_deref() - .map(parse_duration) - .transpose() - .with_context(|| "invalid latency_sla")?; - - let exact_required = parsed.as_ref().map(|p| p.exact_required).unwrap_or(false); - let quantiles = parsed.as_ref().map(|p| p.quantiles.clone()).unwrap_or_default(); - - // Note: `(planner not yet using this)` — these are populated for - // downstream consumers but the planner / cost model still keys - // off `accuracy_sla`, `time_window`, `aggregations`, etc. The - // L4-aware downstream PR will switch the cost model to read - // `spec.accuracy`, the L5 stage allocator to gate on - // `spec.shape`, and the leaf planner to gate on `spec.data`. - let _ = (&spec.accuracy, &spec.shape, &spec.data, - &spec.id, &spec.language, &spec.dollars, - &spec.deployment_model); - - Ok(QueryWorkload { - metric_name, - label_filters: merged_filters, - group_by_labels: all_group_by, - aggregations, - time_window, - repeat_every, - accuracy_sla, - latency_sla, - sketch_type_override: spec.sketch_type, - exact_required, - quantiles, - }) - } - -} - -// ── Duration helpers (used by other modules) ────────────────────────────────── - -/// Parses duration strings like "5m", "1h", "30s", "1h30m", "1h5m30s". -pub fn parse_duration(s: &str) -> anyhow::Result { - let s = s.trim(); - if s.is_empty() { - return Err(anyhow!("empty duration string")); - } - let mut total_secs: u64 = 0; - let mut current_num = String::new(); - for ch in s.chars() { - if ch.is_ascii_digit() { - current_num.push(ch); - } else { - let n: u64 = current_num.parse() - .map_err(|_| anyhow!("invalid number in duration {:?}", s))?; - current_num.clear(); - match ch { - 'h' => total_secs += n * 3600, - 'm' => total_secs += n * 60, - 's' => total_secs += n, - _ => return Err(anyhow!("unknown unit {:?} in duration {:?}", ch, s)), - } - } - } - if !current_num.is_empty() { - return Err(anyhow!("trailing digits without unit in {:?}", s)); - } - Ok(Duration::from_secs(total_secs)) -} - -/// Formats a Duration as a compact string: "5m", "1h30m", "30s". -pub fn format_duration(d: Duration) -> String { - let s = d.as_secs(); - let h = s / 3600; - let m = (s % 3600) / 60; - let sec = s % 60; - let mut out = String::new(); - if h > 0 { out.push_str(&format!("{}h", h)); } - if m > 0 { out.push_str(&format!("{}m", m)); } - if sec > 0 || out.is_empty() { out.push_str(&format!("{}s", sec)); } - out -} - -// ── Private helpers ─────────────────────────────────────────────────────────── - -fn parse_agg_types(raw: &[String]) -> anyhow::Result> { - raw.iter().map(|s| match s.to_lowercase().trim() { - "quantile" => Ok(AggType::Quantile), - "cardinality" => Ok(AggType::Cardinality), - "frequency" => Ok(AggType::Frequency), - other => Err(anyhow!( - "unknown aggregation type {:?} (want: quantile, cardinality, frequency)", other - )), - }).collect() -} - -fn dedup_dims(a: &[String], b: &[String]) -> Vec { - let mut seen = HashSet::new(); - let mut out = Vec::new(); - for v in a.iter().chain(b.iter()) { - if seen.insert(v.clone()) { out.push(v.clone()); } - } - out -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - - fn basic_spec() -> QuerySpec { - QuerySpec { - query_string: None, - metric_name: "request_latency".into(), - label_filters: [("service".into(), "web".into())].into(), - group_by_labels: vec!["host.name".into()], - aggregations: vec!["quantile".into()], - time_window: "5m".into(), - repeat_every: Some("1m".into()), - accuracy_sla: 0.01, - latency_sla: Some("10m".into()), - sketch_type: None, - workload: Default::default(), - // design.md alignment: defaults preserve legacy behaviour. - id: None, - language: None, - accuracy: None, - dollars: None, - deployment_model: None, - shape: QueryShape::default(), - data: DataShape::default(), - } - } - - #[test] - fn valid_spec() { - let w = Analyzer::new().analyze(basic_spec()).unwrap(); - assert_eq!(w.metric_name, "request_latency"); - assert_eq!(w.accuracy_sla, 0.01); - assert_eq!(w.time_window, Duration::from_secs(300)); - assert_eq!(w.repeat_every, Some(Duration::from_secs(60))); - assert_eq!(w.latency_sla, Some(Duration::from_secs(600))); - assert_eq!(w.aggregations, vec![AggType::Quantile]); - } - - #[test] - fn dimension_merge_dedup() { - let mut spec = basic_spec(); - spec.label_filters = [("service".into(), "api".into()), - ("host.name".into(), "h1".into())].into(); - spec.group_by_labels = vec!["host.name".into(), "region".into()]; - let w = Analyzer::new().analyze(spec).unwrap(); - for dim in &["host.name", "region", "service"] { - assert!(w.group_by_labels.contains(&dim.to_string()), "missing {dim}"); - } - // host.name must appear exactly once after dedup - assert_eq!( - w.group_by_labels.iter().filter(|d| d.as_str() == "host.name").count(), 1 - ); - } - - #[test] - fn multiple_aggregations() { - let mut spec = basic_spec(); - spec.aggregations = vec!["cardinality".into(), "frequency".into()]; - let w = Analyzer::new().analyze(spec).unwrap(); - assert_eq!(w.aggregations, vec![AggType::Cardinality, AggType::Frequency]); - } - - #[test] - fn missing_metric_name() { - let mut spec = basic_spec(); - spec.metric_name = "".into(); - assert!(Analyzer::new().analyze(spec).is_err()); - } - - #[test] - fn missing_aggregations() { - let mut spec = basic_spec(); - spec.aggregations = vec![]; - assert!(Analyzer::new().analyze(spec).is_err()); - } - - #[test] - fn invalid_aggregation_type() { - let mut spec = basic_spec(); - spec.aggregations = vec!["histogram".into()]; - assert!(Analyzer::new().analyze(spec).is_err()); - } - - #[test] - fn invalid_duration() { - let mut spec = basic_spec(); - spec.time_window = "not-a-duration".into(); - assert!(Analyzer::new().analyze(spec).is_err()); - } - - #[test] - fn invalid_accuracy_sla() { - for bad in &[-0.1f64, 1.5] { - let mut spec = basic_spec(); - spec.accuracy_sla = *bad; - assert!(Analyzer::new().analyze(spec).is_err(), - "expected error for accuracy_sla={bad}"); - } - } - - #[test] - fn parse_duration_formats() { - assert_eq!(parse_duration("30s").unwrap(), Duration::from_secs(30)); - assert_eq!(parse_duration("5m").unwrap(), Duration::from_secs(300)); - assert_eq!(parse_duration("1h").unwrap(), Duration::from_secs(3600)); - assert_eq!(parse_duration("1h30m").unwrap(), Duration::from_secs(5400)); - assert_eq!(parse_duration("1h5m30s").unwrap(),Duration::from_secs(3930)); - } - - #[test] - fn format_duration_roundtrip() { - for secs in [30u64, 300, 3600, 5400, 3930] { - let d = Duration::from_secs(secs); - let s = format_duration(d); - let parsed = parse_duration(&s).unwrap(); - assert_eq!(parsed, d, "roundtrip failed for {secs}s → {s:?}"); - } - } - - #[test] - fn trailing_digits_error() { - assert!(parse_duration("5").is_err()); - } - - // ── query_string path ───────────────────────────────────────────────────── - - /// Build a minimal QuerySpec driven entirely by a query_string. - fn qs_only(query: &str) -> QuerySpec { - QuerySpec { - query_string: Some(query.into()), - metric_name: "".into(), - label_filters: Default::default(), - group_by_labels: vec![], - aggregations: vec![], - time_window: "".into(), - repeat_every: None, - accuracy_sla: 0.01, - latency_sla: None, - sketch_type: None, - workload: Default::default(), - // design.md alignment: defaults preserve legacy behaviour. - id: None, - language: None, - accuracy: None, - dollars: None, - deployment_model: None, - shape: QueryShape::default(), - data: DataShape::default(), - } - } - - /// PromQL query_string auto-populates metric_name, aggregations, - /// time_window, and quantiles — no explicit fields required. - #[test] - fn query_string_promql_populates_workload() { - let w = Analyzer::new() - .analyze(qs_only("sum by (host) (quantile_over_time(0.99, latency[5m]))")) - .unwrap(); - assert_eq!(w.metric_name, "latency"); - assert_eq!(w.aggregations, vec![AggType::Quantile]); - assert_eq!(w.time_window, Duration::from_secs(300)); - assert_eq!(w.quantiles, vec![0.99]); - assert!(!w.exact_required); - } - - /// SQL query_string auto-populates metric_name, aggregations, - /// and group_by_labels. - #[test] - fn query_string_sql_populates_workload() { - let w = Analyzer::new() - .analyze(qs_only( - "SELECT symbol, COUNT(*) FROM financial_last_trade_price GROUP BY symbol", - )) - .unwrap(); - assert_eq!(w.metric_name, "financial_last_trade_price"); - assert_eq!(w.aggregations, vec![AggType::Frequency]); - assert!(w.group_by_labels.contains(&"symbol".to_string())); - } - - /// Explicit metric_name overrides the name derived from query_string. - #[test] - fn explicit_metric_name_overrides_parsed() { - let mut spec = qs_only("sum by (host) (avg_over_time(cpu[5m]))"); - spec.metric_name = "my_custom_metric".into(); - let w = Analyzer::new().analyze(spec).unwrap(); - assert_eq!(w.metric_name, "my_custom_metric"); - // aggregations still come from parse (avg → DDSketch → Quantile) - assert_eq!(w.aggregations, vec![AggType::Quantile]); - } - - /// Explicit time_window overrides the window derived from query_string. - #[test] - fn explicit_time_window_overrides_parsed() { - let mut spec = qs_only("sum by (host) (avg_over_time(cpu[5m]))"); - spec.time_window = "1h".into(); - let w = Analyzer::new().analyze(spec).unwrap(); - assert_eq!(w.time_window, Duration::from_secs(3600)); - } - - /// Explicit aggregations override those derived from query_string. - #[test] - fn explicit_aggregations_override_parsed() { - let mut spec = qs_only("sum by (host) (avg_over_time(cpu[5m]))"); // → Quantile - spec.aggregations = vec!["cardinality".into()]; - let w = Analyzer::new().analyze(spec).unwrap(); - assert_eq!(w.aggregations, vec![AggType::Cardinality]); - } - - /// sum_over_time is a stateful exact aggregation; exact_required is set. - #[test] - fn query_string_exact_required_propagated() { - let w = Analyzer::new() - .analyze(qs_only("sum by (service) (sum_over_time(request_bytes[1h]))")) - .unwrap(); - assert!(w.exact_required, "sum_over_time must set exact_required"); - assert_eq!(w.aggregations, vec![]); - } - - /// DDSketch quantile φ values are surfaced through the workload. - #[test] - fn query_string_quantiles_populated() { - let w = Analyzer::new() - .analyze(qs_only("sum by (host) (quantile_over_time(0.5, latency[5m]))")) - .unwrap(); - assert_eq!(w.quantiles, vec![0.5]); - } - - /// Existing callers that supply all fields explicitly and omit - /// query_string continue to work unchanged (backward compatibility). - #[test] - fn backward_compat_no_query_string() { - let w = Analyzer::new().analyze(basic_spec()).unwrap(); - assert_eq!(w.metric_name, "request_latency"); - assert_eq!(w.aggregations, vec![AggType::Quantile]); - assert_eq!(w.time_window, Duration::from_secs(300)); - assert!(!w.exact_required); - assert!(w.quantiles.is_empty()); - } - - // ── design.md alignment tests ───────────────────────────────────────────── - - /// Typed `accuracy: Some(Epsilon(0.05))` overrides the legacy - /// `accuracy_sla: 0.99` (which would translate to `Epsilon(0.01)`), - /// and the resolved value flows through to `QueryWorkload.accuracy_sla`. - #[test] - fn typed_accuracy_overrides_legacy_accuracy_sla() { - let mut spec = basic_spec(); - spec.accuracy_sla = 0.99; // legacy: ε = 0.01 - spec.accuracy = Some(AccuracyTarget::Epsilon(0.05)); - let w = Analyzer::new().analyze(spec).unwrap(); - // The resolved 1.0 - 0.05 = 0.95 must reach the QueryWorkload, not - // the legacy 0.99. - assert!((w.accuracy_sla - 0.95).abs() < 1e-9, "got {}", w.accuracy_sla); - } - - /// Typed `accuracy: Some(Exact)` clamps the SLA to 1.0 regardless of - /// the legacy field's value. - #[test] - fn typed_accuracy_exact_clamps_to_one() { - let mut spec = basic_spec(); - spec.accuracy_sla = 0.5; - spec.accuracy = Some(AccuracyTarget::Exact); - let w = Analyzer::new().analyze(spec).unwrap(); - assert_eq!(w.accuracy_sla, 1.0); - } - - /// L1 rejects `(QueryShape::Streaming, DataShape::Batch)` per the - /// `design.md` §6 cross-product table. - #[test] - fn l1_rejects_streaming_over_batch() { - let mut spec = basic_spec(); - spec.shape = QueryShape::Streaming; - spec.data = DataShape::Batch; - let err = Analyzer::new().analyze(spec).unwrap_err().to_string(); - assert!(err.contains("Streaming") && err.contains("Batch"), - "expected the error to name the rejected combination: {err}"); - } - - /// L1 rejects `(QueryShape::Streaming, DataShape::Mutable)` — no - /// retraction-aware sketches in the catalog yet. - #[test] - fn l1_rejects_streaming_over_mutable() { - let mut spec = basic_spec(); - spec.shape = QueryShape::Streaming; - spec.data = DataShape::Mutable; - let err = Analyzer::new().analyze(spec).unwrap_err().to_string(); - assert!(err.contains("Streaming") && err.contains("Mutable"), - "expected the error to name the rejected combination: {err}"); - } - - /// `(QueryShape::Streaming, DataShape::AppendOnlyStream)` — the - /// canonical streaming case — is accepted. - #[test] - fn l1_accepts_streaming_over_append_only_stream() { - let mut spec = basic_spec(); - spec.shape = QueryShape::Streaming; - spec.data = DataShape::AppendOnlyStream; - assert!(Analyzer::new().analyze(spec).is_ok()); - } - - /// JSON without any of the new fields parses correctly via serde — - /// the existing `/api/v1/plan` HTTP API surface keeps working - /// byte-for-byte. Fields default to `None` / `OneShot` / - /// `AppendOnlyStream` per the `#[serde(default)]` annotations. - #[test] - fn json_back_compat_omitting_new_fields() { - let json = r#"{ - "metric_name": "request_latency", - "aggregations": ["quantile"], - "time_window": "5m", - "accuracy_sla": 0.99 - }"#; - let spec: QuerySpec = serde_json::from_str(json).unwrap(); - assert!(spec.id.is_none()); - assert!(spec.language.is_none()); - assert!(spec.accuracy.is_none()); - assert!(spec.dollars.is_none()); - assert!(spec.deployment_model.is_none()); - assert_eq!(spec.shape, QueryShape::OneShot); - assert_eq!(spec.data, DataShape::AppendOnlyStream); - // And the analyzer accepts it. - let w = Analyzer::new().analyze(spec).unwrap(); - assert_eq!(w.metric_name, "request_latency"); - // Legacy accuracy_sla=0.99 round-trips through resolution - // (no typed `accuracy` supplied → translate from legacy → - // Epsilon(0.01) → back to 1 - 0.01 = 0.99). - assert!((w.accuracy_sla - 0.99).abs() < 1e-9, "got {}", w.accuracy_sla); - } - - /// JSON *with* the new fields parses correctly — the wire schema - /// is forward-compatible with callers that supply them. Exercises the - /// adjacently-tagged `AccuracyTarget` form (`kind` + `value`) and the - /// internally-tagged `QueryShape::Periodic` form. - #[test] - fn json_forward_compat_supplying_new_fields() { - let json = r#"{ - "metric_name": "request_latency", - "aggregations": ["quantile"], - "time_window": "5m", - "accuracy_sla": 0.5, - "id": "q-001", - "language": "prom_ql", - "accuracy": { "kind": "epsilon", "value": 0.02 }, - "dollars": 0.001, - "deployment_model": "asaplifecycle", - "shape": { "kind": "periodic", "every": { "secs": 60, "nanos": 0 } }, - "data": "batch" - }"#; - let spec: QuerySpec = serde_json::from_str(json).unwrap(); - assert_eq!(spec.id.as_ref().unwrap().as_str(), "q-001"); - assert_eq!(spec.language, Some(QueryLanguage::PromQL)); - assert_eq!(spec.accuracy, Some(AccuracyTarget::Epsilon(0.02))); - assert_eq!(spec.dollars, Some(0.001)); - assert_eq!(spec.deployment_model.as_deref(), Some("asaplifecycle")); - assert!(matches!(spec.shape, QueryShape::Periodic { .. })); - assert_eq!(spec.data, DataShape::Batch); - // Periodic + Batch is accepted at L1 (scheduled batch report row - // in the design.md cross-product table). - let w = Analyzer::new().analyze(spec).unwrap(); - // typed `accuracy: Epsilon(0.02)` overrode the legacy 0.5 → - // resolved accuracy_sla in the workload is 1.0 - 0.02 = 0.98. - assert!((w.accuracy_sla - 0.98).abs() < 1e-9, "got {}", w.accuracy_sla); - } -} diff --git a/controller/src/backend_client.rs b/controller/src/backend_client.rs deleted file mode 100644 index 2815c5fb..00000000 --- a/controller/src/backend_client.rs +++ /dev/null @@ -1,418 +0,0 @@ -//! HTTP client that pushes a freshly-generated `StreamingConfig` YAML -//! to the ASAPQuery-backend's `POST /api/v1/streaming-config` endpoint. -//! -//! This is the controller-side **producer** of the PR E phase 1 / phase 2 -//! hot-reload contract that landed in ASAPQuery-backend PRs #10 and #12. -//! The replanner calls into this module immediately after generating a -//! new plan so the backend's active `StreamingConfig` is updated without -//! a restart and subsequent queries observe the new aggregation layout. -//! -//! The client is **fire-and-forget at the call site** — the replanner -//! awaits the POST but doesn't block its own return on the outcome. -//! Errors are logged at WARN; the controller is expected to be tolerant -//! of transient backend unavailability because the next replan cycle -//! will try again with the latest plan. - -use std::time::Duration; - -use anyhow::{Context, Result}; -use reqwest::Client; -use tracing::{debug, warn}; - -/// Minimal HTTP client for ASAPQuery-backend's streaming-config endpoint. -/// Built once at controller startup from the `CONTROLLER_BACKEND_ENDPOINT` -/// environment variable and shared via `Arc` with the replanner. -#[derive(Debug, Clone)] -pub struct BackendClient { - endpoint: String, - http: Client, -} - -impl BackendClient { - /// Construct a client pointing at the backend's plan-push endpoint. - /// `endpoint` should be the full URL, e.g. - /// `http://backend.svc:8088/api/v1/streaming-config`. - /// - /// A 5-second timeout bounds the duration a slow or unreachable - /// backend can stall the replanner — consistent with the symmetric - /// 5-second timeout on ASAPQuery-backend's `HttpControllerClient` - /// (the reverse direction in the same loop). - pub fn new(endpoint: impl Into) -> Self { - let http = Client::builder() - .timeout(Duration::from_secs(5)) - .build() - .unwrap_or_else(|_| Client::new()); - Self { - endpoint: endpoint.into(), - http, - } - } - - /// Construct with an explicit `reqwest::Client`. Used by tests that - /// need to inject a mock-server URL without reconfiguring the - /// timeout setup. - pub fn with_http(endpoint: impl Into, http: Client) -> Self { - Self { - endpoint: endpoint.into(), - http, - } - } - - pub fn endpoint(&self) -> &str { - &self.endpoint - } - - /// POST the given `StreamingConfig` YAML to the backend. Returns - /// `Ok(())` on any 2xx status, otherwise an error carrying the - /// status code and response body. The caller (typically - /// [`Replanner::replan_metric`]) logs the error and moves on — the - /// next replan cycle will retry with the latest plan. - pub async fn push_streaming_config(&self, yaml: String) -> Result<()> { - debug!( - endpoint = %self.endpoint, - yaml_bytes = yaml.len(), - "pushing streaming-config YAML to ASAPQuery-backend" - ); - let resp = self - .http - .post(&self.endpoint) - .header("content-type", "application/x-yaml") - .body(yaml) - .send() - .await - .context("failed to POST streaming-config to backend")?; - - let status = resp.status(); - if status.is_success() { - Ok(()) - } else { - let body = resp.text().await.unwrap_or_default(); - Err(anyhow::anyhow!( - "backend returned {} for streaming-config POST: {}", - status, - body - )) - } - } - - /// Phase C (MVP v6) variant of [`Self::push_streaming_config`] - /// that POSTs `application/json`. The typed L5 - /// `emit_backend_config_json` emitter produces a `serde_json::Value` - /// rather than a YAML document, and the ASAPQuery-backend's - /// `/api/v1/streaming-config` endpoint accepts both content types - /// (PR #297 / Phase B documents the JSON shape). Same 2xx-or-error - /// contract as the YAML variant; same fire-and-forget semantics - /// at the call site. - pub async fn post_streaming_config_json(&self, json: String) -> Result<()> { - debug!( - endpoint = %self.endpoint, - json_bytes = json.len(), - "posting streaming-config JSON to ASAPQuery-backend" - ); - let resp = self - .http - .post(&self.endpoint) - .header("content-type", "application/json") - .body(json) - .send() - .await - .context("failed to POST streaming-config JSON to backend")?; - - let status = resp.status(); - if status.is_success() { - Ok(()) - } else { - let body = resp.text().await.unwrap_or_default(); - Err(anyhow::anyhow!( - "backend returned {} for streaming-config JSON POST: {}", - status, - body - )) - } - } - - /// Phase α (MVP) sibling of [`Self::post_streaming_config_json`]: - /// POSTs the controller-emitted `BackendStorageRouting` JSON - /// document to the backend's `POST /api/v1/storage_routing` - /// endpoint. The backend hot-loads the routing table and the next - /// instant query consults the new table. - /// - /// Endpoint resolution: the field [`Self::endpoint`] is the - /// controller's configured streaming-config endpoint (e.g. - /// `http://backend.svc:8088/api/v1/streaming-config`). We rewrite - /// the path component from `/api/v1/streaming-config` to - /// `/api/v1/storage_routing` so operators only configure one - /// `CONTROLLER_BACKEND_ENDPOINT` env var and both pushes land at - /// the same backend host. URLs that don't end in - /// `/api/v1/streaming-config` are passed through unchanged - /// (a test-mode escape hatch — the unit test below builds a - /// mock URL ending in `/storage_routing` directly). - pub async fn post_storage_routing_json(&self, json: String) -> Result<()> { - let url = derive_storage_routing_url(&self.endpoint); - debug!( - endpoint = %url, - json_bytes = json.len(), - "posting storage-routing JSON to ASAPQuery-backend" - ); - let resp = self - .http - .post(&url) - .header("content-type", "application/json") - .body(json) - .send() - .await - .context("failed to POST storage-routing JSON to backend")?; - - let status = resp.status(); - if status.is_success() { - Ok(()) - } else { - let body = resp.text().await.unwrap_or_default(); - Err(anyhow::anyhow!( - "backend returned {} for storage-routing JSON POST: {}", - status, - body - )) - } - } -} - -/// Map a streaming-config endpoint URL to the sibling storage-routing -/// endpoint by rewriting the trailing path component. URLs that don't -/// end with `/api/v1/streaming-config` (or `/api/v1/streaming_config` — -/// either spelling is supported) pass through unchanged so tests can -/// inject a mock-server URL directly. -fn derive_storage_routing_url(endpoint: &str) -> String { - const STREAMING_PATH_DASH: &str = "/api/v1/streaming-config"; - const STREAMING_PATH_UNDERSCORE: &str = "/api/v1/streaming_config"; - const ROUTING_PATH: &str = "/api/v1/storage_routing"; - if let Some(stripped) = endpoint.strip_suffix(STREAMING_PATH_DASH) { - return format!("{stripped}{ROUTING_PATH}"); - } - if let Some(stripped) = endpoint.strip_suffix(STREAMING_PATH_UNDERSCORE) { - return format!("{stripped}{ROUTING_PATH}"); - } - endpoint.to_string() -} - -/// Fire-and-forget convenience helper used by the replanner. Logs -/// errors at WARN and never propagates them — the replanner should -/// never fail an entire replan because the backend was temporarily -/// unreachable. -pub async fn push_or_log(client: &BackendClient, metric: &str, yaml: String) { - match client.push_streaming_config(yaml).await { - Ok(()) => { - debug!(metric, endpoint = %client.endpoint, "streaming-config push succeeded"); - } - Err(e) => { - warn!( - metric, - endpoint = %client.endpoint, - error = %e, - "streaming-config push to ASAPQuery-backend failed; \ - next replan cycle will retry" - ); - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use axum::extract::State; - use axum::routing::post; - use axum::Router; - use std::sync::{Arc as StdArc, Mutex}; - - #[derive(Clone)] - struct SharedSink(StdArc>>); - - async fn start_mock_backend(sink: SharedSink, status: axum::http::StatusCode) -> String { - let app = Router::new() - .route( - "/api/v1/streaming-config", - post( - move |State(sink): State, body: axum::body::Bytes| async move { - let yaml = String::from_utf8_lossy(&body).to_string(); - sink.0.lock().unwrap().push(yaml); - status - }, - ), - ) - .with_state(sink); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - tokio::spawn(async move { - axum::serve(listener, app).await.unwrap(); - }); - tokio::time::sleep(Duration::from_millis(50)).await; - format!("http://{addr}/api/v1/streaming-config") - } - - #[tokio::test] - async fn success_path_round_trips_yaml() { - let sink = SharedSink(StdArc::new(Mutex::new(Vec::new()))); - let url = start_mock_backend(sink.clone(), axum::http::StatusCode::OK).await; - - let client = BackendClient::new(url); - let yaml = "aggregations:\n - aggregationId: 42\n metric: cpu\n".to_string(); - client - .push_streaming_config(yaml.clone()) - .await - .expect("push ok"); - - let received = sink.0.lock().unwrap(); - assert_eq!(received.len(), 1); - assert_eq!(received[0], yaml); - } - - #[tokio::test] - async fn non_2xx_status_is_reported_as_error() { - let sink = SharedSink(StdArc::new(Mutex::new(Vec::new()))); - let url = - start_mock_backend(sink.clone(), axum::http::StatusCode::INTERNAL_SERVER_ERROR).await; - - let client = BackendClient::new(url); - let result = client.push_streaming_config("whatever".to_string()).await; - assert!(result.is_err(), "expected error on 500, got {result:?}"); - let msg = result.unwrap_err().to_string(); - assert!(msg.contains("500"), "error msg should mention 500: {msg}"); - } - - #[tokio::test] - async fn push_or_log_swallows_errors() { - // Point at an unreachable port so the request fails fast. - let client = BackendClient::new("http://127.0.0.1:1/api/v1/streaming-config"); - // Must not panic or propagate — fire-and-forget semantics. - push_or_log(&client, "cpu_usage", "content".to_string()).await; - } - - /// Phase C: the JSON variant POSTs the body verbatim, returns - /// `Ok(())` on a 2xx, and surfaces non-2xx as `Err`. Mock backend - /// captures the body so we can verify it round-trips. - #[tokio::test] - async fn json_post_round_trips_body() { - let sink = SharedSink(StdArc::new(Mutex::new(Vec::new()))); - let url = start_mock_backend(sink.clone(), axum::http::StatusCode::OK).await; - - let client = BackendClient::new(url); - let json = r#"{"aggregations":[{"aggregationId":7,"metric":"latency"}]}"#.to_string(); - client - .post_streaming_config_json(json.clone()) - .await - .expect("json post ok"); - - let received = sink.0.lock().unwrap(); - assert_eq!(received.len(), 1); - assert_eq!(received[0], json); - } - - /// Phase C: non-2xx from the backend surfaces as an error so the - /// caller (handle_plan) can log + move on. - #[tokio::test] - async fn json_post_non_2xx_is_error() { - let sink = SharedSink(StdArc::new(Mutex::new(Vec::new()))); - let url = - start_mock_backend(sink.clone(), axum::http::StatusCode::BAD_REQUEST).await; - - let client = BackendClient::new(url); - let result = client.post_streaming_config_json("{}".to_string()).await; - assert!(result.is_err(), "expected error on 400, got {result:?}"); - let msg = result.unwrap_err().to_string(); - assert!(msg.contains("400"), "error msg should mention 400: {msg}"); - } - - /// Phase α: storage-routing-URL derivation rewrites the path - /// component when the configured endpoint ends in - /// `/api/v1/streaming-config`, leaving everything else untouched. - #[test] - fn storage_routing_url_rewrites_streaming_path() { - assert_eq!( - derive_storage_routing_url("http://backend:8088/api/v1/streaming-config"), - "http://backend:8088/api/v1/storage_routing" - ); - assert_eq!( - derive_storage_routing_url("http://backend:8088/api/v1/streaming_config"), - "http://backend:8088/api/v1/storage_routing" - ); - } - - #[test] - fn storage_routing_url_preserves_unknown_paths_for_tests() { - // Test escape hatch — a mock-server URL pointing directly at - // `/api/v1/storage_routing` already passes through unchanged. - assert_eq!( - derive_storage_routing_url("http://127.0.0.1:1/api/v1/storage_routing"), - "http://127.0.0.1:1/api/v1/storage_routing" - ); - // Unrelated path passes through too — no surprise rewriting. - assert_eq!( - derive_storage_routing_url("http://x/foo"), - "http://x/foo" - ); - } - - /// Phase α: full happy path. A mock backend hosts the storage - /// routing endpoint; the client POSTs the controller-emitted JSON - /// and the body round-trips verbatim. Mirrors `json_post_round_trips_body`. - async fn start_mock_routing_backend( - sink: SharedSink, - status: axum::http::StatusCode, - ) -> String { - let app = Router::new() - .route( - "/api/v1/storage_routing", - post( - move |State(sink): State, body: axum::body::Bytes| async move { - let json = String::from_utf8_lossy(&body).to_string(); - sink.0.lock().unwrap().push(json); - status - }, - ), - ) - .with_state(sink); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - tokio::spawn(async move { - axum::serve(listener, app).await.unwrap(); - }); - tokio::time::sleep(Duration::from_millis(50)).await; - // Return the streaming-config URL — the client will rewrite - // the path before issuing the POST. - format!("http://{addr}/api/v1/streaming-config") - } - - #[tokio::test] - async fn storage_routing_post_round_trips_body_via_url_rewrite() { - let sink = SharedSink(StdArc::new(Mutex::new(Vec::new()))); - let url = start_mock_routing_backend(sink.clone(), axum::http::StatusCode::OK).await; - - let client = BackendClient::new(url); - let json = - r#"{"default_engine":"sketch_warm_tier","metrics":[{"name":"x","targets":[]}]}"# - .to_string(); - client - .post_storage_routing_json(json.clone()) - .await - .expect("routing post ok"); - - let received = sink.0.lock().unwrap(); - assert_eq!(received.len(), 1); - assert_eq!(received[0], json); - } - - #[tokio::test] - async fn storage_routing_post_non_2xx_is_error() { - let sink = SharedSink(StdArc::new(Mutex::new(Vec::new()))); - let url = start_mock_routing_backend( - sink.clone(), - axum::http::StatusCode::INTERNAL_SERVER_ERROR, - ) - .await; - let client = BackendClient::new(url); - let result = client.post_storage_routing_json("{}".to_string()).await; - assert!(result.is_err(), "expected error on 500, got {result:?}"); - let msg = result.unwrap_err().to_string(); - assert!(msg.contains("500"), "error msg should mention 500: {msg}"); - } -} diff --git a/controller/src/config/agent.rs b/controller/src/config/agent.rs deleted file mode 100644 index 699112e8..00000000 --- a/controller/src/config/agent.rs +++ /dev/null @@ -1,558 +0,0 @@ -use anyhow::Context; -use serde::Serialize; -use serde_yaml::{Mapping, Value}; -use std::collections::HashMap; - -use crate::analyzer::format_duration; -use crate::types::*; - -// ── YAML structural types ───────────────────────────────────────────────────── - -#[derive(Serialize)] -struct CollectorYaml { - extensions: HashMap, - receivers: HashMap, - processors: HashMap, - exporters: HashMap, - service: ServiceSection, -} - -#[derive(Serialize)] -struct ServiceSection { - extensions: Vec, - pipelines: HashMap, -} - -#[derive(Serialize)] -struct Pipeline { - receivers: Vec, - processors: Vec, - exporters: Vec, -} - -// ── Public API ──────────────────────────────────────────────────────────────── - -/// Generates an OTel collector YAML string for an agent collector from a plan. -/// -/// The `opamp_endpoint` parameter specifies the OpAMP WebSocket endpoint that -/// the collector should connect to for receiving runtime config updates from the -/// controller. An `extensions.opamp` section is included in the generated YAML -/// so the collector can receive pushed configs without a restart. -pub fn generate_agent_config( - cfg: &AgentCollectorConfig, - opamp_endpoint: &str, -) -> anyhow::Result { - let processor_key = cfg.sketch_type.to_string(); - let processor_val = build_processor_block(cfg); - - // Standard OTLP receiver (gRPC + HTTP) with optional series_id registry. - let mut otlp_map: Mapping = serde_yaml::from_str( - "protocols:\n grpc:\n endpoint: \"0.0.0.0:4317\"\n http:\n endpoint: \"0.0.0.0:4318\"\n", - ).unwrap(); - - otlp_map.insert("enable_series_id".into(), Value::Bool(cfg.enable_series_id)); - if cfg.series_id_ttl_secs > 0 { - otlp_map.insert( - "series_id_ttl".into(), - Value::String(format!("{}s", cfg.series_id_ttl_secs)), - ); - } - let otlp_receiver = Value::Mapping(otlp_map); - - // Build the exporter block from `cfg.data_sink`. The planner - // chooses the sketch + window + projection; *where* the - // sketched data goes is a deployment-scope concern carried - // here. Default is `otlp/backend` because the modified-OTLP - // `Data::Ddsketch` / `KLLSketch` / ... variants only survive - // an OTLP transport — the legacy `prometheus` exporter is - // kept only for raw-scalar pipelines. - let (exporter_key, exporter_val) = build_exporter_block(&cfg.data_sink); - - // OpAMP extension — allows the controller to push config updates at runtime. - let opamp_ext: Value = serde_yaml::from_str(&format!( - "server:\n ws:\n endpoint: \"{opamp_endpoint}\"\n" - )).unwrap(); - - let doc = CollectorYaml { - extensions: [("opamp".to_string(), opamp_ext)].into(), - receivers: [("otlp".to_string(), otlp_receiver)].into(), - processors: [(processor_key.clone(), processor_val)].into(), - exporters: [(exporter_key.clone(), exporter_val)].into(), - service: ServiceSection { - extensions: vec!["opamp".into()], - pipelines: [( - "metrics".to_string(), - Pipeline { - receivers: vec!["otlp".into()], - processors: vec![processor_key], - exporters: vec![exporter_key], - }, - )] - .into(), - }, - }; - - serde_yaml::to_string(&doc).context("serialize agent config") -} - -/// Maps the planner's `AgentDataSink` choice to a (component_id, -/// component_yaml) pair. The component_id is what goes into the -/// `exporters:` map AND the pipeline's `exporters:` list — both -/// references must agree, so it's returned alongside the YAML -/// block. -fn build_exporter_block(sink: &AgentDataSink) -> (String, Value) { - match sink { - AgentDataSink::Otlp { - endpoint, - compression, - } => { - let yaml = format!( - "endpoint: \"{endpoint}\"\ntls:\n insecure: true\ncompression: {compression}\n" - ); - ( - "otlp/backend".to_string(), - serde_yaml::from_str(&yaml).unwrap(), - ) - } - AgentDataSink::PrometheusScrape { endpoint } => { - let yaml = format!("endpoint: \"{endpoint}\"\n"); - ( - "prometheus".to_string(), - serde_yaml::from_str(&yaml).unwrap(), - ) - } - } -} - -fn build_processor_block(cfg: &AgentCollectorConfig) -> Value { - let mut m = Mapping::new(); - - m.insert("mode".into(), Value::String(cfg.mode.to_string())); - m.insert( - "enable_self_monitoring".into(), - Value::Bool(cfg.enable_self_monitoring), - ); - m.insert("transmit_sketch".into(), Value::Bool(cfg.transmit_sketch)); - - if cfg.mode == ProcessorMode::Window { - if let Some(wd) = cfg.window_duration { - m.insert("window_duration".into(), Value::String(format_duration(wd))); - } - } - - if !cfg.aggregate_by.is_empty() { - m.insert("aggregate_by".into(), seq_of_strings(&cfg.aggregate_by)); - } - if !cfg.label_matchers.is_empty() { - // Go processors expect []LabelMatcher{Key, Value}, not flat strings. - let matchers: Vec = cfg.label_matchers.iter().filter_map(|s| { - let (k, v) = s.split_once('=')?; - let mut map = serde_yaml::Mapping::new(); - map.insert("key".into(), Value::String(k.to_string())); - map.insert("value".into(), Value::String(v.to_string())); - Some(Value::Mapping(map)) - }).collect(); - if !matchers.is_empty() { - m.insert("label_matchers".into(), Value::Sequence(matchers)); - } - } - - // Delta transmission: only emit fields each processor's Config actually defines. - // KLL rejects delta_transmission at Validate(); HLL has no delta_threshold key. - if cfg.delta_transmission && cfg.sketch_type != SketchType::KLL { - m.insert("delta_transmission".into(), Value::Bool(true)); - if matches!( - cfg.sketch_type, - SketchType::DDSketch | SketchType::CountSketch | SketchType::CountMinSketch - ) { - m.insert( - "delta_threshold".into(), - Value::Number(cfg.delta_threshold.into()), - ); - } - } - - // Sketch-type-specific params. - match &cfg.sketch_params { - SketchParams::DDSketch { relative_accuracy, quantiles } => { - m.insert("relative_accuracy".into(), Value::Number((*relative_accuracy).into())); - if !quantiles.is_empty() { - m.insert("quantiles".into(), Value::Sequence( - quantiles.iter().map(|q| Value::Number((*q).into())).collect(), - )); - } - } - SketchParams::KLL { k, quantiles } => { - m.insert("k".into(), Value::Number((*k as u64).into())); - if !quantiles.is_empty() { - m.insert("quantiles".into(), Value::Sequence( - quantiles.iter().map(|q| Value::Number((*q).into())).collect(), - )); - } - } - SketchParams::HLL { .. } => { - // hllprocessor uses a fixed HLL precision in code; Config has no precision field. - } - SketchParams::CountSketch { epsilon, delta } => { - m.insert("epsilon".into(), Value::Number((*epsilon).into())); - m.insert("delta".into(), Value::Number((*delta).into())); - } - SketchParams::CountMinSketch { rows, cols, metric_name } => { - m.insert("metric_name".into(), Value::String(metric_name.clone())); - m.insert("rows".into(), Value::Number((*rows as u64).into())); - m.insert("columns".into(), Value::Number((*cols as u64).into())); - } - } - - Value::Mapping(m) -} - -fn seq_of_strings(v: &[String]) -> Value { - Value::Sequence(v.iter().map(|s| Value::String(s.clone())).collect()) -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use std::time::Duration; - - fn ddsketch_cfg() -> AgentCollectorConfig { - AgentCollectorConfig { - output_mode: OutputMode::Sketch, - sketch_type: SketchType::DDSketch, - sketch_params: SketchParams::DDSketch { - relative_accuracy: 0.01, - quantiles: vec![0.5, 0.9, 0.99], - }, - aggregate_by: vec!["host.name".into(), "service".into()], - label_matchers: vec!["env=prod".into()], - window_duration: Some(Duration::from_secs(300)), - mode: ProcessorMode::Window, - enable_self_monitoring: true, - transmit_sketch: true, - drop_original: true, - delta_transmission: false, - delta_threshold: 0.0, - enable_series_id: true, - series_id_ttl_secs: 0, - // Pre-existing fixture tests (`contains_prometheus_exporter`, - // `pipeline_has_receivers_and_exporters`) assert the legacy - // prometheus exporter on :8889 — keep the test semantics by - // pinning the sink, not by changing the default. - data_sink: AgentDataSink::PrometheusScrape { - endpoint: "0.0.0.0:8889".to_string(), - }, - } - } - - #[test] - fn contains_processor_key() { - let yaml = generate_agent_config(&ddsketch_cfg(), "ws://ctrl:4320/v1/opamp").unwrap(); - assert!( - yaml.contains("ddsketch:"), - "YAML should contain 'ddsketch:'\n{yaml}" - ); - assert!( - yaml.contains("enable_self_monitoring: true"), - "YAML should carry enable_self_monitoring\n{yaml}" - ); - } - - #[test] - fn contains_opamp_extension() { - let yaml = generate_agent_config(&ddsketch_cfg(), "ws://ctrl:4320/v1/opamp").unwrap(); - assert!( - yaml.contains("opamp"), - "YAML should include the opamp extension\n{yaml}" - ); - assert!( - yaml.contains("ws://ctrl:4320/v1/opamp"), - "YAML should contain the opamp endpoint\n{yaml}" - ); - } - - #[test] - fn contains_window_duration() { - let yaml = generate_agent_config(&ddsketch_cfg(), "ws://ctrl:4320/v1/opamp").unwrap(); - assert!( - yaml.contains("5m"), - "YAML should contain window_duration\n{yaml}" - ); - } - - #[test] - fn batch_mode_omits_window_duration() { - let mut cfg = ddsketch_cfg(); - cfg.mode = ProcessorMode::Batch; - cfg.window_duration = None; - let yaml = generate_agent_config(&cfg, "ws://ctrl:4320/v1/opamp").unwrap(); - assert!( - !yaml.contains("window_duration"), - "batch mode should not have window_duration\n{yaml}" - ); - } - - #[test] - fn contains_aggregate_by() { - let yaml = generate_agent_config(&ddsketch_cfg(), "ws://ctrl:4320/v1/opamp").unwrap(); - assert!( - yaml.contains("host.name"), - "YAML should contain aggregate_by labels\n{yaml}" - ); - } - - #[test] - fn hll_processor() { - let cfg = AgentCollectorConfig { - sketch_type: SketchType::HLL, - sketch_params: SketchParams::HLL { precision: 14 }, - mode: ProcessorMode::Batch, - window_duration: None, - output_mode: OutputMode::Sketch, - aggregate_by: vec![], - label_matchers: vec![], - enable_self_monitoring: true, - transmit_sketch: true, - drop_original: true, - delta_transmission: false, - delta_threshold: 0.0, - enable_series_id: true, - series_id_ttl_secs: 0, - data_sink: AgentDataSink::default(), - }; - let yaml = generate_agent_config(&cfg, "ws://ctrl:4320/v1/opamp").unwrap(); - assert!(yaml.contains("HLL:"), "YAML should contain HLL processor key\n{yaml}"); - assert!( - yaml.contains("- HLL"), - "pipeline should reference HLL processor\n{yaml}" - ); - assert!( - !yaml.contains("precision"), - "HLL processor YAML must not set precision (not in Config)\n{yaml}" - ); - } - - #[test] - fn countminsketch_processor() { - let cfg = AgentCollectorConfig { - sketch_type: SketchType::CountMinSketch, - sketch_params: SketchParams::CountMinSketch { - rows: 5, - cols: 2048, - metric_name: "test_metric".into(), - }, - mode: ProcessorMode::Batch, - window_duration: None, - output_mode: OutputMode::Sketch, - aggregate_by: vec![], - label_matchers: vec![], - enable_self_monitoring: true, - transmit_sketch: true, - drop_original: true, - delta_transmission: false, - delta_threshold: 0.0, - enable_series_id: true, - series_id_ttl_secs: 0, - data_sink: AgentDataSink::default(), - }; - let yaml = generate_agent_config(&cfg, "ws://ctrl:4320/v1/opamp").unwrap(); - assert!( - yaml.contains("countmin:"), - "YAML should use countmin component id (factory type)\n{yaml}" - ); - } - - #[test] - fn contains_otlp_receiver() { - let yaml = generate_agent_config(&ddsketch_cfg(), "ws://ctrl:4320/v1/opamp").unwrap(); - assert!( - yaml.contains("receivers:"), - "YAML should have receivers section\n{yaml}" - ); - assert!( - yaml.contains("otlp:"), - "YAML should have otlp receiver\n{yaml}" - ); - assert!(yaml.contains("4317"), "YAML should have gRPC port\n{yaml}"); - assert!(yaml.contains("4318"), "YAML should have HTTP port\n{yaml}"); - } - - #[test] - fn contains_prometheus_exporter() { - let yaml = generate_agent_config(&ddsketch_cfg(), "ws://ctrl:4320/v1/opamp").unwrap(); - assert!( - yaml.contains("exporters:"), - "YAML should have exporters section\n{yaml}" - ); - assert!( - yaml.contains("prometheus:"), - "YAML should have prometheus exporter\n{yaml}" - ); - assert!( - yaml.contains("8889"), - "YAML should have prometheus port\n{yaml}" - ); - } - - #[test] - fn pipeline_has_receivers_and_exporters() { - let yaml = generate_agent_config(&ddsketch_cfg(), "ws://ctrl:4320/v1/opamp").unwrap(); - // Ensure the pipeline block references both receiver and exporter keys. - assert!( - yaml.contains("- otlp"), - "pipeline receivers should list otlp\n{yaml}" - ); - assert!( - yaml.contains("- prometheus"), - "pipeline exporters should list prometheus\n{yaml}" - ); - } - - #[test] - fn delta_fields_present_when_enabled() { - let mut cfg = ddsketch_cfg(); - cfg.delta_transmission = true; - cfg.delta_threshold = 1.0; - let yaml = generate_agent_config(&cfg, "ws://ctrl:4320/v1/opamp").unwrap(); - assert!( - yaml.contains("delta_transmission: true"), - "YAML should contain delta_transmission: true\n{yaml}" - ); - assert!( - yaml.contains("delta_threshold"), - "YAML should contain delta_threshold\n{yaml}" - ); - } - - #[test] - fn delta_fields_absent_when_disabled() { - let cfg = ddsketch_cfg(); // delta_transmission: false by default - let yaml = generate_agent_config(&cfg, "ws://ctrl:4320/v1/opamp").unwrap(); - assert!( - !yaml.contains("delta_transmission"), - "YAML must not contain delta_transmission when disabled\n{yaml}" - ); - assert!( - !yaml.contains("delta_threshold"), - "YAML must not contain delta_threshold when disabled\n{yaml}" - ); - } - - #[test] - fn kll_processor() { - let cfg = AgentCollectorConfig { - sketch_type: SketchType::KLL, - sketch_params: SketchParams::KLL { - k: 200, - quantiles: vec![0.5, 0.99], - }, - mode: ProcessorMode::Window, - window_duration: Some(std::time::Duration::from_secs(300)), - output_mode: OutputMode::Sketch, - aggregate_by: vec![], - label_matchers: vec![], - enable_self_monitoring: true, - transmit_sketch: true, - drop_original: true, - delta_transmission: false, - delta_threshold: 0.0, - enable_series_id: true, - series_id_ttl_secs: 0, - data_sink: AgentDataSink::default(), - }; - let yaml = generate_agent_config(&cfg, "ws://ctrl:4320/v1/opamp").unwrap(); - assert!(yaml.contains("KLL:"), "YAML should contain 'KLL:'\n{yaml}"); - assert!(yaml.contains("k:"), "YAML should contain 'k:' param\n{yaml}"); - assert!(!yaml.contains("ddsketch:"), "YAML must not contain wrong processor key\n{yaml}"); - } - - #[test] - fn countsketch_processor() { - let cfg = AgentCollectorConfig { - sketch_type: SketchType::CountSketch, - sketch_params: SketchParams::CountSketch { - epsilon: CountSketchDefaults::default().epsilon, - delta: CountSketchDefaults::default().delta, - }, - mode: ProcessorMode::Batch, - window_duration: None, - output_mode: OutputMode::Sketch, - aggregate_by: vec![], - label_matchers: vec![], - enable_self_monitoring: true, - transmit_sketch: true, - drop_original: true, - delta_transmission: false, - delta_threshold: 0.0, - enable_series_id: true, - series_id_ttl_secs: 0, - data_sink: AgentDataSink::default(), - }; - let yaml = generate_agent_config(&cfg, "ws://ctrl:4320/v1/opamp").unwrap(); - assert!( - yaml.contains("countsketch:"), - "YAML should contain 'countsketch:'\n{yaml}" - ); - assert!( - !yaml.contains("countminsketch:"), - "YAML must not contain 'countminsketch:' for CountSketch\n{yaml}" - ); - } - - /// Verifies that for every sketch type the processor key in the `processors:` - /// section and the key listed under `service.pipelines.metrics.processors` - /// are identical. This guards against the processor map and the pipeline - /// reference going out of sync. - #[test] - fn all_sketch_types_processor_key_matches_pipeline_ref() { - let cases: &[(&str, SketchType, SketchParams)] = &[ - ("ddsketch", SketchType::DDSketch, SketchParams::DDSketch { relative_accuracy: 0.01, quantiles: vec![0.5] }), - ("KLL", SketchType::KLL, SketchParams::KLL { k: 200, quantiles: vec![0.5] }), - ("HLL", SketchType::HLL, SketchParams::HLL { precision: 14 }), - ("countsketch", SketchType::CountSketch, SketchParams::CountSketch { epsilon: CountSketchDefaults::default().epsilon, delta: CountSketchDefaults::default().delta }), - ("countmin", SketchType::CountMinSketch, SketchParams::CountMinSketch { rows: 5, cols: 2048, metric_name: "m".into() }), - ]; - - for (expected_key, sketch_type, sketch_params) in cases { - let cfg = AgentCollectorConfig { - sketch_type: sketch_type.clone(), - sketch_params: sketch_params.clone(), - mode: ProcessorMode::Batch, - window_duration: None, - output_mode: OutputMode::Sketch, - aggregate_by: vec![], - label_matchers: vec![], - enable_self_monitoring: true, - transmit_sketch: true, - drop_original: true, - delta_transmission: false, - delta_threshold: 0.0, - enable_series_id: true, - series_id_ttl_secs: 0, - data_sink: AgentDataSink::default(), - }; - let yaml = generate_agent_config(&cfg, "ws://ctrl:4320/v1/opamp").unwrap(); - - // Processor section key present. - assert!( - yaml.contains(&format!("{expected_key}:")), - "sketch_type={expected_key}: YAML missing processor key '{expected_key}:'\n{yaml}" - ); - // Pipeline processor list references the same key. - assert!( - yaml.contains(&format!("- {expected_key}")), - "sketch_type={expected_key}: pipeline processor list missing '- {expected_key}'\n{yaml}" - ); - // No other sketch type key should appear as a processor. - for (other_key, _, _) in cases { - if other_key == expected_key { continue; } - assert!( - !yaml.contains(&format!("{other_key}:")), - "sketch_type={expected_key}: YAML must not contain foreign key '{other_key}:'\n{yaml}" - ); - } - } - } -} diff --git a/controller/src/config/asapquery_backend.rs b/controller/src/config/asapquery_backend.rs deleted file mode 100644 index 61d1d16e..00000000 --- a/controller/src/config/asapquery_backend.rs +++ /dev/null @@ -1,300 +0,0 @@ -//! Convert a [`CollectionPlan`] into the YAML shape ASAPQuery-backend's -//! `POST /api/v1/streaming-config` endpoint accepts (the same format its -//! `StreamingConfig::from_yaml_data` parser consumes at startup). -//! -//! This is **separate** from `config::backend` (which produces OTel YAML -//! for a backend OTel collector running merge processors). The two -//! consumers are different: -//! -//! * `config::backend` — OTel collector, expects -//! `processors: { ddsketch_merge: {...} }` + `service.pipelines`. -//! * `config::asapquery_backend` (this module) — ASAPQuery-backend -//! query engine, expects -//! `aggregations: [{ aggregationId, aggregationType, metric, labels, -//! parameters, windowSize, windowType, spatialFilter }]`. -//! -//! Both are generated from the same `CollectionPlan` fields but target -//! different services. The replanner pushes the OTel YAML via OpAMP to -//! backend-role collectors and pushes this one via HTTP to the -//! ASAPQuery-backend's `/api/v1/streaming-config` endpoint. - -use std::time::Duration; - -use anyhow::{Context, Result}; - -use crate::types::{AgentCollectorConfig, CollectionPlan, SketchType}; - -/// Stable aggregation ID used when the planner has no explicit id to -/// assign. The ASAPQuery-backend uses `u64` agg IDs; we derive one -/// deterministically from the metric name so the same metric always -/// maps to the same id across successive pushes (otherwise the backend -/// would grow unbounded as each replan introduces a new agg_id). -pub fn deterministic_agg_id(metric: &str) -> u64 { - use std::collections::hash_map::DefaultHasher; - use std::hash::{Hash, Hasher}; - let mut h = DefaultHasher::new(); - metric.hash(&mut h); - // Bias away from 0 so the id space is [1, u64::MAX]; 0 is reserved - // in some of the backend's existing test fixtures as a sentinel. - h.finish().saturating_add(1) -} - -/// Generate the `StreamingConfig` YAML for the ASAPQuery-backend from a -/// single-metric `CollectionPlan`. Produces a one-element `aggregations` -/// list — the backend's endpoint will merge this into its active config -/// (add on conflict, replace on same id). -/// -/// # Errors -/// -/// Returns an error if the plan is missing a window (the backend's -/// config parser rejects zero-window aggregations) or if YAML -/// serialization fails. -pub fn generate_streaming_config_yaml(metric: &str, plan: &CollectionPlan) -> Result { - let agg = &plan.agent_config; - let window_secs = agg - .window_duration - .map(|d: Duration| d.as_secs()) - .unwrap_or(0); - if window_secs == 0 { - anyhow::bail!( - "generate_streaming_config_yaml: plan for metric {metric} has \ - no window_duration; ASAPQuery-backend rejects zero-window aggregations" - ); - } - - // Parameters map: copy sketch-type-specific params (K for KLL, - // epsilon/delta for CountMin, etc.) into the string-keyed YAML map - // the backend expects. We serialize via serde_yaml to pick up - // SketchParams' own Serialize impl and then re-parse into a - // generic Mapping so we can embed it. - let params_yaml = serde_yaml::to_value(&agg.sketch_params) - .context("serialize SketchParams for ASAPQuery streaming config")?; - - let agg_type_str = map_sketch_type_to_agg_type(&agg.sketch_type); - - let aggregation = serde_yaml::Mapping::from_iter([ - ( - serde_yaml::Value::from("aggregationId"), - serde_yaml::Value::from(deterministic_agg_id(metric)), - ), - ( - serde_yaml::Value::from("aggregationType"), - serde_yaml::Value::from(agg_type_str), - ), - ( - serde_yaml::Value::from("aggregationSubType"), - serde_yaml::Value::from(""), - ), - ( - serde_yaml::Value::from("metric"), - serde_yaml::Value::from(metric), - ), - ( - serde_yaml::Value::from("labels"), - labels_mapping(&agg.aggregate_by), - ), - (serde_yaml::Value::from("parameters"), params_yaml), - ( - serde_yaml::Value::from("windowSize"), - serde_yaml::Value::from(window_secs), - ), - ( - serde_yaml::Value::from("windowType"), - serde_yaml::Value::from("tumbling"), - ), - ( - serde_yaml::Value::from("spatialFilter"), - serde_yaml::Value::from(normalize_spatial_filter(&agg.label_matchers)), - ), - ]); - - let top = serde_yaml::Mapping::from_iter([( - serde_yaml::Value::from("aggregations"), - serde_yaml::Value::Sequence(vec![serde_yaml::Value::Mapping(aggregation)]), - )]); - - serde_yaml::to_string(&serde_yaml::Value::Mapping(top)) - .context("serialize ASAPQuery streaming-config YAML") -} - -/// Map the controller's `SketchType` to the backend's -/// `AggregationType::Display` string. These strings must match what the -/// backend's `FromStr for AggregationType` in -/// `promql_utilities::query_logics::enums` accepts — hence the variant -/// names rather than the collector factory names (e.g. `"DatasketchesKLL"` -/// not `"KLL"`). -fn map_sketch_type_to_agg_type(t: &SketchType) -> &'static str { - match t { - SketchType::DDSketch => "DDSketch", - SketchType::KLL => "DatasketchesKLL", - SketchType::HLL => "HLL", - SketchType::CountSketch => "CountSketch", - SketchType::CountMinSketch => "CountMinSketch", - } -} - -/// Build the `labels` sub-mapping the backend expects. All three lists -/// exist because the backend's parser reads them separately for -/// key-value / spatial-rollup distinction; today the controller only -/// tracks `aggregate_by` (grouping), so rollup and aggregated stay -/// empty and are populated in a follow-up when the cost model starts -/// producing richer label metadata. -fn labels_mapping(aggregate_by: &[String]) -> serde_yaml::Value { - serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter([ - ( - serde_yaml::Value::from("grouping"), - serde_yaml::Value::Sequence( - aggregate_by - .iter() - .cloned() - .map(serde_yaml::Value::from) - .collect(), - ), - ), - ( - serde_yaml::Value::from("rollup"), - serde_yaml::Value::Sequence(vec![]), - ), - ( - serde_yaml::Value::from("aggregated"), - serde_yaml::Value::Sequence(vec![]), - ), - ])) -} - -/// Join the controller's `label_matchers` list (each shaped like -/// `"key=value"`) into a single comma-separated string the backend's -/// spatial-filter parser accepts. When the list is empty, returns an -/// empty string (the backend treats that as "no spatial filter"). -fn normalize_spatial_filter(label_matchers: &[String]) -> String { - label_matchers.join(",") -} - -// ─── Unused-warning suppression for types that are referenced only -// inside the unit tests below. This keeps the module self-contained -// even when the rest of the controller crate's cfg(test) surface grows. -#[allow(dead_code)] -fn _type_check(_: &AgentCollectorConfig) {} - -#[cfg(test)] -mod tests { - use super::*; - use crate::types::{ - AgentDataSink, BackendCollectorConfig, CollectionPlan, DeltaDecision, - GatewayCollectorConfig, OutputMode, ProcessorMode, SketchParams, TransmissionCostSummary, - }; - use std::time::Duration; - - fn dummy_plan(sketch_type: SketchType) -> CollectionPlan { - CollectionPlan { - agent_config: AgentCollectorConfig { - output_mode: OutputMode::Sketch, - sketch_type: sketch_type.clone(), - sketch_params: SketchParams::default(), - aggregate_by: vec!["host".to_string(), "service".to_string()], - label_matchers: vec!["env=prod".to_string()], - window_duration: Some(Duration::from_secs(30)), - mode: ProcessorMode::Window, - enable_self_monitoring: false, - transmit_sketch: true, - drop_original: true, - enable_series_id: false, - series_id_ttl_secs: 0, - delta_transmission: false, - delta_threshold: 0.0, - data_sink: AgentDataSink::default(), - }, - gateway_config: GatewayCollectorConfig { passthrough: true }, - backend_config: BackendCollectorConfig { - merge_sketch_type: sketch_type, - group_by: vec![], - }, - precompute: vec![], - valid_until: chrono::Utc::now() + chrono::Duration::seconds(300), - delta_decision: DeltaDecision::default(), - transmission_cost_summary: TransmissionCostSummary::default(), - staged_plan: None, - } - } - - #[test] - fn deterministic_id_is_stable_across_calls() { - assert_eq!( - deterministic_agg_id("cpu_usage"), - deterministic_agg_id("cpu_usage") - ); - assert_ne!( - deterministic_agg_id("cpu_usage"), - deterministic_agg_id("mem_usage") - ); - // Id is biased away from 0 so test fixtures that use 0 as a - // sentinel don't accidentally collide. - assert_ne!(deterministic_agg_id("any"), 0); - } - - #[test] - fn yaml_round_trips_through_serde_yaml() { - let plan = dummy_plan(SketchType::DDSketch); - let yaml = generate_streaming_config_yaml("cpu_usage", &plan).expect("yaml ok"); - let parsed: serde_yaml::Value = serde_yaml::from_str(&yaml).expect("re-parse ok"); - - let aggs = parsed["aggregations"].as_sequence().expect("sequence"); - assert_eq!(aggs.len(), 1); - let a = &aggs[0]; - assert_eq!(a["aggregationType"], serde_yaml::Value::from("DDSketch")); - assert_eq!(a["metric"], serde_yaml::Value::from("cpu_usage")); - assert_eq!(a["windowSize"], serde_yaml::Value::from(30u64)); - assert_eq!(a["windowType"], serde_yaml::Value::from("tumbling")); - assert_eq!(a["spatialFilter"], serde_yaml::Value::from("env=prod")); - - let grouping = a["labels"]["grouping"].as_sequence().expect("grouping seq"); - let grouping: Vec<&str> = grouping.iter().filter_map(|v| v.as_str()).collect(); - assert_eq!(grouping, vec!["host", "service"]); - } - - #[test] - fn maps_all_sketch_types() { - assert_eq!( - map_sketch_type_to_agg_type(&SketchType::DDSketch), - "DDSketch" - ); - assert_eq!( - map_sketch_type_to_agg_type(&SketchType::KLL), - "DatasketchesKLL", - "KLL must map to the backend's enum variant name, not the factory name" - ); - assert_eq!(map_sketch_type_to_agg_type(&SketchType::HLL), "HLL"); - assert_eq!( - map_sketch_type_to_agg_type(&SketchType::CountSketch), - "CountSketch" - ); - assert_eq!( - map_sketch_type_to_agg_type(&SketchType::CountMinSketch), - "CountMinSketch" - ); - } - - #[test] - fn rejects_plan_without_window_duration() { - let mut plan = dummy_plan(SketchType::HLL); - plan.agent_config.window_duration = None; - let err = generate_streaming_config_yaml("m", &plan).expect_err("should error"); - assert!( - err.to_string().contains("window_duration"), - "error should mention window_duration: {err}" - ); - } - - #[test] - fn spatial_filter_joins_label_matchers() { - let mut plan = dummy_plan(SketchType::DDSketch); - plan.agent_config.label_matchers = - vec!["env=prod".to_string(), "region=us-east".to_string()]; - let yaml = generate_streaming_config_yaml("m", &plan).expect("ok"); - let parsed: serde_yaml::Value = serde_yaml::from_str(&yaml).unwrap(); - assert_eq!( - parsed["aggregations"][0]["spatialFilter"], - serde_yaml::Value::from("env=prod,region=us-east") - ); - } -} diff --git a/controller/src/config/backend.rs b/controller/src/config/backend.rs deleted file mode 100644 index fa467675..00000000 --- a/controller/src/config/backend.rs +++ /dev/null @@ -1,120 +0,0 @@ -use anyhow::Context; -use serde_json::json; -use crate::types::*; - -/// Generates an OTel collector YAML string for the backend merge collector. -/// -/// **SP-9**: when `staged.has_dedup` is true a `dedup` processor is inserted -/// before the merge processor in the pipeline, honouring the `Dedup` node -/// assignment from [`crate::planner::stage_split::split_expr_by_stage`]. -pub fn generate_backend_config(cfg: &BackendCollectorConfig, opamp_endpoint: &str) -> anyhow::Result { - generate_backend_config_staged(cfg, None, opamp_endpoint) -} - -/// Extended entry point used by SP-9-aware callers that supply a -/// [`BackendSubPlan`] carrying the dedup flag. -pub fn generate_backend_config_staged( - cfg: &BackendCollectorConfig, - staged: Option<&BackendSubPlan>, - opamp_endpoint: &str, -) -> anyhow::Result { - let merge_key = format!("{}_merge", cfg.merge_sketch_type); - let has_dedup = staged.map(|s| s.has_dedup).unwrap_or(false); - - // Build the processors map and pipeline processor list. - let mut processors = serde_json::Map::new(); - let mut pipeline_processors: Vec = vec![]; - - if has_dedup { - processors.insert( - "dedup".into(), - json!({ "mode": "dedup" }), - ); - pipeline_processors.push(json!("dedup")); - } - - processors.insert( - merge_key.clone(), - json!({ "mode": "merge", "group_by": cfg.group_by }), - ); - pipeline_processors.push(json!(&merge_key)); - - let doc = serde_yaml::to_value(&json!({ - "extensions": { - "opamp": { "server": { "ws": { "endpoint": opamp_endpoint } } } - }, - "processors": processors, - "service": { - "extensions": ["opamp"], - "pipelines": { - "metrics": { "processors": pipeline_processors } - } - } - })) - .context("build backend doc")?; - - serde_yaml::to_string(&doc).context("serialize backend config") -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn contains_merge_key() { - let cfg = BackendCollectorConfig { - merge_sketch_type: SketchType::DDSketch, - group_by: vec!["host.name".into()], - }; - let yaml = generate_backend_config(&cfg, "ws://ctrl:4320/v1/opamp").unwrap(); - assert!(yaml.contains("ddsketch_merge"), "YAML should contain merge key\n{yaml}"); - assert!(yaml.contains("host.name"), "YAML should contain group_by\n{yaml}"); - } - - #[test] - fn hll_merge_key() { - let cfg = BackendCollectorConfig { - merge_sketch_type: SketchType::HLL, - group_by: vec![], - }; - let yaml = generate_backend_config(&cfg, "ws://ctrl:4320/v1/opamp").unwrap(); - assert!(yaml.contains("HLL_merge"), "{yaml}"); - } - - #[test] - fn contains_opamp_endpoint() { - let ep = "ws://custom-ctrl:9000/v1/opamp"; - let cfg = BackendCollectorConfig { merge_sketch_type: SketchType::KLL, group_by: vec![] }; - let yaml = generate_backend_config(&cfg, ep).unwrap(); - assert!(yaml.contains(ep), "YAML should contain endpoint\n{yaml}"); - } - - #[test] - fn dedup_processor_emitted_when_staged_has_dedup() { - let cfg = BackendCollectorConfig { - merge_sketch_type: SketchType::HLL, - group_by: vec!["user_id".into()], - }; - let staged = BackendSubPlan { has_dedup: true, has_merge: true, group_by: vec![] }; - let yaml = generate_backend_config_staged(&cfg, Some(&staged), "ws://ctrl:4320/v1/opamp").unwrap(); - assert!(yaml.contains("dedup:"), "YAML should contain dedup processor\n{yaml}"); - // dedup must appear before merge in the pipeline list - let dedup_pos = yaml.find("- dedup").expect("missing dedup in pipeline"); - let merge_pos = yaml.find("- HLL_merge").expect("missing merge in pipeline"); - assert!(dedup_pos < merge_pos, "dedup must precede merge\n{yaml}"); - } - - #[test] - fn no_dedup_when_staged_has_dedup_false() { - let cfg = BackendCollectorConfig { - merge_sketch_type: SketchType::DDSketch, - group_by: vec![], - }; - let staged = BackendSubPlan { has_dedup: false, has_merge: true, group_by: vec![] }; - let yaml = generate_backend_config_staged(&cfg, Some(&staged), "ws://ctrl:4320/v1/opamp").unwrap(); - assert!(!yaml.contains("dedup"), "YAML must not contain dedup\n{yaml}"); - } -} - diff --git a/controller/src/config/mod.rs b/controller/src/config/mod.rs deleted file mode 100644 index e6939d52..00000000 --- a/controller/src/config/mod.rs +++ /dev/null @@ -1,498 +0,0 @@ -pub mod agent; -pub mod asapquery_backend; -pub mod backend; -pub mod precompute; -pub mod stage_config; -pub mod stage_config_otap; -pub mod stage_config_telegraf; -pub mod workloads; - -pub use agent::generate_agent_config; -pub use asapquery_backend::generate_streaming_config_yaml; -pub use backend::{generate_backend_config, generate_backend_config_staged}; -pub use precompute::{should_precompute, build_precompute_jobs, PrecomputeClient}; -pub use stage_config::{ - emit_backend_config_json, emit_backend_storage_routing, - emit_backend_storage_routing_for_tenant, emit_backend_storage_routing_with_prometheus, - emit_backend_storage_routing_with_prometheus_for_tenant, emit_edge_yaml, emit_gateway_yaml, - DEFAULT_TENANT, -}; -pub use stage_config_otap::emit_otap_dag_yaml; -pub use stage_config_telegraf::emit_telegraf_toml; -pub use workloads::WorkloadRegistry; - -use crate::stage_split::emitter::EdgeStageConfig; -use crate::sketch_algebra::SketchExpr; -use crate::sketch_algebra::params::SketchKind; -use crate::store::WorkloadStore; -use anyhow::Result; - -/// Phase ε.1.5 — which edge runtime an agent identifies as. -/// -/// Today every agent the controller has built for runs the OTel-collector -/// (`AsapOtel`); Phase ε.1.5 adds the two new runtime variants the -/// per-runtime emitters target. The runtime is reported by the agent on -/// OpAMP `on_connect` (header `X-Agent-Runtime`); when absent (legacy -/// agents) the controller defaults to `AsapOtel` so the existing -/// behaviour is preserved. -/// -/// Phase ε.1.5 commits the enum + emit-dispatch function. Threading the -/// runtime through OpAMP `on_connect` and into the typed L5 emit path -/// is a follow-up — the emitters can be exercised in isolation today -/// (the Phase ε.1.5 test suite does exactly that). -/// -/// Naming history: the variants were originally `Sketchcollector` / -/// `Sketchotap` / `Sketchtelegraf`; the rename to `AsapOtel` / -/// `AsapOtap` / `AsapTelegraf` (PR `refactor/rename-edge-runtimes-...`) -/// drops the v0 `sketch*` prefix in favour of the symmetric `asap-*` -/// namespace. `from_header` accepts both forms during transition. -#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "kebab-case")] -pub enum AgentRuntime { - /// Default — OTel-collector contrib build (existing behaviour). - AsapOtel, - /// otap-dataflow Rust runtime. - AsapOtap, - /// Telegraf runtime. - AsapTelegraf, -} - -impl Default for AgentRuntime { - fn default() -> Self { - AgentRuntime::AsapOtel - } -} - -impl AgentRuntime { - /// Parse an `X-Agent-Runtime` header value. Recognises - /// `asap-otel` / `asap-otap` / `asap-telegraf` - /// (case-insensitive); any other value (including the empty string) - /// defaults to `AsapOtel` so legacy agents keep working. - pub fn from_header(value: &str) -> Self { - match value.trim().to_lowercase().as_str() { - "asap-otap" | "otap" => AgentRuntime::AsapOtap, - "asap-telegraf" | "telegraf" => AgentRuntime::AsapTelegraf, - "asap-otel" => AgentRuntime::AsapOtel, - _ => AgentRuntime::AsapOtel, - } - } -} - -/// Phase ε.1.5 — dispatch the edge emit by agent runtime. Mirrors -/// `emit_edge_yaml`'s `(cfg, opamp_endpoint) -> String` shape; the OTAP -/// and Telegraf emitters take an additional optional Prometheus URL -/// override which we pass through `prometheus_url`. -/// -/// `prometheus_url` is the Mode-3 destination override: -/// * `AsapOtel` → ignored (the OTel-collector emitter already -/// reads `${ASAP_PROMETHEUS_OTLP_URL}` at runtime); -/// * `AsapOtap` → OTLP HTTP URL passed to `emit_otap_dag_yaml`; -/// * `AsapTelegraf` → remote-write URL passed to `emit_telegraf_toml`. -pub fn emit_for_runtime( - runtime: AgentRuntime, - cfg: &EdgeStageConfig, - opamp_endpoint: &str, - prometheus_url: Option<&str>, -) -> Result { - match runtime { - AgentRuntime::AsapOtel => emit_edge_yaml(cfg, opamp_endpoint), - AgentRuntime::AsapOtap => emit_otap_dag_yaml(cfg, opamp_endpoint, prometheus_url), - AgentRuntime::AsapTelegraf => emit_telegraf_toml(cfg, prometheus_url), - } -} - -/// 10s flush window for the freshness probes — see the comment in -/// `emit_bootstrap_typed` (and the original PR #333) for the rationale. -/// Smallest window that produces well-formed Prometheus-TSDB blocks -/// while keeping criterion ⑥'s warm-tier p50 ≤ 30s budget. -pub const FRESHNESS_PROBE_WINDOW_SECS: u64 = 10; - -/// 60s window for non-probe workload-registry metrics added to the -/// archive tier so the accuracy reducer's archive engine has ground -/// truth for every replay row. -pub const WORKLOAD_ARCHIVE_WINDOW_SECS: u64 = 60; - -/// The two freshness probes — bootstrap/replan demo plumbing for -/// criterion ⑥. Not user metrics. The replay client polls them via -/// `last_over_time(http_freshness_probe_warm[10s])`; without -/// warm-passthrough routing the DDSketch processor renames them to -/// `_quantile`, and without `gorillas3` archive write the warm engine -/// has nothing to look at. -pub const FRESHNESS_PROBE_METRICS: &[&str] = &[ - "http_freshness_probe_warm", - "http_freshness_probe_archive", -]; - -/// Bootstrap/replan-scope plumbing: extend an Edge stage config with -/// the freshness-probe metrics (`http_freshness_probe_warm` / -/// `http_freshness_probe_archive`) AND the workload-registry archive -/// metrics so the agent's `gorillas3` processor writes them into the -/// Gorilla-S3 / Thanos archive — required for criterion ⑥ -/// (freshness probe routing) and criterion ④ (archive ground truth). -/// -/// Mutates `edge_cfg` in place. Idempotent — metrics already present -/// in `archive_tier_metrics` / `warm_passthrough_metrics` are not -/// duplicated. -/// -/// Originally inlined in `main::emit_bootstrap_typed`; lifted here so -/// the typed-replan path in `replan::Replanner` can apply the same -/// extension without depending on private state in `main.rs`. -/// -/// ## Scope note -/// -/// The live planner stays free to plan per-metric without these -/// defaults bleeding into its output — the helper is only invoked -/// from the bootstrap GET path and the OpAMP-on-connect / replan -/// push paths, both of which are demo-scope contracts. -pub fn extend_edge_with_demo_plumbing( - edge_cfg: &mut EdgeStageConfig, - workload_registry_metrics: impl IntoIterator, -) { - use crate::stage_split::emitter::ArchiveTierMetric; - - // 1. Freshness probes → archive tier with the tight 10s window. - for m in FRESHNESS_PROBE_METRICS.iter() { - if !edge_cfg.archive_tier_metrics.iter().any(|a| a.metric == *m) { - edge_cfg.archive_tier_metrics.push(ArchiveTierMetric { - metric: (*m).to_string(), - window_secs: Some(FRESHNESS_PROBE_WINDOW_SECS), - }); - } - } - - // 2. Freshness probes → warm-passthrough so the DDSketch processor - // doesn't rename them to `_quantile`. - for m in FRESHNESS_PROBE_METRICS.iter() { - if !edge_cfg.warm_passthrough_metrics.iter().any(|s| s == m) { - edge_cfg.warm_passthrough_metrics.push((*m).to_string()); - } - } - - // 3. All non-probe workload-registry metrics → archive tier (60s). - let mut seen: std::collections::HashSet = edge_cfg - .archive_tier_metrics - .iter() - .map(|a| a.metric.clone()) - .collect(); - for metric in workload_registry_metrics { - if seen.insert(metric.clone()) { - edge_cfg.archive_tier_metrics.push(ArchiveTierMetric { - metric, - window_secs: Some(WORKLOAD_ARCHIVE_WINDOW_SECS), - }); - } - } -} - -// ── MVP §46: planner ↔ 5-sketch emitter stitching ────────────────────────────── -// -// PR #339 (planner) classifies a single metric and produces a `SketchExpr` -// pinning a sketch family. PR #340 (emitter) gates the 5-sketch -// routing-connector wire shape on `EdgeStageConfig::metric_to_family` -// being non-empty. Until this stitch shipped, nothing populated the -// HashMap — the typed bootstrap / replan paths emitted single-pipeline -// YAML and the routing-connector path stayed dormant. -// -// `extract_root_sketch_kind` walks a `SketchExpr` tree and returns the -// committed sketch family — looking through `SketchEstimate`, -// `SketchAgg`, `SketchMerge`, `LetBinding`, and `RawAtEdgeSketchAtBackend`. -// `SketchAgg::sketch_type` is the canonical source of truth (the typed -// path's `Bind*` rules drop their family commitment here). -// -// `collect_metric_to_family` is the multi-metric loop: walk the workload -// registry, run `bind_workload_typed` per metric, and collect the -// committed family into the HashMap. Metrics that decline binding — -// `http_requests_total` (raw passthrough), exact-required workloads, -// multi-intent — are skipped, which is exactly the contract the -// `emit_edge_yaml_5sketch_routing` path expects (absent metrics -// fall through to `metrics/raw_passthrough`). - -/// Walk a `SketchExpr` tree and return the first `SketchAgg::sketch_type` -/// (or the `RawAtEdgeSketchAtBackend::family` Mode-2 equivalent). The -/// canonical shape produced by `bind_workload_typed` is -/// `SketchEstimate { child: SketchAgg { sketch_type, … } }`, so this is -/// effectively a one-level descent — but we walk recursively to stay -/// robust against future shape changes (e.g. Bind* rules wrapping -/// in `LetBinding` for fan-in shared sketches). -/// -/// Returns `None` only for trees that carry no sketch commitment -/// (`Logical`-only, unresolved `Ref`, raw Mode-3 archive). These map -/// onto the raw-passthrough default pipeline in the routing emitter, -/// which is correct. -pub fn extract_root_sketch_kind(expr: &SketchExpr) -> Option { - match expr { - SketchExpr::SketchAgg { sketch_type, .. } => Some(sketch_type.clone()), - SketchExpr::RawAtEdgeSketchAtBackend { family, .. } => Some(family.clone()), - SketchExpr::SketchEstimate { child, .. } => extract_root_sketch_kind(child), - SketchExpr::SketchMerge { children, .. } => { - children.iter().find_map(extract_root_sketch_kind) - } - SketchExpr::LetBinding { expr, child, .. } => { - extract_root_sketch_kind(expr).or_else(|| extract_root_sketch_kind(child)) - } - SketchExpr::Logical(_) - | SketchExpr::Ref { .. } - | SketchExpr::RawAtEdgePrometheusArchive { .. } => None, - } -} - -/// Walk every entry in `registry`, look the metric up in `workload_store`, -/// run `planner::rules::bind_workload_typed` per workload, and assemble -/// the `metric_to_family` HashMap that drives the 5-sketch -/// routing-connector emit path in `emit_edge_yaml_5sketch_routing`. -/// -/// Skipped: -/// - Metrics absent from `workload_store` (registry pre-pop failed). -/// - Metrics where `bind_workload_typed` declines (raw passthrough -/// like `http_requests_total`, exact-required, multi-intent). -/// These fall through to `metrics/raw_passthrough` in the emitter, -/// which is the contract for raw / unsketched metrics. -/// -/// The returned map drops directly into `EdgeStageConfig::metric_to_family`. -/// Empty map ⇒ caller falls back to legacy single-pipeline emit (the -/// `is_empty()` gate in `emit_edge_yaml`). -pub fn collect_metric_to_family( - registry: &WorkloadRegistry, - workload_store: &WorkloadStore, -) -> std::collections::HashMap { - let mut out = std::collections::HashMap::new(); - for entry in registry.entries() { - let Some((workload, _wc)) = workload_store.get(&entry.metric_name) else { - continue; - }; - let Some(sketch_expr) = crate::planner::rules::bind_workload_typed(&workload) else { - // `http_requests_total` and other raw-passthrough metrics - // land here — correctly excluded so they fall through to - // the routing connector's default `metrics/raw_passthrough` - // pipeline in the emitter. - continue; - }; - if let Some(kind) = extract_root_sketch_kind(&sketch_expr) { - out.insert(entry.metric_name.clone(), kind); - } - } - out -} - -#[cfg(test)] -mod runtime_tests { - use super::*; - - #[test] - fn agent_runtime_from_header_recognises_three_values() { - assert_eq!(AgentRuntime::from_header("asap-otel"), AgentRuntime::AsapOtel); - assert_eq!(AgentRuntime::from_header("asap-otap"), AgentRuntime::AsapOtap); - assert_eq!(AgentRuntime::from_header("asap-telegraf"), AgentRuntime::AsapTelegraf); - } - - #[test] - fn agent_runtime_from_header_short_aliases() { - assert_eq!(AgentRuntime::from_header("otap"), AgentRuntime::AsapOtap); - assert_eq!(AgentRuntime::from_header("telegraf"), AgentRuntime::AsapTelegraf); - } - - #[test] - fn agent_runtime_from_header_default_is_asap_otel() { - assert_eq!(AgentRuntime::from_header(""), AgentRuntime::AsapOtel); - assert_eq!(AgentRuntime::from_header("garbage"), AgentRuntime::AsapOtel); - } - - #[test] - fn emit_for_runtime_default_matches_emit_edge_yaml() { - use crate::sketch_algebra::params::{DDSketchParams, SketchParams}; - use crate::stage_split::emitter::{EdgeSketchProcessor, ExportTarget}; - use crate::stage_split::stage_id::StageId; - use crate::sketch_algebra::params::SketchKind; - - let cfg = EdgeStageConfig { - source_metric: Some("m".to_string()), - label_filters: Vec::new(), - window_secs: Some(60), - sketch_processors: vec![EdgeSketchProcessor { - processor_name: "ddsketch".to_string(), - sketch_kind: SketchKind::DDSketch, - sketch_params: SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), - aggregation_id: "agg0".to_string(), - }], - exporter_target: ExportTarget::Stage(StageId::Gateway), - prometheus_archive_metrics: Vec::new(), - archive_tier_metrics: Vec::new(), - warm_passthrough_metrics: Vec::new(), - metric_to_family: std::collections::HashMap::new(), - }; - - let collector = emit_for_runtime( - AgentRuntime::AsapOtel, &cfg, "ws://ctrl/v1/opamp", None, - ).expect("collector emit ok"); - let direct = emit_edge_yaml(&cfg, "ws://ctrl/v1/opamp").expect("direct emit ok"); - assert_eq!(collector, direct, "AsapOtel dispatch must equal emit_edge_yaml"); - } - - #[test] - fn emit_for_runtime_otap_yields_dag_yaml() { - use crate::stage_split::emitter::ExportTarget; - use crate::stage_split::stage_id::StageId; - - let cfg = EdgeStageConfig { - source_metric: Some("m".to_string()), - label_filters: Vec::new(), - window_secs: Some(60), - sketch_processors: Vec::new(), - exporter_target: ExportTarget::Stage(StageId::Gateway), - prometheus_archive_metrics: Vec::new(), - archive_tier_metrics: Vec::new(), - warm_passthrough_metrics: Vec::new(), - metric_to_family: std::collections::HashMap::new(), - }; - let yaml = emit_for_runtime( - AgentRuntime::AsapOtap, &cfg, "ws://ctrl/v1/opamp", None, - ).expect("otap emit ok"); - // OTAP-specific token. - assert!(yaml.contains("otel_dataflow/v1"), "expected OTAP DAG version\n{yaml}"); - } - - #[test] - fn emit_for_runtime_telegraf_yields_toml() { - use crate::stage_split::emitter::ExportTarget; - use crate::stage_split::stage_id::StageId; - - let cfg = EdgeStageConfig { - source_metric: Some("m".to_string()), - label_filters: Vec::new(), - window_secs: Some(60), - sketch_processors: Vec::new(), - exporter_target: ExportTarget::Stage(StageId::Gateway), - prometheus_archive_metrics: Vec::new(), - archive_tier_metrics: Vec::new(), - warm_passthrough_metrics: Vec::new(), - metric_to_family: std::collections::HashMap::new(), - }; - let toml = emit_for_runtime( - AgentRuntime::AsapTelegraf, &cfg, "ws://ctrl/v1/opamp", None, - ).expect("telegraf emit ok"); - // Telegraf-specific token. - assert!(toml.contains("[[inputs.opentelemetry]]"), "expected Telegraf TOML header\n{toml}"); - } - - // ── stitching-gap regression: registry walk binds all 6 contract metrics ── - // - // The 6 MVP contract metrics from `deploy/configs/mvp-workload.yaml` must - // every one bind through `collect_metric_to_family` so the routing - // table covers the full 5-sketch (DDSketch / KLL / HLL / CountSketch / - // CountMinSketch) shape, with `http_requests_total` declining to raw. - // - // Reproduces the live demo gap: 3 of 6 (HLL, CountSketch, CMS) silently - // drop because the analyzer pre-population path doesn't propagate - // `sketch_family_override` from the workload YAML into - // `QueryWorkload::sketch_type_override`. - - /// Mimics the pre-population loop in `main()` — turns each - /// `WorkloadEntry` into a `QueryWorkload` via the shared `Analyzer`. - fn populate_store_from_registry( - registry: &WorkloadRegistry, - store: &WorkloadStore, - ) { - use crate::analyzer::{Analyzer, QuerySpec}; - use crate::types; - use crate::types_v2; - let analyzer = Analyzer::new(); - for entry in registry.entries() { - let spec = QuerySpec { - query_string: entry.query_string.clone(), - metric_name: entry.metric_name.clone(), - label_filters: Default::default(), - group_by_labels: vec![], - aggregations: vec!["quantile".into()], - time_window: "5m".into(), - repeat_every: None, - accuracy_sla: entry.accuracy_sla, - latency_sla: None, - sketch_type: entry.sketch_family_override.clone(), - workload: types::WorkloadCharacteristics::default(), - id: None, - language: None, - accuracy: None, - dollars: None, - deployment_model: None, - shape: types_v2::QueryShape::default(), - data: types_v2::DataShape::default(), - }; - if let Ok(wl) = analyzer.analyze(spec) { - store.set(&entry.metric_name, wl, types::WorkloadCharacteristics::default()); - } - } - } - - #[test] - fn collect_metric_to_family_binds_all_six_contract_metrics_from_live_yaml() { - use crate::sketch_algebra::params::SketchKind; - - // The 6 contract metrics reproduced inline (mirrors - // deploy/configs/mvp-workload.yaml entries 1, 5, 6, 7, 8 plus the - // raw-passthrough http_requests_total). Note we use the contract - // metric name `http_latency_ms` (the live YAML uses - // `http_requests_total_latency_ms` which falls back via AggType - // → DDSketch — but it's the metric-name variant that exercises - // classify_demo_metric for the DDSketch row). - let yaml = r#" -- metric_name: http_latency_ms - query_string: "quantile_over_time(0.99, http_latency_ms[1m])" - accuracy_sla: 0.01 - assign_to_role: agent -- metric_name: http_requests_total - query_string: "count(http_requests_total)" - accuracy_sla: 0.0 - assign_to_role: agent -- metric_name: request_size_bytes - query_string: "quantile_over_time(0.99, request_size_bytes[1m])" - accuracy_sla: 0.05 - assign_to_role: agent - sketch_family_override: KLL -- metric_name: unique_users_per_min - query_string: "count(unique_users_per_min)" - accuracy_sla: 0.02 - assign_to_role: agent - sketch_family_override: HLL -- metric_name: top_endpoint_qps - query_string: "topk(5, top_endpoint_qps)" - accuracy_sla: 0.05 - assign_to_role: agent - sketch_family_override: CountSketch -- metric_name: endpoint_request_freq - query_string: "rate(endpoint_request_freq[5m])" - accuracy_sla: 0.05 - assign_to_role: agent - sketch_family_override: CountMinSketch -"#; - let entries: Vec = - serde_yaml::from_str(yaml).expect("parse workload yaml"); - assert_eq!(entries.len(), 6, "all 6 contract metrics must deserialize"); - - let registry = crate::config::workloads::WorkloadRegistry::from_entries(entries); - let store = WorkloadStore::new(); - populate_store_from_registry(®istry, &store); - - let map = collect_metric_to_family(®istry, &store); - - // 5 sketched metrics + http_requests_total (raw, declines binding). - let expected: Vec<(&str, Option)> = vec![ - ("http_latency_ms", Some(SketchKind::DDSketch)), - ("http_requests_total", None), // raw passthrough - ("request_size_bytes", Some(SketchKind::Kll)), - ("unique_users_per_min", Some(SketchKind::Hll)), - ("top_endpoint_qps", Some(SketchKind::CountSketch)), - ("endpoint_request_freq", Some(SketchKind::Cms)), - ]; - for (metric, want) in &expected { - let got = map.get(*metric).cloned(); - assert_eq!( - got, *want, - "metric {metric}: expected {want:?} in routing table, got {got:?}\n\ - full map: {map:?}", - ); - } - // Routing table covers all 5 sketched metrics. - assert_eq!(map.len(), 5, - "routing table should have 5 entries (5 sketches; raw declines), got: {map:?}"); - } -} diff --git a/controller/src/config/precompute.rs b/controller/src/config/precompute.rs deleted file mode 100644 index bd074e60..00000000 --- a/controller/src/config/precompute.rs +++ /dev/null @@ -1,370 +0,0 @@ -use anyhow::Context; -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; -use std::time::Duration; - -use crate::analyzer::format_duration; -use crate::types::*; - -// ── Scheduling rule ─────────────────────────────────────────────────────────── - -/// Returns true when a query should be precomputed. -/// -/// Rule (design doc): precompute when `repeat_every < latency_sla`, meaning -/// the query fires more often than the system can recompute it on demand. -pub fn should_precompute(w: &QueryWorkload) -> bool { - match (w.repeat_every, w.latency_sla) { - (Some(re), Some(ls)) => re < ls, - _ => false, - } -} - -/// Builds the list of precompute jobs for a plan. Returns an empty Vec if the -/// workload does not meet the precompute eligibility criterion. -/// -/// **SP-9**: when `plan.staged_plan` is `Some` and the precompute sub-plan is -/// active, the job's `query_expr` is taken from the AST-derived PromQL -/// serialisation ([`PrecomputeSubPlan::query_expr`]) rather than the hardcoded -/// `quantile_over_time(0.99, …)` template. This allows the precompute engine -/// to evaluate the actual upper sub-tree (e.g. `topk(10, count_over_time(…))`). -/// -/// When no `staged_plan` is present (SP-3 flat path), the legacy -/// `build_query_expr()` template is used as the fallback. -pub fn build_precompute_jobs( - w: &QueryWorkload, - plan: &CollectionPlan, - backend_addr: &str, -) -> Vec { - if !should_precompute(w) { - return vec![]; - } - let granularity = w.repeat_every.unwrap(); // safe: should_precompute checked it - - // SP-9: use the staged plan's PromQL when available and non-empty. - let query_expr = plan - .staged_plan - .as_ref() - .filter(|sp| sp.precompute.active && !sp.precompute.query_expr.is_empty()) - .map(|sp| sp.precompute.query_expr.clone()) - .unwrap_or_else(|| build_query_expr(w)); - - vec![PrecomputeJob { - query_expr, - granularity, - sketch_source: backend_addr.to_string(), - store_path: build_store_path(w), - }] -} - -fn build_query_expr(w: &QueryWorkload) -> String { - let agg_fn = match w.aggregations.first() { - Some(AggType::Cardinality) => "count_distinct_over_time", - Some(AggType::Frequency) => "top_k_over_time", - _ => "quantile_over_time", - }; - let filters: Vec = w - .label_filters - .iter() - .map(|(k, v)| format!("{k}=\"{v}\"")) - .collect(); - let selector = if filters.is_empty() { - w.metric_name.clone() - } else { - format!("{}{{{}}}", w.metric_name, filters.join(",")) - }; - format!( - "{agg_fn}(0.99, {selector}[{}])", - format_duration(w.time_window) - ) -} - -fn build_store_path(w: &QueryWorkload) -> String { - format!( - "precomputed/{}/p99/{}", - w.metric_name, - format_duration(w.time_window) - ) -} - -// ── HTTP client for ASAPQuery precompute API ────────────────────────────────── - -#[derive(Debug, Serialize)] -struct JobRequest { - query: String, - granularity: String, - source: String, - sketch_type: String, - store_path: String, -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct JobResponse { - pub job_id: String, - pub status: String, - pub created_at: Option>, -} - -pub struct PrecomputeClient { - base_url: String, - client: reqwest::Client, -} - -impl PrecomputeClient { - pub fn new(base_url: impl Into) -> Self { - Self { - base_url: base_url.into(), - client: reqwest::Client::builder() - .timeout(Duration::from_secs(10)) - .build() - .expect("reqwest client"), - } - } - - /// Registers a precompute job with the ASAPQuery engine. - pub async fn register(&self, job: &PrecomputeJob) -> anyhow::Result { - let req = JobRequest { - query: job.query_expr.clone(), - granularity: format_duration(job.granularity), - source: job.sketch_source.clone(), - sketch_type: "ddsketch".into(), - store_path: job.store_path.clone(), - }; - let resp = self - .client - .post(format!("{}/api/v1/precompute/jobs", self.base_url)) - .json(&req) - .send() - .await - .context("POST precompute job")?; - - if !resp.status().is_success() { - anyhow::bail!("precompute API returned {}", resp.status()); - } - resp.json::() - .await - .context("decode job response") - } - - /// Removes a precompute job by ID. - pub async fn deregister(&self, job_id: &str) -> anyhow::Result<()> { - let resp = self - .client - .delete(format!("{}/api/v1/precompute/jobs/{job_id}", self.base_url)) - .send() - .await - .context("DELETE precompute job")?; - - if !resp.status().is_success() { - anyhow::bail!("precompute API returned {}", resp.status()); - } - Ok(()) - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use axum::{ - http::StatusCode, - routing::{delete, post}, - Json, Router, - }; - use serde_json::json; - use std::collections::HashMap; - use tokio::net::TcpListener; - - fn w(repeat_every: Option, latency_sla: Option) -> QueryWorkload { - QueryWorkload { - metric_name: "latency".into(), - label_filters: HashMap::new(), - group_by_labels: vec![], - aggregations: vec![AggType::Quantile], - time_window: Duration::from_secs(300), - repeat_every, - accuracy_sla: 0.01, - latency_sla, - sketch_type_override: None, - exact_required: false, - quantiles: vec![], - } - } - - fn dummy_plan() -> CollectionPlan { - CollectionPlan { - agent_config: AgentCollectorConfig { - output_mode: OutputMode::Sketch, - sketch_type: SketchType::DDSketch, - sketch_params: Default::default(), - aggregate_by: vec![], - label_matchers: vec![], - window_duration: None, - mode: ProcessorMode::Batch, - enable_self_monitoring: true, - transmit_sketch: true, - drop_original: true, - delta_transmission: false, - delta_threshold: 0.0, - enable_series_id: true, - series_id_ttl_secs: 0, - data_sink: AgentDataSink::default(), - }, - gateway_config: GatewayCollectorConfig { passthrough: true }, - backend_config: BackendCollectorConfig { - merge_sketch_type: SketchType::DDSketch, - group_by: vec![], - }, - precompute: vec![], - valid_until: chrono::Utc::now(), - delta_decision: DeltaDecision::default(), - transmission_cost_summary: TransmissionCostSummary::default(), - staged_plan: None, - } - } - - #[test] - fn should_precompute_true_when_repeat_lt_latency() { - assert!(should_precompute(&w( - Some(Duration::from_secs(60)), - Some(Duration::from_secs(300)) - ))); - } - - #[test] - fn should_precompute_false_when_repeat_geq_latency() { - assert!(!should_precompute(&w( - Some(Duration::from_secs(300)), - Some(Duration::from_secs(60)) - ))); - } - - #[test] - fn should_precompute_false_when_missing_fields() { - assert!(!should_precompute(&w(None, Some(Duration::from_secs(300))))); - assert!(!should_precompute(&w(Some(Duration::from_secs(60)), None))); - } - - #[test] - fn build_jobs_returns_job_when_eligible() { - let workload = w( - Some(Duration::from_secs(60)), - Some(Duration::from_secs(600)), - ); - let jobs = build_precompute_jobs(&workload, &dummy_plan(), "backend:4317"); - assert_eq!(jobs.len(), 1); - assert_eq!(jobs[0].sketch_source, "backend:4317"); - assert_eq!(jobs[0].granularity, Duration::from_secs(60)); - assert!(jobs[0].query_expr.contains("latency")); - } - - #[test] - fn build_jobs_empty_when_not_eligible() { - let workload = w( - Some(Duration::from_secs(300)), - Some(Duration::from_secs(60)), - ); - let jobs = build_precompute_jobs(&workload, &dummy_plan(), "backend:4317"); - assert!(jobs.is_empty()); - } - - #[test] - fn store_path_contains_metric_and_window() { - let workload = w( - Some(Duration::from_secs(60)), - Some(Duration::from_secs(600)), - ); - let jobs = build_precompute_jobs(&workload, &dummy_plan(), "backend:4317"); - assert!(jobs[0].store_path.contains("latency")); - assert!(jobs[0].store_path.contains("5m")); - } - - #[test] - fn cardinality_uses_count_distinct_expr() { - let mut workload = w( - Some(Duration::from_secs(60)), - Some(Duration::from_secs(600)), - ); - workload.aggregations = vec![AggType::Cardinality]; - let jobs = build_precompute_jobs(&workload, &dummy_plan(), "backend:4317"); - assert!( - jobs[0].query_expr.contains("count_distinct_over_time"), - "got: {}", - jobs[0].query_expr - ); - } - - #[tokio::test] - async fn client_register_success() { - let app = Router::new().route( - "/api/v1/precompute/jobs", - post(|| async { - ( - StatusCode::OK, - Json(json!({ - "job_id": "job-123", - "status": "created", - "created_at": null - })), - ) - }), - ); - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - tokio::spawn(async move { - axum::serve(listener, app).await.unwrap(); - }); - - let client = PrecomputeClient::new(format!("http://{addr}")); - let resp = client - .register(&PrecomputeJob { - query_expr: "quantile_over_time(0.99, latency[5m])".into(), - granularity: Duration::from_secs(60), - sketch_source: "backend:4317".into(), - store_path: "precomputed/latency/p99/5m".into(), - }) - .await - .unwrap(); - assert_eq!(resp.job_id, "job-123"); - } - - #[tokio::test] - async fn client_register_error_on_bad_status() { - let app = Router::new().route( - "/api/v1/precompute/jobs", - post(|| async { StatusCode::INTERNAL_SERVER_ERROR }), - ); - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - tokio::spawn(async move { - axum::serve(listener, app).await.unwrap(); - }); - - let client = PrecomputeClient::new(format!("http://{addr}")); - assert!(client - .register(&PrecomputeJob { - query_expr: "q".into(), - granularity: Duration::from_secs(60), - sketch_source: "s".into(), - store_path: "p".into(), - }) - .await - .is_err()); - } - - #[tokio::test] - async fn client_deregister_success() { - let app = Router::new().route( - "/api/v1/precompute/jobs/:id", - delete(|| async { StatusCode::NO_CONTENT }), - ); - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - tokio::spawn(async move { - axum::serve(listener, app).await.unwrap(); - }); - - let client = PrecomputeClient::new(format!("http://{addr}")); - assert!(client.deregister("job-abc").await.is_ok()); - } -} diff --git a/controller/src/config/stage_config.rs b/controller/src/config/stage_config.rs deleted file mode 100644 index 5fde9e35..00000000 --- a/controller/src/config/stage_config.rs +++ /dev/null @@ -1,3020 +0,0 @@ -//! Phase B (MVP v6) — turn a typed L5 [`StageConfig`] map (produced by -//! [`crate::stage_split::ThreeStageEmitter`]) into the **wire bytes** the -//! three executors actually consume: -//! -//! - [`emit_edge_yaml`] → OTel-collector YAML for the edge agent (OTLP -//! receiver → per-sketch processor(s) → OTLP exporter to gateway). -//! - [`emit_gateway_yaml`] → OTel-collector YAML for the gateway -//! aggregator (OTLP receiver → per-family `*merge` processor(s) → OTLP -//! exporter to backend). -//! - [`emit_backend_config_json`] → JSON document matching the -//! ASAPQuery-backend `POST /api/v1/streaming-config` API surface — same -//! shape that [`crate::config::asapquery_backend::generate_streaming_config_yaml`] -//! builds today, just from the typed [`BackendStageConfig`] instead of -//! a `CollectionPlan`. -//! - [`emit_backend_storage_routing`] → JSON document matching the -//! ASAPQuery-backend `POST /api/v1/storage_routing` API surface — -//! per-metric query-shape → engine routing table (Phase α). Sources -//! the per-metric sketch families from the typed [`BackendStageConfig`] -//! inputs and turns them into `(metric, [target])` rows the backend's -//! HTTP query handler consults via `BackendStorageRouting::lookup_with_shape`. -//! -//! These four functions are deliberately **stage-shaped**, not -//! plan-shaped: the typed L5 emitter has already split the SketchExpr -//! across edge / gateway / backend, so each function only sees the slice -//! that's relevant to its executor. The legacy emitters in -//! [`crate::config::agent`] / [`crate::config::backend`] still operate -//! on the legacy `AgentCollectorConfig` / `BackendCollectorConfig` — -//! Phase C will gate-flip the demo overlay onto these typed emitters. -//! -//! All three are pure transformations: no I/O, no env lookup. The -//! `opamp_endpoint` parameter is the controller's WebSocket URL the -//! emitted YAML's `extensions.opamp` block must point at; the caller -//! threads it through from `AppState::opamp_endpoint`. - -use anyhow::{Context, Result}; -use serde::Serialize; -use serde_json::{json, Value as JsonValue}; -use serde_yaml::{Mapping, Value}; -use std::collections::HashMap; - -use crate::sketch_algebra::params::{SketchKind, SketchParams}; -use crate::sketch_algebra::sketch_expr::EstimateOp; -use crate::stage_split::emitter::{ - AggregationInput, ArchiveTierMetric, BackendAggregation, BackendReadout, BackendStageConfig, - EdgeSketchProcessor, EdgeStageConfig, ExportTarget, GatewayMergeProcessor, GatewayStageConfig, - PrometheusArchiveMetric, -}; -use crate::stage_split::stage_id::StageId; - -// ── YAML structural types ───────────────────────────────────────────────────── -// -// These mirror the structural types in `config::agent`. We keep a -// private copy here rather than re-exporting because the L5 typed path -// has slightly different shape constraints (e.g. no `series_id_ttl` on -// the receiver block — that's a wire-layer concern Phase G+ owns). - -#[derive(Serialize)] -struct CollectorYaml { - extensions: HashMap, - receivers: HashMap, - processors: HashMap, - /// OTel collector v0.106+ ships the `routing` component as a - /// **connector**, not a processor (`routingprocessor` was - /// deprecated and removed). Connectors live in their own - /// top-level block and are referenced as both an exporter (entry - /// pipeline) and a receiver (each downstream pipeline). - /// Empty for legacy single-pipeline / Mode-3 / warm-passthrough - /// emit paths — preserved by `skip_serializing_if` so the YAML - /// shape doesn't gain an empty `connectors: {}` block. - #[serde(skip_serializing_if = "HashMap::is_empty")] - connectors: HashMap, - exporters: HashMap, - service: ServiceSection, -} - -#[derive(Serialize)] -struct ServiceSection { - extensions: Vec, - pipelines: HashMap, -} - -#[derive(Serialize)] -struct Pipeline { - receivers: Vec, - processors: Vec, - exporters: Vec, -} - -// ── Public API ──────────────────────────────────────────────────────────────── - -/// Build the OTel-collector YAML for an edge agent from the typed L5 -/// [`EdgeStageConfig`] payload. -/// -/// The `opamp_endpoint` is embedded under `extensions.opamp.server.ws.endpoint` -/// so the agent can receive runtime config updates without restart. -/// -/// The emitter does NOT resolve `ExportTarget::Stage(_)` to a concrete -/// network address; Phase C plumbs a `DeploymentConstraints` resolver -/// that maps the symbolic stage role to e.g. `gateway:4317`. Until -/// then, we emit a documented placeholder (`gateway:4317`) so the YAML -/// is syntactically valid and round-trips through Otel's loader for -/// integration tests. -pub fn emit_edge_yaml(cfg: &EdgeStageConfig, opamp_endpoint: &str) -> Result { - // ── MVP §46: 5-sketch routing-connector dispatch ─────────────────────── - // - // When the planner has populated `cfg.metric_to_family` (the per-metric - // → SketchKind table sourced from the workload spec), we switch to the - // canonical 5-sketch routing-connector wire shape: all referenced - // sketch processors live at the top level, the OTel `routing` - // *connector* (NOT the deprecated routing processor) lives under - // `connectors:`, and a fan-out of per-family pipelines (DDSketch / - // KLL / HLL / CountSketch / CountMinSketch) plus a `raw_passthrough` - // default each consume from the connector. This is the shape the - // asap-otel binary's builder-config registers for OTel collector - // v0.106+ where `routingprocessor` was removed. - // - // Empty `metric_to_family` ⇒ legacy single-pipeline / Mode-3 / - // warm-passthrough emit paths kick in (preserved verbatim below). - if !cfg.metric_to_family.is_empty() { - return emit_edge_yaml_5sketch_routing(cfg, opamp_endpoint); - } - - // ── Receivers ───────────────────────────────────────────────────────────── - // Edge agents accept OTLP gRPC on 4317 + HTTP on 4318. Phase B does - // not yet plumb an alternate port through `EdgeStageConfig`; if/when - // that field is added, swap the literal here for a `cfg.otlp_port` - // read. - let otlp_receiver: Value = serde_yaml::from_str( - "protocols:\n grpc:\n endpoint: \"0.0.0.0:4317\"\n max_recv_msg_size_mib: 64\n http:\n endpoint: \"0.0.0.0:4318\"\n", - ) - .context("parse static OTLP receiver block")?; - - // ── Processors ──────────────────────────────────────────────────────────── - // One processor per `EdgeSketchProcessor`. Names come straight from - // `EdgeSketchProcessor::processor_name` (already resolved by - // `emitter::edge_processor_name`) and the param block is built from - // the typed `SketchParams` payload. - let mut processors: HashMap = HashMap::new(); - let mut sketch_pipeline_processors: Vec = Vec::new(); - for sp in &cfg.sketch_processors { - let block = build_edge_processor_block( - sp, - cfg.window_secs, - &cfg.label_filters, - cfg.source_metric.as_deref(), - ); - // Use the processor_name verbatim as the YAML key — matches the - // factory `Type` strings the patched OTel-contrib build registers - // (see `opentelemetry-collector-contrib-patch/processor/*processor/factory.go`). - processors.insert(sp.processor_name.clone(), block); - sketch_pipeline_processors.push(sp.processor_name.clone()); - } - - // ── Phase 3.2.5 Bug (a): Gorilla-S3 archive processor block ────────────── - // When the plan includes any archive-tier metric (a freshness-probe - // archive metric, a `RawAtEdgePrometheusArchive` Mode-3 metric, or - // any other metric the routing table claims `gorilla_s3_archive` - // for), the agent's pipeline MUST run the `gorillas3` processor so - // the metric's samples land in MinIO. Without this, freshness probes - // (and any other archive-bound metric) never reach the cold tier and - // the warm-tier engine's `last_over_time(...)` returns empty. - // - // Config matches `deploy/configs/asap-otel-agent-b6-asap-single-sketch.yaml` - // — `block_format: prometheus_tsdb` so the Thanos store-gateway can - // read the emitted blocks; `drop_original: false` so the metric also - // flows downstream to the warm-tier sketch / OTLP exporter; the - // `window_interval` is the smallest `window_secs` declared on any - // archive-tier metric (defaults to 60s). - let has_archive_tier = !cfg.archive_tier_metrics.is_empty(); - if has_archive_tier { - let window_secs: u64 = cfg - .archive_tier_metrics - .iter() - .filter_map(|m| m.window_secs) - .min() - .unwrap_or(60); - let gorillas3_yaml = format!( - "window_interval: {window_secs}s\n\ -drop_original: false\n\ -endpoint: \"${{ASAP_MINIO_ENDPOINT:-http://minio:9000}}\"\n\ -bucket: \"${{ASAP_GORILLA_BUCKET:-asap-gorilla}}\"\n\ -region: us-east-1\n\ -use_ssl: false\n\ -access_key_id: \"${{ASAP_MINIO_ACCESS_KEY:-asap}}\"\n\ -secret_access_key: \"${{ASAP_MINIO_SECRET_KEY:-asap-local-only}}\"\n\ -prefix_template: \"{{tenant}}/{{metric}}/{{YYYY}}/{{MM}}/{{DD}}/{{HH}}/\"\n\ -tenant: \"${{ASAP_TENANT:-default}}\"\n\ -max_retries: 3\n\ -retry_backoff: 1s\n\ -upload_timeout: 30s\n\ -block_format: prometheus_tsdb\n\ -tsdb_bucket: \"${{ASAP_GORILLA_TSDB_BUCKET:-asap-gorilla-tsdb}}\"\n\ -tsdb_block_duration: {window_secs}s\n", - ); - let gorillas3: Value = - serde_yaml::from_str(&gorillas3_yaml).context("parse gorillas3 processor block")?; - processors.insert("gorillas3".to_string(), gorillas3); - } - - // Pipeline-processor list for the warm-tier path. Order matches - // `asap-otel-agent-b6-asap-single-sketch.yaml`: gorillas3 runs FIRST - // so the cold-tier write happens on the raw sample BEFORE the sketch - // processor mutates / suffix-renames the metric stream. - let warm_tier_processors: Vec = { - let mut v = Vec::new(); - if has_archive_tier { - v.push("gorillas3".to_string()); - } - v.extend(sketch_pipeline_processors.iter().cloned()); - v - }; - // Pipeline-processor list for the warm-passthrough path (Bug b): - // gorillas3 still runs (the metric still wants to land in the - // archive) but the sketch processor is bypassed so the metric name - // is preserved end-to-end. Empty when no archive tier and no - // sketches — passthrough = receiver → exporter. - let warm_passthrough_processors: Vec = { - let mut v = Vec::new(); - if has_archive_tier { - v.push("gorillas3".to_string()); - } - v - }; - - // ── Exporters ───────────────────────────────────────────────────────────── - // Edge always exports to the gateway. ExportTarget gets resolved to - // a concrete endpoint here (Phase B): symbolic stages map to - // documented hostnames the demo overlay (Phase C) will provision. - let (exporter_key, exporter_val) = build_otlp_exporter("gateway", &cfg.exporter_target); - - let mut exporters: HashMap = [(exporter_key.clone(), exporter_val)].into(); - let mut pipelines: HashMap = HashMap::new(); - - let has_prometheus_archive = !cfg.prometheus_archive_metrics.is_empty(); - let has_warm_passthrough = !cfg.warm_passthrough_metrics.is_empty(); - - // ── Phase ε.1 Mode 3 / Phase 3.2.5 Bug (b) — per-pipeline routing ───── - // Two routing axes can fire from a single edge agent: - // - // * Phase ε.1: Mode-3 metrics carry `asap.mode = prometheus_archive` - // as a data-point attribute and dispatch to the Prometheus OTLP - // receiver via a separate `otlphttp/prometheus` exporter. - // * Phase 3.2.5 Bug (b): warm-passthrough metrics (the freshness - // probes) need to bypass the family-specific sketch processor so - // the metric name is preserved end-to-end. They dispatch by - // metric name, NOT by `asap.mode` (so we don't have to teach the - // fake-exporter to set an extra attribute on top of the name). - // - // When ONLY the Phase ε.1 Mode-3 axis is active we emit the legacy - // `from_attribute: asap.mode` form to keep the wire shape stable. - // When the warm-passthrough axis is active (alone or together with - // Mode 3) we emit the OTTL-statement form (`route() where ...`) - // which lets a single routing processor dispatch by both axes from - // a single table. - if has_prometheus_archive { - // Exporter: OTLP HTTP to Prometheus's native receiver. The path - // is the canonical `/api/v1/otlp/v1/metrics`. The OTel collector's - // `otlphttp` exporter uses a `metrics_endpoint` field for the - // full URL (the `endpoint` field auto-appends `/v1/metrics` per - // OTel SDK convention; we use `metrics_endpoint` to be explicit - // and match the Prom path verbatim). - let prom_exporter_yaml = "metrics_endpoint: \"${ASAP_PROMETHEUS_OTLP_URL:-http://prometheus:9090/api/v1/otlp/v1/metrics}\"\nencoding: proto\ntls:\n insecure: true\n"; - let prom_exporter: Value = serde_yaml::from_str(prom_exporter_yaml) - .context("parse otlphttp/prometheus exporter block")?; - exporters.insert("otlphttp/prometheus".to_string(), prom_exporter); - } - - if has_warm_passthrough { - // ── Phase 3.2.5 Bug (b): warm-passthrough routing ─────────────────── - // The freshness probes are timestamp counters by design — the - // wire value `unix_ts_ms_of_emission` IS the freshness signal, - // so they MUST flow through the warm tier with their original - // metric name preserved. The DDSketch processor's `_quantile` - // suffix would rename `http_freshness_probe_warm` to - // `http_freshness_probe_warm_quantile` and break the replay - // client's `last_over_time(http_freshness_probe_warm[10s])` - // query. - // - // The fix: a `routing` processor with OTTL `route()` statements - // dispatches by metric name. Listed metrics route to - // `metrics/warm_passthrough` (gorillas3 → exporter, NO sketch); - // everything else takes the regular `metrics/warm_tier` path - // (gorillas3 → sketches → exporter). Phase ε.1's Mode-3 entry - // (matching `attributes["asap.mode"]`) is folded into the same - // table when prometheus_archive is also configured. - let mut table_entries: Vec = Vec::new(); - for metric in &cfg.warm_passthrough_metrics { - table_entries.push(format!( - " - statement: 'route() where metric.name == \"{metric}\"'\n pipelines: [metrics/warm_passthrough]" - )); - } - if has_prometheus_archive { - table_entries.push( - " - statement: 'route() where attributes[\"asap.mode\"] == \"prometheus_archive\"'\n pipelines: [metrics/prometheus_archive]".to_string(), - ); - } - let routing_yaml = format!( - "default_pipelines: [metrics/warm_tier]\ntable:\n{}\n", - table_entries.join("\n"), - ); - let routing: Value = serde_yaml::from_str(&routing_yaml) - .context("parse routing processor block (OTTL form)")?; - processors.insert("routing".to_string(), routing); - - pipelines.insert( - "metrics/warm_tier".to_string(), - Pipeline { - receivers: vec!["otlp".into()], - processors: warm_tier_processors.clone(), - exporters: vec![exporter_key.clone()], - }, - ); - pipelines.insert( - "metrics/warm_passthrough".to_string(), - Pipeline { - receivers: vec!["otlp".into()], - processors: warm_passthrough_processors.clone(), - exporters: vec![exporter_key.clone()], - }, - ); - if has_prometheus_archive { - pipelines.insert( - "metrics/prometheus_archive".to_string(), - Pipeline { - receivers: vec!["otlp".into()], - processors: Vec::new(), - exporters: vec!["otlphttp/prometheus".to_string()], - }, - ); - } - let mut entry_exporters = vec![exporter_key.clone()]; - if has_prometheus_archive { - entry_exporters.push("otlphttp/prometheus".to_string()); - } - pipelines.insert( - "metrics".to_string(), - Pipeline { - receivers: vec!["otlp".into()], - processors: vec!["routing".to_string()], - exporters: entry_exporters, - }, - ); - } else if has_prometheus_archive { - // Legacy Phase ε.1 routing — `from_attribute: asap.mode`. - // Preserved as-is so the wire shape stays stable for the - // (warm_passthrough_metrics empty) cases that already exist. - let routing_yaml = "from_attribute: asap.mode\ndefault_pipelines: [metrics/warm_tier]\ntable:\n - value: prometheus_archive\n pipelines: [metrics/prometheus_archive]\n"; - let routing: Value = - serde_yaml::from_str(routing_yaml).context("parse routing processor block")?; - processors.insert("routing".to_string(), routing); - - // Two named pipelines: - // `metrics/warm_tier` — gorillas3 (if archive) + - // sketch processors → otlp/backend - // `metrics/prometheus_archive` — passthrough → otlphttp/prometheus - pipelines.insert( - "metrics/warm_tier".to_string(), - Pipeline { - receivers: vec!["otlp".into()], - processors: warm_tier_processors.clone(), - exporters: vec![exporter_key.clone()], - }, - ); - pipelines.insert( - "metrics/prometheus_archive".to_string(), - Pipeline { - receivers: vec!["otlp".into()], - processors: Vec::new(), - exporters: vec!["otlphttp/prometheus".to_string()], - }, - ); - // Main `metrics` pipeline keeps the receiver + routing only — - // this is what the OTel routing connector pattern expects (one - // entry pipeline that fans out via the routing processor's - // table). - pipelines.insert( - "metrics".to_string(), - Pipeline { - receivers: vec!["otlp".into()], - processors: vec!["routing".to_string()], - exporters: vec![exporter_key.clone(), "otlphttp/prometheus".to_string()], - }, - ); - } else { - // No routing — single pipeline with the warm-tier processor - // chain (gorillas3 if archive_tier_metrics non-empty, then - // sketches). - pipelines.insert( - "metrics".to_string(), - Pipeline { - receivers: vec!["otlp".into()], - processors: warm_tier_processors, - exporters: vec![exporter_key], - }, - ); - } - - // ── OpAMP extension ─────────────────────────────────────────────────────── - let opamp_ext: Value = serde_yaml::from_str(&format!( - "server:\n ws:\n endpoint: \"{opamp_endpoint}\"\n" - )) - .context("parse opamp extension block")?; - - // ── Top-level YAML ──────────────────────────────────────────────────────── - let doc = CollectorYaml { - extensions: [("opamp".to_string(), opamp_ext)].into(), - receivers: [("otlp".to_string(), otlp_receiver)].into(), - processors, - // Legacy emit paths don't use the routing connector — see the - // MVP §46 dispatch at the top of `emit_edge_yaml`. - connectors: HashMap::new(), - exporters, - service: ServiceSection { - extensions: vec!["opamp".into()], - pipelines, - }, - }; - - serde_yaml::to_string(&doc).context("serialize edge stage config") -} - -/// Build the OTel-collector YAML for a gateway aggregator from the -/// typed L5 [`GatewayStageConfig`] payload. -/// -/// The gateway runs one `merge` processor per -/// `GatewayMergeProcessor` entry — these are the patched merge -/// processors in `opentelemetry-collector-contrib-patch/processor/`. -pub fn emit_gateway_yaml(cfg: &GatewayStageConfig, opamp_endpoint: &str) -> Result { - // Receiver — port from cfg, both gRPC + HTTP. - let port = cfg.otlp_receiver_port; - let otlp_receiver: Value = serde_yaml::from_str(&format!( - "protocols:\n grpc:\n endpoint: \"0.0.0.0:{port}\"\n max_recv_msg_size_mib: 64\n http:\n endpoint: \"0.0.0.0:{}\"\n", - port + 1, - )) - .context("parse gateway OTLP receiver block")?; - - // Processors — one merge processor per merge entry. Naming - // convention matches the patched contrib build: - // * SketchKind::DDSketch → `ddsketchmerge` - // * SketchKind::Kll → `kllmerge` - // * SketchKind::Hll → `hllmerge` - // * SketchKind::Cms → `countminsketchmerge` - // * SketchKind::CountSketch → `countsketchmerge` - // - // We honour `GatewayMergeProcessor::processor_name` if non-empty - // (the typed emitter today populates it as `"sketchmergeprocessor"` - // — a placeholder until Phase C flips factory names per-family), - // otherwise we derive the family-specific name from `sketch_kind`. - let mut processors: HashMap = HashMap::new(); - let mut pipeline_processors: Vec = Vec::new(); - for mp in &cfg.merge_processors { - let key = gateway_merge_processor_name(mp); - let block = build_gateway_merge_block(mp); - processors.insert(key.clone(), block); - pipeline_processors.push(key); - } - - // Exporter — backend OTLP. - let (exporter_key, exporter_val) = build_otlp_exporter("backend", &cfg.exporter_target); - - let opamp_ext: Value = serde_yaml::from_str(&format!( - "server:\n ws:\n endpoint: \"{opamp_endpoint}\"\n" - )) - .context("parse opamp extension block")?; - - let doc = CollectorYaml { - extensions: [("opamp".to_string(), opamp_ext)].into(), - receivers: [("otlp".to_string(), otlp_receiver)].into(), - processors, - // Gateway stage doesn't use the routing connector. - connectors: HashMap::new(), - exporters: [(exporter_key.clone(), exporter_val)].into(), - service: ServiceSection { - extensions: vec!["opamp".into()], - pipelines: [( - "metrics".to_string(), - Pipeline { - receivers: vec!["otlp".into()], - processors: pipeline_processors, - exporters: vec![exporter_key], - }, - )] - .into(), - }, - }; - - serde_yaml::to_string(&doc).context("serialize gateway stage config") -} - -/// Build the JSON document the ASAPQuery-backend's -/// `POST /api/v1/streaming-config` endpoint accepts, sourced from the -/// typed L5 [`BackendStageConfig`]. -/// -/// Output shape mirrors the YAML shape produced by -/// [`crate::config::asapquery_backend::generate_streaming_config_yaml`]: -/// a top-level `aggregations` array of -/// `{ aggregationId, aggregationType, metric, parameters, ... }` rows. -/// We additionally surface a parallel `readouts` array so the backend's -/// query engine can prepare per-readout dispatch entries up-front (the -/// existing YAML form has no readouts list because the legacy planner -/// materialises one aggregation per metric and infers readouts from the -/// PromQL query at execution time; Phase B's typed `BackendStageConfig` -/// carries the readouts explicitly, so we ship them too — backends that -/// don't recognise the field will ignore it without erroring). -pub fn emit_backend_config_json(cfg: &BackendStageConfig) -> Result { - let aggregations: Vec = cfg - .aggregations - .iter() - .map(build_backend_aggregation_json) - .collect(); - - let readouts: Vec = cfg - .readouts - .iter() - .map(build_backend_readout_json) - .collect(); - - Ok(json!({ - "aggregations": aggregations, - "readouts": readouts, - })) -} - -/// Phase α (MVP): build the JSON document the ASAPQuery-backend's -/// `POST /api/v1/storage_routing` endpoint accepts, sourced from the -/// typed L5 [`BackendStageConfig`] payloads emitted by [`crate::planner::stage_split`]. -/// -/// `metric_plans` is the list of `(metric_name, &BackendStageConfig)` -/// pairs the controller has produced this planning cycle — one entry -/// per workload that ran through the typed L5 path. Each entry yields -/// one `metrics:` row in the emitted JSON. `default_engine` is the -/// fallback for any metric the backend's HTTP handler observes that the -/// controller did not plan for. -/// -/// ## Schema -/// -/// Output mirrors the existing `deploy/configs/backend-storage-routing.yaml` -/// schema (the `routes:` form), serialised as JSON: -/// -/// ```json -/// { -/// "default_engine": "sketch_warm_tier", -/// "metrics": [ -/// { "name": "http_requests_total", -/// "targets": [ -/// { "engine": "thanos_archive", -/// "applies_to_query_shape": ["count", "topk", "rate_post_hoc", -/// "histogram_quantile", "delta", "absent"] }, -/// { "engine": "sketch_warm_tier" } -/// ] -/// } -/// ] -/// } -/// ``` -/// -/// ## Classification rules (Phase α) -/// -/// For each `(metric, BackendStageConfig)` we derive a target list by -/// inspecting the L4 sketch families landed at the backend: -/// -/// * **DDSketch / KLL** present → warm-tier serves `quantile` shape; -/// warm-tier is the default for everything the archive doesn't claim. -/// * **HLL** present → warm-tier serves `count` shape (cardinality -/// readout). NOTE: with HLL planned, `count` does NOT route to archive -/// — the warm-tier sketch is lossier-but-cheaper than archive scan and -/// the controller already chose to spend the bandwidth on it. -/// * **Count-Sketch** present → warm-tier serves `topk` shape (the -/// sketch's whole purpose). -/// * **CountMinSketch** present → warm-tier serves `point_count` / -/// `count` shape (the CMS's `Estimate` readout). -/// -/// The `thanos_archive` target is always added with the **archive-eligible -/// shape list** — those PromQL shapes that no warm-tier sketch can -/// answer at all (`histogram_quantile`, `delta`, `deriv`, `absent`, -/// post-hoc / un-planned ranges). When a sketch-eligible shape is also -/// in the archive's claim list (e.g. `count` when no HLL was planned) -/// it is added so the archive picks it up as a fallback. -/// -/// Phase α is conservative: we always emit BOTH a warm-tier default -/// slot AND a thanos archive slot for every planned metric, so v7 -/// dual-routing semantics are preserved by construction. Future phases -/// (β / γ) may prune the archive slot for metrics the cost model -/// prices out of cold storage. -pub fn emit_backend_storage_routing( - metric_plans: &[(String, &BackendStageConfig)], -) -> Result { - emit_backend_storage_routing_for_tenant(DEFAULT_TENANT, metric_plans) -} - -/// Tenant id used when the deploy is single-tenant. Mirrors the -/// backend's `crate::routing::DEFAULT_TENANT` (defined in -/// `ASAPQuery-backend/asap-query-engine/src/routing/backend_storage_routing.rs`) -/// — kept as a literal here so the controller doesn't take a build-time -/// dependency on the backend crate just for one constant. -pub const DEFAULT_TENANT: &str = "default"; - -/// Per-tenant follow-up to PR #333 — emit a `BackendStorageRouting` -/// JSON document scoped to a specific tenant. The single-tenant -/// [`emit_backend_storage_routing`] entry point delegates to this -/// with [`DEFAULT_TENANT`], preserving the existing single-tenant -/// emit contract. -/// -/// The emitted JSON adds a top-level `tenant: ""` field. The -/// backend's `BackendStorageRouting::from_json_payload` parser -/// reads this field (defaulting to `"default"` when absent) and -/// the `POST /api/v1/storage_routing` swap handler routes the swap -/// to the named tenant's slot. Multi-tenant deployments emit one -/// JSON per tenant; single-tenant deployments keep emitting with -/// the default tenant and need no controller-side change. -pub fn emit_backend_storage_routing_for_tenant( - tenant: &str, - metric_plans: &[(String, &BackendStageConfig)], -) -> Result { - let mut metrics_json: Vec = Vec::with_capacity(metric_plans.len()); - for (metric_name, backend_cfg) in metric_plans { - metrics_json.push(build_routing_entry(metric_name, backend_cfg)); - } - Ok(json!({ - "tenant": tenant, - "default_engine": "sketch_warm_tier", - "metrics": metrics_json, - })) -} - -/// Phase ε.1 — same as [`emit_backend_storage_routing`] but also -/// emits `prometheus_remote` engine entries for Mode 3 metrics. -/// -/// Mode-3 metrics have NO `BackendStageConfig` entry (the backend doesn't -/// own the storage; Prometheus does). They surface here as plain metric -/// names paired with a single `prometheus_remote` target. The backend's -/// HTTP query handler consults the routing table at request time and -/// HTTP-forwards Mode-3 queries to -/// `${ASAP_PROMETHEUS_QUERY_URL:-http://prometheus:9090}/api/v1/query`. -/// -/// Phase ε.2 implements the `prometheus_remote` engine on the backend -/// (the HTTP forwarder); Phase ε.1 only commits the routing wire shape. -/// -/// `mode3_metrics` is the list of metric names the planner routed to -/// Prometheus archive this cycle. Each yields a single-target row with -/// `engine: prometheus_remote` and no shape filter (Prom answers -/// everything for these metrics, exact ε = 0). -pub fn emit_backend_storage_routing_with_prometheus( - metric_plans: &[(String, &BackendStageConfig)], - mode3_metrics: &[String], -) -> Result { - emit_backend_storage_routing_with_prometheus_for_tenant( - DEFAULT_TENANT, - metric_plans, - mode3_metrics, - ) -} - -/// Per-tenant variant of [`emit_backend_storage_routing_with_prometheus`]. -/// Mirrors [`emit_backend_storage_routing_for_tenant`] — adds a -/// top-level `tenant: ""` field; defaults preserve the existing -/// single-tenant emit shape. -pub fn emit_backend_storage_routing_with_prometheus_for_tenant( - tenant: &str, - metric_plans: &[(String, &BackendStageConfig)], - mode3_metrics: &[String], -) -> Result { - let mut metrics_json: Vec = - Vec::with_capacity(metric_plans.len() + mode3_metrics.len()); - for (metric_name, backend_cfg) in metric_plans { - metrics_json.push(build_routing_entry(metric_name, backend_cfg)); - } - for metric_name in mode3_metrics { - // Mode 3 — Prometheus owns the storage. Single target, - // engine=prometheus_remote, no shape filter (all PromQL shapes - // route through the backend's HTTP forwarder). - metrics_json.push(json!({ - "name": metric_name, - "targets": [ - { "engine": "prometheus_remote" } - ], - "asap_mode": "prometheus_archive", - })); - } - Ok(json!({ - "tenant": tenant, - "default_engine": "sketch_warm_tier", - "metrics": metrics_json, - })) -} - -// ── Internals ───────────────────────────────────────────────────────────────── - -/// Build the JSON `metrics:` entry for one (metric, BackendStageConfig) -/// pair — picks per-shape targets from the L4 sketch families the plan -/// landed at the backend. -/// -/// Returns a JSON object of shape: -/// ```text -/// { "name": , "targets": [, ...] } -/// ``` -/// where each `` is either `{ "engine": , "applies_to_query_shape": [...] }` -/// or `{ "engine": }` for the default slot. -fn build_routing_entry(metric_name: &str, cfg: &BackendStageConfig) -> JsonValue { - let kinds: Vec = cfg - .aggregations - .iter() - .map(|a| a.sketch_kind.clone()) - .collect(); - - // Sketch-eligible shapes — the warm tier serves these natively - // because we planned a sketch for them. - let mut warm_shapes: Vec<&'static str> = Vec::new(); - let has_quantile_sketch = kinds - .iter() - .any(|k| matches!(k, SketchKind::DDSketch | SketchKind::Kll)); - if has_quantile_sketch { - warm_shapes.push("quantile"); - warm_shapes.push("quantile_over_time"); - } - let has_hll = kinds.iter().any(|k| matches!(k, SketchKind::Hll)); - if has_hll { - warm_shapes.push("count"); - } - let has_count_sketch = kinds.iter().any(|k| matches!(k, SketchKind::CountSketch)); - if has_count_sketch { - warm_shapes.push("topk"); - } - let has_cms = kinds.iter().any(|k| matches!(k, SketchKind::Cms)); - if has_cms { - // CMS's `Estimate` readout serves point-count / count queries. - // If HLL also planned, `count` is already in the list — push - // only when not already there (keep order stable). - if !warm_shapes.contains(&"count") { - warm_shapes.push("count"); - } - } - // Sketch-planned `rate / sum / avg / min / max` over the planned - // ranges — every sketch family the planner emits also tracks the - // range aggregation needed to answer these from the warm tier - // (the gateway merge processor produces a windowed accumulator). - if !kinds.is_empty() { - warm_shapes.push("rate"); - warm_shapes.push("sum"); - warm_shapes.push("avg"); - warm_shapes.push("min"); - warm_shapes.push("max"); - } - - // Archive-eligible shapes — Thanos / cold archive answers these - // because no warm-tier sketch can. - // - // Classification rule (surprised-me bullet for the report): `topk` - // and `count` route to archive only when NO matching sketch was - // planned. With Count-Sketch the warm tier answers `topk` via the - // CountSketch's heap-augmented Estimate; with HLL the warm tier - // answers `count` via the cardinality estimate. Pruning the - // archive's claim list is what makes Phase α a planner-driven - // routing table rather than a static "everything goes to archive" - // failover. - let mut archive_shapes: Vec<&'static str> = Vec::new(); - archive_shapes.push("histogram_quantile"); - archive_shapes.push("delta"); - archive_shapes.push("deriv"); - archive_shapes.push("absent"); - archive_shapes.push("rate_post_hoc"); - if !has_count_sketch { - archive_shapes.push("topk"); - } - if !has_hll && !has_cms { - archive_shapes.push("count"); - } - - // Emit the warm-tier default slot first (no filter — catches every - // shape the archive doesn't claim), then the archive slot with the - // explicit-shape claim list. Ordering matches the existing - // `deploy/configs/backend-storage-routing.yaml` convention. The - // backend's `lookup_with_shape` is two-pass: explicit-shape match - // wins (so `count` / `topk` / etc. land on archive when listed - // there), default slot otherwise (so `quantile` / `sum` / etc. - // land on warm). - // - // We do NOT attach `applies_to_query_shape` to the warm slot — - // attaching it would turn warm into a shape-specific target and - // any unanticipated shape (e.g. `LastOverTime` on a metric where - // the operator added a probe after planning) would fall through - // to the archive's first-target fallback, which is the wrong - // failure mode. Warm = default; archive = the specific shapes - // archive serves better. - let mut targets: Vec = Vec::new(); - targets.push(json!({ - "engine": "sketch_warm_tier", - })); - if !archive_shapes.is_empty() { - targets.push(json!({ - "engine": "thanos_archive", - "applies_to_query_shape": archive_shapes, - })); - } - - // The warm-shape list is informational — surface it on a side - // field for operators / tests to spot-check what the controller - // decided the warm tier serves natively. The backend ignores - // unknown fields (`#[serde(default)]` on the parser side). - let mut entry = json!({ - "name": metric_name, - "targets": targets, - }); - if !warm_shapes.is_empty() { - entry["warm_tier_native_shapes"] = json!(warm_shapes); - } - entry -} - -// ── MVP §46: 5-sketch routing-connector edge YAML emitter ───────────────── -// -// CRITICAL CORRECTNESS NOTE (call out as a real bugfix, not a refactor): -// the legacy `emit_edge_yaml` placed `routing` under `processors:`. That -// is WRONG for OTel collector v0.106+ — the routing component was -// deprecated as a processor and re-shipped as a *connector*. The -// `routingprocessor` factory was removed in collector-contrib v0.106 -// and the asap-otel binary's `builder-config.yaml` registers -// `routingconnector` instead. Emitting the old shape produces a YAML -// that fails `confmap.Provider` validation on the agent at boot: -// `error decoding 'processors': unknown type: "routing"`. -// -// This function emits the canonical connector-form layout — see the -// MVP §46 contract: -// -// receivers: { otlp } -// processors: { gorillas3?, batch, ddsketch, KLL, HLL, -// countsketch, countmin } -// connectors: { routing: { default_pipelines: [metrics/raw_passthrough], -// table: [ ... per-metric OTTL conditions ... ] } } -// exporters: { otlp/backend, otlphttp/prometheus? } -// -// service.pipelines: -// metrics: (entry — receivers: [otlp], -// exporters: [routing]) -// metrics/raw_passthrough: (default — receivers: [routing], -// processors: [gorillas3?, batch], -// exporters: [otlp/backend]) -// metrics/{ddsketch,kll,hll,countsketch,countminsketch}_path: -// (per-family — receivers: [routing], -// processors: [gorillas3?, -// processor, -// batch], -// exporters: [otlp/backend]) -// -// `gorillas3` runs FIRST in every per-sketch pipeline (when an -// archive tier is declared) so the raw sample lands in the cold -// archive BEFORE the family-specific sketch processor mutates the -// stream — same invariant the legacy emit path enforces. -// -// Phase ε.1 Mode-3 metrics (`prometheus_archive_metrics`) and Bug (b) -// `warm_passthrough_metrics` (the freshness probes) are folded into -// the routing table's `table:` and route to the `metrics/raw_passthrough` -// pipeline — they intentionally bypass every sketch processor. -fn emit_edge_yaml_5sketch_routing(cfg: &EdgeStageConfig, opamp_endpoint: &str) -> Result { - use crate::sketch_algebra::params::SketchKind; - - let otlp_receiver: Value = serde_yaml::from_str( - "protocols:\n grpc:\n endpoint: \"0.0.0.0:4317\"\n max_recv_msg_size_mib: 64\n http:\n endpoint: \"0.0.0.0:4318\"\n", - ) - .context("parse static OTLP receiver block")?; - - // ── Processors ───────────────────────────────────────────────────────── - // - // We always load all 5 sketch processors regardless of which metrics - // route to them — the planner agent's contract is that the agent - // can be retargeted at runtime via OpAMP without re-building, so a - // future plan that maps a new metric to (say) HLL must work without - // a config push that touches `processors:`. - let mut processors: HashMap = HashMap::new(); - - // Build per-family processor blocks. We pull from - // `cfg.sketch_processors` when an entry exists for that family - // (so the params flow through), otherwise we - // synthesise a default-param block so the YAML always carries - // all 5 processor keys. - let mut family_to_proc: HashMap = HashMap::new(); - for sp in &cfg.sketch_processors { - family_to_proc.insert(sp.sketch_kind.clone(), sp); - } - - for kind in [ - SketchKind::DDSketch, - SketchKind::Kll, - SketchKind::Hll, - SketchKind::CountSketch, - SketchKind::Cms, - ] { - let processor_name = sketch_kind_to_processor_name(&kind); - let metric_name_hint = cfg - .metric_to_family - .iter() - .filter_map(|(metric, mapped)| { - if mapped == &kind { - Some(metric.as_str()) - } else { - None - } - }) - .min(); - let block = if let Some(sp) = family_to_proc.get(&kind) { - build_edge_processor_block(sp, cfg.window_secs, &cfg.label_filters, metric_name_hint) - } else { - build_default_edge_processor_block(&kind, cfg.window_secs, metric_name_hint) - }; - processors.insert(processor_name.to_string(), block); - } - - // ── gorillas3 archive processor ──────────────────────────────────────── - let has_archive_tier = !cfg.archive_tier_metrics.is_empty(); - if has_archive_tier { - let window_secs: u64 = cfg - .archive_tier_metrics - .iter() - .filter_map(|m| m.window_secs) - .min() - .unwrap_or(60); - let gorillas3_yaml = format!( - "window_interval: {window_secs}s\n\ -drop_original: false\n\ -endpoint: \"${{ASAP_MINIO_ENDPOINT:-http://minio:9000}}\"\n\ -bucket: \"${{ASAP_GORILLA_BUCKET:-asap-gorilla}}\"\n\ -region: us-east-1\n\ -use_ssl: false\n\ -access_key_id: \"${{ASAP_MINIO_ACCESS_KEY:-asap}}\"\n\ -secret_access_key: \"${{ASAP_MINIO_SECRET_KEY:-asap-local-only}}\"\n\ -prefix_template: \"{{tenant}}/{{metric}}/{{YYYY}}/{{MM}}/{{DD}}/{{HH}}/\"\n\ -tenant: \"${{ASAP_TENANT:-default}}\"\n\ -max_retries: 3\n\ -retry_backoff: 1s\n\ -upload_timeout: 30s\n\ -block_format: prometheus_tsdb\n\ -tsdb_bucket: \"${{ASAP_GORILLA_TSDB_BUCKET:-asap-gorilla-tsdb}}\"\n\ -tsdb_block_duration: {window_secs}s\n", - ); - let gorillas3: Value = serde_yaml::from_str(&gorillas3_yaml) - .context("parse gorillas3 processor block (5-sketch routing)")?; - processors.insert("gorillas3".to_string(), gorillas3); - } - - // batch processor — every per-family pipeline ends in batch so the - // gateway sees properly framed OTLP. Defaults match - // `deploy/configs/asap-otel-agent-b6-asap-single-sketch.yaml`. - let batch_block: Value = serde_yaml::from_str("send_batch_size: 1024\ntimeout: 1s\n") - .context("parse batch processor block")?; - processors.insert("batch".to_string(), batch_block); - - // memory_limiter processor — backpressure BEFORE gorillas3 so the - // collector refuses incoming batches when RSS crosses the soft - // threshold instead of OOM-killing the agent. Follow-up to PR #355 - // (gorillas3 archive write fix): even with `window_interval: 5s` - // the agent was OOM-killed (exit 137) ~3 min into sustained load - // because six per-family in-memory windowState buffers can overshoot - // the 1.5 GiB cgroup ceiling at peak. Threshold = 1280 MiB / 256 MiB - // spike (≈ 80 % / 17 % of cgroup), mirrors gateway shape but scaled - // to the agent's smaller cgroup. MUST be the first processor in - // every per-sketch pipeline (see `make_sketch_pipeline` below) — - // limiting AFTER gorillas3 would mean the buffer has already - // accreted on heap by the time the limiter rejects. - let memory_limiter_block: Value = - serde_yaml::from_str("check_interval: 1s\nlimit_mib: 1280\nspike_limit_mib: 256\n") - .context("parse memory_limiter processor block")?; - processors.insert("memory_limiter".to_string(), memory_limiter_block); - - // ── Exporters ────────────────────────────────────────────────────────── - let (exporter_key, exporter_val) = build_otlp_exporter("gateway", &cfg.exporter_target); - let mut exporters: HashMap = [(exporter_key.clone(), exporter_val)].into(); - - let has_prometheus_archive = !cfg.prometheus_archive_metrics.is_empty(); - if has_prometheus_archive { - let prom_exporter_yaml = "metrics_endpoint: \"${ASAP_PROMETHEUS_OTLP_URL:-http://prometheus:9090/api/v1/otlp/v1/metrics}\"\nencoding: proto\ntls:\n insecure: true\n"; - let prom_exporter: Value = serde_yaml::from_str(prom_exporter_yaml) - .context("parse otlphttp/prometheus exporter block")?; - exporters.insert("otlphttp/prometheus".to_string(), prom_exporter); - } - - // ── Routing connector ────────────────────────────────────────────────── - // - // Build the OTTL route table. Iterate the planner's - // `metric_to_family` map in deterministic order (sorted by metric - // name) so the YAML is stable across runs — `HashMap` iteration is - // not order-stable. - let mut metric_family_pairs: Vec<(&String, &SketchKind)> = - cfg.metric_to_family.iter().collect(); - metric_family_pairs.sort_by(|a, b| a.0.cmp(b.0)); - - let mut table_entries: Vec = Vec::new(); - let mut referenced_pipelines: std::collections::BTreeSet = - std::collections::BTreeSet::new(); - - for (metric, kind) in &metric_family_pairs { - let pipeline = sketch_kind_to_pipeline_name(kind); - table_entries.push(format!( - " - context: metric\n condition: 'name == \"{metric}\"'\n pipelines: [{pipeline}]" - )); - referenced_pipelines.insert(pipeline.to_string()); - } - - // Phase 3.2.5 Bug (b) — warm-passthrough freshness probes route to - // raw_passthrough (no sketch processor mutates the metric name). - for metric in &cfg.warm_passthrough_metrics { - table_entries.push(format!( - " - context: metric\n condition: 'name == \"{metric}\"'\n pipelines: [metrics/raw_passthrough]" - )); - } - - // Phase ε.1 — Mode 3 prometheus-archive routing folds in via the - // `asap.mode` attribute axis. The dedicated - // `metrics/prometheus_archive` pipeline ships the metric to - // Prometheus's native OTLP receiver via `otlphttp/prometheus`. - if has_prometheus_archive { - table_entries.push( - " - context: datapoint\n condition: 'attributes[\"asap.mode\"] == \"prometheus_archive\"'\n pipelines: [metrics/prometheus_archive]" - .to_string(), - ); - } - - let routing_yaml = format!( - "default_pipelines: [metrics/raw_passthrough]\ntable:\n{}\n", - table_entries.join("\n"), - ); - let routing_block: Value = - serde_yaml::from_str(&routing_yaml).context("parse routing connector block (5-sketch)")?; - let mut connectors: HashMap = HashMap::new(); - connectors.insert("routing".to_string(), routing_block); - - // ── Pipeline assembly ────────────────────────────────────────────────── - // - // Helper: per-family pipeline = - // `[memory_limiter, gorillas3?, processor, batch]`. - // memory_limiter runs FIRST so backpressure rejects incoming batches - // BEFORE gorillas3 buffers them into windowState. gorillas3 then - // does the cold-tier write on raw samples BEFORE the sketch - // processor mutates / suffix-renames the stream. - let make_sketch_pipeline = |family_proc: &str| -> Pipeline { - let mut procs: Vec = Vec::new(); - procs.push("memory_limiter".to_string()); - if has_archive_tier { - procs.push("gorillas3".to_string()); - } - procs.push(family_proc.to_string()); - procs.push("batch".to_string()); - Pipeline { - receivers: vec!["routing".into()], - processors: procs, - exporters: vec![exporter_key.clone()], - } - }; - - let mut pipelines: HashMap = HashMap::new(); - - // Entry pipeline — receivers: [otlp], exporters: [routing] - // (`routing` here is the connector, used as exporter for the entry - // stage). NO processors on the entry pipeline; the connector is - // responsible for fan-out. - pipelines.insert( - "metrics".to_string(), - Pipeline { - receivers: vec!["otlp".into()], - processors: Vec::new(), - exporters: vec!["routing".to_string()], - }, - ); - - // Default raw_passthrough — - // `[memory_limiter, gorillas3?, batch]`. NO sketch processor — the - // raw counters land at the gateway verbatim. This is also the - // destination of warm_passthrough metrics (freshness probes). - // memory_limiter runs first so backpressure applies to the default - // route too. - let raw_passthrough = { - let mut procs: Vec = Vec::new(); - procs.push("memory_limiter".to_string()); - if has_archive_tier { - procs.push("gorillas3".to_string()); - } - procs.push("batch".to_string()); - Pipeline { - receivers: vec!["routing".into()], - processors: procs, - exporters: vec![exporter_key.clone()], - } - }; - pipelines.insert("metrics/raw_passthrough".to_string(), raw_passthrough); - - // Always emit all 5 per-family pipelines so the agent's pipeline - // graph is closed regardless of which families the table currently - // references — keeps the runtime swap (planner re-emits with a - // different `metric_to_family`) zero-touch on the pipeline graph. - for kind in [ - SketchKind::DDSketch, - SketchKind::Kll, - SketchKind::Hll, - SketchKind::CountSketch, - SketchKind::Cms, - ] { - let proc_name = sketch_kind_to_processor_name(&kind); - let pipeline_name = sketch_kind_to_pipeline_name(&kind); - pipelines.insert(pipeline_name.to_string(), make_sketch_pipeline(proc_name)); - } - - // Phase ε.1 — Mode 3 prometheus-archive pipeline (raw passthrough - // to the Prometheus OTLP exporter). No sketch processors; only the - // Prometheus exporter target is referenced. - if has_prometheus_archive { - pipelines.insert( - "metrics/prometheus_archive".to_string(), - Pipeline { - receivers: vec!["routing".into()], - processors: Vec::new(), - exporters: vec!["otlphttp/prometheus".to_string()], - }, - ); - } - - // ── OpAMP extension ──────────────────────────────────────────────────── - let opamp_ext: Value = serde_yaml::from_str(&format!( - "server:\n ws:\n endpoint: \"{opamp_endpoint}\"\n" - )) - .context("parse opamp extension block")?; - - let doc = CollectorYaml { - extensions: [("opamp".to_string(), opamp_ext)].into(), - receivers: [("otlp".to_string(), otlp_receiver)].into(), - processors, - connectors, - exporters, - service: ServiceSection { - extensions: vec!["opamp".into()], - pipelines, - }, - }; - - serde_yaml::to_string(&doc).context("serialize edge stage config (5-sketch)") -} - -/// Map a `SketchKind` to the OTel processor name registered by the -/// patched contrib build's factory. Keep in sync with -/// `crate::stage_split::emitter::edge_processor_name`. -fn sketch_kind_to_processor_name(kind: &SketchKind) -> &'static str { - match kind { - SketchKind::DDSketch => "ddsketch", - SketchKind::Kll => "KLL", - SketchKind::Hll => "HLL", - SketchKind::CountSketch => "countsketch", - SketchKind::Cms => "countmin", - } -} - -/// Map a `SketchKind` to its per-family pipeline name in the routing -/// connector layout. -fn sketch_kind_to_pipeline_name(kind: &SketchKind) -> &'static str { - match kind { - SketchKind::DDSketch => "metrics/ddsketch_path", - SketchKind::Kll => "metrics/kll_path", - SketchKind::Hll => "metrics/hll_path", - SketchKind::CountSketch => "metrics/countsketch_path", - SketchKind::Cms => "metrics/countminsketch_path", - } -} - -/// Build a default-parameter processor block for a `SketchKind` when -/// the planner's `metric_to_family` references a family that -/// `cfg.sketch_processors` didn't enumerate. Defaults match the catalog -/// values used by the planner's L4 rules so the wire shape is what the -/// rest of the system expects when a metric is later re-routed onto -/// this family. -fn build_default_edge_processor_block( - kind: &SketchKind, - window_secs: Option, - metric_name_hint: Option<&str>, -) -> Value { - use crate::sketch_algebra::params::{ - CmsParams, CountSketchParams, DDSketchParams, HllParams, KllParams, - }; - let params = match kind { - SketchKind::DDSketch => SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), - SketchKind::Kll => SketchParams::Kll(KllParams { k: 200 }), - SketchKind::Hll => SketchParams::Hll(HllParams { precision: 14 }), - SketchKind::CountSketch => SketchParams::CountSketch(CountSketchParams { - w: 2048, - d: 5, - with_heap: true, - }), - SketchKind::Cms => SketchParams::Cms(CmsParams { w: 4096, d: 4 }), - }; - let synthetic = EdgeSketchProcessor { - processor_name: sketch_kind_to_processor_name(kind).to_string(), - sketch_kind: kind.clone(), - sketch_params: params, - aggregation_id: format!("agg_default_{}", sketch_kind_tag(kind)), - }; - build_edge_processor_block(&synthetic, window_secs, &[], metric_name_hint) -} - -/// Resolve an `ExportTarget` to a concrete `endpoint:port` string. Phase -/// B uses documented placeholder hostnames (`gateway:4317`, -/// `backend:4317`) for symbolic stages — Phase C plumbs a real -/// `DeploymentConstraints::executors()` resolver. -fn resolve_export_endpoint(default_host: &str, target: &ExportTarget) -> String { - match target { - ExportTarget::Endpoint(s) => s.clone(), - ExportTarget::Stage(StageId::Edge) => "edge:4317".to_string(), - ExportTarget::Stage(StageId::Gateway) => format!("{default_host}:4317"), - ExportTarget::Stage(StageId::Backend) => format!("{default_host}:4317"), - } -} - -/// Build the `(component_id, yaml)` pair for an OTLP exporter pointed -/// at the supplied symbolic / concrete target. `default_host` is the -/// host portion used when the target is a symbolic stage role. -fn build_otlp_exporter(default_host: &str, target: &ExportTarget) -> (String, Value) { - let endpoint = resolve_export_endpoint(default_host, target); - let yaml = format!("endpoint: \"{endpoint}\"\ntls:\n insecure: true\ncompression: none\n",); - ( - "otlp/backend".to_string(), - serde_yaml::from_str(&yaml).expect("inline OTLP exporter yaml is valid"), - ) -} - -/// Build the per-edge-processor parameter block. Mirrors the param -/// surface of `crate::config::agent::build_processor_block` but reads -/// from the typed `EdgeSketchProcessor` + ambient `EdgeStageConfig` -/// fields rather than the legacy `AgentCollectorConfig`. -fn build_edge_processor_block( - sp: &EdgeSketchProcessor, - window_secs: Option, - label_filters: &[(String, String)], - metric_name_hint: Option<&str>, -) -> Value { - let mut m = Mapping::new(); - - // Mode — `window` whenever a window landed on edge, else `batch`. - if let Some(w) = window_secs { - m.insert("mode".into(), Value::String("window".to_string())); - m.insert("window_duration".into(), Value::String(format!("{w}s"))); - } else { - m.insert("mode".into(), Value::String("batch".to_string())); - } - m.insert("transmit_sketch".into(), Value::Bool(true)); - m.insert("enable_self_monitoring".into(), Value::Bool(true)); - - // Label matchers — same `[{key, value}]` shape the legacy agent - // emitter uses (Go processor expects `[]LabelMatcher{Key, Value}`). - if !label_filters.is_empty() { - let matchers: Vec = label_filters - .iter() - .map(|(k, v)| { - let mut e = Mapping::new(); - e.insert("key".into(), Value::String(k.clone())); - e.insert("value".into(), Value::String(v.clone())); - Value::Mapping(e) - }) - .collect(); - m.insert("label_matchers".into(), Value::Sequence(matchers)); - } - - // Family-specific params. - // - // `delta_transmission` is set to `true` for the four families - // that support sparse delta encoding (DDSketch, HLL, CountSketch, - // Count-Min). KLL deliberately does NOT get the flag — KLL uses - // randomised compaction and is not additively mergeable, so its - // wire payload is always full state. The KLL processor's - // `Config.Validate` rejects `delta_transmission: true` with an - // error rather than silently falling back; emitting the flag - // would break agent boot. See `Implementation.tex` ("KLL has - // no delta variant and matches its full cost") and - // `kllprocessor/config.go::Config.Validate`. - // - // The flag matches the four factories' `DeltaTransmission: true` - // defaults (see `factory.go` in each processor); we still emit - // it explicitly so the wire YAML doesn't depend on a factory - // default that could regress to full-state in a future build. - match &sp.sketch_params { - SketchParams::Kll(p) => { - m.insert("k".into(), Value::Number((p.k as u64).into())); - // No delta_transmission for KLL: see comment above. - } - SketchParams::DDSketch(p) => { - m.insert("relative_accuracy".into(), Value::Number(p.alpha.into())); - m.insert("delta_transmission".into(), Value::Bool(true)); - } - SketchParams::Hll(_p) => { - // HLL takes no precision knob in its Config (the - // patched build hard-codes p=14); nothing further to set. - m.insert("encoding".into(), Value::String("msgpack".into())); - m.insert("delta_transmission".into(), Value::Bool(true)); - } - SketchParams::Cms(p) => { - m.insert( - "metric_name".into(), - Value::String( - metric_name_hint - .unwrap_or("endpoint_request_freq") - .to_string(), - ), - ); - m.insert("rows".into(), Value::Number((p.d as u64).into())); - m.insert("columns".into(), Value::Number((p.w as u64).into())); - m.insert("encoding".into(), Value::String("msgpack".into())); - m.insert("delta_transmission".into(), Value::Bool(true)); - } - SketchParams::CountSketch(p) => { - // Translate (w, d) to the legacy (epsilon, delta) surface - // that the patched countsketch processor's Config accepts — - // matches `crate::sketch_algebra::params::SketchParams::to_legacy`. - let epsilon = std::f64::consts::E / (p.w as f64); - let delta = 2f64.powi(-(p.d as i32)); - m.insert("epsilon".into(), Value::Number(epsilon.into())); - m.insert("delta".into(), Value::Number(delta.into())); - m.insert("encoding".into(), Value::String("msgpack".into())); - m.insert("delta_transmission".into(), Value::Bool(true)); - } - } - - Value::Mapping(m) -} - -/// Compute the gateway-side merge processor name for a `GatewayMergeProcessor`. -/// -/// Today the typed emitter populates every entry's `processor_name` -/// with the placeholder `"sketchmergeprocessor"`; the patched contrib -/// build instead has per-family merge processors: -/// `kllmerge`, `ddsketchmerge`, `hllmerge`, `countminsketchmerge`, -/// `countsketchmerge`. We map the kind to the family-specific name -/// here so the emitted YAML round-trips through the patched build. -fn gateway_merge_processor_name(mp: &GatewayMergeProcessor) -> String { - match mp.sketch_kind { - SketchKind::Kll => "kllmerge".to_string(), - SketchKind::DDSketch => "ddsketchmerge".to_string(), - SketchKind::Hll => "hllmerge".to_string(), - SketchKind::Cms => "countminsketchmerge".to_string(), - SketchKind::CountSketch => "countsketchmerge".to_string(), - } -} - -/// Build the per-merge-processor parameter block for the gateway YAML. -fn build_gateway_merge_block(mp: &GatewayMergeProcessor) -> Value { - let mut m = Mapping::new(); - m.insert("mode".into(), Value::String("merge".to_string())); - m.insert( - "aggregation_id".into(), - Value::String(mp.aggregation_id.clone()), - ); - m.insert( - "sketch_kind".into(), - Value::String(sketch_kind_tag(&mp.sketch_kind).to_string()), - ); - Value::Mapping(m) -} - -/// Build one aggregation row in the backend streaming-config JSON. -fn build_backend_aggregation_json(agg: &BackendAggregation) -> JsonValue { - let parameters = sketch_params_to_json(&agg.sketch_params); - // Phase ε.1 — surface `aggregation_input` so the backend's - // `StreamingConfig` consumer knows whether the wire payload is a - // pre-built sketch envelope (Mode 1) or raw OTLP samples the backend - // builds the sketch from at ingest (Mode 2). Phase ε.2 adds the - // raw-input ingest path; Phase ε.1 only commits the wire shape. - let aggregation_input = match agg.aggregation_input { - AggregationInput::SketchEnvelope => "sketch_envelope", - AggregationInput::Raw => "raw", - }; - json!({ - "aggregationId": agg.aggregation_id, - "aggregationType": sketch_kind_to_backend_type(&agg.sketch_kind), - "parameters": parameters, - "aggregationInput": aggregation_input, - }) -} - -/// Build one readout row in the backend streaming-config JSON. -fn build_backend_readout_json(r: &BackendReadout) -> JsonValue { - match &r.op { - EstimateOp::Quantile { q } => json!({ - "aggregationId": r.aggregation_id, - "op": "quantile", - "q": q, - }), - EstimateOp::Cardinality => json!({ - "aggregationId": r.aggregation_id, - "op": "cardinality", - }), - EstimateOp::PointCount { key } => json!({ - "aggregationId": r.aggregation_id, - "op": "point_count", - "key": key, - }), - EstimateOp::TopK { k } => json!({ - "aggregationId": r.aggregation_id, - "op": "topk", - "k": k, - }), - } -} - -/// Map a `SketchKind` to the backend's `AggregationType::Display` string -/// — the same mapping -/// [`crate::config::asapquery_backend::map_sketch_type_to_agg_type`] uses -/// (the strings must match `AggregationType::FromStr` in the backend's -/// `promql_utilities::query_logics::enums`). -fn sketch_kind_to_backend_type(kind: &SketchKind) -> &'static str { - match kind { - SketchKind::DDSketch => "DDSketch", - SketchKind::Kll => "DatasketchesKLL", - SketchKind::Hll => "HLL", - SketchKind::CountSketch => "CountSketch", - SketchKind::Cms => "CountMinSketch", - } -} - -/// Stable lowercase tag for a `SketchKind` — used as a passthrough -/// `sketch_kind` field in YAML so downstream consumers can dispatch -/// without round-tripping through serde. -fn sketch_kind_tag(kind: &SketchKind) -> &'static str { - match kind { - SketchKind::Kll => "kll", - SketchKind::DDSketch => "ddsketch", - SketchKind::Hll => "hll", - SketchKind::Cms => "cms", - SketchKind::CountSketch => "count_sketch", - } -} - -/// Serialize a `SketchParams` payload to a flat JSON object the backend -/// can read directly without round-tripping through the controller's -/// internally-tagged enum form. -fn sketch_params_to_json(p: &SketchParams) -> JsonValue { - match p { - SketchParams::Kll(p) => json!({ "k": p.k }), - SketchParams::DDSketch(p) => json!({ "alpha": p.alpha }), - SketchParams::Hll(p) => json!({ "precision": p.precision }), - SketchParams::Cms(p) => json!({ "w": p.w, "d": p.d }), - SketchParams::CountSketch(p) => { - json!({ "w": p.w, "d": p.d, "with_heap": p.with_heap }) - } - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use crate::sketch_algebra::params::{ - CmsParams, CountSketchParams, DDSketchParams, HllParams, KllParams, - }; - - fn ddsketch_edge_cfg() -> EdgeStageConfig { - EdgeStageConfig { - source_metric: Some("http_request_duration_seconds".to_string()), - label_filters: vec![("service".to_string(), "api".to_string())], - window_secs: Some(60), - sketch_processors: vec![EdgeSketchProcessor { - processor_name: "ddsketch".to_string(), - sketch_kind: SketchKind::DDSketch, - sketch_params: SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), - aggregation_id: "agg0".to_string(), - }], - exporter_target: ExportTarget::Stage(StageId::Gateway), - prometheus_archive_metrics: Vec::new(), - archive_tier_metrics: Vec::new(), - warm_passthrough_metrics: Vec::new(), - metric_to_family: HashMap::new(), - } - } - - #[test] - fn edge_yaml_contains_processor_and_pipeline_refs() { - let yaml = emit_edge_yaml(&ddsketch_edge_cfg(), "ws://ctrl:4320/v1/opamp") - .expect("emit_edge_yaml ok"); - - // Receiver block. - assert!( - yaml.contains("receivers:"), - "missing receivers section\n{yaml}" - ); - assert!(yaml.contains("otlp:"), "missing otlp receiver key\n{yaml}"); - assert!(yaml.contains("4317"), "missing gRPC port\n{yaml}"); - - // Processor key + pipeline ref. - assert!(yaml.contains("ddsketch:"), "missing ddsketch key\n{yaml}"); - assert!( - yaml.contains("- ddsketch"), - "pipeline must reference ddsketch\n{yaml}" - ); - - // Window + label filter surfaced. - assert!( - yaml.contains("window_duration: 60s"), - "missing window_duration\n{yaml}" - ); - assert!(yaml.contains("relative_accuracy"), "missing alpha\n{yaml}"); - assert!( - yaml.contains("key: service"), - "missing label matcher key\n{yaml}" - ); - assert!( - yaml.contains("value: api"), - "missing label matcher value\n{yaml}" - ); - assert!( - !yaml.contains("aggregation_id:") && !yaml.contains("sketch_kind:"), - "edge processor config must not emit planning-only fields rejected by OTel configs\n{yaml}" - ); - - // Exporter — gateway. - assert!(yaml.contains("otlp/backend:"), "missing exporter\n{yaml}"); - assert!( - yaml.contains("gateway:4317"), - "exporter should target gateway\n{yaml}" - ); - - // OpAMP extension carries the controller endpoint. - assert!( - yaml.contains("ws://ctrl:4320/v1/opamp"), - "missing opamp endpoint\n{yaml}" - ); - } - - #[test] - fn edge_yaml_kll_uses_k_param() { - let mut cfg = ddsketch_edge_cfg(); - cfg.sketch_processors[0] = EdgeSketchProcessor { - processor_name: "KLL".to_string(), - sketch_kind: SketchKind::Kll, - sketch_params: SketchParams::Kll(KllParams { k: 200 }), - aggregation_id: "agg7".to_string(), - }; - let yaml = emit_edge_yaml(&cfg, "ws://c/").expect("emit ok"); - assert!(yaml.contains("KLL:"), "{yaml}"); - assert!(yaml.contains("k: 200"), "{yaml}"); - assert!( - !yaml.contains("relative_accuracy"), - "KLL must not carry alpha\n{yaml}" - ); - assert!( - !yaml.contains("encoding:"), - "KLL Config does not accept encoding\n{yaml}" - ); - // KLL has no delta variant: the KLL's `Config.Validate` - // rejects `delta_transmission: true`. Make sure we don't emit - // the flag (a future regression that flips it on globally would - // break agent boot for KLL). - assert!( - !yaml.contains("delta_transmission"), - "KLL emit must NOT carry delta_transmission\n{yaml}" - ); - } - - #[test] - fn edge_yaml_emits_delta_transmission_for_supported_families() { - // DDSketch / HLL / CountSketch / Count-Min all support sparse - // delta encoding — the controller emits `delta_transmission: - // true` so the per-window wire footprint is the bucket / cell - // diff, not the full sketch state. KLL deliberately omits the - // flag (see `edge_yaml_kll_uses_k_param`). - for (kind, processor_name, params) in [ - ( - SketchKind::DDSketch, - "ddsketch", - SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), - ), - ( - SketchKind::Hll, - "HLL", - SketchParams::Hll(HllParams { precision: 14 }), - ), - ( - SketchKind::CountSketch, - "countsketch", - SketchParams::CountSketch(CountSketchParams { - w: 2048, - d: 5, - with_heap: true, - }), - ), - ( - SketchKind::Cms, - "countmin", - SketchParams::Cms(CmsParams { w: 4096, d: 4 }), - ), - ] { - let mut cfg = ddsketch_edge_cfg(); - cfg.sketch_processors[0] = EdgeSketchProcessor { - processor_name: processor_name.to_string(), - sketch_kind: kind, - sketch_params: params, - aggregation_id: "agg-delta".to_string(), - }; - let yaml = emit_edge_yaml(&cfg, "ws://c/").expect("emit ok"); - assert!( - yaml.contains("delta_transmission: true"), - "{processor_name:?} emit must carry delta_transmission: true\n{yaml}" - ); - } - } - - #[test] - fn edge_yaml_countmin_includes_required_metric_name() { - let mut cfg = ddsketch_edge_cfg(); - cfg.source_metric = Some("endpoint_request_freq".to_string()); - cfg.sketch_processors[0] = EdgeSketchProcessor { - processor_name: "countmin".to_string(), - sketch_kind: SketchKind::Cms, - sketch_params: SketchParams::Cms(CmsParams { w: 4096, d: 4 }), - aggregation_id: "agg-cms".to_string(), - }; - let yaml = emit_edge_yaml(&cfg, "ws://c/").expect("emit ok"); - assert!(yaml.contains("countmin:"), "{yaml}"); - assert!( - yaml.contains("metric_name: endpoint_request_freq"), - "{yaml}" - ); - } - - #[test] - fn edge_yaml_batch_mode_when_no_window() { - let mut cfg = ddsketch_edge_cfg(); - cfg.window_secs = None; - let yaml = emit_edge_yaml(&cfg, "ws://c/").expect("emit ok"); - assert!(yaml.contains("mode: batch"), "{yaml}"); - assert!( - !yaml.contains("window_duration"), - "batch mode must not have window_duration\n{yaml}" - ); - } - - fn ddsketch_gateway_cfg() -> GatewayStageConfig { - GatewayStageConfig { - otlp_receiver_port: 4317, - merge_processors: vec![GatewayMergeProcessor { - processor_name: "sketchmergeprocessor".to_string(), - sketch_kind: SketchKind::DDSketch, - aggregation_id: "agg0".to_string(), - }], - exporter_target: ExportTarget::Stage(StageId::Backend), - } - } - - #[test] - fn gateway_yaml_uses_family_specific_merge_name() { - let yaml = emit_gateway_yaml(&ddsketch_gateway_cfg(), "ws://ctrl:4320/v1/opamp") - .expect("emit_gateway_yaml ok"); - - // Family-specific merge name (NOT the placeholder). - assert!(yaml.contains("ddsketchmerge:"), "{yaml}"); - assert!(yaml.contains("- ddsketchmerge"), "{yaml}"); - assert!( - !yaml.contains("sketchmergeprocessor"), - "placeholder must be replaced\n{yaml}" - ); - - // Receiver bound to declared port. - assert!(yaml.contains("0.0.0.0:4317"), "{yaml}"); - - // Aggregation id threaded through. - assert!(yaml.contains("aggregation_id: agg0"), "{yaml}"); - - // Exporter targets backend. - assert!(yaml.contains("backend:4317"), "{yaml}"); - - // OpAMP endpoint embedded. - assert!(yaml.contains("ws://ctrl:4320/v1/opamp"), "{yaml}"); - } - - #[test] - fn gateway_yaml_emits_one_processor_per_merge_entry() { - let cfg = GatewayStageConfig { - otlp_receiver_port: 4317, - merge_processors: vec![ - GatewayMergeProcessor { - processor_name: "x".into(), - sketch_kind: SketchKind::Kll, - aggregation_id: "agg0".into(), - }, - GatewayMergeProcessor { - processor_name: "x".into(), - sketch_kind: SketchKind::Hll, - aggregation_id: "agg1".into(), - }, - ], - exporter_target: ExportTarget::Stage(StageId::Backend), - }; - let yaml = emit_gateway_yaml(&cfg, "ws://c/").expect("emit ok"); - assert!(yaml.contains("kllmerge:"), "{yaml}"); - assert!(yaml.contains("hllmerge:"), "{yaml}"); - assert!( - yaml.contains("- kllmerge"), - "pipeline missing kll merge\n{yaml}" - ); - assert!( - yaml.contains("- hllmerge"), - "pipeline missing hll merge\n{yaml}" - ); - } - - #[test] - fn backend_json_round_trips_aggregations_and_readouts() { - let cfg = BackendStageConfig { - aggregations: vec![ - BackendAggregation { - aggregation_id: "agg0".into(), - sketch_kind: SketchKind::DDSketch, - sketch_params: SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), - aggregation_input: AggregationInput::SketchEnvelope, - }, - BackendAggregation { - aggregation_id: "agg1".into(), - sketch_kind: SketchKind::Hll, - sketch_params: SketchParams::Hll(HllParams { precision: 14 }), - aggregation_input: AggregationInput::SketchEnvelope, - }, - ], - readouts: vec![ - BackendReadout { - aggregation_id: "agg0".into(), - op: EstimateOp::Quantile { q: 0.99 }, - }, - BackendReadout { - aggregation_id: "agg1".into(), - op: EstimateOp::Cardinality, - }, - ], - }; - let v = emit_backend_config_json(&cfg).expect("emit ok"); - - let aggs = v["aggregations"].as_array().expect("aggregations array"); - assert_eq!(aggs.len(), 2, "{v}"); - assert_eq!(aggs[0]["aggregationId"], "agg0"); - assert_eq!(aggs[0]["aggregationType"], "DDSketch"); - assert_eq!(aggs[0]["parameters"]["alpha"], 0.01); - assert_eq!(aggs[1]["aggregationType"], "HLL"); - assert_eq!(aggs[1]["parameters"]["precision"], 14); - - let reads = v["readouts"].as_array().expect("readouts array"); - assert_eq!(reads.len(), 2, "{v}"); - assert_eq!(reads[0]["op"], "quantile"); - assert_eq!(reads[0]["q"], 0.99); - assert_eq!(reads[1]["op"], "cardinality"); - } - - #[test] - fn backend_json_handles_topk_and_pointcount_readouts() { - let cfg = BackendStageConfig { - aggregations: vec![ - BackendAggregation { - aggregation_id: "agg0".into(), - sketch_kind: SketchKind::CountSketch, - sketch_params: SketchParams::CountSketch(CountSketchParams { - w: 2048, - d: 5, - with_heap: true, - }), - aggregation_input: AggregationInput::SketchEnvelope, - }, - BackendAggregation { - aggregation_id: "agg1".into(), - sketch_kind: SketchKind::Cms, - sketch_params: SketchParams::Cms(CmsParams { w: 4096, d: 4 }), - aggregation_input: AggregationInput::SketchEnvelope, - }, - ], - readouts: vec![ - BackendReadout { - aggregation_id: "agg0".into(), - op: EstimateOp::TopK { k: 10 }, - }, - BackendReadout { - aggregation_id: "agg1".into(), - op: EstimateOp::PointCount { - key: "user_42".into(), - }, - }, - ], - }; - let v = emit_backend_config_json(&cfg).expect("emit ok"); - let reads = v["readouts"].as_array().unwrap(); - assert_eq!(reads[0]["op"], "topk"); - assert_eq!(reads[0]["k"], 10); - assert_eq!(reads[1]["op"], "point_count"); - assert_eq!(reads[1]["key"], "user_42"); - - let aggs = v["aggregations"].as_array().unwrap(); - assert_eq!(aggs[0]["aggregationType"], "CountSketch"); - assert_eq!(aggs[0]["parameters"]["with_heap"], true); - assert_eq!(aggs[1]["aggregationType"], "CountMinSketch"); - assert_eq!(aggs[1]["parameters"]["w"], 4096); - } - - #[test] - fn export_target_endpoint_is_passed_through_verbatim() { - let mut cfg = ddsketch_edge_cfg(); - cfg.exporter_target = ExportTarget::Endpoint("custom-gw:5317".into()); - let yaml = emit_edge_yaml(&cfg, "ws://c/").expect("emit ok"); - assert!(yaml.contains("custom-gw:5317"), "{yaml}"); - } - - // ── Phase α: BackendStorageRouting emitter tests ────────────────────── - - /// Helper: build a single-aggregation BackendStageConfig of the - /// requested kind. `aggregation_id` is hard-coded — the routing - /// emitter doesn't care about it. - fn backend_cfg_with_kind(kind: SketchKind) -> BackendStageConfig { - let params = match kind { - SketchKind::DDSketch => SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), - SketchKind::Kll => SketchParams::Kll(KllParams { k: 200 }), - SketchKind::Hll => SketchParams::Hll(HllParams { precision: 14 }), - SketchKind::Cms => SketchParams::Cms(CmsParams { w: 4096, d: 4 }), - SketchKind::CountSketch => SketchParams::CountSketch(CountSketchParams { - w: 2048, - d: 5, - with_heap: true, - }), - }; - BackendStageConfig { - aggregations: vec![BackendAggregation { - aggregation_id: "agg0".into(), - sketch_kind: kind.clone(), - sketch_params: params, - aggregation_input: AggregationInput::SketchEnvelope, - }], - readouts: vec![BackendReadout { - aggregation_id: "agg0".into(), - op: match kind { - SketchKind::DDSketch | SketchKind::Kll => EstimateOp::Quantile { q: 0.99 }, - SketchKind::Hll => EstimateOp::Cardinality, - SketchKind::CountSketch => EstimateOp::TopK { k: 10 }, - SketchKind::Cms => EstimateOp::PointCount { - key: "user_42".into(), - }, - }, - }], - } - } - - #[test] - fn storage_routing_emits_default_engine_and_metrics_array() { - let ddsketch = backend_cfg_with_kind(SketchKind::DDSketch); - let plans: Vec<(String, &BackendStageConfig)> = - vec![("http_request_duration_seconds".to_string(), &ddsketch)]; - let v = emit_backend_storage_routing(&plans).expect("emit ok"); - assert_eq!(v["default_engine"], "sketch_warm_tier"); - let metrics = v["metrics"].as_array().expect("metrics array"); - assert_eq!(metrics.len(), 1); - assert_eq!(metrics[0]["name"], "http_request_duration_seconds"); - } - - // ── Per-tenant routing emit tests (follow-up to PR #333) ────────────── - - /// Single-tenant entry point — the convenience - /// [`emit_backend_storage_routing`] alias must keep emitting the - /// `default` tenant id so existing single-tenant deploys are - /// byte-compatible (modulo the new `tenant` field appearing). - #[test] - fn storage_routing_default_tenant_for_single_tenant_emit() { - let ddsketch = backend_cfg_with_kind(SketchKind::DDSketch); - let v = emit_backend_storage_routing(&[("latency".into(), &ddsketch)]).expect("emit ok"); - assert_eq!(v["tenant"], DEFAULT_TENANT); - } - - /// Per-tenant entry point — explicit `tenant` arg lands in the - /// emitted JSON's top-level `tenant` field. Other fields are - /// unchanged from the single-tenant emit, so the backend's - /// per-tenant swap routes to the named tenant's slot via the - /// body-tenant precedence rule. - #[test] - fn storage_routing_for_tenant_emits_explicit_tenant_field() { - let ddsketch = backend_cfg_with_kind(SketchKind::DDSketch); - let v = - emit_backend_storage_routing_for_tenant("tenant-a", &[("latency".into(), &ddsketch)]) - .expect("emit ok"); - assert_eq!(v["tenant"], "tenant-a"); - assert_eq!(v["default_engine"], "sketch_warm_tier"); - // Single metric, single warm + archive target shape — the - // per-tenant emit doesn't change the metric-side shape. - let metrics = v["metrics"].as_array().expect("metrics array"); - assert_eq!(metrics.len(), 1); - assert_eq!(metrics[0]["name"], "latency"); - } - - /// Per-tenant variant of the prometheus-aware emit — tenant - /// scope must thread through Mode-3 metrics too. - #[test] - fn storage_routing_with_prometheus_for_tenant_emits_explicit_tenant_field() { - let mode3 = vec!["http_requests_total".to_string()]; - let v = emit_backend_storage_routing_with_prometheus_for_tenant("tenant-b", &[], &mode3) - .expect("emit ok"); - assert_eq!(v["tenant"], "tenant-b"); - assert_eq!(v["metrics"][0]["name"], "http_requests_total"); - assert_eq!(v["metrics"][0]["targets"][0]["engine"], "prometheus_remote"); - } - - #[test] - fn storage_routing_ddsketch_warm_serves_quantile_archive_serves_others() { - let ddsketch = backend_cfg_with_kind(SketchKind::DDSketch); - let v = emit_backend_storage_routing(&[("latency".into(), &ddsketch)]).expect("emit ok"); - let metric = &v["metrics"][0]; - let targets = metric["targets"].as_array().expect("targets array"); - - // Default slot — warm tier, no filter. - assert_eq!(targets[0]["engine"], "sketch_warm_tier"); - assert!( - targets[0].get("applies_to_query_shape").is_none(), - "warm slot must be the default (no filter); got {targets:?}" - ); - - // Archive slot — must carry the predictable archive shapes. - assert_eq!(targets[1]["engine"], "thanos_archive"); - let archive_shapes: Vec = targets[1]["applies_to_query_shape"] - .as_array() - .unwrap() - .iter() - .map(|s| s.as_str().unwrap().to_string()) - .collect(); - assert!(archive_shapes.contains(&"histogram_quantile".to_string())); - assert!(archive_shapes.contains(&"delta".to_string())); - assert!(archive_shapes.contains(&"absent".to_string())); - assert!(archive_shapes.contains(&"rate_post_hoc".to_string())); - // DDSketch planned → `topk` and `count` not warm-tier-eligible - // (only quantile is). Both stay in archive's claim list. - assert!(archive_shapes.contains(&"topk".to_string())); - assert!(archive_shapes.contains(&"count".to_string())); - - // Warm-tier native shapes surfaced for spot-check. - let warm_native: Vec = metric["warm_tier_native_shapes"] - .as_array() - .unwrap() - .iter() - .map(|s| s.as_str().unwrap().to_string()) - .collect(); - assert!(warm_native.contains(&"quantile".to_string())); - assert!(warm_native.contains(&"quantile_over_time".to_string())); - } - - #[test] - fn storage_routing_count_sketch_pulls_topk_off_archive() { - let cs = backend_cfg_with_kind(SketchKind::CountSketch); - let v = emit_backend_storage_routing(&[("requests".into(), &cs)]).expect("emit ok"); - let archive_shapes: Vec = v["metrics"][0]["targets"][1]["applies_to_query_shape"] - .as_array() - .unwrap() - .iter() - .map(|s| s.as_str().unwrap().to_string()) - .collect(); - // Count-Sketch planned → warm tier serves `topk`, archive - // claim list must NOT include topk. - assert!( - !archive_shapes.contains(&"topk".to_string()), - "Count-Sketch planned ⇒ topk must drop off the archive list; got {archive_shapes:?}" - ); - // `count` still routes to archive (no HLL / CMS). - assert!(archive_shapes.contains(&"count".to_string())); - } - - #[test] - fn storage_routing_hll_pulls_count_off_archive() { - let hll = backend_cfg_with_kind(SketchKind::Hll); - let v = emit_backend_storage_routing(&[("active_users".into(), &hll)]).expect("emit ok"); - let archive_shapes: Vec = v["metrics"][0]["targets"][1]["applies_to_query_shape"] - .as_array() - .unwrap() - .iter() - .map(|s| s.as_str().unwrap().to_string()) - .collect(); - // HLL planned → warm tier serves `count` (cardinality); - // archive claim list must NOT include count. `topk` still - // routes to archive (no Count-Sketch). - assert!( - !archive_shapes.contains(&"count".to_string()), - "HLL planned ⇒ count must drop off the archive list; got {archive_shapes:?}" - ); - assert!(archive_shapes.contains(&"topk".to_string())); - } - - #[test] - fn storage_routing_three_metric_snapshot_stable() { - // Snapshot test: three metrics with three different sketch - // families. The serialized form must be deterministic across - // runs (HashMap iteration order can drift, but our impl - // stages everything through a Vec so order matches input - // order). - let ddsketch = backend_cfg_with_kind(SketchKind::DDSketch); - let hll = backend_cfg_with_kind(SketchKind::Hll); - let cs = backend_cfg_with_kind(SketchKind::CountSketch); - let plans: Vec<(String, &BackendStageConfig)> = vec![ - ("http_requests_total".into(), &cs), - ("active_users".into(), &hll), - ("request_latency_seconds".into(), &ddsketch), - ]; - let v = emit_backend_storage_routing(&plans).expect("emit ok"); - let s = serde_json::to_string_pretty(&v).expect("ser"); - - // Pretty-print the snapshot for easy regression diffing. - // Per-tenant follow-up to PR #333: the top-level `tenant` - // field is now emitted (defaults to `"default"` for the - // single-tenant entry point). The `serde_json::Value` map - // serialises keys alphabetically, so `tenant` lands at the - // end of the document. - let expected = r#"{ - "default_engine": "sketch_warm_tier", - "metrics": [ - { - "name": "http_requests_total", - "targets": [ - { - "engine": "sketch_warm_tier" - }, - { - "applies_to_query_shape": [ - "histogram_quantile", - "delta", - "deriv", - "absent", - "rate_post_hoc", - "count" - ], - "engine": "thanos_archive" - } - ], - "warm_tier_native_shapes": [ - "topk", - "rate", - "sum", - "avg", - "min", - "max" - ] - }, - { - "name": "active_users", - "targets": [ - { - "engine": "sketch_warm_tier" - }, - { - "applies_to_query_shape": [ - "histogram_quantile", - "delta", - "deriv", - "absent", - "rate_post_hoc", - "topk" - ], - "engine": "thanos_archive" - } - ], - "warm_tier_native_shapes": [ - "count", - "rate", - "sum", - "avg", - "min", - "max" - ] - }, - { - "name": "request_latency_seconds", - "targets": [ - { - "engine": "sketch_warm_tier" - }, - { - "applies_to_query_shape": [ - "histogram_quantile", - "delta", - "deriv", - "absent", - "rate_post_hoc", - "topk", - "count" - ], - "engine": "thanos_archive" - } - ], - "warm_tier_native_shapes": [ - "quantile", - "quantile_over_time", - "rate", - "sum", - "avg", - "min", - "max" - ] - } - ], - "tenant": "default" -}"#; - assert_eq!(s, expected, "snapshot mismatch:\n{s}"); - } - - #[test] - fn storage_routing_empty_input_emits_empty_metrics_array() { - let v = emit_backend_storage_routing(&[]).expect("emit ok"); - assert_eq!(v["default_engine"], "sketch_warm_tier"); - assert_eq!(v["metrics"].as_array().unwrap().len(), 0); - } - - #[test] - fn storage_routing_empty_aggregations_still_emits_archive_default() { - // A plan with no aggregations (degenerate; should not happen - // in practice but we don't want to panic). The metric still - // lands in the table as archive-only — no warm-tier-native - // shapes, no warm-tier annotation field. - let cfg = BackendStageConfig { - aggregations: vec![], - readouts: vec![], - }; - let v = emit_backend_storage_routing(&[("orphan".into(), &cfg)]).expect("emit ok"); - let metric = &v["metrics"][0]; - assert_eq!(metric["name"], "orphan"); - // No warm_tier_native_shapes side field. - assert!(metric.get("warm_tier_native_shapes").is_none()); - // Targets: warm-tier default + archive default-shape list. - let targets = metric["targets"].as_array().unwrap(); - assert_eq!(targets[0]["engine"], "sketch_warm_tier"); - assert_eq!(targets[1]["engine"], "thanos_archive"); - } - - // ── Phase β: emit_backend_config_json snapshot for new pattern coverage ── - // - // The new archive-only L3 intents (HistogramQuantile, Absent, Delta, …) - // bind to `SketchExpr::Logical` rather than producing a `BackendAggregation`, - // so they correctly stay OUT of the warm-tier StreamingConfig the - // backend's SimpleEngine receives. Phase α wires the archive routing - // entry separately. This snapshot pins that contract. - - /// Snapshot: an empty `BackendStageConfig` produces the canonical - /// `{"aggregations": [], "readouts": []}` shape — what the backend - /// receives when every intent in the workload is archive-only. - #[test] - fn phase_b_empty_warm_tier_snapshot_for_all_archive_only_workload() { - let cfg = BackendStageConfig { - aggregations: vec![], - readouts: vec![], - }; - let v = emit_backend_config_json(&cfg).expect("emit ok"); - let s = serde_json::to_string(&v).unwrap(); - assert_eq!(s, r#"{"aggregations":[],"readouts":[]}"#); - } - - /// Snapshot: every Phase β warm-tier-bound intent (KLL/DDSketch - /// quantile, HLL cardinality, CMS frequency, CountSketch topk) maps to - /// a stable `aggregationType` string the backend's `AggregationType:: - /// FromStr` recognises. This is the contract the L4 → L5 → backend - /// pipeline relies on; pinning it here so a sketch-kind rename can't - /// silently break the backend. - #[test] - fn phase_b_backend_agg_type_strings_for_every_sketch_kind() { - let cases = vec![ - (SketchKind::Kll, "DatasketchesKLL"), - (SketchKind::DDSketch, "DDSketch"), - (SketchKind::Hll, "HLL"), - (SketchKind::Cms, "CountMinSketch"), - (SketchKind::CountSketch, "CountSketch"), - ]; - for (kind, expected) in cases { - assert_eq!( - sketch_kind_to_backend_type(&kind), - expected, - "sketch_kind_to_backend_type({kind:?}) drift — backend FromStr will reject" - ); - } - } - - /// Snapshot: aggregations + readouts together exhibit the - /// id-aliasing the backend uses to wire readouts back to their - /// producing aggregation. Pins the sort order + key names. Phase β - /// uses this as the wire-format anchor for the wider intent set — - /// the JSON shape is intent-orthogonal, so adding new intents to L3 - /// can't drift this off so long as they bind through SketchKind / - /// SketchParams. - #[test] - fn phase_b_backend_json_aggregation_readout_alias_snapshot() { - let cfg = BackendStageConfig { - aggregations: vec![BackendAggregation { - aggregation_id: "phase_b_agg0".into(), - sketch_kind: SketchKind::Kll, - sketch_params: SketchParams::Kll(KllParams { k: 200 }), - aggregation_input: AggregationInput::SketchEnvelope, - }], - readouts: vec![BackendReadout { - aggregation_id: "phase_b_agg0".into(), - op: EstimateOp::Quantile { q: 0.99 }, - }], - }; - let v = emit_backend_config_json(&cfg).expect("emit ok"); - // The id surfaces on both the agg and the readout, with the same - // key name — the backend looks the readout up by `aggregationId`. - assert_eq!(v["aggregations"][0]["aggregationId"], "phase_b_agg0"); - assert_eq!(v["readouts"][0]["aggregationId"], "phase_b_agg0"); - assert_eq!(v["aggregations"][0]["aggregationType"], "DatasketchesKLL"); - assert_eq!(v["aggregations"][0]["parameters"]["k"], 200); - assert_eq!(v["readouts"][0]["op"], "quantile"); - assert_eq!(v["readouts"][0]["q"], 0.99); - } - - // ── Phase ε.1: three-mode wire shape tests ──────────────────────────── - - /// Mode 1 (sketch at edge) keeps the existing aggregation_input - /// default — `sketch_envelope` — so legacy plans round-trip - /// unchanged. - #[test] - fn phase_eps1_mode1_aggregation_input_is_sketch_envelope() { - let cfg = BackendStageConfig { - aggregations: vec![BackendAggregation { - aggregation_id: "agg0".into(), - sketch_kind: SketchKind::DDSketch, - sketch_params: SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), - aggregation_input: AggregationInput::SketchEnvelope, - }], - readouts: vec![], - }; - let v = emit_backend_config_json(&cfg).expect("emit ok"); - assert_eq!(v["aggregations"][0]["aggregationInput"], "sketch_envelope"); - } - - /// Mode 2 (raw at edge → sketch at backend) sets - /// `aggregation_input: raw` so the backend builds the sketch from - /// raw OTLP samples at ingest. Phase ε.2 implements the raw-input - /// ingest path on the backend. - #[test] - fn phase_eps1_mode2_aggregation_input_is_raw() { - let cfg = BackendStageConfig { - aggregations: vec![BackendAggregation { - aggregation_id: "agg0".into(), - sketch_kind: SketchKind::DDSketch, - sketch_params: SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), - aggregation_input: AggregationInput::Raw, - }], - readouts: vec![], - }; - let v = emit_backend_config_json(&cfg).expect("emit ok"); - assert_eq!(v["aggregations"][0]["aggregationInput"], "raw"); - } - - /// Mode 3 (Prometheus archive) — the routing emitter adds a - /// `prometheus_remote` engine target for the metric. The backend's - /// HTTP query handler HTTP-forwards the matching PromQL queries to - /// `${ASAP_PROMETHEUS_QUERY_URL}/api/v1/query`. Phase ε.2 registers - /// the engine on the backend. - #[test] - fn phase_eps1_mode3_storage_routing_emits_prometheus_remote() { - // No backend-side aggregations for mode 3 — Prometheus owns it. - let mode3 = vec!["http_requests_total".to_string()]; - let v = emit_backend_storage_routing_with_prometheus(&[], &mode3).expect("emit ok"); - let metrics = v["metrics"].as_array().unwrap(); - assert_eq!(metrics.len(), 1); - assert_eq!(metrics[0]["name"], "http_requests_total"); - let targets = metrics[0]["targets"].as_array().unwrap(); - assert_eq!(targets.len(), 1); - assert_eq!(targets[0]["engine"], "prometheus_remote"); - // No shape filter — Prometheus serves every PromQL shape. - assert!(targets[0].get("applies_to_query_shape").is_none()); - // `asap_mode` annotation surfaces so operators can see why a - // metric routes off warm tier. - assert_eq!(metrics[0]["asap_mode"], "prometheus_archive"); - } - - /// Mode 1 + Mode 3 mixed in one cycle — warm-tier metric AND - /// Prometheus-archive metric coexist in one routing JSON. - #[test] - fn phase_eps1_mixed_mode1_and_mode3_share_one_routing_table() { - let ddsketch = backend_cfg_with_kind(SketchKind::DDSketch); - let plans: Vec<(String, &BackendStageConfig)> = vec![("latency_seconds".into(), &ddsketch)]; - let mode3 = vec!["http_requests_total".to_string()]; - let v = emit_backend_storage_routing_with_prometheus(&plans, &mode3).expect("emit ok"); - let metrics = v["metrics"].as_array().unwrap(); - assert_eq!(metrics.len(), 2); - assert_eq!(metrics[0]["name"], "latency_seconds"); - // Mode-1 entry — full warm/archive routing. - let m1_targets = metrics[0]["targets"].as_array().unwrap(); - assert_eq!(m1_targets[0]["engine"], "sketch_warm_tier"); - assert_eq!(m1_targets[1]["engine"], "thanos_archive"); - // Mode-3 entry — single prometheus_remote target. - assert_eq!(metrics[1]["name"], "http_requests_total"); - let m3_targets = metrics[1]["targets"].as_array().unwrap(); - assert_eq!(m3_targets.len(), 1); - assert_eq!(m3_targets[0]["engine"], "prometheus_remote"); - } - - /// Mode 3 emit_edge_yaml — produces a YAML with `otlphttp/prometheus` - /// exporter pointing at `/api/v1/otlp/v1/metrics`, plus the routing - /// processor that dispatches per-metric on `asap.mode`. - #[test] - fn phase_eps1_mode3_edge_yaml_has_otlphttp_prometheus_exporter() { - let cfg = EdgeStageConfig { - source_metric: Some("http_requests_total".to_string()), - label_filters: Vec::new(), - window_secs: Some(60), - sketch_processors: Vec::new(), - exporter_target: ExportTarget::Stage(StageId::Gateway), - prometheus_archive_metrics: vec![PrometheusArchiveMetric { - metric: "http_requests_total".to_string(), - window_secs: Some(60), - label_proj: vec!["service.name".to_string()], - }], - // RawAtEdgePrometheusArchive auto-populates the archive - // tier list as well (Phase 3.2.5): the Mode-3 metric also - // lands in the Gorilla-S3 archive so the warm-tier engine - // can serve last_over_time(...) queries. - archive_tier_metrics: vec![ArchiveTierMetric { - metric: "http_requests_total".to_string(), - window_secs: Some(60), - }], - warm_passthrough_metrics: Vec::new(), - metric_to_family: HashMap::new(), - }; - let yaml = emit_edge_yaml(&cfg, "ws://c/").expect("emit ok"); - - // Exporter — Prometheus's native OTLP receiver, full path. - assert!( - yaml.contains("otlphttp/prometheus:"), - "missing otlphttp/prometheus exporter\n{yaml}" - ); - assert!( - yaml.contains("/api/v1/otlp/v1/metrics"), - "exporter should hit Prometheus's native OTLP path\n{yaml}" - ); - assert!( - yaml.contains("ASAP_PROMETHEUS_OTLP_URL"), - "endpoint should be env-overridable for the deploy team\n{yaml}" - ); - // `encoding: proto` — the Prometheus OTLP receiver expects - // protobuf-encoded OTLP HTTP, not JSON. - assert!( - yaml.contains("encoding: proto"), - "encoding should be proto\n{yaml}" - ); - - // Routing processor — dispatches by `asap.mode`. - assert!( - yaml.contains("routing:"), - "missing routing processor\n{yaml}" - ); - assert!( - yaml.contains("from_attribute: asap.mode"), - "routing should dispatch by asap.mode\n{yaml}" - ); - assert!( - yaml.contains("prometheus_archive"), - "routing must match prometheus_archive value\n{yaml}" - ); - - // Two named pipelines + the routing entry pipeline. - assert!( - yaml.contains("metrics/prometheus_archive:"), - "missing metrics/prometheus_archive pipeline\n{yaml}" - ); - assert!( - yaml.contains("metrics/warm_tier:"), - "missing metrics/warm_tier pipeline\n{yaml}" - ); - } - - /// When no Mode 3 metrics are configured, the edge YAML stays - /// single-pipeline (no routing processor, no otlphttp/prometheus - /// exporter) — preserves the existing Phase β layout for backward - /// compatibility. - #[test] - fn phase_eps1_no_mode3_edge_yaml_unchanged_from_phase_b() { - let cfg = ddsketch_edge_cfg(); - let yaml = emit_edge_yaml(&cfg, "ws://c/").expect("emit ok"); - assert!( - !yaml.contains("otlphttp/prometheus"), - "no Mode 3 → no otlphttp/prometheus\n{yaml}" - ); - assert!( - !yaml.contains("metrics/prometheus_archive"), - "no Mode 3 → no archive pipeline\n{yaml}" - ); - assert!( - !yaml.contains("metrics/warm_tier"), - "no Mode 3 → main pipeline keeps the legacy `metrics:` name\n{yaml}" - ); - // Phase 3.2.5 — without archive_tier_metrics no gorillas3 block. - assert!( - !yaml.contains("gorillas3"), - "no archive tier → no gorillas3 processor\n{yaml}" - ); - } - - // ── Phase 3.2.5 Bug (a) — gorillas3 in the emitted edge YAML ──────────── - - /// Bug (a): when at least one archive-tier metric is configured the - /// emitted YAML MUST include the `gorillas3` processor block + the - /// processor MUST be in the warm-tier pipeline. Without this freshness - /// probes (and any other archive-bound metric) never reach MinIO so - /// the warm-tier engine's `last_over_time(...)` returns empty. - #[test] - fn phase_3_2_5_bug_a_archive_tier_metrics_emit_gorillas3_processor() { - let mut cfg = ddsketch_edge_cfg(); - cfg.archive_tier_metrics = vec![ArchiveTierMetric { - metric: "http_freshness_probe_archive".to_string(), - window_secs: Some(5), - }]; - let yaml = emit_edge_yaml(&cfg, "ws://c/").expect("emit ok"); - - // Processor block surfaced at the top level. - assert!( - yaml.contains("gorillas3:"), - "missing gorillas3 processor block\n{yaml}" - ); - // Critical knobs the gorillas3processor's Config requires + the - // ones the demo overlay inherits via env override. - assert!( - yaml.contains("block_format: prometheus_tsdb"), - "gorillas3 must emit prometheus_tsdb blocks for the Thanos sidecar\n{yaml}" - ); - assert!( - yaml.contains("tsdb_bucket"), - "gorillas3 needs a TSDBBucket so the Thanos store-gateway can read the blocks\n{yaml}" - ); - assert!( - yaml.contains("ASAP_MINIO_ENDPOINT"), - "endpoint should be env-overridable for the deploy team\n{yaml}" - ); - // `drop_original: false` so the metric ALSO flows downstream - // through the warm-tier sketch / OTLP exporter (without this - // the warm tier never sees the metric). - assert!( - yaml.contains("drop_original: false"), - "drop_original must be false so warm-tier sketches still see the metric\n{yaml}" - ); - // Processor name in the pipeline list. - assert!( - yaml.contains("- gorillas3"), - "gorillas3 must appear in the warm-tier pipeline processors\n{yaml}" - ); - // window_interval picked up from the smallest declared - // window_secs — 5 here, matching the freshness-probe spec. - assert!( - yaml.contains("window_interval: 5s"), - "gorillas3 window_interval must reflect the smallest archive-tier window\n{yaml}" - ); - } - - /// Bug (a) corollary: gorillas3 runs BEFORE the sketch processor in - /// the warm-tier pipeline so the cold-tier write happens on raw - /// samples — mirrors `asap-otel-agent-b6-asap-single-sketch.yaml`'s - /// canonical `[gorillas3, ddsketch, batch]` ordering. - #[test] - fn phase_3_2_5_bug_a_gorillas3_runs_before_sketch_in_pipeline() { - let mut cfg = ddsketch_edge_cfg(); - cfg.archive_tier_metrics = vec![ArchiveTierMetric { - metric: "http_freshness_probe_archive".to_string(), - window_secs: Some(5), - }]; - let yaml = emit_edge_yaml(&cfg, "ws://c/").expect("emit ok"); - - // Find the pipeline processor list — should contain gorillas3 - // ahead of ddsketch in the serialized order. Robust - // search: locate the `processors:` block under the metrics - // pipeline and check substring positions. - let pipeline_idx = yaml.find("metrics:\n").unwrap_or_default(); - let after_pipeline = &yaml[pipeline_idx..]; - let g_idx = after_pipeline - .find("- gorillas3") - .expect("- gorillas3 missing in pipeline"); - let s_idx = after_pipeline - .find("- ddsketch") - .expect("- ddsketch missing in pipeline"); - assert!( - g_idx < s_idx, - "gorillas3 must come BEFORE ddsketch in the warm pipeline\n{yaml}" - ); - } - - // ── Phase 3.2.5 Bug (b) — warm-tier passthrough routing ───────────────── - - /// Bug (b): freshness probes (and other counters whose value IS - /// the signal) must bypass the family-specific sketch processor so - /// the metric name is preserved end-to-end. The L5 emitter adds a - /// `routing` processor with OTTL `route()` statements that dispatch - /// listed metrics to a `metrics/warm_passthrough` pipeline; everything - /// else takes `metrics/warm_tier` as before. - #[test] - fn phase_3_2_5_bug_b_warm_passthrough_routes_around_sketch() { - let mut cfg = ddsketch_edge_cfg(); - cfg.archive_tier_metrics = vec![ArchiveTierMetric { - metric: "http_freshness_probe_warm".to_string(), - window_secs: Some(1), - }]; - cfg.warm_passthrough_metrics = vec!["http_freshness_probe_warm".to_string()]; - let yaml = emit_edge_yaml(&cfg, "ws://c/").expect("emit ok"); - - // Routing processor present, dispatches by metric name (OTTL form). - assert!( - yaml.contains("routing:"), - "missing routing processor\n{yaml}" - ); - assert!( - yaml.contains("route() where metric.name == \"http_freshness_probe_warm\""), - "routing must match on metric.name\n{yaml}" - ); - assert!( - yaml.contains("metrics/warm_passthrough"), - "warm_passthrough pipeline target must be referenced\n{yaml}" - ); - - // Both pipelines exist. - assert!( - yaml.contains("metrics/warm_tier:"), - "missing metrics/warm_tier pipeline\n{yaml}" - ); - assert!( - yaml.contains("metrics/warm_passthrough:"), - "missing metrics/warm_passthrough pipeline\n{yaml}" - ); - - // Critical assertion: the warm_passthrough pipeline does NOT - // reference the family-specific sketch processor — that's the - // whole point of routing around DDSketch. - let passthrough_idx = yaml - .find("metrics/warm_passthrough:") - .expect("warm_passthrough section not found"); - // Slice to the next pipeline (or end of file). - let after = &yaml[passthrough_idx..]; - let next_pipeline_offset = after[1..] - .find("metrics/") - .map(|x| x + 1) - .unwrap_or(after.len()); - let passthrough_section = &after[..next_pipeline_offset]; - assert!( - !passthrough_section.contains("ddsketch"), - "warm_passthrough pipeline must NOT include ddsketch (the bug we're fixing)\n{yaml}" - ); - // ... but it SHOULD still include gorillas3 so the metric - // lands in the archive (the warm engine queries it from - // there). - assert!( - passthrough_section.contains("gorillas3"), - "warm_passthrough pipeline still routes through gorillas3 for archive write\n{yaml}" - ); - } - - /// Bug (b) corollary: warm_passthrough composes cleanly with the - /// Phase ε.1 prometheus_archive routing — single routing processor - /// with both an `asap.mode` and a `metric.name` table entry. - #[test] - fn phase_3_2_5_bug_b_warm_passthrough_composes_with_prometheus_archive() { - let mut cfg = ddsketch_edge_cfg(); - cfg.archive_tier_metrics = vec![ArchiveTierMetric { - metric: "http_freshness_probe_warm".to_string(), - window_secs: Some(1), - }]; - cfg.warm_passthrough_metrics = vec!["http_freshness_probe_warm".to_string()]; - cfg.prometheus_archive_metrics = vec![PrometheusArchiveMetric { - metric: "http_requests_total".to_string(), - window_secs: Some(60), - label_proj: Vec::new(), - }]; - let yaml = emit_edge_yaml(&cfg, "ws://c/").expect("emit ok"); - - // OTTL form gives us a single routing processor that handles - // both dispatch axes. - assert!( - yaml.contains("route() where metric.name"), - "must dispatch by metric name (warm_passthrough)\n{yaml}" - ); - assert!( - yaml.contains("attributes[\\\"asap.mode\\\"]") - || yaml.contains("attributes['asap.mode']") - || yaml.contains("attributes[\"asap.mode\"]"), - "must dispatch by asap.mode (prometheus_archive)\n{yaml}" - ); - assert!( - yaml.contains("metrics/prometheus_archive:"), - "prometheus_archive pipeline still emitted\n{yaml}" - ); - assert!( - yaml.contains("metrics/warm_passthrough:"), - "warm_passthrough pipeline still emitted\n{yaml}" - ); - assert!( - yaml.contains("metrics/warm_tier:"), - "warm_tier (default) pipeline still emitted\n{yaml}" - ); - } - - // ── MVP §46: 5-sketch routing-connector edge YAML emit tests ────────── - // - // The new emit path activates when `cfg.metric_to_family` is - // non-empty. These tests pin: - // * All 5 sketch processors in `processors:` regardless of which - // metrics route to them (runtime swap → zero pipeline graph - // change). - // * `routing` in `connectors:` (NOT `processors:`) — the real - // bugfix; `routingprocessor` was removed in OTel-collector - // v0.106 so emitting it would fail agent boot. - // * All 6 named pipelines: entry `metrics:` + 5 per-family - // paths + `metrics/raw_passthrough` default. - // * Each per-sketch pipeline starts with `gorillas3` when an - // archive tier is declared (cold-tier write happens BEFORE - // sketch mutation). - // * Freshness-probe (warm-passthrough) routing folds into - // `metrics/raw_passthrough` so the metric name is preserved - // end-to-end. - - /// Helper: build a 5-metric `EdgeStageConfig` covering every sketch - /// family per the canonical workload-spec table in MVP §46. - fn five_sketch_edge_cfg() -> EdgeStageConfig { - let mut metric_to_family: HashMap = HashMap::new(); - metric_to_family.insert("http_requests_total_latency_ms".into(), SketchKind::DDSketch); - metric_to_family.insert("request_size_bytes".into(), SketchKind::Kll); - metric_to_family.insert("unique_users_per_min".into(), SketchKind::Hll); - metric_to_family.insert("top_endpoint_qps".into(), SketchKind::CountSketch); - metric_to_family.insert("endpoint_request_freq".into(), SketchKind::Cms); - // `http_requests_total` is intentionally NOT in this map — it - // falls through to the `metrics/raw_passthrough` default. - EdgeStageConfig { - source_metric: None, - label_filters: Vec::new(), - window_secs: Some(60), - sketch_processors: Vec::new(), - exporter_target: ExportTarget::Stage(StageId::Gateway), - prometheus_archive_metrics: Vec::new(), - archive_tier_metrics: Vec::new(), - warm_passthrough_metrics: Vec::new(), - metric_to_family, - } - } - - #[test] - fn mvp46_emit_loads_all_5_sketch_processors() { - let cfg = five_sketch_edge_cfg(); - let yaml = emit_edge_yaml(&cfg, "ws://c/").expect("emit ok"); - for proc in ["ddsketch", "KLL", "HLL", "countsketch", "countmin"] { - assert!( - yaml.contains(&format!("{proc}:")), - "missing top-level processor key {proc}\n{yaml}" - ); - } - } - - #[test] - fn mvp46_routing_lives_in_connectors_not_processors() { - // The real bugfix: OTel collector v0.106+ removed - // `routingprocessor`; the routing component is now a - // `routingconnector`. We MUST emit it under `connectors:`. - let cfg = five_sketch_edge_cfg(); - let yaml = emit_edge_yaml(&cfg, "ws://c/").expect("emit ok"); - - // Connectors block exists with a `routing:` entry. - assert!( - yaml.contains("connectors:"), - "missing top-level connectors block\n{yaml}" - ); - let connectors_idx = yaml.find("connectors:").expect("connectors:"); - let after_conn = &yaml[connectors_idx..]; - // Find the next top-level section (one of receivers, processors, - // exporters, service, extensions) — `routing:` must appear before - // it. - let routing_idx = after_conn - .find("routing:") - .expect("routing: not found after connectors:"); - // Heuristically check that `routing:` appears in the connectors - // block, not later under `service.pipelines` (where it'd appear - // as `- routing` not `routing:`). - let next_section = ["exporters:", "service:"] - .iter() - .filter_map(|s| after_conn.find(s)) - .min() - .unwrap_or(after_conn.len()); - assert!( - routing_idx < next_section, - "routing: must appear inside connectors block, not later\n{yaml}" - ); - - // Critical negative assertion: `routing` is NOT under - // `processors:`. The processors block lists only the sketch - // processors + gorillas3? + batch. - let processors_idx = yaml.find("processors:").expect("processors:"); - let proc_end = yaml[processors_idx..] - .find("\nconnectors:") - .or_else(|| yaml[processors_idx..].find("\nexporters:")) - .map(|x| processors_idx + x) - .unwrap_or(yaml.len()); - let processors_section = &yaml[processors_idx..proc_end]; - assert!( - !processors_section.contains("routing:"), - "routing must NOT live under processors: (the v0.106 bug we're fixing)\n{processors_section}" - ); - } - - #[test] - fn mvp46_emits_all_6_named_pipelines() { - let cfg = five_sketch_edge_cfg(); - let yaml = emit_edge_yaml(&cfg, "ws://c/").expect("emit ok"); - for pl in [ - // Entry pipeline. - "metrics:", - // Default raw-passthrough. - "metrics/raw_passthrough:", - // 5 per-family pipelines. - "metrics/ddsketch_path:", - "metrics/kll_path:", - "metrics/hll_path:", - "metrics/countsketch_path:", - "metrics/countminsketch_path:", - ] { - assert!(yaml.contains(pl), "missing pipeline entry {pl}\n{yaml}"); - } - } - - #[test] - fn mvp46_entry_pipeline_routes_to_connector_not_processor() { - let cfg = five_sketch_edge_cfg(); - let yaml = emit_edge_yaml(&cfg, "ws://c/").expect("emit ok"); - // Find the entry `metrics:` pipeline section (under - // service.pipelines) and verify it has `exporters: [routing]` - // and no processors list (or empty). - let pipelines_idx = yaml.find("pipelines:").expect("pipelines block"); - let after = &yaml[pipelines_idx..]; - // First `metrics:` (NOT `metrics/...`) section is the entry. - // Look for " metrics:\n" pattern. - let entry_marker = " metrics:\n"; - let entry_idx = after.find(entry_marker).expect("metrics: entry"); - let entry_section_end = after[entry_idx + entry_marker.len()..] - .find(" metrics/") - .map(|x| entry_idx + entry_marker.len() + x) - .unwrap_or(after.len()); - let entry_section = &after[entry_idx..entry_section_end]; - // `exporters: [routing]` — but serde_yaml may render the list - // long-form; tolerate both `- routing` and `[routing]`. - assert!( - entry_section.contains("- routing") || entry_section.contains("[routing]"), - "entry pipeline must export to the routing connector\n{entry_section}" - ); - } - - #[test] - fn mvp46_per_sketch_pipelines_have_gorillas3_first_when_archive_declared() { - let mut cfg = five_sketch_edge_cfg(); - // Declare an archive-tier metric so gorillas3 is emitted. - cfg.archive_tier_metrics = vec![ArchiveTierMetric { - metric: "http_requests_total_latency_ms".into(), - window_secs: Some(60), - }]; - let yaml = emit_edge_yaml(&cfg, "ws://c/").expect("emit ok"); - - // gorillas3 processor block present. - assert!( - yaml.contains("gorillas3:"), - "missing gorillas3 block\n{yaml}" - ); - assert!(yaml.contains("block_format: prometheus_tsdb"), "{yaml}"); - - // Each per-sketch pipeline starts with gorillas3 BEFORE the - // family processor. We slice the YAML per-pipeline section and - // check the relative order. - for (pipeline, family_proc) in [ - ("metrics/ddsketch_path:", "ddsketch"), - ("metrics/kll_path:", "KLL"), - ("metrics/hll_path:", "HLL"), - ("metrics/countsketch_path:", "countsketch"), - ("metrics/countminsketch_path:", "countmin"), - ] { - let p_idx = yaml.find(pipeline).expect(pipeline); - // Section runs to the next `metrics/` header or end. - let after = &yaml[p_idx..]; - let next_offset = after[1..] - .find(" metrics") - .map(|x| x + 1) - .unwrap_or(after.len()); - let section = &after[..next_offset]; - let g_idx = section - .find("- gorillas3") - .unwrap_or_else(|| panic!("gorillas3 missing in {pipeline}\n{section}")); - let f_idx = section - .find(&format!("- {family_proc}")) - .unwrap_or_else(|| panic!("{family_proc} missing in {pipeline}\n{section}")); - assert!( - g_idx < f_idx, - "gorillas3 must come BEFORE {family_proc} in {pipeline}\n{section}" - ); - } - } - - #[test] - fn mvp46_per_sketch_pipelines_have_memory_limiter_first() { - // Follow-up to PR #355: every per-sketch pipeline (and the - // default raw_passthrough) MUST list `memory_limiter` as the - // FIRST processor so backpressure refuses incoming batches - // BEFORE gorillas3 buffers them — the previous shape OOM-killed - // the agent at ~3 min under sustained load. - let mut cfg = five_sketch_edge_cfg(); - cfg.archive_tier_metrics = vec![ArchiveTierMetric { - metric: "http_requests_total_latency_ms".into(), - window_secs: Some(60), - }]; - let yaml = emit_edge_yaml(&cfg, "ws://c/").expect("emit ok"); - - // memory_limiter processor block present with the chosen - // threshold (1280 MiB ≈ 80 % of agent's 1536 MiB cgroup). - assert!( - yaml.contains("memory_limiter:"), - "missing top-level memory_limiter processor block\n{yaml}" - ); - assert!( - yaml.contains("limit_mib: 1280"), - "memory_limiter must pin limit_mib: 1280 (agent cgroup is 1536 MiB)\n{yaml}" - ); - assert!( - yaml.contains("spike_limit_mib: 256"), - "memory_limiter must pin spike_limit_mib: 256\n{yaml}" - ); - - // Each per-sketch pipeline (and raw_passthrough) lists - // memory_limiter as the FIRST processor — slice each section - // and assert relative ordering. - for (pipeline, family_proc) in [ - ("metrics/raw_passthrough:", "gorillas3"), - ("metrics/ddsketch_path:", "gorillas3"), - ("metrics/kll_path:", "gorillas3"), - ("metrics/hll_path:", "gorillas3"), - ("metrics/countsketch_path:", "gorillas3"), - ("metrics/countminsketch_path:", "gorillas3"), - ] { - let p_idx = yaml.find(pipeline).expect(pipeline); - let after = &yaml[p_idx..]; - let next_offset = after[1..] - .find(" metrics") - .map(|x| x + 1) - .unwrap_or(after.len()); - let section = &after[..next_offset]; - let m_idx = section - .find("- memory_limiter") - .unwrap_or_else(|| panic!("memory_limiter missing in {pipeline}\n{section}")); - let f_idx = section - .find(&format!("- {family_proc}")) - .unwrap_or_else(|| panic!("{family_proc} missing in {pipeline}\n{section}")); - assert!( - m_idx < f_idx, - "memory_limiter must come BEFORE {family_proc} in {pipeline}\n{section}" - ); - } - } - - #[test] - fn mvp46_routing_table_dispatches_per_metric_to_correct_family() { - let cfg = five_sketch_edge_cfg(); - let yaml = emit_edge_yaml(&cfg, "ws://c/").expect("emit ok"); - // Every metric in the contract dispatches via routingconnector OTTL - // conditions. - // to its family pipeline. serde_yaml may render sequences - // either inline (`[metrics/x]`) or block-form (`- metrics/x`) - // depending on width; tolerate both. - for (metric, pipeline) in [ - ("http_requests_total_latency_ms", "metrics/ddsketch_path"), - ("request_size_bytes", "metrics/kll_path"), - ("unique_users_per_min", "metrics/hll_path"), - ("top_endpoint_qps", "metrics/countsketch_path"), - ("endpoint_request_freq", "metrics/countminsketch_path"), - ] { - let needle = format!("name == \"{metric}\""); - let n_idx = yaml - .find(&needle) - .unwrap_or_else(|| panic!("missing routing condition for {metric}\n{yaml}")); - let near = &yaml[n_idx..n_idx.saturating_add(256).min(yaml.len())]; - let inline = format!("[{pipeline}]"); - let block = format!("- {pipeline}"); - assert!( - near.contains(&inline) || near.contains(&block), - "{metric} should route to {pipeline}; got\n{near}" - ); - } - } - - #[test] - fn mvp46_default_pipeline_is_raw_passthrough() { - let cfg = five_sketch_edge_cfg(); - let yaml = emit_edge_yaml(&cfg, "ws://c/").expect("emit ok"); - // Tolerate inline-vs-block list rendering — serde_yaml chooses - // based on width. - let inline = "default_pipelines: [metrics/raw_passthrough]"; - let block = "default_pipelines:\n - metrics/raw_passthrough"; - let block2 = "default_pipelines:\n - metrics/raw_passthrough"; - assert!( - yaml.contains(inline) || yaml.contains(block) || yaml.contains(block2), - "routing must default to raw_passthrough so http_requests_total\ - (and any unrouted metric) falls through without sketching\n{yaml}" - ); - } - - #[test] - fn mvp46_warm_passthrough_routes_to_raw_passthrough_pipeline() { - // Freshness probes (Phase 3.2.5 Bug b) must bypass every sketch - // processor — they route to `metrics/raw_passthrough` so the - // metric name is preserved end-to-end. - let mut cfg = five_sketch_edge_cfg(); - cfg.archive_tier_metrics = vec![ArchiveTierMetric { - metric: "http_freshness_probe_warm".into(), - window_secs: Some(1), - }]; - cfg.warm_passthrough_metrics = vec!["http_freshness_probe_warm".into()]; - let yaml = emit_edge_yaml(&cfg, "ws://c/").expect("emit ok"); - - let needle = "name == \"http_freshness_probe_warm\""; - let idx = yaml - .find(needle) - .unwrap_or_else(|| panic!("missing freshness-probe route\n{yaml}")); - let near = &yaml[idx..idx.saturating_add(256).min(yaml.len())]; - // serde_yaml renders sequences inline or block-form; tolerate both. - assert!( - near.contains("[metrics/raw_passthrough]") - || near.contains("- metrics/raw_passthrough"), - "warm_passthrough metric must route to raw_passthrough\n{near}" - ); - - // raw_passthrough pipeline must NOT include any family-specific - // sketch processor (the whole point of the bypass). - let pl_idx = yaml - .find("metrics/raw_passthrough:") - .expect("raw_passthrough pipeline"); - let after = &yaml[pl_idx..]; - let next_offset = after[1..] - .find(" metrics") - .map(|x| x + 1) - .unwrap_or(after.len()); - let section = &after[..next_offset]; - for forbidden in ["ddsketch", "KLL", "HLL", "countsketch", "countmin"] { - assert!( - !section.contains(forbidden), - "raw_passthrough must NOT include {forbidden}\n{section}" - ); - } - // ... but gorillas3 still runs (the metric still wants to land - // in the cold archive). - assert!( - section.contains("- gorillas3"), - "raw_passthrough still routes through gorillas3 for archive write\n{section}" - ); - } - - #[test] - fn mvp46_per_sketch_pipelines_use_routing_as_receiver() { - // The connector is referenced as both an exporter (entry - // pipeline) and a receiver (each per-family pipeline). This - // pins the receiver-side wiring. - let cfg = five_sketch_edge_cfg(); - let yaml = emit_edge_yaml(&cfg, "ws://c/").expect("emit ok"); - for pipeline in [ - "metrics/ddsketch_path:", - "metrics/kll_path:", - "metrics/hll_path:", - "metrics/countsketch_path:", - "metrics/countminsketch_path:", - "metrics/raw_passthrough:", - ] { - let p_idx = yaml.find(pipeline).expect(pipeline); - let after = &yaml[p_idx..]; - let next_offset = after[1..] - .find(" metrics") - .map(|x| x + 1) - .unwrap_or(after.len()); - let section = &after[..next_offset]; - assert!( - section.contains("- routing") || section.contains("[routing]"), - "{pipeline} must consume from the routing connector\n{section}" - ); - } - } - - #[test] - fn mvp46_empty_metric_to_family_falls_back_to_legacy_emit() { - // Backward-compat invariant: when the planner hasn't populated - // metric_to_family, the emitter must produce the legacy - // single-pipeline shape (no connectors block, no per-family - // pipelines). - let cfg = ddsketch_edge_cfg(); - assert!( - cfg.metric_to_family.is_empty(), - "ddsketch_edge_cfg fixture must keep metric_to_family empty" - ); - let yaml = emit_edge_yaml(&cfg, "ws://c/").expect("emit ok"); - // No connectors block. - assert!( - !yaml.contains("connectors:"), - "legacy emit must NOT add connectors block\n{yaml}" - ); - // No 5-sketch pipelines. - assert!( - !yaml.contains("metrics/ddsketch_path"), - "legacy emit keeps single-pipeline shape\n{yaml}" - ); - assert!( - !yaml.contains("metrics/raw_passthrough"), - "legacy emit keeps single-pipeline shape\n{yaml}" - ); - } - - #[test] - fn mvp46_composes_with_prometheus_archive_mode3() { - // Mode 3 (Prometheus archive) folds into the same routing - // connector table — the `metrics/prometheus_archive` pipeline - // is added as an additional fan-out target. - let mut cfg = five_sketch_edge_cfg(); - cfg.prometheus_archive_metrics = vec![PrometheusArchiveMetric { - metric: "http_requests_total".into(), - window_secs: Some(60), - label_proj: vec!["service.name".into()], - }]; - let yaml = emit_edge_yaml(&cfg, "ws://c/").expect("emit ok"); - - assert!( - yaml.contains("metrics/prometheus_archive:"), - "Mode-3 pipeline must be added\n{yaml}" - ); - assert!( - yaml.contains("otlphttp/prometheus:"), - "Mode-3 exporter must be added\n{yaml}" - ); - assert!( - yaml.contains("attributes[\\\"asap.mode\\\"]") - || yaml.contains("attributes['asap.mode']") - || yaml.contains("attributes[\"asap.mode\"]"), - "routing table must dispatch by asap.mode for Mode 3\n{yaml}" - ); - } -} diff --git a/controller/src/config/stage_config_otap.rs b/controller/src/config/stage_config_otap.rs deleted file mode 100644 index 85fa47c5..00000000 --- a/controller/src/config/stage_config_otap.rs +++ /dev/null @@ -1,525 +0,0 @@ -//! Phase ε.1.5 — OTAP Dataflow DAG YAML emitter (per-runtime mirror of -//! [`super::stage_config::emit_edge_yaml`]). -//! -//! `asap-otap` uses the otap-dataflow Rust runtime; its config surface is -//! a DAG YAML where `nodes..type` is a registered plugin URN -//! (e.g. `receiver:otlp`, `exporter:otlp_http`, -//! `urn:otel:exporter:otlp_http`). The OTLP HTTP exporter ships in -//! `otel-arrow/rust/otap-dataflow/crates/core-nodes/src/exporters/otlp_http_exporter/` -//! and registers under the URN `urn:otel:exporter:otlp_http` via -//! `linkme`'s `distributed_slice(OTAP_EXPORTER_FACTORIES)`. -//! -//! Three modes (mirror the [`crate::planner::wire_cost::BindMode`] enum -//! Phase ε.1 introduced): -//! -//! 1. `SketchAtEdge` — DAG includes a sketch processor node between the -//! OTLP receiver and the OTLP gRPC exporter to the gateway. (The -//! `asap_sketches` plugin lives in the otap-patch tree; we wire its -//! type URN here without depending on its source.) -//! 2. `RawAtEdgeSketchAtBackend` — passthrough DAG: receiver → exporter. -//! No sketch processor. Egress is OTLP gRPC to the gateway, which -//! builds sketches at backend ingest. -//! 3. `RawAtEdgePrometheusArchive` — passthrough DAG: receiver → OTLP -//! HTTP exporter pointed at Prometheus's native OTLP receiver -//! (`/api/v1/otlp/v1/metrics`). -//! -//! The function consumes the same [`EdgeStageConfig`] the OTel-collector -//! emitter does — Phase ε.1.5 keeps the typed L5 plan as the single -//! source of truth across all three runtimes. The emitter dispatches per -//! `EdgeStageConfig` via the same `prometheus_archive_metrics` / -//! `sketch_processors` signals the OTel-collector emitter uses (Mode 3 -//! ↔ `prometheus_archive_metrics` non-empty; Mode 1 ↔ `sketch_processors` -//! non-empty; Mode 2 ↔ both empty + the bind decision lives in the -//! upstream stage_split — the edge YAML for Mode 2 is identical to a -//! plain raw passthrough at this layer). - -use anyhow::{Context, Result}; -use serde::Serialize; -use serde_yaml::{Mapping, Value}; -use std::collections::BTreeMap; - -use crate::sketch_algebra::params::{SketchKind, SketchParams}; -use crate::stage_split::emitter::{EdgeStageConfig, EdgeSketchProcessor, ExportTarget}; -use crate::stage_split::stage_id::StageId; - -/// Default URL for Prometheus's native OTLP HTTP receiver. -/// Matches `super::stage_config::emit_edge_yaml`'s placeholder so the -/// three runtime emitters agree on the wire endpoint. -pub const DEFAULT_PROMETHEUS_OTLP_URL: &str = - "http://prometheus:9090/api/v1/otlp/v1/metrics"; - -/// URN of the OTLP HTTP exporter registered by -/// `otel-arrow/rust/otap-dataflow/crates/core-nodes/src/exporters/otlp_http_exporter/`. -const URN_OTLP_HTTP_EXPORTER: &str = "exporter:otlp_http"; - -/// URN of the OTLP gRPC exporter registered by -/// `otel-arrow/rust/otap-dataflow/crates/core-nodes/src/exporters/otlp_grpc_exporter/`. -const URN_OTLP_GRPC_EXPORTER: &str = "exporter:otlp_grpc"; - -/// URN of the OTLP receiver (gRPC + HTTP). -const URN_OTLP_RECEIVER: &str = "receiver:otlp"; - -/// URN of the asap_sketches processor registered in the otap-patch tree. -/// Phase ε.1.5 wires the URN abstractly; the binary side lands the -/// plugin in `otap-patch/plugins/asap_sketches/` per -/// [`docs/design-asap-otap-rust-integration.md`]. -const URN_ASAP_SKETCHES_PROCESSOR: &str = "processor:asap_sketches"; - -// ── DAG YAML structural types ──────────────────────────────────────────────── -// -// These mirror the otap-dataflow `engine`/`groups`/`pipelines`/`nodes` -// schema in `otel-arrow/rust/otap-dataflow/configs/*.yaml`. We keep a -// minimal set to round-trip; the full schema (channel_capacity policies, -// engine settings, etc.) is left at struct defaults — Phase ε.1.5 only -// commits the wiring shape, not policy. - -#[derive(Serialize)] -struct OtapDag { - version: String, - engine: BTreeMap, - groups: BTreeMap, -} - -#[derive(Serialize)] -struct Group { - pipelines: BTreeMap, -} - -#[derive(Serialize)] -struct PipelineDef { - nodes: BTreeMap, - connections: Vec, -} - -#[derive(Serialize)] -struct NodeDef { - #[serde(rename = "type")] - kind: String, - config: Value, -} - -#[derive(Serialize)] -struct Connection { - from: String, - to: String, -} - -// ── Public API ─────────────────────────────────────────────────────────────── - -/// Build the OTAP-Dataflow DAG YAML for the `asap-otap` runtime from a -/// typed L5 [`EdgeStageConfig`]. -/// -/// `opamp_endpoint` is the controller's WebSocket URL; reserved for a -/// future `extension:opamp` node when the otap-dataflow runtime grows -/// OpAMP support (today the otap-dataflow `engine` block has no -/// extension model, so we accept the param for shape-parity with -/// [`super::stage_config::emit_edge_yaml`] and ignore it). -/// -/// `prometheus_otlp_url` overrides the default Prometheus OTLP HTTP -/// endpoint when present (for Mode 3 metrics). `None` falls back to -/// [`DEFAULT_PROMETHEUS_OTLP_URL`]. -pub fn emit_otap_dag_yaml( - cfg: &EdgeStageConfig, - _opamp_endpoint: &str, - prometheus_otlp_url: Option<&str>, -) -> Result { - let mut nodes: BTreeMap = BTreeMap::new(); - let mut connections: Vec = Vec::new(); - - // ── OTLP receiver ──────────────────────────────────────────────────────── - // Both gRPC + HTTP listeners — matches `emit_edge_yaml`'s shape so - // the three runtimes accept the same upstream traffic. - let receiver_cfg: Value = serde_yaml::from_str( - "protocols:\n grpc:\n listening_addr: \"0.0.0.0:4317\"\n http:\n listening_addr: \"0.0.0.0:4318\"\n", - ) - .context("parse OTAP otlp receiver block")?; - nodes.insert( - "receiver".to_string(), - NodeDef { - kind: URN_OTLP_RECEIVER.to_string(), - config: receiver_cfg, - }, - ); - - // ── Mode dispatch ──────────────────────────────────────────────────────── - let has_prometheus_archive = !cfg.prometheus_archive_metrics.is_empty(); - let has_sketch = !cfg.sketch_processors.is_empty(); - - if has_prometheus_archive { - // Mode 3 — Prometheus archive: passthrough → otlp_http exporter - // pointed at Prometheus's native OTLP receiver. We do NOT also - // emit a sketch node; Mode 3 metrics are the whole edge stream - // for that pipeline. (When a single agent is hosting Mode 3 + - // Mode 1/2 metrics simultaneously, the upstream typed splitter - // produces two `EdgeStageConfig`s — one per mode bucket — and - // we emit two pipelines side-by-side via the otap-dataflow - // multi-pipeline `pipelines:` map. Phase ε.1.5 ships the - // single-pipeline case; the multi-pipeline case is an upstream - // splitter concern.) - let prom_url = - prometheus_otlp_url.unwrap_or(DEFAULT_PROMETHEUS_OTLP_URL); - let exp_cfg = build_otlp_http_exporter_config(prom_url); - nodes.insert( - "exporter".to_string(), - NodeDef { - kind: URN_OTLP_HTTP_EXPORTER.to_string(), - config: exp_cfg, - }, - ); - connections.push(Connection { - from: "receiver".to_string(), - to: "exporter".to_string(), - }); - } else if has_sketch { - // Mode 1 — sketch at edge. Insert one processor per - // `EdgeSketchProcessor`; chain them serially between receiver - // and the gateway-bound OTLP gRPC exporter. - let mut prev = "receiver".to_string(); - for (i, sp) in cfg.sketch_processors.iter().enumerate() { - let name = format!("sketch_{i}"); - nodes.insert( - name.clone(), - NodeDef { - kind: URN_ASAP_SKETCHES_PROCESSOR.to_string(), - config: build_asap_sketches_config(sp, cfg.window_secs), - }, - ); - connections.push(Connection { - from: prev.clone(), - to: name.clone(), - }); - prev = name; - } - let endpoint = resolve_export_endpoint("gateway", &cfg.exporter_target); - nodes.insert( - "exporter".to_string(), - NodeDef { - kind: URN_OTLP_GRPC_EXPORTER.to_string(), - config: build_otlp_grpc_exporter_config(&endpoint), - }, - ); - connections.push(Connection { - from: prev, - to: "exporter".to_string(), - }); - } else { - // Mode 2 — raw at edge → sketch at backend. Passthrough DAG. - let endpoint = resolve_export_endpoint("gateway", &cfg.exporter_target); - nodes.insert( - "exporter".to_string(), - NodeDef { - kind: URN_OTLP_GRPC_EXPORTER.to_string(), - config: build_otlp_grpc_exporter_config(&endpoint), - }, - ); - connections.push(Connection { - from: "receiver".to_string(), - to: "exporter".to_string(), - }); - } - - let mut pipelines = BTreeMap::new(); - pipelines.insert( - "main".to_string(), - PipelineDef { nodes, connections }, - ); - - let mut groups = BTreeMap::new(); - groups.insert("default".to_string(), Group { pipelines }); - - let dag = OtapDag { - version: "otel_dataflow/v1".to_string(), - engine: BTreeMap::new(), - groups, - }; - - serde_yaml::to_string(&dag).context("serialize OTAP DAG YAML") -} - -// ── Internals ──────────────────────────────────────────────────────────────── - -fn resolve_export_endpoint(default_host: &str, target: &ExportTarget) -> String { - match target { - ExportTarget::Endpoint(s) => s.clone(), - ExportTarget::Stage(StageId::Edge) => "edge:4317".to_string(), - ExportTarget::Stage(StageId::Gateway) => format!("{default_host}:4317"), - ExportTarget::Stage(StageId::Backend) => format!("{default_host}:4317"), - } -} - -/// Build the OTLP HTTP exporter `config:` block. Matches the -/// `crates/core-nodes/src/exporters/otlp_http_exporter/config.rs` schema -/// — `endpoint` (base URL) plus an explicit `metrics_endpoint` so the -/// Prometheus path `/api/v1/otlp/v1/metrics` round-trips verbatim. -fn build_otlp_http_exporter_config(metrics_url: &str) -> Value { - // Derive the bare endpoint from the metrics URL: drop the path. For - // typical inputs this is `http://prometheus:9090`. - let base = match metrics_url.find("/api/") { - Some(i) => &metrics_url[..i], - None => metrics_url, - }; - let yaml = format!( - "endpoint: \"{base}\"\nmetrics_endpoint: \"{metrics_url}\"\nhttp:\n request_timeout: \"30s\"\nclient_pool_size: 1\n", - ); - serde_yaml::from_str(&yaml).expect("inline OTLP HTTP exporter config is valid YAML") -} - -/// Build the OTLP gRPC exporter `config:` block. The otap-dataflow -/// `otlp_grpc` exporter uses `grpc_endpoint` as the field name (see -/// `configs/otlp-otlp.yaml`). -fn build_otlp_grpc_exporter_config(endpoint: &str) -> Value { - let url = if endpoint.starts_with("http://") || endpoint.starts_with("https://") { - endpoint.to_string() - } else { - format!("http://{endpoint}") - }; - let yaml = format!("grpc_endpoint: \"{url}\"\ntimeout: \"15s\"\n"); - serde_yaml::from_str(&yaml).expect("inline OTLP gRPC exporter config is valid YAML") -} - -/// Build the per-edge-processor `asap_sketches` config block. Mirrors -/// the same fields the OTel-collector emitter writes -/// (`super::stage_config::build_edge_processor_block`) so the binary -/// side can share a single schema across the OTel + OTAP runtimes. -fn build_asap_sketches_config(sp: &EdgeSketchProcessor, window_secs: Option) -> Value { - let mut m = Mapping::new(); - if let Some(w) = window_secs { - m.insert("mode".into(), Value::String("window".to_string())); - m.insert("window_duration".into(), Value::String(format!("{w}s"))); - } else { - m.insert("mode".into(), Value::String("batch".to_string())); - } - m.insert( - "aggregation_id".into(), - Value::String(sp.aggregation_id.clone()), - ); - m.insert("sketch_kind".into(), Value::String(sketch_kind_tag(&sp.sketch_kind).into())); - match &sp.sketch_params { - SketchParams::Kll(p) => { - m.insert("k".into(), Value::Number((p.k as u64).into())); - } - SketchParams::DDSketch(p) => { - m.insert("relative_accuracy".into(), Value::Number(p.alpha.into())); - m.insert("delta_transmission".into(), Value::Bool(true)); - } - SketchParams::Hll(_p) => { - m.insert("delta_transmission".into(), Value::Bool(true)); - } - SketchParams::Cms(p) => { - m.insert("rows".into(), Value::Number((p.d as u64).into())); - m.insert("columns".into(), Value::Number((p.w as u64).into())); - m.insert("delta_transmission".into(), Value::Bool(true)); - } - SketchParams::CountSketch(p) => { - let epsilon = std::f64::consts::E / (p.w as f64); - let delta = 2f64.powi(-(p.d as i32)); - m.insert("epsilon".into(), Value::Number(epsilon.into())); - m.insert("delta".into(), Value::Number(delta.into())); - m.insert("delta_transmission".into(), Value::Bool(true)); - } - } - Value::Mapping(m) -} - -fn sketch_kind_tag(kind: &SketchKind) -> &'static str { - match kind { - SketchKind::Kll => "kll", - SketchKind::DDSketch => "ddsketch", - SketchKind::Hll => "hll", - SketchKind::Cms => "cms", - SketchKind::CountSketch => "count_sketch", - } -} - -// ── Tests ──────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use crate::sketch_algebra::params::DDSketchParams; - use crate::stage_split::emitter::{EdgeSketchProcessor, PrometheusArchiveMetric}; - - /// Minimal struct-stub used to validate the emitted DAG parses as the - /// otap-dataflow schema. We don't pull in the otap-df-config crate - /// here (it would add an enormous dependency footprint to the - /// controller); instead we verify the top-level shape (`version`, - /// `groups`, `pipelines`, `nodes`, `connections`) round-trips. - #[derive(Debug, serde::Deserialize)] - struct OtapDagStub { - version: String, - #[allow(dead_code)] - engine: serde_yaml::Value, - groups: BTreeMap, - } - - #[derive(Debug, serde::Deserialize)] - struct GroupStub { - pipelines: BTreeMap, - } - - #[derive(Debug, serde::Deserialize)] - struct PipelineStub { - nodes: BTreeMap, - connections: Vec, - } - - #[derive(Debug, serde::Deserialize)] - struct NodeStub { - #[serde(rename = "type")] - kind: String, - #[allow(dead_code)] - config: serde_yaml::Value, - } - - #[derive(Debug, serde::Deserialize)] - struct ConnectionStub { - from: String, - to: String, - } - - fn ddsketch_edge_cfg_mode1() -> EdgeStageConfig { - EdgeStageConfig { - source_metric: Some("http_request_duration_seconds".to_string()), - label_filters: Vec::new(), - window_secs: Some(60), - sketch_processors: vec![EdgeSketchProcessor { - processor_name: "ddsketch".to_string(), - sketch_kind: SketchKind::DDSketch, - sketch_params: SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), - aggregation_id: "agg0".to_string(), - }], - exporter_target: ExportTarget::Stage(StageId::Gateway), - prometheus_archive_metrics: Vec::new(), - archive_tier_metrics: Vec::new(), - warm_passthrough_metrics: Vec::new(), - metric_to_family: std::collections::HashMap::new(), - } - } - - fn raw_edge_cfg_mode2() -> EdgeStageConfig { - EdgeStageConfig { - source_metric: Some("http_request_duration_seconds".to_string()), - label_filters: Vec::new(), - window_secs: Some(60), - sketch_processors: Vec::new(), - exporter_target: ExportTarget::Stage(StageId::Gateway), - prometheus_archive_metrics: Vec::new(), - archive_tier_metrics: Vec::new(), - warm_passthrough_metrics: Vec::new(), - metric_to_family: std::collections::HashMap::new(), - } - } - - fn prom_edge_cfg_mode3() -> EdgeStageConfig { - EdgeStageConfig { - source_metric: Some("http_request_duration_seconds".to_string()), - label_filters: Vec::new(), - window_secs: Some(60), - sketch_processors: Vec::new(), - exporter_target: ExportTarget::Stage(StageId::Gateway), - prometheus_archive_metrics: vec![PrometheusArchiveMetric { - metric: "http_request_duration_seconds".to_string(), - window_secs: Some(60), - label_proj: vec!["service.name".to_string()], - }], - archive_tier_metrics: Vec::new(), - warm_passthrough_metrics: Vec::new(), - metric_to_family: std::collections::HashMap::new(), - } - } - - /// Mode 1 snapshot — sketch at edge: receiver → asap_sketches → - /// otlp_grpc exporter to gateway. - #[test] - fn otap_dag_mode1_sketch_at_edge_shape() { - let yaml = - emit_otap_dag_yaml(&ddsketch_edge_cfg_mode1(), "ws://ctrl/v1/opamp", None) - .expect("emit_otap_dag_yaml ok"); - let dag: OtapDagStub = serde_yaml::from_str(&yaml).expect("DAG parses"); - assert_eq!(dag.version, "otel_dataflow/v1"); - let pipe = dag.groups.get("default").unwrap().pipelines.get("main").unwrap(); - // Receiver + sketch + exporter == 3 nodes. - assert_eq!(pipe.nodes.len(), 3, "expected 3 nodes\n{yaml}"); - assert_eq!(pipe.nodes.get("receiver").unwrap().kind, URN_OTLP_RECEIVER); - assert_eq!( - pipe.nodes.get("sketch_0").unwrap().kind, - URN_ASAP_SKETCHES_PROCESSOR - ); - assert_eq!(pipe.nodes.get("exporter").unwrap().kind, URN_OTLP_GRPC_EXPORTER); - // Connections: receiver → sketch_0 → exporter. - assert_eq!(pipe.connections.len(), 2); - assert_eq!(pipe.connections[0].from, "receiver"); - assert_eq!(pipe.connections[0].to, "sketch_0"); - assert_eq!(pipe.connections[1].from, "sketch_0"); - assert_eq!(pipe.connections[1].to, "exporter"); - // Endpoint contains gateway:4317. - assert!(yaml.contains("gateway:4317"), "missing gateway endpoint\n{yaml}"); - } - - /// Mode 2 snapshot — raw at edge: receiver → otlp_grpc exporter. - /// No sketch node; the gateway / backend will build sketches. - #[test] - fn otap_dag_mode2_raw_at_edge_shape() { - let yaml = emit_otap_dag_yaml(&raw_edge_cfg_mode2(), "ws://ctrl/v1/opamp", None) - .expect("emit_otap_dag_yaml ok"); - let dag: OtapDagStub = serde_yaml::from_str(&yaml).expect("DAG parses"); - let pipe = dag.groups.get("default").unwrap().pipelines.get("main").unwrap(); - assert_eq!(pipe.nodes.len(), 2, "expected receiver + exporter only\n{yaml}"); - assert_eq!(pipe.nodes.get("receiver").unwrap().kind, URN_OTLP_RECEIVER); - assert_eq!(pipe.nodes.get("exporter").unwrap().kind, URN_OTLP_GRPC_EXPORTER); - // Direct connection. - assert_eq!(pipe.connections.len(), 1); - assert_eq!(pipe.connections[0].from, "receiver"); - assert_eq!(pipe.connections[0].to, "exporter"); - // No sketch processor in YAML. - assert!( - !yaml.contains(URN_ASAP_SKETCHES_PROCESSOR), - "Mode 2 must not include a sketch processor\n{yaml}" - ); - } - - /// Mode 3 snapshot — Prometheus archive: receiver → otlp_http - /// exporter pointed at `/api/v1/otlp/v1/metrics`. - #[test] - fn otap_dag_mode3_prometheus_archive_shape() { - let yaml = emit_otap_dag_yaml(&prom_edge_cfg_mode3(), "ws://ctrl/v1/opamp", None) - .expect("emit_otap_dag_yaml ok"); - let dag: OtapDagStub = serde_yaml::from_str(&yaml).expect("DAG parses"); - let pipe = dag.groups.get("default").unwrap().pipelines.get("main").unwrap(); - assert_eq!(pipe.nodes.len(), 2); - assert_eq!(pipe.nodes.get("exporter").unwrap().kind, URN_OTLP_HTTP_EXPORTER); - // Path round-trips verbatim. - assert!( - yaml.contains("/api/v1/otlp/v1/metrics"), - "missing Prometheus OTLP path\n{yaml}" - ); - // No sketch processor. - assert!( - !yaml.contains(URN_ASAP_SKETCHES_PROCESSOR), - "Mode 3 must not include a sketch processor\n{yaml}" - ); - } - - /// Mode 3 with override URL — caller can redirect to a non-default - /// Prometheus instance (`https://prom-prod:9090/...`). - #[test] - fn otap_dag_mode3_url_override() { - let yaml = emit_otap_dag_yaml( - &prom_edge_cfg_mode3(), - "ws://ctrl/v1/opamp", - Some("https://prom-prod:9090/api/v1/otlp/v1/metrics"), - ) - .expect("emit_otap_dag_yaml ok"); - assert!( - yaml.contains("https://prom-prod:9090/api/v1/otlp/v1/metrics"), - "override URL not propagated\n{yaml}" - ); - // The base endpoint should drop the path. (serde_yaml elides - // quotes around scalar strings that don't need them, so we - // match the unquoted form.) - assert!( - yaml.contains("endpoint: https://prom-prod:9090\n"), - "base endpoint not derived\n{yaml}" - ); - } -} diff --git a/controller/src/config/stage_config_telegraf.rs b/controller/src/config/stage_config_telegraf.rs deleted file mode 100644 index 5542525b..00000000 --- a/controller/src/config/stage_config_telegraf.rs +++ /dev/null @@ -1,493 +0,0 @@ -//! Phase ε.1.5 — Telegraf TOML emitter (per-runtime mirror of -//! [`super::stage_config::emit_edge_yaml`]). -//! -//! `asap-telegraf` is the Telegraf-runtime variant of the ASAP edge -//! agent. Telegraf consumes TOML; the relevant plugins are: -//! -//! * `inputs.opentelemetry` — OTLP gRPC / HTTP receiver (port 4317 / -//! 4318) — provides the upstream OTLP stream. -//! * `processors.allsketches` — the Telegraf-side streaming sketch -//! processor patched into `telegraf-patch/processors/`. Mirror of the -//! OTel-collector `*sketchprocessor` family. -//! * `outputs.opentelemetry` — OTLP **gRPC** exporter to the gateway -//! (Mode 1 / Mode 2). Telegraf's shipped plugin is gRPC-only — see -//! `telegraf/plugins/outputs/opentelemetry/opentelemetry.go`. There is -//! no `protocol = "http/protobuf"` field in the upstream plugin. -//! * `outputs.http` — generic HTTP POST output. For Mode 3 we use this -//! with the `prometheusremotewrite` serializer so the agent can ship -//! raw samples to Prometheus's remote-write endpoint -//! (`/api/v1/write`). This is a documented Phase ε.1.5 deviation -//! from the OTel-collector path's `otlphttp/prometheus` exporter: -//! Telegraf has no OTLP-HTTP serializer, but Prometheus's -//! remote-write endpoint accepts the same physical archive that the -//! OTLP receiver writes to, so the **archive contents end up -//! identical**. The wire framing differs; the storage outcome does -//! not. -//! -//! Three modes (mirror the [`crate::planner::wire_cost::BindMode`] enum -//! Phase ε.1 introduced): -//! -//! 1. `SketchAtEdge` — `[[processors.allsketches]]` between -//! `[[inputs.opentelemetry]]` and `[[outputs.opentelemetry]]`. -//! 2. `RawAtEdgeSketchAtBackend` — passthrough: input → output. No -//! sketch processor at the edge. -//! 3. `RawAtEdgePrometheusArchive` — passthrough: input → -//! `[[outputs.http]]` with `data_format = "prometheusremotewrite"` -//! pointed at Prometheus's `/api/v1/write`. - -use anyhow::{Context, Result}; - -use crate::sketch_algebra::params::{SketchKind, SketchParams}; -use crate::stage_split::emitter::{EdgeStageConfig, EdgeSketchProcessor, ExportTarget}; -use crate::stage_split::stage_id::StageId; - -/// Default Prometheus remote-write URL for Mode 3 — Telegraf doesn't -/// support OTLP-HTTP egress, so we land in the same Prometheus archive -/// via remote-write instead. The URL maps to the same Prometheus instance -/// the OTel-collector emitter targets via OTLP HTTP — Prometheus accepts -/// both ingest paths and stores into the same TSDB. -pub const DEFAULT_PROMETHEUS_REMOTE_WRITE_URL: &str = - "http://prometheus:9090/api/v1/write"; - -// ── Public API ─────────────────────────────────────────────────────────────── - -/// Build the Telegraf TOML for the `asap-telegraf` runtime from a -/// typed L5 [`EdgeStageConfig`]. -/// -/// `prometheus_remote_write_url` overrides the default Prometheus -/// remote-write endpoint when present (for Mode 3 metrics). `None` -/// falls back to [`DEFAULT_PROMETHEUS_REMOTE_WRITE_URL`]. -pub fn emit_telegraf_toml( - cfg: &EdgeStageConfig, - prometheus_remote_write_url: Option<&str>, -) -> Result { - let mut out = String::new(); - out.push_str("# Generated by ASAP controller (Phase ε.1.5).\n"); - out.push_str("# Mode-2/3 detection follows the EdgeStageConfig signals.\n\n"); - - // ── inputs.opentelemetry ───────────────────────────────────────────────── - // Telegraf's OTLP input listens on the default OTLP ports. - out.push_str("[[inputs.opentelemetry]]\n"); - out.push_str(" service_address = \"0.0.0.0:4317\"\n"); - out.push_str(" http_service_address = \"0.0.0.0:4318\"\n"); - out.push_str(" timeout = \"5s\"\n"); - out.push('\n'); - - // ── Mode dispatch ──────────────────────────────────────────────────────── - let has_prometheus_archive = !cfg.prometheus_archive_metrics.is_empty(); - let has_sketch = !cfg.sketch_processors.is_empty(); - - if has_prometheus_archive { - // Mode 3 — Prometheus archive: passthrough, then remote-write - // to Prometheus. We do NOT include `[[processors.allsketches]]`. - let url = - prometheus_remote_write_url.unwrap_or(DEFAULT_PROMETHEUS_REMOTE_WRITE_URL); - emit_outputs_http_remote_write(&mut out, url); - } else if has_sketch { - // Mode 1 — sketch at edge. One `[[processors.allsketches]]` per - // sketch processor; outputs.opentelemetry to gateway. - for sp in &cfg.sketch_processors { - emit_processors_allsketches(&mut out, sp, cfg.window_secs); - } - let endpoint = resolve_export_endpoint("gateway", &cfg.exporter_target); - emit_outputs_opentelemetry(&mut out, &endpoint); - } else { - // Mode 2 — raw at edge. Passthrough; outputs.opentelemetry - // ships raw OTLP to the gateway. - let endpoint = resolve_export_endpoint("gateway", &cfg.exporter_target); - emit_outputs_opentelemetry(&mut out, &endpoint); - } - - // Validate that what we emitted parses as TOML — catches malformed - // table headers / quoted strings before the agent boots. - let _: toml_minimal::Document = toml_minimal::Document::parse(&out) - .with_context(|| format!("generated Telegraf TOML failed minimal parse: {out}"))?; - - Ok(out) -} - -// ── Internals ──────────────────────────────────────────────────────────────── - -fn resolve_export_endpoint(default_host: &str, target: &ExportTarget) -> String { - match target { - ExportTarget::Endpoint(s) => s.clone(), - ExportTarget::Stage(StageId::Edge) => "edge:4317".to_string(), - ExportTarget::Stage(StageId::Gateway) => format!("{default_host}:4317"), - ExportTarget::Stage(StageId::Backend) => format!("{default_host}:4317"), - } -} - -fn emit_outputs_opentelemetry(out: &mut String, endpoint: &str) { - // `outputs.opentelemetry` is gRPC-only; `service_address` takes - // `host:port` (no scheme). Compression defaults to gzip. - out.push_str("[[outputs.opentelemetry]]\n"); - out.push_str(&format!(" service_address = \"{endpoint}\"\n")); - out.push_str(" timeout = \"5s\"\n"); - out.push_str(" compression = \"gzip\"\n"); - out.push('\n'); -} - -fn emit_outputs_http_remote_write(out: &mut String, url: &str) { - // Telegraf's `outputs.http` POSTs the serialized batch to the URL. - // `data_format = "prometheusremotewrite"` selects the remote-write - // serializer shipped in `telegraf/plugins/serializers/prometheusremotewrite/`. - out.push_str("[[outputs.http]]\n"); - out.push_str(&format!(" url = \"{url}\"\n")); - out.push_str(" method = \"POST\"\n"); - out.push_str(" data_format = \"prometheusremotewrite\"\n"); - out.push_str(" content_encoding = \"snappy\"\n"); - out.push_str(" [outputs.http.headers]\n"); - out.push_str(" Content-Type = \"application/x-protobuf\"\n"); - out.push_str(" X-Prometheus-Remote-Write-Version = \"0.1.0\"\n"); - out.push('\n'); -} - -fn emit_processors_allsketches( - out: &mut String, - sp: &EdgeSketchProcessor, - window_secs: Option, -) { - out.push_str("[[processors.allsketches]]\n"); - let mode = if window_secs.is_some() { "window" } else { "batch" }; - out.push_str(&format!(" mode = \"{mode}\"\n")); - if let Some(w) = window_secs { - out.push_str(&format!(" window_duration = \"{w}s\"\n")); - } - out.push_str(&format!( - " aggregation_id = \"{}\"\n", - sp.aggregation_id - )); - out.push_str(&format!( - " sketch_kind = \"{}\"\n", - sketch_kind_tag(&sp.sketch_kind) - )); - match &sp.sketch_params { - SketchParams::Kll(p) => { - out.push_str(&format!(" k = {}\n", p.k)); - } - SketchParams::DDSketch(p) => { - out.push_str(&format!(" relative_accuracy = {}\n", p.alpha)); - out.push_str(" delta_transmission = true\n"); - } - SketchParams::Hll(_p) => { - out.push_str(" delta_transmission = true\n"); - } - SketchParams::Cms(p) => { - out.push_str(&format!(" rows = {}\n", p.d)); - out.push_str(&format!(" columns = {}\n", p.w)); - out.push_str(" delta_transmission = true\n"); - } - SketchParams::CountSketch(p) => { - let epsilon = std::f64::consts::E / (p.w as f64); - let delta = 2f64.powi(-(p.d as i32)); - out.push_str(&format!(" epsilon = {epsilon}\n")); - out.push_str(&format!(" delta = {delta}\n")); - out.push_str(" delta_transmission = true\n"); - } - } - out.push('\n'); -} - -fn sketch_kind_tag(kind: &SketchKind) -> &'static str { - match kind { - SketchKind::Kll => "kll", - SketchKind::DDSketch => "ddsketch", - SketchKind::Hll => "hll", - SketchKind::Cms => "cms", - SketchKind::CountSketch => "count_sketch", - } -} - -// ── Minimal TOML parser stub ───────────────────────────────────────────────── -// -// The controller crate doesn't depend on a full `toml` crate (the -// existing surface is YAML / JSON only). To validate that our emitted -// TOML is syntactically valid, we ship a tiny purpose-built validator -// that recognizes the subset Telegraf consumes: `[[table.array]]` / -// `[table]` headers, `key = "string"`, `key = number`, `key = bool`, -// indented sub-tables, and `# comments`. This is conservative — it -// rejects malformed table headers / unbalanced quotes / etc., which -// is the failure mode we care about catching before agent boot. - -mod toml_minimal { - use anyhow::{anyhow, Result}; - - #[derive(Debug)] - pub struct Document; - - impl Document { - pub fn parse(src: &str) -> Result { - for (lineno, raw) in src.lines().enumerate() { - let line = strip_comment(raw).trim(); - if line.is_empty() { - continue; - } - if line.starts_with('[') { - if !is_balanced_brackets(line) { - return Err(anyhow!( - "line {}: unbalanced [ in table header: {raw}", - lineno + 1 - )); - } - continue; - } - // key = value - let Some(eq) = line.find('=') else { - return Err(anyhow!( - "line {}: expected `key = value`, got: {raw}", - lineno + 1 - )); - }; - let key = line[..eq].trim(); - let val = line[eq + 1..].trim(); - if key.is_empty() { - return Err(anyhow!("line {}: empty key in: {raw}", lineno + 1)); - } - if val.is_empty() { - return Err(anyhow!("line {}: empty value in: {raw}", lineno + 1)); - } - // Validate value: string (balanced quotes), bool, or - // unquoted scalar (number / fraction). - if val.starts_with('"') { - if !val.ends_with('"') || val.len() < 2 { - return Err(anyhow!( - "line {}: unbalanced \" in: {raw}", - lineno + 1 - )); - } - } - } - Ok(Document) - } - } - - fn strip_comment(line: &str) -> &str { - // Strip a trailing `# ...` comment, but not when inside quotes. - let mut in_str = false; - for (i, c) in line.char_indices() { - match c { - '"' => in_str = !in_str, - '#' if !in_str => return &line[..i], - _ => {} - } - } - line - } - - fn is_balanced_brackets(s: &str) -> bool { - let mut depth = 0i32; - for c in s.chars() { - match c { - '[' => depth += 1, - ']' => depth -= 1, - _ => {} - } - if depth < 0 { - return false; - } - } - depth == 0 - } -} - -// ── Tests ──────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use crate::sketch_algebra::params::{DDSketchParams, KllParams}; - use crate::stage_split::emitter::{EdgeSketchProcessor, PrometheusArchiveMetric}; - - fn ddsketch_edge_cfg_mode1() -> EdgeStageConfig { - EdgeStageConfig { - source_metric: Some("http_request_duration_seconds".to_string()), - label_filters: Vec::new(), - window_secs: Some(60), - sketch_processors: vec![EdgeSketchProcessor { - processor_name: "ddsketch".to_string(), - sketch_kind: SketchKind::DDSketch, - sketch_params: SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), - aggregation_id: "agg0".to_string(), - }], - exporter_target: ExportTarget::Stage(StageId::Gateway), - prometheus_archive_metrics: Vec::new(), - archive_tier_metrics: Vec::new(), - warm_passthrough_metrics: Vec::new(), - metric_to_family: std::collections::HashMap::new(), - } - } - - fn raw_edge_cfg_mode2() -> EdgeStageConfig { - EdgeStageConfig { - source_metric: Some("http_request_duration_seconds".to_string()), - label_filters: Vec::new(), - window_secs: Some(60), - sketch_processors: Vec::new(), - exporter_target: ExportTarget::Stage(StageId::Gateway), - prometheus_archive_metrics: Vec::new(), - archive_tier_metrics: Vec::new(), - warm_passthrough_metrics: Vec::new(), - metric_to_family: std::collections::HashMap::new(), - } - } - - fn prom_edge_cfg_mode3() -> EdgeStageConfig { - EdgeStageConfig { - source_metric: Some("http_request_duration_seconds".to_string()), - label_filters: Vec::new(), - window_secs: Some(60), - sketch_processors: Vec::new(), - exporter_target: ExportTarget::Stage(StageId::Gateway), - prometheus_archive_metrics: vec![PrometheusArchiveMetric { - metric: "http_request_duration_seconds".to_string(), - window_secs: Some(60), - label_proj: vec!["service.name".to_string()], - }], - archive_tier_metrics: Vec::new(), - warm_passthrough_metrics: Vec::new(), - metric_to_family: std::collections::HashMap::new(), - } - } - - /// Mode 1 snapshot — sketch at edge. - /// `[[inputs.opentelemetry]]` + `[[processors.allsketches]]` + - /// `[[outputs.opentelemetry]]`. - #[test] - fn telegraf_toml_mode1_sketch_at_edge_shape() { - let toml = emit_telegraf_toml(&ddsketch_edge_cfg_mode1(), None) - .expect("emit_telegraf_toml ok"); - assert!(toml.contains("[[inputs.opentelemetry]]"), "missing input\n{toml}"); - assert!( - toml.contains("[[processors.allsketches]]"), - "missing sketch processor\n{toml}" - ); - assert!( - toml.contains("[[outputs.opentelemetry]]"), - "missing output\n{toml}" - ); - assert!( - toml.contains("service_address = \"gateway:4317\""), - "missing gateway endpoint\n{toml}" - ); - // Sketch params preserved. - assert!(toml.contains("relative_accuracy = 0.01"), "missing alpha\n{toml}"); - assert!(toml.contains("sketch_kind = \"ddsketch\""), "wrong kind\n{toml}"); - } - - /// Mode 2 snapshot — raw at edge. No `[[processors.allsketches]]`. - /// `[[outputs.opentelemetry]]` ships raw OTLP to the gateway. - #[test] - fn telegraf_toml_mode2_raw_at_edge_shape() { - let toml = emit_telegraf_toml(&raw_edge_cfg_mode2(), None) - .expect("emit_telegraf_toml ok"); - assert!(toml.contains("[[inputs.opentelemetry]]"), "missing input\n{toml}"); - assert!( - !toml.contains("[[processors.allsketches]]"), - "Mode 2 must not include sketch processor\n{toml}" - ); - assert!( - toml.contains("[[outputs.opentelemetry]]"), - "missing output\n{toml}" - ); - assert!( - toml.contains("service_address = \"gateway:4317\""), - "missing gateway endpoint\n{toml}" - ); - } - - /// Mode 3 snapshot — Prometheus archive. `[[outputs.http]]` POSTs - /// to Prometheus's remote-write endpoint `/api/v1/write` (Telegraf - /// has no OTLP-HTTP serializer; remote-write lands in the same - /// Prometheus TSDB the OTel-collector path lands in via OTLP HTTP). - #[test] - fn telegraf_toml_mode3_prometheus_archive_shape() { - let toml = emit_telegraf_toml(&prom_edge_cfg_mode3(), None) - .expect("emit_telegraf_toml ok"); - assert!(toml.contains("[[inputs.opentelemetry]]"), "missing input\n{toml}"); - assert!( - !toml.contains("[[processors.allsketches]]"), - "Mode 3 must not include sketch processor\n{toml}" - ); - assert!( - toml.contains("[[outputs.http]]"), - "missing http output\n{toml}" - ); - assert!( - toml.contains("url = \"http://prometheus:9090/api/v1/write\""), - "missing Prometheus remote-write URL\n{toml}" - ); - assert!( - toml.contains("data_format = \"prometheusremotewrite\""), - "missing remote-write serializer\n{toml}" - ); - // No outputs.opentelemetry — Mode 3 is a passthrough to - // Prometheus, not the gateway. - assert!( - !toml.contains("[[outputs.opentelemetry]]"), - "Mode 3 must not also export to gateway\n{toml}" - ); - } - - /// Mode 3 with override URL — caller can redirect to a different - /// Prometheus instance. - #[test] - fn telegraf_toml_mode3_url_override() { - let toml = emit_telegraf_toml( - &prom_edge_cfg_mode3(), - Some("https://prom-prod:9090/api/v1/write"), - ) - .expect("emit_telegraf_toml ok"); - assert!( - toml.contains("https://prom-prod:9090/api/v1/write"), - "override URL not propagated\n{toml}" - ); - } - - /// Catch malformed output early — emit_telegraf_toml should produce - /// TOML that round-trips through our minimal validator. - #[test] - fn telegraf_toml_all_modes_parse() { - for (name, cfg) in [ - ("mode1", ddsketch_edge_cfg_mode1()), - ("mode2", raw_edge_cfg_mode2()), - ("mode3", prom_edge_cfg_mode3()), - ] { - let toml = emit_telegraf_toml(&cfg, None) - .unwrap_or_else(|e| panic!("emit failed for {name}: {e}")); - // Parsing happens inside emit_telegraf_toml; if we got Ok, - // parsing succeeded. Spot-check a handful of expected - // tokens defensively. - assert!(toml.contains("inputs.opentelemetry"), "{name}: missing input header"); - } - } - - /// KLL params — k is preserved, no delta_transmission flag (KLL has - /// no delta variant per Implementation.tex). - #[test] - fn telegraf_toml_mode1_kll_no_delta_flag() { - let cfg = EdgeStageConfig { - source_metric: Some("metric".to_string()), - label_filters: Vec::new(), - window_secs: Some(60), - sketch_processors: vec![EdgeSketchProcessor { - processor_name: "KLL".to_string(), - sketch_kind: SketchKind::Kll, - sketch_params: SketchParams::Kll(KllParams { k: 200 }), - aggregation_id: "agg0".to_string(), - }], - exporter_target: ExportTarget::Stage(StageId::Gateway), - prometheus_archive_metrics: Vec::new(), - archive_tier_metrics: Vec::new(), - warm_passthrough_metrics: Vec::new(), - metric_to_family: std::collections::HashMap::new(), - }; - let toml = emit_telegraf_toml(&cfg, None).expect("emit ok"); - assert!(toml.contains("k = 200"), "k not propagated\n{toml}"); - assert!(toml.contains("sketch_kind = \"kll\""), "kind\n{toml}"); - // KLL has no delta variant — flag must be absent. - assert!( - !toml.contains("delta_transmission"), - "KLL must not have delta_transmission\n{toml}" - ); - } -} diff --git a/controller/src/config/workloads.rs b/controller/src/config/workloads.rs deleted file mode 100644 index 725949c6..00000000 --- a/controller/src/config/workloads.rs +++ /dev/null @@ -1,297 +0,0 @@ -//! Declarative workload registration. -//! -//! Loads a YAML file describing workloads and their assignments so the -//! controller can pre-populate the plan store and assign workloads to -//! agents on connect without requiring an explicit HTTP `POST /api/v1/plan`. - -use serde::{Deserialize, Deserializer, Serialize}; -use tracing::{info, warn}; - -use crate::types::SketchType; - -/// A single workload entry from the workloads YAML file. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct WorkloadEntry { - /// Metric name this workload targets (e.g. `http_request_duration_seconds`). - pub metric_name: String, - /// PromQL / SQL query string for the planner. - #[serde(default)] - pub query_string: Option, - /// Required accuracy SLA (0.0 – 1.0). - #[serde(default = "default_accuracy_sla")] - pub accuracy_sla: f64, - /// Role that should receive this workload (e.g. `"agent"`, `"backend"`). - #[serde(default = "default_role")] - pub assign_to_role: String, - /// Optional explicit sketch family override. When set, the planner pins - /// this family for the metric (modulo `(sketch, statistic)` validity - /// per `sketch_algebra::capability_matching::is_valid_pair`). Threaded - /// into `QueryWorkload::sketch_type_override` by the registry pre-pop - /// path so the typed L4 binding (`bind_workload_typed`) honours it. - /// - /// MVP-§46 contract entries 5–8 in `deploy/configs/mvp-workload.yaml` - /// rely on this field to pin HLL / CountSketch / CountMinSketch - /// against metrics whose name-classified statistic class is - /// `Cardinality` / `TopK` / `Frequency`. - #[serde(default, deserialize_with = "deserialize_sketch_family")] - pub sketch_family_override: Option, - /// Optional storage tier hint (e.g. `"warm"`, `"archive"`). Round-trips - /// silently for now — kept here so the YAML schema matches the - /// capability_matching agent's expected shape (no rename step at - /// integration). Not yet read by the planner. - #[serde(default)] - pub target_path: Option, -} - -fn default_accuracy_sla() -> f64 { 0.01 } -fn default_role() -> String { "agent".into() } - -/// Case-insensitive `SketchType` deserialiser. The wire YAML in -/// `deploy/configs/mvp-workload.yaml` spells the variants in mixed case -/// (`KLL`, `HLL`, `CountSketch`, `CountMinSketch`, `DDSketch`) to match -/// the capability-matching agent's schema, while `SketchType`'s -/// `#[serde(rename_all = "lowercase")]` would otherwise reject those -/// strings. Accepts both spellings. -fn deserialize_sketch_family<'de, D>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - let opt: Option = Option::deserialize(deserializer)?; - let Some(s) = opt else { return Ok(None) }; - let kind = match s.trim().to_ascii_lowercase().as_str() { - "ddsketch" => SketchType::DDSketch, - "kll" => SketchType::KLL, - "hll" => SketchType::HLL, - "countsketch" => SketchType::CountSketch, - "countminsketch" | "countmin" | "cms" => SketchType::CountMinSketch, - other => return Err(serde::de::Error::custom(format!( - "unknown sketch_family_override `{other}`; expected one of \ - DDSketch / KLL / HLL / CountSketch / CountMinSketch" - ))), - }; - Ok(Some(kind)) -} - -/// Registry of declarative workloads loaded from a YAML file. -#[derive(Debug, Clone)] -pub struct WorkloadRegistry { - entries: Vec, -} - -impl WorkloadRegistry { - /// Load from a YAML file. Returns an empty registry on any error. - pub fn load(path: &str) -> Self { - match std::fs::read_to_string(path) { - Ok(contents) => match serde_yaml::from_str::>(&contents) { - Ok(entries) => { - info!(path, count = entries.len(), "loaded workload registry"); - Self { entries } - } - Err(e) => { - warn!(path, error = %e, "invalid workloads YAML; using empty registry"); - Self { entries: vec![] } - } - }, - Err(_) => { - info!(path, "workloads file not found; using empty registry"); - Self { entries: vec![] } - } - } - } - - /// Create an empty registry (no file). - pub fn empty() -> Self { - Self { entries: vec![] } - } - - /// Create a registry from in-memory entries (useful for tests and - /// programmatic construction). - pub fn from_entries(entries: Vec) -> Self { - Self { entries } - } - - /// Returns all workload entries. - pub fn entries(&self) -> &[WorkloadEntry] { - &self.entries - } - - /// Returns workload entries assigned to a given role. - pub fn for_role(&self, role: &str) -> Vec<&WorkloadEntry> { - self.entries.iter() - .filter(|e| e.assign_to_role.eq_ignore_ascii_case(role)) - .collect() - } - - /// Returns the first workload entry for a given role, if any. - pub fn first_for_role(&self, role: &str) -> Option<&WorkloadEntry> { - self.entries.iter() - .find(|e| e.assign_to_role.eq_ignore_ascii_case(role)) - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn load_empty_on_missing_file() { - let reg = WorkloadRegistry::load("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/nonexistent/workloads.yaml"); - assert!(reg.entries().is_empty()); - } - - #[test] - fn empty_registry() { - let reg = WorkloadRegistry::empty(); - assert!(reg.entries().is_empty()); - assert!(reg.first_for_role("agent").is_none()); - } - - #[test] - fn deserialize_entries() { - let yaml = r#" -- metric_name: latency - query_string: "histogram_quantile(0.99, rate(http_duration_bucket[5m]))" - accuracy_sla: 0.01 - assign_to_role: agent -- metric_name: error_count - accuracy_sla: 0.05 - assign_to_role: backend -"#; - let entries: Vec = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(entries.len(), 2); - assert_eq!(entries[0].metric_name, "latency"); - assert_eq!(entries[0].assign_to_role, "agent"); - assert!(entries[0].query_string.is_some()); - assert_eq!(entries[1].metric_name, "error_count"); - assert!(entries[1].query_string.is_none()); - } - - #[test] - fn for_role_filters_correctly() { - let reg = WorkloadRegistry { - entries: vec![ - WorkloadEntry { - metric_name: "a".into(), - query_string: None, - accuracy_sla: 0.01, - assign_to_role: "agent".into(), - sketch_family_override: None, - target_path: None, - }, - WorkloadEntry { - metric_name: "b".into(), - query_string: None, - accuracy_sla: 0.05, - assign_to_role: "backend".into(), - sketch_family_override: None, - target_path: None, - }, - WorkloadEntry { - metric_name: "c".into(), - query_string: None, - accuracy_sla: 0.02, - assign_to_role: "agent".into(), - sketch_family_override: None, - target_path: None, - }, - ], - }; - assert_eq!(reg.for_role("agent").len(), 2); - assert_eq!(reg.for_role("backend").len(), 1); - assert_eq!(reg.first_for_role("agent").unwrap().metric_name, "a"); - } - - #[test] - fn deserialize_sketch_family_override_mixed_case() { - // The live wire YAML in `deploy/configs/mvp-workload.yaml` spells - // the override values in mixed case (KLL / HLL / CountSketch / - // CountMinSketch). Verify deserialization picks them up — without - // this, MVP §46 entries 5–8 silently drop their family override - // (the original stitching-gap symptom). - let yaml = r#" -- metric_name: a - sketch_family_override: KLL -- metric_name: b - sketch_family_override: HLL -- metric_name: c - sketch_family_override: CountSketch -- metric_name: d - sketch_family_override: CountMinSketch -- metric_name: e - sketch_family_override: DDSketch -- metric_name: f -"#; - let entries: Vec = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(entries.len(), 6); - assert_eq!(entries[0].sketch_family_override, Some(SketchType::KLL)); - assert_eq!(entries[1].sketch_family_override, Some(SketchType::HLL)); - assert_eq!(entries[2].sketch_family_override, Some(SketchType::CountSketch)); - assert_eq!(entries[3].sketch_family_override, Some(SketchType::CountMinSketch)); - assert_eq!(entries[4].sketch_family_override, Some(SketchType::DDSketch)); - assert_eq!(entries[5].sketch_family_override, None); - } - - #[test] - fn deserialize_sketch_family_override_lowercase_aliases() { - // Lowercase / kebab-case spellings also accepted, plus the two - // CMS aliases (`countmin`, `cms`). - let yaml = r#" -- metric_name: a - sketch_family_override: ddsketch -- metric_name: b - sketch_family_override: countmin -- metric_name: c - sketch_family_override: cms -"#; - let entries: Vec = serde_yaml::from_str(yaml).unwrap(); - assert_eq!(entries[0].sketch_family_override, Some(SketchType::DDSketch)); - assert_eq!(entries[1].sketch_family_override, Some(SketchType::CountMinSketch)); - assert_eq!(entries[2].sketch_family_override, Some(SketchType::CountMinSketch)); - } - - #[test] - fn live_mvp_workload_yaml_loads_with_overrides() { - // Smoke-test the live deploy file. Confirms entries 5–8 carry - // their `sketch_family_override` after deserialization (the - // original stitching gap was this field being silently ignored - // by `serde`'s unknown-field default behaviour). - use std::path::PathBuf; - let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - path.pop(); - path.push("deploy/configs/mvp-workload.yaml"); - if !path.exists() { - // Live file not in this checkout; skip silently. - return; - } - let registry = WorkloadRegistry::load(path.to_str().unwrap()); - let by_name: std::collections::HashMap<&str, &WorkloadEntry> = - registry.entries().iter().map(|e| (e.metric_name.as_str(), e)).collect(); - - assert_eq!( - by_name.get("request_size_bytes") - .and_then(|e| e.sketch_family_override.clone()), - Some(SketchType::KLL), - "request_size_bytes must carry KLL override", - ); - assert_eq!( - by_name.get("unique_users_per_min") - .and_then(|e| e.sketch_family_override.clone()), - Some(SketchType::HLL), - "unique_users_per_min must carry HLL override", - ); - assert_eq!( - by_name.get("top_endpoint_qps") - .and_then(|e| e.sketch_family_override.clone()), - Some(SketchType::CountSketch), - "top_endpoint_qps must carry CountSketch override", - ); - assert_eq!( - by_name.get("endpoint_request_freq") - .and_then(|e| e.sketch_family_override.clone()), - Some(SketchType::CountMinSketch), - "endpoint_request_freq must carry CountMinSketch override", - ); - } -} diff --git a/controller/src/intent_algebra/agg_intent.rs b/controller/src/intent_algebra/agg_intent.rs deleted file mode 100644 index d3ef36e1..00000000 --- a/controller/src/intent_algebra/agg_intent.rs +++ /dev/null @@ -1,610 +0,0 @@ -//! Layer 3 aggregation-intent vocabulary. -//! -//! Per `controller/docs/design.md` §6 "`AggIntent` — what to compute, not -//! how" (around line ~468). L3 carries intent ("compute a quantile to -//! ε=0.01 accuracy"). The choice between `HashAgg` / `SortAgg` / -//! `SketchAgg(KLL{k=200})` is made by L4 cost-aware rules, not encoded -//! here. -//! -//! Intent vs operator distinction. `AggIntent::TopK` is an *intent* (a -//! dedicated heavy-hitter sketch primitive — SpaceSaving, CMS-with-heap -//! — computes it in a single pass). The generic `Sort + Limit` operator -//! pair survives in `QueryExpr` for non-heavy-hitter cases (`ORDER BY -//! name LIMIT 10`). L1→L2→L3 lowering picks one or the other -//! deterministically. -//! -//! No `QuantileOverTime` intent. The window is fully captured by the -//! surrounding `QueryExpr::Window` node; the quantile *operation* is the -//! same regardless. PromQL `quantile_over_time(0.99, m[5m])` lowers to -//! `Window{size=5m} → Aggregate{aggs:[Quantile{q=0.99}]}`. -//! -//! `Rate` and `Increase` survive that argument because they include -//! PromQL's counter-reset adjustment, a non-trivial transformation that -//! exact `Sum` does not perform. - -#![allow(dead_code)] - -use std::time::Duration; - -use serde::{Deserialize, Serialize}; - -use crate::intent_algebra::schema::{Column, DataType}; -use crate::types_v2::AccuracyTarget; - -/// "What to compute" at L3 — vocabulary the planner pivots on. See module -/// doc for the intent vs operator distinction. -/// -/// Variants intentionally mirror `design.md` §6 line ~468; data-model- -/// agnostic intents come first, time-series-streaming derivatives -/// (`Rate` / `Increase`) come last. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum AggIntent { - // ── Data-model-agnostic ────────────────────────────────────────────── - /// COUNT(*) / `count` — number of rows / samples per group. - /// `accuracy: Exact` selects an exact counter; `Epsilon` / `EpsilonDelta` - /// unlock CMS / linear-counting sketch families. - Count { - accuracy: AccuracyTarget, - }, - /// SUM(col). Always exact at L3 — no approximation intent for `Sum` - /// in the catalog (`design.md` §6 line ~485). - Sum, - /// Per-group minimum — exact at L3. - Min, - /// Per-group maximum — exact at L3. - Max, - /// Arithmetic mean. Exact at L3; sketch backends fold this onto a - /// `Quantile{q=0.5}` only when the cost model allows the relaxation. - Avg, - /// Compute the φ-th quantile (0 ≤ q ≤ 1) to the given accuracy. - /// Sketch families: KLL, DDSketch, t-digest. - Quantile { - q: f64, - accuracy: AccuracyTarget, - }, - /// Heavy-hitter top-k. Distinct from generic `Sort + Limit` because - /// a dedicated sketch primitive (SpaceSaving, CMS-with-heap, - /// Misra-Gries) computes it as a single operation. L1→L2→L3 lowering - /// produces this when it recognises `topk(k, …)` (PromQL) or - /// `ORDER BY count DESC LIMIT k` (SQL). - TopK { - k: usize, - accuracy: AccuracyTarget, - }, - /// COUNT DISTINCT — number of distinct values in the input column, - /// to the given accuracy. Sketch families: HLL, theta-sketch. - Cardinality { - accuracy: AccuracyTarget, - }, - /// Frequency of a key in the input — `count(*) WHERE key = k` modeled - /// as a sketch query. Sketch families: CMS, count-min-log. - Frequency { - accuracy: AccuracyTarget, - }, - - // ── Time-series streaming derivatives ──────────────────────────────── - /// Per-second average derivative with PromQL's counter-reset - /// adjustment. Specialized — exact `Sum / Count over Window` does - /// NOT serve this intent. - Rate { - window: Duration, - }, - /// Cumulative increase over the given window with counter-reset - /// adjustment. Specialized — see `Rate` above. - Increase { - window: Duration, - }, - - // ── Archive-only intents (Phase β migration) ───────────────────────── - // Intents below have no warm-tier sketch family today; the L4 binder - // emits a `SketchExpr::Logical` pass-through and the L5 emitter routes - // them to the cold archive tier (Gorilla / Thanos). Adding a streaming - // sketch family for any of these is a follow-up — the L3 vocabulary - // captures the intent so the routing decision is layered above intent. - // - /// `histogram_quantile(φ, le_bucketed_metric)`. Operates on Prometheus - /// histogram buckets — semantically a quantile readout but the input - /// shape (per-bucket counter) requires bucket-aware aggregation that - /// the existing KLL / DDSketch rules don't model. Archive-only today. - HistogramQuantile { - q: f64, - }, - /// `absent(vector_selector)` — 1 iff the selector matched no series in - /// the evaluation window, no value otherwise. Routed to archive: the - /// engine answers it directly off the index. - Absent, - /// `present_over_time(m[range])` — 1 iff the selector had at least one - /// sample in the window. Inverse of `Absent`. Archive-routed. - Present, - /// `delta(m[range])` — last − first sample within the window, NO - /// counter-reset adjustment. Distinct from [`AggIntent::Increase`]. - Delta { - window: Duration, - }, - /// `deriv(m[range])` — per-second derivative via simple linear - /// regression. Archive-routed (no streaming sketch). - Deriv { - window: Duration, - }, - /// `predict_linear(m[range], t)` — linear-regression prediction `t` - /// seconds into the future. Archive-routed. - PredictLinear { - window: Duration, - ahead: Duration, - }, - /// `holt_winters(m[range], sf, tf)` — exponential-smoothing forecast. - /// Archive-routed. - HoltWinters { - window: Duration, - smoothing_factor: f64, - trend_factor: f64, - }, - /// `idelta(m[range])` — `last − second_to_last`, instant delta. No - /// streaming sketch. - Idelta { - window: Duration, - }, - /// `irate(m[range])` — instant per-second rate computed from the last - /// two samples. Counter-reset adjusted but evaluated point-wise; not - /// the same as the streaming [`AggIntent::Rate`]. - Irate { - window: Duration, - }, - /// `resets(m[range])` — count of counter resets over the window. - /// Archive-routed. - Resets { - window: Duration, - }, - /// `changes(m[range])` — count of value changes over the window. - /// Archive-routed. - Changes { - window: Duration, - }, -} - -impl AggIntent { - /// True iff this intent has no warm-tier (streaming sketch) binding - /// today. `false` means a `Bind*` rule may match. `true` means the - /// L5 emitter routes the intent to the cold-store / archive tier. - /// - /// Per Phase β orchestrator spec, the new `HistogramQuantile` plus the - /// PromQL functions that were previously refused outright by - /// `asap-planner-rs::single_query::is_supported()` (everything outside - /// the 5 patterns) all return `true` here. Adding a sketch family for - /// any of them is a future PR — flipping the flag to `false` is the - /// single point of change. - pub fn archive_only(&self) -> bool { - matches!( - self, - AggIntent::HistogramQuantile { .. } - | AggIntent::Absent - | AggIntent::Present - | AggIntent::Delta { .. } - | AggIntent::Deriv { .. } - | AggIntent::PredictLinear { .. } - | AggIntent::HoltWinters { .. } - | AggIntent::Idelta { .. } - | AggIntent::Irate { .. } - | AggIntent::Resets { .. } - | AggIntent::Changes { .. } - ) - } -} - -impl AggIntent { - /// Output column name + type produced by this intent when applied to - /// `input`. Used by `QueryExpr::Aggregate`'s schema-derivation rule - /// (`design.md` §6 schema-flow table: "one new column per entry in - /// `aggs`, each named and typed by `AggIntent::output_type(input_field)`"). - /// - /// PromQL convention: aggregate column name = intent kind (`count`, - /// `quantile_0_99`, …) so consumers can locate it without an alias - /// lookup. - pub fn output_column(&self, input: &Column) -> Column { - match self { - AggIntent::Count { .. } => Column { - name: "count".into(), - dtype: DataType::Int64, - nullable: false, - }, - AggIntent::Sum => Column { - name: "sum".into(), - dtype: input.dtype.clone(), - nullable: false, - }, - AggIntent::Min => Column { - name: "min".into(), - dtype: input.dtype.clone(), - nullable: input.nullable, - }, - AggIntent::Max => Column { - name: "max".into(), - dtype: input.dtype.clone(), - nullable: input.nullable, - }, - AggIntent::Avg => Column { - name: "avg".into(), - dtype: DataType::Float64, - nullable: false, - }, - AggIntent::Quantile { q, .. } => Column { - name: format!("quantile_{}", quantile_suffix(*q)), - dtype: DataType::Float64, - nullable: false, - }, - AggIntent::TopK { k, .. } => Column { - name: format!("topk_{k}"), - // TopK output is a struct/list per row; modeled as Utf8 - // for L3 (the L4 sketch-bound IR upgrades the dtype). - dtype: DataType::Utf8, - nullable: false, - }, - AggIntent::Cardinality { .. } => Column { - name: "cardinality".into(), - dtype: DataType::Int64, - nullable: false, - }, - AggIntent::Frequency { .. } => Column { - name: "frequency".into(), - dtype: DataType::Int64, - nullable: false, - }, - AggIntent::Rate { .. } => Column { - name: "rate".into(), - dtype: DataType::Float64, - nullable: false, - }, - AggIntent::Increase { .. } => Column { - name: "increase".into(), - dtype: DataType::Float64, - nullable: false, - }, - // ── Archive-only intents (Phase β) ──────────────────────────── - // Each carries a stable column name keyed on the intent kind so - // the StreamingConfig emitter and Phase α routing entry can - // locate them. All are Float64 except the boolean Absent / - // Present, which surface as Int64 (1 / 0) per PromQL convention. - AggIntent::HistogramQuantile { q } => Column { - name: format!("histogram_quantile_{}", quantile_suffix(*q)), - dtype: DataType::Float64, - nullable: false, - }, - AggIntent::Absent => Column { - name: "absent".into(), - dtype: DataType::Int64, - nullable: false, - }, - AggIntent::Present => Column { - name: "present".into(), - dtype: DataType::Int64, - nullable: false, - }, - AggIntent::Delta { .. } => Column { - name: "delta".into(), - dtype: DataType::Float64, - nullable: false, - }, - AggIntent::Deriv { .. } => Column { - name: "deriv".into(), - dtype: DataType::Float64, - nullable: false, - }, - AggIntent::PredictLinear { .. } => Column { - name: "predict_linear".into(), - dtype: DataType::Float64, - nullable: false, - }, - AggIntent::HoltWinters { .. } => Column { - name: "holt_winters".into(), - dtype: DataType::Float64, - nullable: false, - }, - AggIntent::Idelta { .. } => Column { - name: "idelta".into(), - dtype: DataType::Float64, - nullable: false, - }, - AggIntent::Irate { .. } => Column { - name: "irate".into(), - dtype: DataType::Float64, - nullable: false, - }, - AggIntent::Resets { .. } => Column { - name: "resets".into(), - dtype: DataType::Int64, - nullable: false, - }, - AggIntent::Changes { .. } => Column { - name: "changes".into(), - dtype: DataType::Int64, - nullable: false, - }, - } - } -} - -/// `0.99` → `"0_99"`, `0.5` → `"0_5"`. Used by `Quantile` output naming -/// so `quantile_0_99` is a valid identifier downstream. -fn quantile_suffix(q: f64) -> String { - let mut s = format!("{q}"); - if let Some(stripped) = s.strip_prefix('-') { - s = format!("neg_{stripped}"); - } - s.replace('.', "_") -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use crate::intent_algebra::schema::{Column, DataType}; - - fn col(name: &str, dtype: DataType) -> Column { - Column { - name: name.into(), - dtype, - nullable: false, - } - } - - #[test] - fn agg_intent_serde_roundtrip() { - let cases = vec![ - AggIntent::Count { - accuracy: AccuracyTarget::Exact, - }, - AggIntent::Sum, - AggIntent::Min, - AggIntent::Max, - AggIntent::Avg, - AggIntent::Quantile { - q: 0.99, - accuracy: AccuracyTarget::Epsilon(0.01), - }, - AggIntent::TopK { - k: 10, - accuracy: AccuracyTarget::Epsilon(0.05), - }, - AggIntent::Cardinality { - accuracy: AccuracyTarget::EpsilonDelta { - eps: 0.01, - delta: 0.001, - }, - }, - AggIntent::Frequency { - accuracy: AccuracyTarget::Epsilon(0.01), - }, - AggIntent::Rate { - window: Duration::from_secs(60), - }, - AggIntent::Increase { - window: Duration::from_secs(300), - }, - ]; - for variant in cases { - let json = serde_json::to_string(&variant).unwrap(); - let back: AggIntent = serde_json::from_str(&json).unwrap(); - assert_eq!(variant, back, "round-trip failed for {variant:?}"); - } - } - - #[test] - fn output_column_names_are_intent_keyed() { - let v = col("value", DataType::Float64); - assert_eq!( - AggIntent::Count { - accuracy: AccuracyTarget::Exact, - } - .output_column(&v) - .name, - "count" - ); - assert_eq!(AggIntent::Sum.output_column(&v).name, "sum"); - assert_eq!( - AggIntent::Quantile { - q: 0.99, - accuracy: AccuracyTarget::Epsilon(0.01), - } - .output_column(&v) - .name, - "quantile_0_99" - ); - assert_eq!( - AggIntent::TopK { - k: 5, - accuracy: AccuracyTarget::Exact, - } - .output_column(&v) - .name, - "topk_5" - ); - } - - #[test] - fn quantile_output_is_float64() { - let v = col("value", DataType::Int64); - let out = AggIntent::Quantile { - q: 0.5, - accuracy: AccuracyTarget::Epsilon(0.01), - } - .output_column(&v); - assert!(matches!(out.dtype, DataType::Float64)); - } - - #[test] - fn sum_preserves_input_dtype() { - let int_col = col("c", DataType::Int64); - let float_col = col("c", DataType::Float64); - assert!(matches!( - AggIntent::Sum.output_column(&int_col).dtype, - DataType::Int64 - )); - assert!(matches!( - AggIntent::Sum.output_column(&float_col).dtype, - DataType::Float64 - )); - } - - // ── Phase β archive-only intent tests ───────────────────────────────── - - /// Every intent the legacy `asap-planner-rs::single_query::is_supported` - /// previously refused now lifts to L3 with `archive_only() == true`. - /// The negative cases are the warm-tier-bound intents — they must - /// continue to return false, otherwise the L4 binder would short-circuit - /// them to the cold tier. - #[test] - fn archive_only_flag_partitions_intents() { - // Archive-only — every Phase β migration target. - let archive: Vec = vec![ - AggIntent::HistogramQuantile { q: 0.99 }, - AggIntent::Absent, - AggIntent::Present, - AggIntent::Delta { - window: Duration::from_secs(60), - }, - AggIntent::Deriv { - window: Duration::from_secs(60), - }, - AggIntent::PredictLinear { - window: Duration::from_secs(300), - ahead: Duration::from_secs(60), - }, - AggIntent::HoltWinters { - window: Duration::from_secs(300), - smoothing_factor: 0.3, - trend_factor: 0.3, - }, - AggIntent::Idelta { - window: Duration::from_secs(60), - }, - AggIntent::Irate { - window: Duration::from_secs(60), - }, - AggIntent::Resets { - window: Duration::from_secs(300), - }, - AggIntent::Changes { - window: Duration::from_secs(300), - }, - ]; - for v in archive { - assert!( - v.archive_only(), - "{v:?} should be archive-only after Phase β migration" - ); - } - - // Warm-tier — must NOT be flagged archive-only or the L4 binder - // breaks. - let warm: Vec = vec![ - AggIntent::Count { - accuracy: AccuracyTarget::Exact, - }, - AggIntent::Sum, - AggIntent::Min, - AggIntent::Max, - AggIntent::Avg, - AggIntent::Quantile { - q: 0.99, - accuracy: AccuracyTarget::Epsilon(0.01), - }, - AggIntent::TopK { - k: 10, - accuracy: AccuracyTarget::Epsilon(0.05), - }, - AggIntent::Cardinality { - accuracy: AccuracyTarget::EpsilonDelta { - eps: 0.01, - delta: 0.001, - }, - }, - AggIntent::Frequency { - accuracy: AccuracyTarget::Epsilon(0.01), - }, - AggIntent::Rate { - window: Duration::from_secs(60), - }, - AggIntent::Increase { - window: Duration::from_secs(300), - }, - ]; - for v in warm { - assert!( - !v.archive_only(), - "{v:?} is warm-tier and must not be archive-only" - ); - } - } - - #[test] - fn archive_only_intent_serde_roundtrip() { - let cases = vec![ - AggIntent::HistogramQuantile { q: 0.99 }, - AggIntent::Absent, - AggIntent::Present, - AggIntent::Delta { - window: Duration::from_secs(60), - }, - AggIntent::Deriv { - window: Duration::from_secs(60), - }, - AggIntent::PredictLinear { - window: Duration::from_secs(300), - ahead: Duration::from_secs(60), - }, - AggIntent::HoltWinters { - window: Duration::from_secs(300), - smoothing_factor: 0.3, - trend_factor: 0.3, - }, - AggIntent::Idelta { - window: Duration::from_secs(60), - }, - AggIntent::Irate { - window: Duration::from_secs(60), - }, - AggIntent::Resets { - window: Duration::from_secs(300), - }, - AggIntent::Changes { - window: Duration::from_secs(300), - }, - ]; - for v in cases { - let json = serde_json::to_string(&v).unwrap(); - let back: AggIntent = serde_json::from_str(&json).unwrap(); - assert_eq!(v, back, "round-trip failed for {v:?}"); - } - } - - #[test] - fn archive_only_output_column_names() { - let v = col("value", DataType::Float64); - assert_eq!( - AggIntent::HistogramQuantile { q: 0.99 } - .output_column(&v) - .name, - "histogram_quantile_0_99" - ); - assert_eq!(AggIntent::Absent.output_column(&v).name, "absent"); - assert_eq!(AggIntent::Present.output_column(&v).name, "present"); - assert_eq!( - AggIntent::Delta { - window: Duration::from_secs(60) - } - .output_column(&v) - .name, - "delta" - ); - assert_eq!( - AggIntent::Resets { - window: Duration::from_secs(60) - } - .output_column(&v) - .name, - "resets" - ); - } -} diff --git a/controller/src/intent_algebra/cse.rs b/controller/src/intent_algebra/cse.rs deleted file mode 100644 index d370e3e5..00000000 --- a/controller/src/intent_algebra/cse.rs +++ /dev/null @@ -1,327 +0,0 @@ -//! Workload-level Common Sub-Expression Elimination. -//! -//! Per `controller/docs/design.md` §6 batched-queries example (line ~1256 -//! through ~1320). Multi-root planning hoists shared sub-DAGs into -//! `LetBinding`s so the cost model can credit the producer once. -//! -//! Phase F lands the **gate + a basic implementation** that handles the -//! literal "≥2 root queries with identical sub-expressions" case from the -//! design — sufficient to make the workload-cost path observable end-to- -//! end. The fully-general CSE algorithm (alpha-equivalence across -//! `LetBinding` rebinding, schema-merge across compatible-but-not-identical -//! shapes, cross-binding nested CSE) is deferred per design.md §6 line -//! ~562 — it is a downstream optimisation pass, not part of the IR -//! contract Phase F is delivering. -//! -//! Legality is gated by [`cse_reuse_is_legal`](super::schema::cse_reuse_is_legal): -//! a candidate sub-DAG only becomes a `LetBinding` when its output schema -//! has at least one `unique_keys` set (§6 line ~1356 — the field is -//! load-bearing for this pass). - -#![allow(dead_code)] - -use std::collections::HashMap; - -use crate::intent_algebra::query_expr::QueryExpr; -use crate::intent_algebra::schema::cse_reuse_is_legal; -use crate::types_v2::{BindingName, QueryId}; - -/// Multi-root container produced by the CSE pass — mirrors the shape of -/// `types_v2::WorkloadPlan` (§6 batched-queries example) but uses the -/// real `intent_algebra::QueryExpr` rather than the JSON wire-shape -/// `QueryExprPlaceholder` string. -/// -/// When `types_v2::WorkloadPlan` swaps the placeholder for the live -/// `QueryExpr`, this type collapses into that one without an API break. -#[derive(Debug, Clone, PartialEq)] -pub struct CseWorkloadPlan { - /// Named shared producers, hoisted by `dedupe_subtrees`. Each is - /// referenced by ≥2 roots via `QueryExpr::Ref`. - pub bindings: Vec<(BindingName, QueryExpr)>, - /// One root per input query, in input order. - pub roots: Vec<(QueryId, QueryExpr)>, -} - -/// Hoist sub-expressions that are *structurally identical* across ≥2 -/// roots into shared `LetBinding`s, leaving each root with `Ref` sites -/// where the duplicate sub-tree used to live. Per design.md §6 line -/// ~1272 ("a workload-level CSE pass `core::lower::workload::dedupe_subtrees`"). -/// -/// **Phase F scope.** Implements the basic case: identifies sub-trees -/// that appear verbatim (structural equality via `PartialEq`) in ≥2 root -/// inputs and hoists them. Schema-equivalent-but-not-identical sub-trees, -/// alpha-equivalence over inner `LetBinding`s, and recursive nested CSE -/// are deferred — they are the optimisation half of the pass and live -/// downstream of this PR. -/// -/// **Legality.** A candidate sub-tree is hoisted only when -/// `cse_reuse_is_legal(&candidate.output_schema(), consumer_count)` -/// returns `Ok(())`. Sub-trees whose output schema lacks `unique_keys` -/// are left in place per design.md §6 line ~1356 (the deduper must be -/// conservative when it can't prove row identity). -pub fn dedupe_subtrees(roots: Vec<(QueryId, QueryExpr)>) -> CseWorkloadPlan { - // Empty / single-root cases: no reuse possible. Return the inputs - // verbatim with no bindings. - if roots.len() < 2 { - return CseWorkloadPlan { - bindings: vec![], - roots, - }; - } - - // Phase F: identify candidate sub-trees that appear as the immediate - // child of an `Aggregate` in ≥2 roots. The batched-queries example - // shape — multiple `Aggregate`s sharing one `Window`-child producer - // — is the case Phase F lights up; richer detection is downstream. - let mut candidate_counts: HashMap = HashMap::new(); - for (_, root) in &roots { - if let QueryExpr::Aggregate { child, .. } = root { - // Skip already-aliased children (a `Ref` is not a candidate - // for hoisting; it's already pointing at a binding). - if matches!(**child, QueryExpr::Ref { .. }) { - continue; - } - // Use the Debug representation as a structural-key proxy. - // Cheap to compute and matches `PartialEq` for - // `QueryExpr` → adequate for the Phase F basic case. - let key = format!("{child:?}"); - let entry = candidate_counts - .entry(key) - .or_insert_with(|| ((**child).clone(), 0)); - entry.1 += 1; - } - } - - // Pick the most-shared legal candidate. Phase F hoists at most one - // binding per call; the "hoist all eligible candidates" generalisation - // is a follow-up. Choosing the most-shared first matches the design's - // priority — biggest reuse first. - let mut chosen: Option<(QueryExpr, usize)> = None; - for (_key, (expr, count)) in candidate_counts.into_iter() { - if count < 2 { - continue; - } - // Legality gate: producer schema must have `unique_keys` for - // ≥2 consumers to share it (design.md §6 line ~1356). - let Ok(out_schema) = expr.output_schema() else { - continue; - }; - if cse_reuse_is_legal(&out_schema, count).is_err() { - continue; - } - // Bigger fan-in wins; ties broken arbitrarily (HashMap order). - match &chosen { - Some((_, best_count)) if *best_count >= count => {} - _ => chosen = Some((expr, count)), - } - } - - let Some((shared_expr, _count)) = chosen else { - // No eligible candidate — leave roots untouched. - return CseWorkloadPlan { - bindings: vec![], - roots, - }; - }; - - // Rewrite each root: where the Aggregate's child equals - // `shared_expr`, replace with `Ref { name: "shared_0" }`. - let binding_name = BindingName::new("shared_0"); - let mut rewritten: Vec<(QueryId, QueryExpr)> = Vec::with_capacity(roots.len()); - for (qid, root) in roots { - let new_root = match root { - QueryExpr::Aggregate { - by, - aggs, - having, - child, - } if *child == shared_expr => QueryExpr::Aggregate { - by, - aggs, - having, - child: Box::new(QueryExpr::Ref { - name: binding_name.clone(), - }), - }, - other => other, - }; - rewritten.push((qid, new_root)); - } - - CseWorkloadPlan { - bindings: vec![(binding_name, shared_expr)], - roots: rewritten, - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use crate::intent_algebra::agg_intent::AggIntent; - use crate::intent_algebra::query_expr::{LabelFilter, Source, WindowKind}; - use crate::intent_algebra::schema::{Column, DataType, Schema}; - use crate::types_v2::AccuracyTarget; - use std::time::Duration; - - fn col(name: &str, dtype: DataType) -> Column { - Column { - name: name.into(), - dtype, - nullable: false, - } - } - - fn ts_scan() -> QueryExpr { - QueryExpr::Scan { - source: Source::TimeSeries { - metric: "http_request_duration_seconds".into(), - }, - label_filters: vec![LabelFilter { - label: "service".into(), - equals: "api".into(), - }], - schema: Schema::with_time_index( - vec![ - col("ts", DataType::Timestamp), - col("service", DataType::Utf8), - col("value", DataType::Float64), - ], - 0, - vec![vec![0, 1]], - ), - } - } - - fn windowed_scan() -> QueryExpr { - QueryExpr::Window { - kind: WindowKind::Sliding, - size: Duration::from_secs(300), - slide: None, - child: Box::new(ts_scan()), - } - } - - /// Empty input → empty output (no bindings, no roots). - #[test] - fn dedupe_subtrees_empty_input() { - let out = dedupe_subtrees(vec![]); - assert!(out.bindings.is_empty()); - assert!(out.roots.is_empty()); - } - - /// Single-root input → no reuse possible, returned verbatim. - #[test] - fn dedupe_subtrees_single_root_passthrough() { - let q = QueryExpr::Aggregate { - by: vec![1], - aggs: vec![AggIntent::Quantile { - q: 0.99, - accuracy: AccuracyTarget::Epsilon(0.01), - }], - having: None, - child: Box::new(windowed_scan()), - }; - let out = dedupe_subtrees(vec![(QueryId::new("q1"), q.clone())]); - assert!(out.bindings.is_empty()); - assert_eq!(out.roots.len(), 1); - assert_eq!(out.roots[0].1, q); - } - - /// design.md §6 batched-queries example basic case: two queries with - /// identical `Window` sub-trees — the deduper hoists the shared - /// producer into a binding and rewrites each root to reference it. - #[test] - fn dedupe_subtrees_basic() { - let q1 = QueryExpr::Aggregate { - by: vec![1], - aggs: vec![AggIntent::Quantile { - q: 0.99, - accuracy: AccuracyTarget::Epsilon(0.01), - }], - having: None, - child: Box::new(windowed_scan()), - }; - let q2 = QueryExpr::Aggregate { - by: vec![1], - aggs: vec![AggIntent::Quantile { - q: 0.95, - accuracy: AccuracyTarget::Epsilon(0.01), - }], - having: None, - child: Box::new(windowed_scan()), - }; - - let out = dedupe_subtrees(vec![ - (QueryId::new("q1"), q1), - (QueryId::new("q2"), q2), - ]); - - // One binding hoisted, two roots rewritten to `Ref { name: "shared_0" }`. - assert_eq!(out.bindings.len(), 1); - assert_eq!(out.bindings[0].0, BindingName::new("shared_0")); - assert_eq!(out.bindings[0].1, windowed_scan()); - - for (_, root) in &out.roots { - match root { - QueryExpr::Aggregate { child, .. } => assert_eq!( - **child, - QueryExpr::Ref { - name: BindingName::new("shared_0"), - }, - "Aggregate child should be a Ref to the hoisted binding" - ), - other => panic!("expected Aggregate root, got {other:?}"), - } - } - } - - /// Two roots with *different* sub-expressions — no shared producer, - /// no binding hoisted, roots returned untouched. - #[test] - fn dedupe_subtrees_no_shared_subexpr() { - let q1 = QueryExpr::Aggregate { - by: vec![], - aggs: vec![AggIntent::Sum], - having: None, - child: Box::new(windowed_scan()), - }; - // q2 uses a different scan (different metric) — Debug repr - // differs → no hoisting. - let other_scan = QueryExpr::Scan { - source: Source::TimeSeries { - metric: "different_metric".into(), - }, - label_filters: vec![], - schema: Schema::with_time_index( - vec![ - col("ts", DataType::Timestamp), - col("service", DataType::Utf8), - col("value", DataType::Float64), - ], - 0, - vec![vec![0, 1]], - ), - }; - let q2 = QueryExpr::Aggregate { - by: vec![], - aggs: vec![AggIntent::Max], - having: None, - child: Box::new(QueryExpr::Window { - kind: WindowKind::Sliding, - size: Duration::from_secs(300), - slide: None, - child: Box::new(other_scan), - }), - }; - - let out = dedupe_subtrees(vec![ - (QueryId::new("q1"), q1.clone()), - (QueryId::new("q2"), q2.clone()), - ]); - assert!(out.bindings.is_empty(), "no shared subexpr → no binding"); - assert_eq!(out.roots[0].1, q1); - assert_eq!(out.roots[1].1, q2); - } -} diff --git a/controller/src/intent_algebra/lower.rs b/controller/src/intent_algebra/lower.rs deleted file mode 100644 index 8912e6d5..00000000 --- a/controller/src/intent_algebra/lower.rs +++ /dev/null @@ -1,352 +0,0 @@ -//! Lowering from `query_parser::ParsedQuery` → L3 [`QueryExpr`]. -//! -//! Phase B exposes this as a standalone function so callers can opt into -//! the L3 IR without touching the existing `Analyzer::analyze` → -//! `QueryWorkload` pipeline. Wiring the analyzer to *also* emit a -//! `QueryExpr` is the follow-up phase's job. -//! -//! Per-language scope. The `parse_query` entry point in -//! `query_parser::mod` already dispatches PromQL vs SQL; by the time we -//! see a `ParsedQuery` the language distinction has been collapsed onto -//! the flat summary. For PromQL (the only language this crate consumes -//! per the orchestrator spec), the lowering is: -//! -//! ```text -//! Scan{TimeSeries} → [Window?] → Aggregate{ by, [intent_per_aggregation] } -//! ``` -//! -//! The window is emitted only when `ParsedQuery.time_window` is non-zero -//! and the query has at least one aggregation (a bare metric selector -//! lowers to a `Scan` alone). Workload-level CSE that produces fan-in -//! (multi-root with `LetBinding` / `Ref`) is a follow-up; this lowers -//! each query independently. - -#![allow(dead_code)] - -use std::collections::HashMap; -use thiserror::Error; - -use crate::intent_algebra::agg_intent::AggIntent; -use crate::intent_algebra::query_expr::{ - LabelFilter, QueryExpr, QueryExprError, Source, WindowKind, -}; -use crate::intent_algebra::schema::{Column, ColumnId, DataType, Schema}; -use crate::query_parser::ParsedQuery; -use crate::types::AggType; -use crate::types_v2::AccuracyTarget; - -/// Errors returned by [`lower_parsed_query`]. -#[derive(Debug, Error)] -pub enum LoweringError { - /// `ParsedQuery.metric_name` was empty — nothing to scan. - #[error("empty metric name in parsed query")] - EmptyMetricName, - /// `Aggregate` schema-derivation invariant failed inside the lowered - /// tree. Indicates the lowering logic produced an inconsistent shape; - /// surfaced rather than panicked. - #[error("post-lower schema check failed: {0}")] - PostLowerSchema(#[from] QueryExprError), -} - -/// Lower a `ParsedQuery` into the L3 `QueryExpr` DAG. Single-rooted -/// (one query in, one root out). Workload-level CSE is a follow-up. -/// -/// The accuracy target is supplied separately because `ParsedQuery` -/// carries the legacy `accuracy_sla: f64` only on the `QueryWorkload` -/// downstream of analyzer; passing it explicitly keeps this function -/// orthogonal to the analyzer wiring. -pub fn lower_parsed_query( - parsed: &ParsedQuery, - accuracy: AccuracyTarget, -) -> Result { - if parsed.metric_name.trim().is_empty() { - return Err(LoweringError::EmptyMetricName); - } - - // ── Scan ───────────────────────────────────────────────────────────── - let scan_schema = scan_schema_for(parsed); - let label_filters = parsed - .label_filters - .iter() - .map(|(k, v)| LabelFilter { - label: k.clone(), - equals: v.clone(), - }) - .collect::>(); - let mut node = QueryExpr::Scan { - source: Source::TimeSeries { - metric: parsed.metric_name.clone(), - }, - label_filters, - schema: scan_schema.clone(), - }; - - // Bare metric selector (no aggregations, no window) — return the Scan - // directly. PromQL `up{job="api"}` lowers here. - if parsed.aggregations.is_empty() && parsed.time_window.is_zero() { - return Ok(node); - } - - // ── Window (optional) ─────────────────────────────────────────────── - if !parsed.time_window.is_zero() { - node = QueryExpr::Window { - // PromQL `[5m]` is canonically Sliding (per design.md §6 - // line ~308: "PromQL `[5m]` and streaming windows lower - // here"). SQL `TUMBLE` would lower to `Tumbling` — out of - // scope for the PromQL-only Phase B. - kind: WindowKind::Sliding, - size: parsed.time_window, - slide: None, - child: Box::new(node), - }; - } - - // ── Aggregate (optional, when intents present) ───────────────────── - if !parsed.aggregations.is_empty() { - let aggs = build_intents(parsed, accuracy); - let by = group_by_column_ids(&scan_schema, &parsed.group_by_labels); - node = QueryExpr::Aggregate { - by, - aggs, - having: None, - child: Box::new(node), - }; - } - - // Validate the lowered tree has a derivable output schema. Catches - // `Window` without time_index, out-of-range `by` ids, etc. The - // discard is intentional — we only want the validation side-effect. - let _ = node.output_schema()?; - - Ok(node) -} - -/// Synthesise the output schema of the `Scan` for this `ParsedQuery`. -/// -/// PromQL convention: a leaf produces `(ts, value, *labels)` with `ts` at -/// position 0 (the time index) and `value` at position 1. Group-by -/// labels and equality-filter labels are projected into positions 2..N -/// in alphabetical order (deterministic). The unique-key set is -/// `[ts, *labels]` — one sample per (timestamp, label-tuple) pair. -fn scan_schema_for(parsed: &ParsedQuery) -> Schema { - let mut columns = vec![ - Column { - name: "ts".into(), - dtype: DataType::Timestamp, - nullable: false, - }, - Column { - name: "value".into(), - dtype: DataType::Float64, - nullable: false, - }, - ]; - // Collect label names from group_by + filters, dedup + sort for - // determinism. - let mut labels: Vec = parsed.group_by_labels.to_vec(); - for k in parsed.label_filters.keys() { - if !labels.contains(k) { - labels.push(k.clone()); - } - } - labels.sort(); - for label in &labels { - columns.push(Column { - name: label.clone(), - dtype: DataType::Utf8, - nullable: false, - }); - } - // unique_keys = [ts, *labels] — one sample per (ts, label-tuple). - let mut uk: Vec = vec![0]; // ts - for i in 2..columns.len() { - uk.push(i); - } - Schema::with_time_index(columns, 0, vec![uk]) -} - -/// Resolve `group_by_labels` (names) to column ids in the scan schema. -/// Labels missing from the schema are silently dropped — they wouldn't -/// have been preserved by the scan anyway. -fn group_by_column_ids(schema: &Schema, group_by_labels: &[String]) -> Vec { - group_by_labels - .iter() - .filter_map(|name| schema.column_id(name)) - .collect() -} - -/// Translate the legacy `(AggType, Vec, exact_required)` triple -/// from `ParsedQuery` into typed `AggIntent` entries. -/// -/// Each `AggType::Quantile` fans out to one `AggIntent::Quantile{q}` per -/// `parsed.quantiles` entry — `quantile_over_time(0.95)` and -/// `quantile_over_time(0.99)` co-occurring on the same metric produce -/// two intents on the same `Aggregate`. -fn build_intents(parsed: &ParsedQuery, accuracy: AccuracyTarget) -> Vec { - let effective_accuracy = if parsed.exact_required { - AccuracyTarget::Exact - } else { - accuracy.clone() - }; - - // Track which quantiles we've already emitted to avoid duplicates - // when multiple legacy `AggType` rows imply the same φ (e.g. Min → - // q=0.0, Max → q=1.0 historically lived under `AggType::Quantile`). - let mut emitted_q = HashMap::::new(); - let mut out = Vec::new(); - for agg in &parsed.aggregations { - match agg { - AggType::Quantile => { - if parsed.quantiles.is_empty() { - out.push(AggIntent::Quantile { - q: 0.5, - accuracy: effective_accuracy.clone(), - }); - } else { - for &q in &parsed.quantiles { - let key = (q * 1e9) as u64; - if emitted_q.insert(key, true).is_none() { - out.push(AggIntent::Quantile { - q, - accuracy: effective_accuracy.clone(), - }); - } - } - } - } - AggType::Cardinality => out.push(AggIntent::Cardinality { - accuracy: effective_accuracy.clone(), - }), - AggType::Frequency => out.push(AggIntent::Frequency { - accuracy: effective_accuracy.clone(), - }), - } - } - if out.is_empty() && parsed.exact_required { - // PromQL `sum`, `rate`, … — no sketch intent fires; the lowering - // surfaces this as `AggIntent::Sum` so the L3 schema is - // well-formed (an `Aggregate` with empty aggs is meaningless). - out.push(AggIntent::Sum); - } - out -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use crate::intent_algebra::query_expr::QueryExpr; - use crate::query_parser::parse_query; - - #[test] - fn lower_promql_basic() { - let parsed = parse_query( - "quantile_over_time(0.99, http_request_duration_seconds{service=\"api\"}[5m])", - ) - .expect("parse"); - let expr = lower_parsed_query(&parsed, AccuracyTarget::Epsilon(0.01)).expect("lower"); - - // Root: Aggregate { aggs: [Quantile{0.99, ε=0.01}] } - match &expr { - QueryExpr::Aggregate { aggs, child, .. } => { - assert_eq!(aggs.len(), 1); - match &aggs[0] { - AggIntent::Quantile { q, accuracy } => { - assert!((*q - 0.99).abs() < 1e-9); - assert_eq!(*accuracy, AccuracyTarget::Epsilon(0.01)); - } - other => panic!("expected Quantile intent, got {other:?}"), - } - // Mid: Window - match child.as_ref() { - QueryExpr::Window { kind, size, child, .. } => { - assert_eq!(*kind, WindowKind::Sliding); - assert_eq!(*size, std::time::Duration::from_secs(300)); - // Leaf: Scan - match child.as_ref() { - QueryExpr::Scan { - source, - label_filters, - schema, - } => { - match source { - Source::TimeSeries { metric } => { - assert_eq!(metric, "http_request_duration_seconds") - } - _ => panic!("expected TimeSeries source"), - } - assert_eq!(label_filters.len(), 1); - assert_eq!(label_filters[0].label, "service"); - assert_eq!(label_filters[0].equals, "api"); - // Schema: ts (time_index), value, service - assert_eq!(schema.time_index, Some(0)); - assert_eq!(schema.columns[0].name, "ts"); - assert_eq!(schema.columns[1].name, "value"); - assert!(schema.has_unique_key()); - } - other => panic!("expected Scan, got {other:?}"), - } - } - other => panic!("expected Window, got {other:?}"), - } - } - other => panic!("expected Aggregate at root, got {other:?}"), - } - - // The lowered tree's output schema is derivable end-to-end. - let schema = expr.output_schema().expect("output_schema"); - assert!( - schema.columns.iter().any(|c| c.name == "quantile_0_99"), - "output schema should contain quantile_0_99 column, got {:?}", - schema.columns - ); - } - - #[test] - fn lower_promql_with_group_by() { - let parsed = parse_query( - "sum by (host) (quantile_over_time(0.95, latency{env=\"prod\"}[1m]))", - ) - .expect("parse"); - let expr = lower_parsed_query(&parsed, AccuracyTarget::Epsilon(0.05)).expect("lower"); - let schema = expr.output_schema().expect("output_schema"); - // `host` is a group-by → it lands in unique_keys at position 0 - // in the output (positional projection of `by`). - assert!(schema.has_unique_key()); - assert!(schema.columns.iter().any(|c| c.name == "host")); - } - - #[test] - fn lower_promql_cardinality() { - let parsed = parse_query( - "count by (user_id) (count_over_time(active_users{env=\"prod\"}[5m]))", - ) - .expect("parse"); - let expr = lower_parsed_query(&parsed, AccuracyTarget::Epsilon(0.01)).expect("lower"); - let schema = expr.output_schema().expect("output_schema"); - // Cardinality intent → output column named `cardinality` of dtype Int64. - let card = schema - .columns - .iter() - .find(|c| c.name == "cardinality") - .expect("cardinality column should be present"); - assert!(matches!(card.dtype, DataType::Int64)); - } - - #[test] - fn lower_empty_metric_errors() { - let parsed = ParsedQuery { - metric_name: String::new(), - aggregations: vec![], - group_by_labels: vec![], - label_filters: HashMap::new(), - time_window: std::time::Duration::ZERO, - exact_required: false, - quantiles: vec![], - hint: None, - }; - let err = lower_parsed_query(&parsed, AccuracyTarget::Exact).unwrap_err(); - assert!(matches!(err, LoweringError::EmptyMetricName)); - } -} diff --git a/controller/src/intent_algebra/mod.rs b/controller/src/intent_algebra/mod.rs deleted file mode 100644 index 0927ef77..00000000 --- a/controller/src/intent_algebra/mod.rs +++ /dev/null @@ -1,90 +0,0 @@ -//! Layer 3 IR — `core::intent_algebra` per `controller/docs/design.md` §6. -//! -//! ## Phase β cross-reference: asap-planner-rs PromQL patterns -//! -//! `ASAPQuery-backend/asap-planner-rs/src/planner/patterns.rs` defines five -//! PromQL `PromQLPattern` shapes. Every one of those shapes maps onto an -//! [`AggIntent`] kind here — this is the surface the controller's L3 layer -//! exposes so Phase γ can delete the asap-planner-rs binary without losing -//! coverage: -//! -//! | asap-planner-rs pattern (`patterns.rs`) | Controller L3 equivalent | -//! |---|---| -//! | `ONLY_TEMPORAL` quantile (`quantile_over_time(φ, m[range])`) | [`AggIntent::Quantile`] under [`QueryExpr::Window`] | -//! | `ONLY_TEMPORAL` funcs (`{sum,count,avg,min,max}_over_time`, `rate`, `increase`) | [`AggIntent::Sum`] / [`AggIntent::Count`] / [`AggIntent::Avg`] / [`AggIntent::Min`] / [`AggIntent::Max`] under `Window`, plus [`AggIntent::Rate`] / [`AggIntent::Increase`] for the counter-reset variants | -//! | `ONLY_SPATIAL` (`agg_op(metric)`) | `Aggregate{by, [intent]}` over a bare `Scan` (no `Window`) — the spatial `agg_op` is the [`AggIntent`] | -//! | `ONE_TEMPORAL_ONE_SPATIAL` (`agg_op(temporal_func(m[range]))`) | combined `Aggregate{by, [intent]}` over a `Window` — single-rooted L3 captures both axes natively | -//! | `histogram_quantile(φ, …)` (not a `patterns.rs` entry but the legacy planner refused these) | [`AggIntent::HistogramQuantile`] — flagged archive-only via [`AggIntent::archive_only`] | -//! -//! Phase β additionally lifts these archive-only intents from the legacy -//! planner's "unsupported" branch into the L3 vocabulary so they get a -//! StreamingConfig entry (routed to the cold tier rather than the warm -//! sketch tier): [`AggIntent::HistogramQuantile`], [`AggIntent::Absent`], -//! [`AggIntent::Present`], [`AggIntent::Delta`], [`AggIntent::Deriv`], -//! [`AggIntent::PredictLinear`], [`AggIntent::HoltWinters`], -//! [`AggIntent::Idelta`], [`AggIntent::Irate`], [`AggIntent::Resets`], -//! [`AggIntent::Changes`]. -//! -//! ## Phase B introduces the L3 vocabulary the planner pivots on: -//! -//! - [`AggIntent`] — what to compute, not how (no sketch types here; -//! sketch binding is L4). -//! - [`QueryExpr`] — the L3 algebra DAG (intent-only, language-orthogonal, -//! deployment-independent). Single-rooted per query; multi-root -//! workload-level CSE lives one layer up in `types_v2::WorkloadPlan`. -//! - [`Schema`] — typed schema flowing on every L3 edge. `unique_keys` is -//! the load-bearing field for CSE legality (`design.md` §6 line ~1284). -//! - [`lower_parsed_query`] — `query_parser::ParsedQuery` → [`QueryExpr`] -//! single-query lowering. -//! -//! Phase F adds the CSE surface that consumes `Schema::unique_keys`: -//! -//! - [`cse_reuse_is_legal`] — gatekeeper. Two `QueryExpr::Ref` consumers -//! may share a `LetBinding` only when the producer's output schema -//! has at least one `unique_keys` set. This is the proof point that -//! `unique_keys` is load-bearing. -//! - [`dedupe_subtrees`] — basic workload-level CSE pass that hoists -//! structurally-identical sub-trees into shared `LetBinding`s -//! (`design.md` §6 batched-queries example, ~line 1256). The full -//! alpha-equivalence + nested-CSE algorithm is downstream. -//! -//! Scope reduction. The PR ships the variants the DC + PromQL deployment -//! actually needs (`Scan`, `Window`, `Aggregate`, `LetBinding`, `Ref`). -//! The full `design.md` §6 list is larger (`Filter`, `Project`, -//! `Partition`, `Distinct`, `Merge`, `Join`, `SetOp`, `Sort`, `Limit`, -//! `Subquery`, `WindowFunc`, `BinaryOp`); they are deferred to follow-up -//! phases so each variant lands with a planner consumer rather than as -//! dead code. Adding more is purely additive. -//! -//! Wire-up state. Nothing in `analyzer::Analyzer` or `planner/` consumes -//! these types yet — that's a downstream PR. Phase B exposes the IR so -//! that wiring becomes a focused change rather than a co-emission of new -//! types + new consumers. Phase F's `cse_reuse_is_legal` and -//! `dedupe_subtrees` are similarly defined here for the planner to grow -//! into; the cost-model side that consumes them lives in -//! `planner::cost_model::workload_cost`. - -// The intent_algebra module is the new L3 surface — its re-exports are -// the public API that downstream phases will consume. Until Phase C -// (analyzer wiring) lands, none of these symbols have an in-tree call -// site, so the "unused" lints would fire on every build. Suppressing -// them keeps the lint baseline clean. `dead_code` covers the per-variant -// fields and per-impl helpers; `unused_imports` covers the re-export -// surface itself. -#![allow(dead_code, unused_imports)] - -pub mod agg_intent; -pub mod cse; -pub mod lower; -pub mod query_expr; -pub mod schema; - -// Re-exports for the canonical surface — `crate::intent_algebra::*` for -// downstream callers that don't want to chase sub-module paths. -pub use agg_intent::AggIntent; -pub use cse::{dedupe_subtrees, CseWorkloadPlan}; -pub use lower::{lower_parsed_query, LoweringError}; -pub use query_expr::{ - BindingScope, HavingPredicate, LabelFilter, QueryExpr, QueryExprError, Source, WindowKind, -}; -pub use schema::{cse_reuse_is_legal, Column, ColumnId, CseError, DataType, Schema}; diff --git a/controller/src/intent_algebra/query_expr.rs b/controller/src/intent_algebra/query_expr.rs deleted file mode 100644 index 0f2929c9..00000000 --- a/controller/src/intent_algebra/query_expr.rs +++ /dev/null @@ -1,464 +0,0 @@ -//! Layer 3 IR — `QueryExpr` DAG (intent-only, language-orthogonal, -//! deployment-independent). -//! -//! Per `controller/docs/design.md` §6 "`core::intent_algebra` — Layer 3" -//! (around line ~269). Pure intent at this layer: no language-specific -//! operators (no `HistogramQuantile`, no `PromQLSubquery`), no sketch -//! types, no sketch parameters, no physical operator choice. -//! -//! Single-root tree per query. Multi-root DAGs (cross-query CSE fan-in) -//! live one level above in `WorkloadPlan` (`types_v2::WorkloadPlan`). -//! Within-query CTE / let-binding fan-in *is* expressible here via -//! [`QueryExpr::LetBinding`] + [`QueryExpr::Ref`]. -//! -//! Variant set (Phase B subset): `Scan`, `Window`, `Aggregate`, -//! `LetBinding`, `Ref`. The full design.md list is larger (`Filter`, -//! `Project`, `Partition`, `Distinct`, `Merge`, `Join`, `SetOp`, `Sort`, -//! `Limit`, `Subquery`, `WindowFunc`, `BinaryOp`); they are deferred to -//! follow-up phases as the planner grows consumers for them. The shape -//! defined here is forward-compatible — adding more variants is purely -//! additive. - -#![allow(dead_code)] - -use std::collections::HashMap; -use std::time::Duration; - -use serde::{Deserialize, Serialize}; -use thiserror::Error; - -use crate::intent_algebra::agg_intent::AggIntent; -use crate::intent_algebra::schema::{Column, ColumnId, DataType, Schema}; -use crate::types_v2::BindingName; - -/// Errors produced when the L3 IR is constructed or its output schema is -/// derived. Surfaced by the lowering function and any caller that walks -/// the DAG. -#[derive(Debug, Error)] -pub enum QueryExprError { - /// `Ref(name)` did not resolve against any in-scope `LetBinding`. - #[error("unresolved ref: {0}")] - UnresolvedRef(String), - /// `Aggregate { by, .. }` referenced a column position that is not - /// in the input schema. Caught at schema-derivation time per the - /// design's locally-checkable invariant. - #[error("by-column id {0} out of range (input has {1} columns)")] - InvalidGroupByColumn(ColumnId, usize), - /// `Window` requires a `time_index` field on its input schema — - /// `design.md` §6 schema-flow table. - #[error("Window requires a time_index on input schema")] - WindowMissingTimeIndex, -} - -/// Streaming / time-window kind. PromQL `[5m]` is `Sliding`; SQL `TUMBLE` -/// is `Tumbling`; PromQL has no native `Session` window so it stays -/// unused for the DC + PromQL scope of this PR. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum WindowKind { - Tumbling, - Sliding, - Session, -} - -/// Source of a `Scan`. Phase B ships `TimeSeries` (the only shape DC + -/// PromQL needs); `Table` is sketched out so future deployment models -/// (asap-fusion / OLAP) plug in without an enum-shape rev. Recursive -/// `Source::Join` is design.md §6 line ~378 territory and stays out of -/// scope for now. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum Source { - /// PromQL / DC lifecycle leaf — a metric stream identified by name + - /// optional label filters; produces `(timestamp, value, *labels)` - /// columns. - TimeSeries { - metric: String, - }, - /// Tabular leaf — reserved for asap-fusion. Carries the table name; - /// columns ride on the supplied `Schema`. - Table { - table_ref: String, - }, -} - -/// Equality label filter on a `Scan`. PromQL `{service="api"}` → one of -/// these; richer match operators (`!=`, `=~`, `!~`) live in `Filter`'s -/// generic predicate per design.md §6 line ~296 and are deferred to the -/// follow-up phase that adds the `Filter` variant. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct LabelFilter { - pub label: String, - pub equals: String, -} - -/// Optional HAVING-style predicate on `Aggregate`. Modeled as an opaque -/// expression string at L3 — Phase B doesn't have a typed predicate IR -/// yet; adding one is a separate PR (would also introduce the `Filter` -/// variant per design.md §6). -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct HavingPredicate(pub String); - -/// L3 algebra node. See module doc for the variant subset rationale. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(tag = "node", rename_all = "snake_case")] -pub enum QueryExpr { - /// Outermost leaf — a metric stream / table read. `schema` is the - /// authoritative output of this scan, supplied by the lowering pass - /// (which consults the source / DB schema catalog at L1→L2 time). - Scan { - source: Source, - #[serde(default)] - label_filters: Vec, - schema: Schema, - }, - /// Streaming / time-window. Defines the lifecycle (flush / reset - /// bounds) of any aggregate in its sub-tree. - Window { - kind: WindowKind, - size: Duration, - #[serde(default)] - slide: Option, - child: Box, - }, - /// γ + α — GROUP BY + aggregate intents. `by` are positional - /// references into `child.output_schema().columns`; `aggs` carry - /// `AggIntent` (no sketch types — that's L4). - Aggregate { - by: Vec, - aggs: Vec, - #[serde(default)] - having: Option, - child: Box, - }, - /// SQL `WITH name AS (expr) SELECT ... FROM name` / PromQL recording- - /// rule binding. Names a sub-expression; references via `Ref(name)`. - /// Output schema = `child`'s output schema. - LetBinding { - name: BindingName, - expr: Box, - child: Box, - }, - /// Reference a `LetBinding` by name. Resolved at plan time; output - /// schema = the named binding's expression's output schema. - Ref { - name: BindingName, - }, -} - -impl QueryExpr { - /// Compute the output schema of this node. Walks the tree, resolving - /// `Ref` against `LetBinding`s in scope. Errors propagate per - /// [`QueryExprError`]. - /// - /// Callers that want the schema of the *root* of a single query call - /// `expr.output_schema(&BindingScope::default())`. - pub fn output_schema(&self) -> Result { - self.output_schema_in(&BindingScope::default()) - } - - /// Variant of [`Self::output_schema`] that takes an explicit binding - /// scope. Used internally during DAG walks; exposed for callers that - /// pre-populate bindings from a workload-level container. - pub fn output_schema_in(&self, scope: &BindingScope) -> Result { - match self { - QueryExpr::Scan { schema, .. } => Ok(schema.clone()), - QueryExpr::Window { child, .. } => { - let in_schema = child.output_schema_in(scope)?; - if in_schema.time_index.is_none() { - return Err(QueryExprError::WindowMissingTimeIndex); - } - // Window propagates row identity → carries unique_keys - // verbatim. Synthetic `window_id` / `window_start/end` - // columns (design.md §6 schema-flow row 6) are deferred - // until the planner consumes them. - Ok(in_schema) - } - QueryExpr::Aggregate { - by, aggs, child, .. - } => { - let in_schema = child.output_schema_in(scope)?; - // by-column ids must be in range. - let mut out_cols: Vec = Vec::with_capacity(by.len() + aggs.len()); - for &id in by { - let c = in_schema.columns.get(id).ok_or( - QueryExprError::InvalidGroupByColumn(id, in_schema.columns.len()), - )?; - out_cols.push(c.clone()); - } - // One new column per intent. PromQL convention: - // intent applied to the synthetic `value` column when - // present; otherwise to the first non-grouped column. - let value_col_idx = in_schema - .column_id("value") - .or_else(|| (0..in_schema.columns.len()).find(|i| !by.contains(i))); - let probe = value_col_idx - .and_then(|i| in_schema.columns.get(i)) - .cloned() - .unwrap_or(Column { - name: "value".into(), - dtype: DataType::Float64, - nullable: false, - }); - for intent in aggs { - out_cols.push(intent.output_column(&probe)); - } - // Output unique_keys = [by]. The group-by column tuple - // is unique in the output by construction (design.md §6 - // schema-flow table). - let unique_keys = if by.is_empty() { - Vec::new() - } else { - vec![(0..by.len()).collect()] - }; - // Aggregate strips the time axis — output is one row per - // group, not one row per timestamp. - Ok(Schema { - columns: out_cols, - time_index: None, - unique_keys, - }) - } - QueryExpr::LetBinding { name, expr, child } => { - // Bind `name` to `expr`'s output schema, then evaluate - // `child` in the extended scope. - let bound = expr.output_schema_in(scope)?; - let extended = scope.with(name.clone(), bound); - child.output_schema_in(&extended) - } - QueryExpr::Ref { name } => scope - .lookup(name) - .cloned() - .ok_or_else(|| QueryExprError::UnresolvedRef(name.as_str().into())), - } - } -} - -/// Lexical scope for `LetBinding` / `Ref` resolution. A persistent map -/// from binding name to the bound expression's output schema. Built -/// during the schema-derivation walk; the caller usually starts with -/// [`BindingScope::default()`]. -#[derive(Debug, Default, Clone)] -pub struct BindingScope { - bindings: HashMap, -} - -impl BindingScope { - /// Empty scope — no in-scope bindings. - pub fn new() -> Self { - Self::default() - } - - /// Return a new scope with `name` bound to `schema`. The original - /// scope is left unchanged (functional style — keeps recursion - /// shadow semantics correct). - pub fn with(&self, name: BindingName, schema: Schema) -> Self { - let mut bindings = self.bindings.clone(); - bindings.insert(name.as_str().into(), schema); - Self { bindings } - } - - /// Look up `name` in the current scope. `None` if unbound. - pub fn lookup(&self, name: &BindingName) -> Option<&Schema> { - self.bindings.get(name.as_str()) - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use crate::intent_algebra::schema::{Column, DataType}; - use crate::types_v2::AccuracyTarget; - - fn col(name: &str, dtype: DataType) -> Column { - Column { - name: name.into(), - dtype, - nullable: false, - } - } - - fn ts_scan() -> QueryExpr { - QueryExpr::Scan { - source: Source::TimeSeries { - metric: "http_request_duration_seconds".into(), - }, - label_filters: vec![LabelFilter { - label: "service".into(), - equals: "api".into(), - }], - schema: Schema::with_time_index( - vec![ - col("ts", DataType::Timestamp), - col("service", DataType::Utf8), - col("value", DataType::Float64), - ], - 0, - vec![vec![0, 1]], - ), - } - } - - #[test] - fn query_expr_simple_aggregate() { - let expr = QueryExpr::Aggregate { - by: vec![1], // service - aggs: vec![AggIntent::Quantile { - q: 0.99, - accuracy: AccuracyTarget::Epsilon(0.01), - }], - having: None, - child: Box::new(QueryExpr::Window { - kind: WindowKind::Sliding, - size: Duration::from_secs(300), - slide: None, - child: Box::new(ts_scan()), - }), - }; - let schema = expr.output_schema().unwrap(); - // Output: [service, quantile_0_99] - assert_eq!(schema.columns.len(), 2); - assert_eq!(schema.columns[0].name, "service"); - assert_eq!(schema.columns[1].name, "quantile_0_99"); - // unique_keys = [by] — the by columns project to positions [0..by.len()) - // in the output schema. - assert_eq!(schema.unique_keys, vec![vec![0]]); - // Aggregate strips the time axis. - assert!(schema.time_index.is_none()); - } - - #[test] - fn query_expr_let_binding_ref() { - // LetBinding{name="w", expr=Window over Scan, - // child=Aggregate{ child=Ref{"w"} }} - let expr = QueryExpr::LetBinding { - name: BindingName::new("w"), - expr: Box::new(QueryExpr::Window { - kind: WindowKind::Sliding, - size: Duration::from_secs(300), - slide: None, - child: Box::new(ts_scan()), - }), - child: Box::new(QueryExpr::Aggregate { - by: vec![1], - aggs: vec![AggIntent::Max], - having: None, - child: Box::new(QueryExpr::Ref { - name: BindingName::new("w"), - }), - }), - }; - let schema = expr.output_schema().unwrap(); - assert_eq!(schema.columns[0].name, "service"); - assert_eq!(schema.columns[1].name, "max"); - assert_eq!(schema.unique_keys, vec![vec![0]]); - } - - #[test] - fn query_expr_unresolved_ref_errors() { - let expr = QueryExpr::Ref { - name: BindingName::new("nope"), - }; - let err = expr.output_schema().unwrap_err(); - assert!(matches!(err, QueryExprError::UnresolvedRef(s) if s == "nope")); - } - - #[test] - fn query_expr_window_requires_time_index() { - let bad_scan = QueryExpr::Scan { - source: Source::Table { - table_ref: "t".into(), - }, - label_filters: vec![], - // Tabular scan with no time index. - schema: Schema::new(vec![col("a", DataType::Int64)]), - }; - let expr = QueryExpr::Window { - kind: WindowKind::Tumbling, - size: Duration::from_secs(60), - slide: None, - child: Box::new(bad_scan), - }; - let err = expr.output_schema().unwrap_err(); - assert!(matches!(err, QueryExprError::WindowMissingTimeIndex)); - } - - #[test] - fn query_expr_aggregate_invalid_by_column() { - let expr = QueryExpr::Aggregate { - by: vec![99], - aggs: vec![AggIntent::Sum], - having: None, - child: Box::new(ts_scan()), - }; - let err = expr.output_schema().unwrap_err(); - assert!(matches!(err, QueryExprError::InvalidGroupByColumn(99, _))); - } - - #[test] - fn query_expr_serde_roundtrip() { - let expr = QueryExpr::Aggregate { - by: vec![1], - aggs: vec![AggIntent::Quantile { - q: 0.99, - accuracy: AccuracyTarget::Epsilon(0.01), - }], - having: None, - child: Box::new(ts_scan()), - }; - let json = serde_json::to_string(&expr).unwrap(); - let back: QueryExpr = serde_json::from_str(&json).unwrap(); - assert_eq!(expr, back); - } - - /// `unique_keys` is the load-bearing CSE-legality hook (design.md §6 - /// line ~1284). Two `Ref` consumers can share a producer iff the - /// producer's output schema has at least one provable unique-key set; - /// without it, the deduper has to be conservative and reuse drops on - /// the floor. - #[test] - fn cse_substitution_legal_only_with_unique_keys() { - // Producer 1: Scan → Window → Aggregate. Aggregate produces - // `unique_keys = [by]` (a provable unique key). Two `Ref` - // consumers can legally share this. - let producer_with_uk = QueryExpr::Aggregate { - by: vec![1], - aggs: vec![AggIntent::Sum], - having: None, - child: Box::new(QueryExpr::Window { - kind: WindowKind::Sliding, - size: Duration::from_secs(300), - slide: None, - child: Box::new(ts_scan()), - }), - }; - let s1 = producer_with_uk.output_schema().unwrap(); - assert!( - s1.has_unique_key(), - "Aggregate must emit unique_keys = [by] per design.md §6 schema-flow" - ); - - // Producer 2: bare Scan with NO unique key declared. CSE deduper - // would have to refuse to share this without further proof. - let producer_without_uk = QueryExpr::Scan { - source: Source::Table { - table_ref: "t".into(), - }, - label_filters: vec![], - schema: Schema::new(vec![col("a", DataType::Int64)]), - }; - let s2 = producer_without_uk.output_schema().unwrap(); - assert!( - !s2.has_unique_key(), - "no unique_keys → CSE deduper must conservatively refuse to share" - ); - - // The asymmetry is the design's claim: unique_keys is what makes - // CSE substitution legal. Encoded here as a unit invariant so - // downstream rewrites of the schema-flow rules can't silently - // break it. - assert_ne!(s1.has_unique_key(), s2.has_unique_key()); - } -} diff --git a/controller/src/intent_algebra/schema.rs b/controller/src/intent_algebra/schema.rs deleted file mode 100644 index 2e9aadd6..00000000 --- a/controller/src/intent_algebra/schema.rs +++ /dev/null @@ -1,336 +0,0 @@ -//! Layer 3 schema flow — every L3 edge carries a typed `Schema`. -//! -//! Per `controller/docs/design.md` §6 "Schema flow — every L3 edge carries -//! a typed schema". The DAG is type-checked: a node's output schema is a -//! function of its inputs and parameters and is verifiable independently -//! of the surrounding context. -//! -//! `Schema::unique_keys` is the load-bearing field for the workload-level -//! CSE pass (`design.md` §6 "DAG, not tree" + the batched-queries example -//! around line ~1284). Two `QueryExpr::Ref` consumers can share a producer -//! only when its output schema is provably stable across reads — the -//! unique-key metadata is what lets the deduper assert that. -//! -//! Single-query plans don't read this field; it lives here so the metadata -//! is available the moment workload-aware planning lands without requiring -//! an L3-wide schema change. - -#![allow(dead_code)] - -use serde::{Deserialize, Serialize}; - -/// Index into [`Schema::columns`] used everywhere a column position is -/// referenced (group-by keys, unique-key sets, the time axis index). -/// -/// Aliased to `usize` to match `design.md`'s `Vec>` for -/// `unique_keys`. Kept as a named type so downstream code can pattern on -/// the intent ("this is a column position, not just any number"). -pub type ColumnId = usize; - -/// One column in a [`Schema`]. Mirrors `design.md` §6 `Field` — -/// `name + dtype + nullable`. -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub struct Column { - /// Column name as it appears in the producer's output. PromQL leaves - /// produce label-name + the synthetic `value` / `timestamp` columns; - /// SQL leaves carry their `information_schema` names. - pub name: String, - /// Column data type. Kept narrow at L3 (`Int64` / `Float64` / `Utf8` - /// / `Bool` / `Timestamp`); `Sketch(...)` is an L4-only addition per - /// design.md §6.4 and is intentionally absent here. - pub dtype: DataType, - /// Whether NULL values are allowed in this column. PromQL value - /// columns are non-nullable; SQL columns inherit their DDL nullability. - pub nullable: bool, -} - -/// L3 column data types. Deliberately narrow: no sketch state at this -/// layer (see `design.md` §6.4 for the L4 `DataType::Sketch(...)` -/// extension). -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum DataType { - /// 64-bit signed integer. Counter columns, group-cardinality outputs. - Int64, - /// 64-bit IEEE-754 float. Quantile / Avg / Sum-over-floats output. - Float64, - /// UTF-8 string. PromQL label values, SQL `VARCHAR` / `TEXT`. - Utf8, - /// Boolean — predicate output, `unless` / `and` / `or` PromQL ops. - Bool, - /// Wall-clock timestamp. PromQL leaves carry exactly one of these - /// (the `time_index` column); SQL leaves may or may not. - Timestamp, -} - -/// Per-edge L3 schema. Flowing between any two L3 operators, on every -/// node's input and output. -/// -/// `unique_keys` is metadata for reuse-aware planning: each inner `Vec` -/// is a set of column indices that together uniquely identify rows. The -/// outer `Vec` allows multiple unique-key sets (primary key + another -/// unique constraint). Populated by per-node input/output spec — -/// `Aggregate { by, .. }` emits `unique_keys = [by]`; `Distinct { cols }` -/// adds `cols`; most other nodes pass through. -/// -/// **Consumed by**: workload-level CSE (`CostModel::workload_cost` in the -/// design, not yet shipped). The single-query path, the `Bind*` rules, -/// push-down, and L5 emitters do not read this field. -#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] -pub struct Schema { - /// Columns flowing on this edge, in positional order. - pub columns: Vec, - /// Index into `columns` for the time axis, if any. PromQL leaves - /// always carry one; SQL leaves may or may not. - #[serde(default)] - pub time_index: Option, - /// Unique-key sets — each inner vec is a tuple of column indices - /// that together uniquely identifies a row. Empty `Vec` means - /// "no provable unique constraint" (the conservative default). - #[serde(default)] - pub unique_keys: Vec>, -} - -impl Schema { - /// Construct a `Schema` from columns alone — no time index, no - /// unique-key constraint. Used by `Scan` over a tabular source - /// when the catalog supplies no primary-key metadata. - pub fn new(columns: Vec) -> Self { - Self { - columns, - time_index: None, - unique_keys: Vec::new(), - } - } - - /// Construct a `Scan`-style schema with explicit `time_index` + - /// inferred unique keys (e.g. PromQL leaves: `[time_index, label_set]`). - pub fn with_time_index( - columns: Vec, - time_index: ColumnId, - unique_keys: Vec>, - ) -> Self { - Self { - columns, - time_index: Some(time_index), - unique_keys, - } - } - - /// Look up a column by name. `None` if not present. - pub fn column_id(&self, name: &str) -> Option { - self.columns.iter().position(|c| c.name == name) - } - - /// Whether this schema has *any* provable unique key. The CSE pass - /// reads this to decide whether two `Ref` consumers can safely share - /// a producer (see `design.md` §6 line ~1284 + the unit test in - /// `tests::cse_substitution_legal_only_with_unique_keys`). - pub fn has_unique_key(&self) -> bool { - !self.unique_keys.is_empty() - } - - /// Append `cols` as an additional unique-key set if not already present. - /// Used by `Distinct { cols }` per design.md §6 schema-flow table: - /// "the input schema with `unique_keys` tightened to include `cols`". - pub fn add_unique_key(&mut self, cols: Vec) { - if !self.unique_keys.contains(&cols) { - self.unique_keys.push(cols); - } - } -} - -// ── CSE legality (the load-bearing consumer of `unique_keys`) ──────────────── -// -// Phase F per `controller/docs/design.md` §6 Schema flow + the batched- -// queries example (§6 line ~1320): -// -// "CSE legality leans on `Schema::unique_keys` (§6 Schema flow): two -// `QueryExpr::Ref` consumers can share a producer only when its -// output schema is provably stable across reads — the unique-key -// metadata is what lets the deduper assert that without re-running -// the producer's logic." -// -// `cse_reuse_is_legal` is the gatekeeper. The workload-level CSE pass -// (`intent_algebra::cse::dedupe_subtrees`) consults it before emitting -// a `LetBinding` to share a producer between ≥2 `Ref` consumers. - -use thiserror::Error; - -/// Errors returned by [`cse_reuse_is_legal`] when shared-producer reuse -/// would violate the design's stability invariant. -#[derive(Debug, Error, PartialEq, Eq)] -pub enum CseError { - /// Producer schema lacks any `unique_keys` set — row identity is - /// not provably stable across reads, so two `Ref` consumers cannot - /// safely share it. The deduper falls back to per-consumer - /// recomputation. Per design.md §6 line ~1356. - #[error( - "shared-producer CSE refused: producer schema has no unique_keys \ - (design.md §6 schema-flow — without a provable unique key the \ - deduper cannot assert row identity across reads)" - )] - NoUniqueKeys, - /// Trivially-callable case: only one consumer means no reuse to - /// gate. Returned so the caller can short-circuit instead of - /// emitting a degenerate `LetBinding`. - #[error("CSE not applicable: {0} consumer(s) — need ≥ 2 for shared-producer reuse")] - InsufficientConsumers(usize), -} - -/// Two `QueryExpr::Ref` nodes can share a producer (same `LetBinding`) -/// only when the producer's output schema has stable per-row identity — -/// i.e. `Schema::unique_keys` is non-empty. This is the gatekeeper: -/// returns `Ok(())` if shared-producer reuse is legal, otherwise `Err`. -/// -/// Per design.md §6 line ~1356 — `unique_keys` is what makes CSE -/// provably correct. The deduper consults this before emitting a -/// `LetBinding`, and `CostModel::workload_cost` only credits a shared -/// binding when this gate has fired green. -/// -/// `consumer_count` is the number of `QueryExpr::Ref { name }` sites the -/// deduper has identified for the candidate binding. Single-consumer -/// cases short-circuit with `InsufficientConsumers` — a `LetBinding` -/// with one `Ref` is just a no-op alias and shouldn't be hoisted. -pub fn cse_reuse_is_legal( - producer_schema: &Schema, - consumer_count: usize, -) -> Result<(), CseError> { - if consumer_count < 2 { - return Err(CseError::InsufficientConsumers(consumer_count)); - } - if !producer_schema.has_unique_key() { - return Err(CseError::NoUniqueKeys); - } - Ok(()) -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - - fn col(name: &str, dtype: DataType) -> Column { - Column { - name: name.into(), - dtype, - nullable: false, - } - } - - /// `cse_reuse_is_legal` accepts a producer schema with at least one - /// `unique_keys` set + ≥2 consumers. This is the design.md §6 - /// "load-bearing" green path. - #[test] - fn cse_reuse_legal_when_unique_keys_set() { - let producer = Schema::with_time_index( - vec![ - col("ts", DataType::Timestamp), - col("service", DataType::Utf8), - col("value", DataType::Float64), - ], - 0, - vec![vec![0, 1]], - ); - assert_eq!(cse_reuse_is_legal(&producer, 2), Ok(())); - assert_eq!(cse_reuse_is_legal(&producer, 5), Ok(())); - } - - /// Schema without `unique_keys` is the conservative-default case — - /// the deduper must refuse to share it. Pins design.md §6 line - /// ~1356 ("Without it, the deduper has to be conservative and reuse - /// drops on the floor"). - #[test] - fn cse_reuse_illegal_when_unique_keys_empty() { - let producer = Schema::new(vec![ - col("a", DataType::Int64), - col("b", DataType::Float64), - ]); - assert_eq!( - cse_reuse_is_legal(&producer, 2), - Err(CseError::NoUniqueKeys) - ); - } - - /// Single-consumer case is short-circuited — no `LetBinding` should - /// be emitted for one `Ref` because there's no reuse to credit. - #[test] - fn cse_reuse_rejects_single_consumer() { - let producer = Schema::with_time_index( - vec![col("ts", DataType::Timestamp), col("v", DataType::Float64)], - 0, - vec![vec![0]], - ); - assert_eq!( - cse_reuse_is_legal(&producer, 1), - Err(CseError::InsufficientConsumers(1)) - ); - assert_eq!( - cse_reuse_is_legal(&producer, 0), - Err(CseError::InsufficientConsumers(0)) - ); - } - - /// Empty `unique_keys` rejection takes precedence over the consumer - /// count check only when both pass — but here we verify the - /// insufficient-consumers branch fires first (a defensive ordering - /// so callers see the clearer error when they get the call wrong). - #[test] - fn cse_reuse_consumer_check_precedes_unique_key_check() { - let producer = Schema::new(vec![col("a", DataType::Int64)]); - // Both conditions fail; consumer check is reported. - assert_eq!( - cse_reuse_is_legal(&producer, 1), - Err(CseError::InsufficientConsumers(1)) - ); - } - - - #[test] - fn schema_new_has_no_time_or_unique_key() { - let s = Schema::new(vec![col("k", DataType::Utf8), col("v", DataType::Float64)]); - assert!(s.time_index.is_none()); - assert!(!s.has_unique_key()); - assert_eq!(s.column_id("k"), Some(0)); - assert_eq!(s.column_id("v"), Some(1)); - assert_eq!(s.column_id("missing"), None); - } - - #[test] - fn schema_with_time_index_populates_metadata() { - let s = Schema::with_time_index( - vec![ - col("ts", DataType::Timestamp), - col("service", DataType::Utf8), - col("value", DataType::Float64), - ], - 0, - vec![vec![0, 1]], - ); - assert_eq!(s.time_index, Some(0)); - assert!(s.has_unique_key()); - assert_eq!(s.unique_keys, vec![vec![0, 1]]); - } - - #[test] - fn add_unique_key_dedupes() { - let mut s = Schema::new(vec![col("a", DataType::Utf8), col("b", DataType::Utf8)]); - s.add_unique_key(vec![0]); - s.add_unique_key(vec![0]); - s.add_unique_key(vec![0, 1]); - assert_eq!(s.unique_keys, vec![vec![0], vec![0, 1]]); - } - - #[test] - fn schema_serde_roundtrip() { - let s = Schema::with_time_index( - vec![col("ts", DataType::Timestamp), col("value", DataType::Float64)], - 0, - vec![vec![0]], - ); - let json = serde_json::to_string(&s).unwrap(); - let back: Schema = serde_json::from_str(&json).unwrap(); - assert_eq!(s, back); - } -} diff --git a/controller/src/language_logical_plan/lower.rs b/controller/src/language_logical_plan/lower.rs deleted file mode 100644 index d7316126..00000000 --- a/controller/src/language_logical_plan/lower.rs +++ /dev/null @@ -1,43 +0,0 @@ -//! L1 → L2 lowering — produces a [`LanguageLogicalPlan`] from a -//! [`crate::query_language::LanguageAst`]. -//! -//! Today only the PromQL backend has a working L1, so only the PromQL -//! variant of `LanguageAst` lowers to a real L2 tree. Other variants -//! return [`LoweringError::UnsupportedLanguage`] cleanly so callers -//! can surface a uniform error. - -use crate::query_language::{language_ast::LanguageAst, promql::PromQLAst}; -use crate::types_v2::QueryLanguage; - -use super::plan::{LanguageLogicalPlan, LanguageLogicalPlanSummary}; - -/// Errors surfaced by the L1 → L2 lowering pass. -#[derive(Debug, thiserror::Error)] -pub enum LoweringError { - /// The L1 AST is for a language whose L2 lowering is not yet implemented. - #[error("L2 lowering not implemented for {0:?}")] - UnsupportedLanguage(QueryLanguage), -} - -/// Lower an L1 [`LanguageAst`] into an L2 [`LanguageLogicalPlan`]. -/// -/// The PromQL backend's L2 representation is the existing -/// `algebra::expr::QueryExpr` tree (already produced by -/// `query_parser::parse_query_expr` and stashed inside [`PromQLAst`]). -/// We just re-tag it as the PromQL L2 variant and project the flat -/// summary; no re-parsing or re-walking is required. -pub fn lower_to_logical_plan( - ast: &LanguageAst, -) -> Result { - match ast { - LanguageAst::PromQL(p) => Ok(lower_promql(p)), - } -} - -fn lower_promql(ast: &PromQLAst) -> LanguageLogicalPlan { - LanguageLogicalPlan::PromQL { - source: ast.source.clone(), - tree: ast.expr.clone(), - summary: LanguageLogicalPlanSummary::from_parsed_query(&ast.summary), - } -} diff --git a/controller/src/language_logical_plan/mod.rs b/controller/src/language_logical_plan/mod.rs deleted file mode 100644 index c7f502f0..00000000 --- a/controller/src/language_logical_plan/mod.rs +++ /dev/null @@ -1,36 +0,0 @@ -// Layer 2 scaffolding ships ahead of any in-tree call site (the -// downstream `pipeline` driver lands in a later phase). Dead-code -// warnings are silenced here, not on individual items, so the public -// surface is uncluttered. -#![allow(dead_code, unused_imports)] - -//! Layer 2 — `language_logical_plan`. -//! -//! Per `controller/docs/design.md` §6 `core::logical_plan`, L2 is the -//! per-language algebra tree that preserves language-specific -//! semantics (PromQL instant vs range vector, SQL window frames, -//! Elastic buckets) before the L3 normalisation pass collapses -//! everything into the language-orthogonal [`crate::algebra::expr::QueryExpr`]. -//! -//! # DC deployment scope -//! -//! - PromQL → real L2 (carries the existing `QueryExpr` tree from -//! `query_parser::parse_query_expr` plus a flat summary projection). -//! - Other languages → not yet implemented; lowering errors cleanly. -//! -//! # Layer split -//! -//! - L1 ([`crate::query_language`]) — `&str` → [`crate::query_language::LanguageAst`]. -//! - L2 ([`Self`]) — [`crate::query_language::LanguageAst`] → -//! [`LanguageLogicalPlan`]. -//! - L3 (`crate::intent_algebra`, Phase B) — [`LanguageLogicalPlan`] -//! → language-orthogonal [`crate::algebra::expr::QueryExpr`]. - -pub mod lower; -pub mod plan; - -pub use lower::{lower_to_logical_plan, LoweringError}; -pub use plan::{LanguageLogicalPlan, LanguageLogicalPlanSummary}; - -#[cfg(test)] -mod tests; diff --git a/controller/src/language_logical_plan/plan.rs b/controller/src/language_logical_plan/plan.rs deleted file mode 100644 index 40dd267a..00000000 --- a/controller/src/language_logical_plan/plan.rs +++ /dev/null @@ -1,128 +0,0 @@ -//! [`LanguageLogicalPlan`] — the L2 canonical representation. -//! -//! Per `controller/docs/design.md` §6 `core::logical_plan`, L2 is a -//! **per-language algebra tree** preserving language-specific -//! semantics (PromQL instant vs range vector, SQL window frames, -//! Elastic buckets) — `Aggregate { AggFunc }`, `Window`, `Filter`, -//! `Sort`, `Limit`. **No sketch names yet**. -//! -//! In DC's existing tree the equivalent representation is the -//! [`crate::algebra::expr::QueryExpr`] tree produced by -//! `query_parser::parse_query_expr`. To stay consistent with the -//! design.md L2 contract while not duplicating the algebra: -//! -//! - The `PromQL` variant of [`LanguageLogicalPlan`] carries the -//! already-built `QueryExpr` tree (which is the L2 representation -//! for the PromQL language) **plus** the flat `ParsedQuery` -//! summary needed by downstream analyzer/planner. -//! - Other languages get their own variants when implemented. -//! -//! The `intent_algebra` (L3) layer (Phase B, separate worktree) is -//! responsible for normalising this language-specific tree into the -//! language-orthogonal `QueryExpr` shape (e.g. dropping -//! `HistogramQuantile`, `PromQLSubquery`). - -use std::collections::HashMap; -use std::time::Duration; - -use crate::algebra::expr::QueryExpr; -use crate::query_parser::{ParsedQuery, QueryHint}; -use crate::types::AggType; -use crate::types_v2::QueryLanguage; - -/// L2 canonical container — one variant per source language. -/// -/// Each variant carries: -/// - the raw source string (for diagnostics + replan correlation), -/// - the language-specific algebra tree, -/// - a flat [`LanguageLogicalPlanSummary`] view that downstream -/// non-tree consumers (analyzer / planner / replan) can read -/// without walking the tree themselves. -#[derive(Debug, Clone)] -pub enum LanguageLogicalPlan { - /// PromQL L2. Tree = `QueryExpr` from `query_parser::parse_query_expr`. - PromQL { - /// Original PromQL source. - source: String, - /// Algebra tree — the PromQL-flavored L2 shape. - tree: QueryExpr, - /// Flat summary projection of the tree. - summary: LanguageLogicalPlanSummary, - }, - // Future variants (kept commented to surface intent): - // Sql { source: String, tree: SqlTree, summary: ... }, - // DataFusion { source: String, tree: DfPlan, summary: ... }, - // ElasticDsl { source: String, tree: EsTree, summary: ... }, -} - -impl LanguageLogicalPlan { - /// The originating [`QueryLanguage`] for this plan. - pub fn language(&self) -> QueryLanguage { - match self { - LanguageLogicalPlan::PromQL { .. } => QueryLanguage::PromQL, - } - } - - /// Original source string this plan was lowered from. - pub fn source(&self) -> &str { - match self { - LanguageLogicalPlan::PromQL { source, .. } => source, - } - } - - /// Borrow the flat summary projection. - pub fn summary(&self) -> &LanguageLogicalPlanSummary { - match self { - LanguageLogicalPlan::PromQL { summary, .. } => summary, - } - } - - /// Borrow the algebra tree if the language is PromQL. - pub fn as_promql_tree(&self) -> Option<&QueryExpr> { - match self { - LanguageLogicalPlan::PromQL { tree, .. } => Some(tree), - } - } -} - -/// Flat, language-orthogonal projection of the L2 plan. This is the -/// shape downstream non-tree consumers (the legacy analyzer + planner -/// in DC) want. -/// -/// It mirrors the fields on [`crate::query_parser::ParsedQuery`] so -/// adapters between the two layers stay trivial. -#[derive(Debug, Clone, Default)] -pub struct LanguageLogicalPlanSummary { - /// Metric / table name extracted from the L2 tree. - pub metric_name: String, - /// Aggregation types present in the tree. - pub aggregations: Vec, - /// `GROUP BY` / `by (dims)` keys. - pub group_by_labels: Vec, - /// Equality label / `WHERE` filters. - pub label_filters: HashMap, - /// Window size (PromQL `[w]` / SQL `TUMBLE`). - pub time_window: Duration, - /// Set when the query needs per-sample exact computation. - pub exact_required: bool, - /// Quantile φ values implied by the query. - pub quantiles: Vec, - /// Optional named pattern hint (DEBS classifier). - pub hint: Option, -} - -impl LanguageLogicalPlanSummary { - /// Lift a legacy [`ParsedQuery`] into a [`LanguageLogicalPlanSummary`]. - pub fn from_parsed_query(pq: &ParsedQuery) -> Self { - Self { - metric_name: pq.metric_name.clone(), - aggregations: pq.aggregations.clone(), - group_by_labels: pq.group_by_labels.clone(), - label_filters: pq.label_filters.clone(), - time_window: pq.time_window, - exact_required: pq.exact_required, - quantiles: pq.quantiles.clone(), - hint: pq.hint.clone(), - } - } -} diff --git a/controller/src/language_logical_plan/tests.rs b/controller/src/language_logical_plan/tests.rs deleted file mode 100644 index 930cfdb2..00000000 --- a/controller/src/language_logical_plan/tests.rs +++ /dev/null @@ -1,86 +0,0 @@ -//! Tests for L2 lowering (`language_logical_plan`). - -use super::*; -use crate::algebra::expr::{AggIntent, QueryExpr}; -use crate::query_language::{Language, LanguageAst, PromQLLanguage}; -use crate::types::AggType; -use crate::types_v2::QueryLanguage; - -fn parse_promql(src: &str) -> LanguageAst { - PromQLLanguage.parse(src).expect("PromQL parse should succeed") -} - -#[test] -fn lower_promql_to_logical_plan_basic() { - let ast = parse_promql("up"); - let plan = lower_to_logical_plan(&ast).expect("lowering should succeed"); - assert_eq!(plan.language(), QueryLanguage::PromQL); - assert_eq!(plan.source(), "up"); - let s = plan.summary(); - assert_eq!(s.metric_name, "up"); -} - -#[test] -fn lower_promql_quantile_preserves_summary() { - let ast = parse_promql( - "quantile_over_time(0.99, http_request_duration{env=\"prod\"}[5m])", - ); - let plan = lower_to_logical_plan(&ast).unwrap(); - let s = plan.summary(); - assert_eq!(s.metric_name, "http_request_duration"); - assert!(s.aggregations.contains(&AggType::Quantile)); - assert_eq!(s.quantiles, vec![0.99]); - assert_eq!(s.label_filters.get("env").map(|v| v.as_str()), Some("prod")); - assert_eq!(s.time_window, std::time::Duration::from_secs(5 * 60)); -} - -#[test] -fn lower_promql_keeps_algebra_tree_for_l3() { - // The PromQL L2 tree IS the existing `QueryExpr`; assert that the - // tree shape matches what `parse_query_expr` would have produced. - let ast = parse_promql( - "quantile_over_time(0.99, http_request_duration{env=\"prod\"}[5m])", - ); - let plan = lower_to_logical_plan(&ast).unwrap(); - let tree = plan.as_promql_tree().expect("PromQL plan"); - assert!(matches!( - tree, - QueryExpr::WindowedAgg { agg: AggIntent::Quantile { .. }, .. } - )); -} - -#[test] -fn lower_promql_topk_extracts_groupby() { - let ast = parse_promql( - "topk by (service) (10, count_over_time(requests{env=\"prod\"}[1m]))", - ); - let plan = lower_to_logical_plan(&ast).unwrap(); - let s = plan.summary(); - assert_eq!(s.metric_name, "requests"); - assert!(s.group_by_labels.contains(&"service".to_string())); -} - -#[test] -fn lower_unsupported_language_errors_cleanly() { - // Build a stub LanguageAst variant by hand — we cannot construct - // SQL/DataFusion/ElasticDsl variants because LanguageAst doesn't - // expose those yet (PromQL is the only variant). So the equivalent - // smoke test is: stub backends fail at L1 with `Unimplemented`, and - // the type system rules out passing them to L2 lowering. We assert - // that contract here. - let err = crate::query_language::SqlLanguage.parse("SELECT 1").unwrap_err(); - assert!(matches!( - err, - crate::query_language::ParseError::Unimplemented(_) - )); -} - -#[test] -fn lowering_error_for_unsupported_language_is_displayable() { - // Direct construction of the error variant — verifies the Display - // impl renders something useful. Future variants (Sql/etc.) will - // exercise this path organically once their L1 backends ship. - let err = LoweringError::UnsupportedLanguage(QueryLanguage::Sql); - let msg = format!("{err}"); - assert!(msg.contains("Sql"), "got: {msg}"); -} diff --git a/controller/src/main.rs b/controller/src/main.rs deleted file mode 100644 index 5eb878a8..00000000 --- a/controller/src/main.rs +++ /dev/null @@ -1,2636 +0,0 @@ -mod accuracy; -mod algebra; -mod analyzer; -mod backend_client; -mod config; -mod intent_algebra; -mod language_logical_plan; -mod metrics_exposer; -mod monitor; -mod opamp; -mod planner; -mod query_language; -mod query_parser; -mod replan; -mod runtime_samples; -mod sketch_algebra; -mod stage_split; -mod store; -mod types; -mod types_v2; - -use std::collections::HashMap; -use std::sync::Arc; -use std::time::Duration; -use tokio::sync::Mutex; -use axum::{ - extract::{Path, State}, - http::{HeaderMap, StatusCode}, - response::IntoResponse, - routing::{get, post}, - Json, Router, -}; -use serde_json::json; -use tracing::{info, warn}; - -use algebra::{QueryOptimizer, SketchAllocator}; -use analyzer::{Analyzer, QuerySpec}; -use config::{generate_agent_config, generate_backend_config, build_precompute_jobs}; -use config::WorkloadRegistry; -use config::{AgentRuntime, emit_for_runtime}; -use types::AgentCollectorConfig; -use config::generate_backend_config_staged; -use monitor::{Endpoint, Scraper, ScrapedData, Thresholds, Violation}; -use opamp::{AgentRole, OpampServer, RemoteConfig}; -use planner::{CostModelPlanner, BaselinePlanner, ObjectiveWeights, OnlineMetricsStore, init_online_store, pareto_frontier, select_best}; -use planner::online_cost_model; -use planner::stage_split::split_expr_by_stage; -use planner::tco; -use algebra::physical::physical_plan_to_staged; -use query_parser::parse_query_expr; -use replan::Replanner; -use stage_split::BackendStageConfig; -use store::{PlanStore, WorkloadStore}; -use types::StageResourceBudgets; - -// ── Shared state ────────────────────────────────────────────────────────────── - -#[derive(Clone)] -struct AppState { - analyzer: Arc, - planner: Arc, - store: Arc, - workload_store: Arc, - opamp: Arc, - scraper: Arc, - replanner: Arc, - online_store: OnlineMetricsStore, - opamp_endpoint: String, - workload_registry: Arc, - /// Bounded ring buffer for runtime-sample push batches from - /// agents' `sketch-runtime::PushExporter`. Read by decision - /// loops in the replanner. - runtime_samples: Arc, - /// Phase C (MVP v6): shared `BackendClient` for posting - /// `StreamingConfig` JSON / YAML to the ASAPQuery-backend's - /// `POST /api/v1/streaming-config` endpoint. Phase B had this - /// only on the `Replanner`, so the typed L5 stage_split path in - /// `handle_plan` could only `info!`-log the backend JSON it - /// emitted. Sharing via `Arc` lets `AppState` and `Replanner` - /// both push without owning a duplicate client. `None` when - /// `CONTROLLER_BACKEND_ENDPOINT` is unset, matching the - /// pre-existing fire-and-forget contract. - backend_client: Option>, - /// Per-metric `BackendStageConfig` cache used to emit a - /// **cumulative** `BackendStorageRouting` JSON document on every - /// per-metric replan. - /// - /// Why this exists: `POST /api/v1/storage_routing` on the backend - /// is an atomic per-tenant SWAP — every push replaces the whole - /// tenant's routing table. The controller's pre-existing per-metric - /// post path emits a single-element `metrics:[…]` document per - /// `handle_plan` call, so when N metrics replan in sequence only - /// the last metric's entry survives in the backend's routing table. - /// That defaults the other N-1 metrics to `sketch_warm_tier`, which - /// has no warm-tier sketch state for archive-shape queries - /// (`count`, `topk`, `rate_post_hoc`, `histogram_quantile`, - /// `delta`, `deriv`, `absent`) → the backend returns empty / 404 → - /// the demo's accuracy reducer logs `archive_miss` for those metrics - /// even though gorillas3 wrote their TSDB blocks to MinIO and Thanos - /// has them indexed. - /// - /// The cache is a `HashMap` keyed - /// by metric name. On every plan-emit cycle we update the entry for - /// the metric being planned and re-emit the storage-routing JSON - /// from the union of all currently-known plans, then push the - /// cumulative table. The next cycle's swap then preserves every - /// previously-seen metric's routing entry. - backend_routing_cache: Arc>>, -} - -// ── Entry point ─────────────────────────────────────────────────────────────── - -#[tokio::main] -async fn main() { - tracing_subscriber::fmt::init(); - - let api_addr = std::env::var("CONTROLLER_ADDR") - .unwrap_or_else(|_| "0.0.0.0:8080".into()); - let opamp_addr = std::env::var("CONTROLLER_OPAMP_ADDR") - .unwrap_or_else(|_| "0.0.0.0:4320".into()); - let opamp_ep = std::env::var("CONTROLLER_OPAMP_ENDPOINT") - .unwrap_or_else(|_| "ws://controller:4320/v1/opamp".into()); - let scrape_interval = Duration::from_secs( - std::env::var("CONTROLLER_SCRAPE_INTERVAL_SECS") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(60u64), - ); - let backend_endpoint = std::env::var("CONTROLLER_BACKEND_ENDPOINT").ok(); - - // ── SP-5: Online EMA cost store ─────────────────────────────────────────── - let online_store = init_online_store(); - - // ── SP-8: Prometheus scraper ────────────────────────────────────────────── - // Violations are forwarded to the Replanner (built below). - // We use an Arc>>> as a late-binding cell so - // the scraper can hold a reference even though the Replanner is built after it. - let replanner_cell: Arc>>> = - Arc::new(tokio::sync::RwLock::new(None)); - let registry_cell: Arc>>> = - Arc::new(tokio::sync::RwLock::new(None)); - - let scraper: Arc = { - let ema = Arc::clone(&online_store); - let cell = Arc::clone(&replanner_cell); - Arc::new( - Scraper::new( - vec![], - Thresholds::default(), - Arc::new(move |v: Violation| { - warn!(agent = %v.agent_id, kind = %v.kind, - observed = v.observed, threshold = v.threshold, - "SLA violation detected — triggering re-plan"); - let cell = Arc::clone(&cell); - let agent_id = v.agent_id.clone(); - tokio::spawn(async move { - if let Some(r) = cell.read().await.as_ref() { - r.handle_violation(&agent_id).await; - } - }); - }), - scrape_interval, - ) - .with_on_metrics(Arc::new(move |data: ScrapedData| { - // Update EMA only when we know the sketch type and have a - // CPU-per-sample estimate (requires at least 2 scrapes). - if let (Some(st), Some(cpu)) = (data.sketch_type, data.cpu_micros_per_sample) { - let ema = Arc::clone(&ema); - tokio::spawn(async move { - planner::online_cost_model::update( - &ema, &st, data.sketch_size_bytes, cpu, - ).await; - }); - } - })), - ) - }; - - // ── OpAMP server with connect/disconnect hooks ──────────────────────────── - let opamp_srv: Arc = { - let sc = Arc::clone(&scraper); - let sd = Arc::clone(&scraper); - let connect_cell = Arc::clone(&replanner_cell); - let connect_registry = Arc::clone(®istry_cell); - Arc::new( - OpampServer::new() - .with_on_connect(move |agent_id, _role| { - // Convention: agent metrics endpoint at http:///metrics. - // Collectors should set their agent-id to ":" so this - // resolves correctly, or override CONTROLLER_METRICS_PATH. - let url = format!("http://{agent_id}/metrics"); - let sc = Arc::clone(&sc); - let id_copy = agent_id.clone(); - let cell = Arc::clone(&connect_cell); - let reg = Arc::clone(&connect_registry); - let aid = agent_id.clone(); - tokio::spawn(async move { - sc.add_endpoint(Endpoint::new(id_copy, url)).await; - if let Some(r) = cell.read().await.as_ref() { - // Push the current plan config if this agent has a prior assignment. - let pushed = r.push_config_to_agent(&aid).await; - // If the agent has no prior assignment, assign it a workload - // from the registry (if available). - if !pushed { - if let Some(registry) = reg.read().await.as_ref() { - if let Some(entry) = registry.first_for_role("agent") { - r.register_agent(&aid, &entry.metric_name).await; - r.push_config_to_agent(&aid).await; - } - } - } - } - }); - }) - .with_on_disconnect(move |agent_id| { - let sd = Arc::clone(&sd); - tokio::spawn(async move { - sd.remove_endpoint(&agent_id).await; - }); - }), - ) - }; - - // ── Sketch defaults (YAML-configurable) ──────────────────────────────── - let sketch_defaults_path = std::env::var("CONTROLLER_SKETCH_DEFAULTS") - .unwrap_or_else(|_| "sketch_params_default.yml".into()); - let sketch_defaults = types::SketchDefaults::load(&sketch_defaults_path); - info!(path = %sketch_defaults_path, "loaded sketch defaults"); - - // ── BaselinePlanner backed by live EMA data ───────────────────────────── - // Runs the full cost-model optimisation once per metric on the first - // request, then locks in that plan as the baseline. The Replanner resets - // and re-optimises on SLA violation or plan expiry. - let planner = Arc::new(BaselinePlanner::new( - CostModelPlanner::new() - .with_sketch_defaults(sketch_defaults) - .with_online_store(Arc::clone(&online_store)), - )); - - let plan_store = Arc::new(PlanStore::new()); - let workload_store = Arc::new(WorkloadStore::new()); - - // ── Declarative workload registry ──────────────────────────────────────── - let workloads_path = std::env::var("CONTROLLER_WORKLOADS") - .unwrap_or_else(|_| "workloads.yaml".into()); - let workload_registry = Arc::new(WorkloadRegistry::load(&workloads_path)); - - // Pre-populate PlanStore from the registry so agents get a config immediately. - // - // Critical: thread `sketch_family_override` from each registry entry - // into the QuerySpec's `sketch_type` field — that's what populates - // `QueryWorkload::sketch_type_override`, which the typed planner - // (`bind_workload_typed`) reads to honour MVP §46 entries 5–8 (HLL / - // CountSketch / CountMinSketch). Without this stitch the workloads - // round-trip through the analyzer with a None override and the - // capability-matched default fires, but for the metrics whose - // statistic class doesn't match an `AggIntent` synthesizer (TopK in - // particular for `top_endpoint_qps`) the metric-name fallback in - // `classify_demo_metric` becomes the only path — and it works fine - // when the override is also threaded as a belt-and-braces guarantee. - { - let analyzer = Analyzer::new(); - for entry in workload_registry.entries() { - let spec = analyzer::QuerySpec { - query_string: entry.query_string.clone(), - metric_name: entry.metric_name.clone(), - label_filters: Default::default(), - group_by_labels: vec![], - aggregations: vec!["quantile".into()], - time_window: "5m".into(), - repeat_every: None, - accuracy_sla: entry.accuracy_sla, - latency_sla: None, - sketch_type: entry.sketch_family_override.clone(), - workload: types::WorkloadCharacteristics::default(), - // design.md alignment: defaults preserve legacy behaviour. - id: None, - language: None, - accuracy: None, - dollars: None, - deployment_model: None, - shape: types_v2::QueryShape::default(), - data: types_v2::DataShape::default(), - }; - match analyzer.analyze(spec) { - Ok(wl) => { - let wc = types::WorkloadCharacteristics::default(); - let plan = planner.plan(&wl, Some(&wc)); - let metric_name = wl.metric_name.clone(); - plan_store.set(&metric_name, plan); - workload_store.set(&metric_name, wl, wc); - } - Err(e) => { - warn!(metric = %entry.metric_name, error = %e, - "failed to pre-populate plan from workload registry"); - } - } - } - } - - // ── Phase C: shared BackendClient ───────────────────────────────────────── - // Built once at startup; shared between Replanner (existing path — - // pushes the StreamingConfig YAML on every successful replan) and - // AppState (Phase C — pushes the typed L5 backend JSON emitted by - // `emit_backend_config_json` from `handle_plan`). `None` when - // `CONTROLLER_BACKEND_ENDPOINT` is unset preserves the - // fire-and-forget "skip silently" contract from Phase B. - let backend_client_shared: Option> = - backend_endpoint.as_ref().map(|endpoint| { - info!( - endpoint = %endpoint, - "ASAPQuery-backend StreamingConfig push enabled" - ); - Arc::new(backend_client::BackendClient::new(endpoint.clone())) - }); - if backend_client_shared.is_none() { - info!( - "ASAPQuery-backend StreamingConfig push disabled \ - (set CONTROLLER_BACKEND_ENDPOINT= to enable)" - ); - } - - // ── Replanner — closes the SP-8 feedback loop ───────────────────────────── - let replanner = { - let mut r = Replanner::new( - Arc::clone(&planner), - Arc::clone(&plan_store), - Arc::clone(&workload_store), - Arc::clone(&opamp_srv), - Arc::clone(&scraper), - opamp_ep.clone(), - ); - if let Some(client) = backend_client_shared.as_ref() { - r = r.with_backend_client(Arc::clone(client)); - } - // Wire the workload registry so the typed-emit path - // (`USE_TYPED_STAGE_SPLIT`) can extend its edge stage config - // with the same archive-tier metrics the bootstrap GET path - // applies via `emit_bootstrap_typed`. - r = r.with_workload_registry(Arc::clone(&workload_registry)); - Arc::new(r) - }; - // Bind the late-binding cells so callbacks can reach the replanner and registry. - *replanner_cell.write().await = Some(Arc::clone(&replanner)); - *registry_cell.write().await = Some(Arc::clone(&workload_registry)); - - let replan_interval = Duration::from_secs( - std::env::var("CONTROLLER_REPLAN_INTERVAL_SECS") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(300u64), // re-check plan expiry every 5 minutes - ); - - let runtime_samples_store = runtime_samples::RuntimeSamplesStore::new(1024); - let backend_routing_cache: Arc>> = - Arc::new(Mutex::new(HashMap::new())); - let state = AppState { - analyzer: Arc::new(Analyzer::new()), - planner, - store: Arc::clone(&plan_store), - workload_store: Arc::clone(&workload_store), - opamp: Arc::clone(&opamp_srv), - scraper: Arc::clone(&scraper), - replanner: Arc::clone(&replanner), - online_store: Arc::clone(&online_store), - opamp_endpoint: opamp_ep, - workload_registry: Arc::clone(&workload_registry), - runtime_samples: Arc::clone(&runtime_samples_store), - backend_client: backend_client_shared, - backend_routing_cache: Arc::clone(&backend_routing_cache), - }; - - // ── Background tasks ────────────────────────────────────────────────────── - tokio::spawn(Arc::clone(&scraper).run()); - tokio::spawn(Arc::clone(&replanner).run_expiry_ticker(replan_interval)); - - // ── OpAMP WebSocket listener ────────────────────────────────────────────── - let opamp_router = Router::new() - .route("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/v1/opamp", get(OpampServer::ws_handler)) - .with_state(Arc::clone(&opamp_srv)); - - tokio::spawn(async move { - let listener = tokio::net::TcpListener::bind(&opamp_addr).await.unwrap(); - info!("OpAMP server listening on {opamp_addr}"); - axum::serve(listener, opamp_router).await.unwrap(); - }); - - // ── gRPC runtime-samples service (was HTTP+JSONL) ───────────────────────── - // Agents' `sketch-runtime::GrpcExporter` call - // `asap.runtime.v1.RuntimeSamples.Push` on this port. See - // commit message for the HTTP → gRPC pivot rationale. - let runtime_samples_state = Arc::clone(&state.runtime_samples); - let grpc_addr = std::env::var("CONTROLLER_GRPC_ADDR") - .unwrap_or_else(|_| "0.0.0.0:4321".into()); - let grpc_store = Arc::clone(&runtime_samples_state); - tokio::spawn(async move { - let addr: std::net::SocketAddr = grpc_addr.parse().expect("CONTROLLER_GRPC_ADDR"); - info!("runtime-samples gRPC server listening on {addr}"); - let svc = runtime_samples::RuntimeSamplesService::new(grpc_store).into_server(); - if let Err(e) = tonic::transport::Server::builder() - .add_service(svc) - .serve(addr) - .await - { - tracing::error!(error = %e, "runtime-samples gRPC server exited"); - } - }); - - // /metrics — same exposer as before, unchanged. Prom scrapes - // the controller HTTP port; gRPC is the push side only. - let metrics_registry = metrics_exposer::MetricsRegistry::new(); - let metrics_state = metrics_exposer::MetricsState { - registry: Arc::clone(&metrics_registry), - store: Arc::clone(&runtime_samples_state), - stats: runtime_samples_state.stats_handle(), - plan_store: Some(Arc::clone(&plan_store)), - }; - let metrics_router = Router::new() - .route( - "/metrics", - axum::routing::get(metrics_exposer::handle_metrics), - ) - .with_state(metrics_state); - - let app = Router::new() - .route("/api/v1/plan", post(handle_plan)) - .route("/api/v1/plan/pareto", post(handle_pareto)) - .route("/api/v1/plan/:metric", get(handle_get_plan)) - .route("/api/v1/plan/:metric/rollback", post(handle_rollback)) - .route("/api/v1/plan/:metric/diff", get(handle_plan_diff)) - .route("/api/v1/agents", get(handle_agents)) - .route("/api/v1/config/:metric", get(handle_get_config)) - .route("/api/v1/collector-config/agent", get(handle_bootstrap_agent_config)) - .route("/api/v1/collector-config/backend", get(handle_bootstrap_backend_config)) - .route("/api/v1/cost-model", get(handle_cost_model)) - .route("/api/v1/tco", post(handle_tco)) - .with_state(state) - .merge(metrics_router); - - let listener = tokio::net::TcpListener::bind(&api_addr).await.unwrap(); - info!("controller API listening on {api_addr}"); - axum::serve(listener, app).await.unwrap(); -} - -// ── Handlers ────────────────────────────────────────────────────────────────── - -async fn handle_plan( - State(st): State, - Json(spec): Json, -) -> impl IntoResponse { - let wc = spec.workload.clone(); - let query_string = spec.query_string.clone(); - let workload = match st.analyzer.analyze(spec) { - Ok(w) => w, - Err(e) => return (StatusCode::UNPROCESSABLE_ENTITY, e.to_string()).into_response(), - }; - - let mut plan = st.planner.plan(&workload, Some(&wc)); - - // ── SP-9: single QueryExpr pipeline — parse → optimise → stage-split ───── - // When query_string is present, run the full algebra pipeline and attach - // the StagedPlan. The SP-3 flat assignment remains the fallback when no - // query_string is supplied. - if let Some(ref qs) = query_string { - match parse_query_expr(qs) { - Err(e) => warn!(query = %qs, error = %e, "parse_query_expr failed; skipping staged_plan"), - Ok(qe) => { - let raw_bps = plan.transmission_cost_summary.raw_bytes_per_sec; - let budgets = StageResourceBudgets::from_workload_chars(&wc); - let constraints = algebra::optimizer::DeploymentConstraints::from_budgets(&budgets); - let (opt_qe, _) = QueryOptimizer::with_constraints(raw_bps, constraints).optimize(qe); - let (staged, _physical_tree) = physical_plan_to_staged(&opt_qe, &budgets); - plan.staged_plan = Some(staged); - } - } - } - - plan.precompute = build_precompute_jobs(&workload, &plan, "backend:4317"); - st.store.set(&workload.metric_name, plan.clone()); - // Persist workload so the replanner can re-run plan() without the original spec. - let wc_for_algebra = wc.clone(); - st.workload_store.set(&workload.metric_name, workload.clone(), wc); - - // ── Push agent config to agent-role collectors ──────────────────────────── - if let Ok(agent_yaml) = generate_agent_config(&plan.agent_config, &st.opamp_endpoint) { - let hash = short_hash(&agent_yaml); - st.opamp.push_to_role( - AgentRole::Agent, - RemoteConfig { config_hash: hash, yaml: agent_yaml }, - ).await; - } - - // ── Push backend config to backend-role collectors ──────────────────────── - // SP-9: pass the BackendSubPlan so the YAML gains a dedup processor when needed. - let backend_staged = plan.staged_plan.as_ref().map(|sp| &sp.backend); - if let Ok(backend_yaml) = generate_backend_config_staged( - &plan.backend_config, backend_staged, &st.opamp_endpoint, - ) { - let hash = short_hash(&backend_yaml); - st.opamp.push_to_role( - AgentRole::Backend, - RemoteConfig { config_hash: hash, yaml: backend_yaml }, - ).await; - } - - // ── Phase B (MVP v6): typed L5 stage_split → per-stage emitter ──────────── - // Behind the `USE_TYPED_STAGE_SPLIT` env-var gate so existing - // controller behaviour is unchanged unless explicitly opted in. - // When enabled, the workload is bound to a `SketchExpr`, the typed - // L5 path produces a `HashMap`, and each - // per-stage config is materialised into wire bytes via the emitters - // in `config::stage_config`. Phase C will plumb deployment-aware - // endpoint resolution + a real backend POST. - if planner::stage_split::typed_stage_split_enabled() { - if let Some(sketch_expr) = planner::rules::bind_workload_typed(&workload) { - if let Some(configs) = planner::stage_split::split_typed_three_stage(&sketch_expr) { - for (stage_id, stage_cfg) in configs { - match stage_cfg { - crate::stage_split::StageConfig::Edge(edge) => { - match config::emit_edge_yaml(&edge, &st.opamp_endpoint) { - Ok(yaml) => { - let hash = short_hash(&yaml); - info!( - stage = "edge", bytes = yaml.len(), - "[USE_TYPED_STAGE_SPLIT] pushing typed edge YAML" - ); - st.opamp.push_to_role( - AgentRole::Agent, - RemoteConfig { config_hash: hash, yaml }, - ).await; - } - Err(e) => warn!(error = %e, "emit_edge_yaml failed"), - } - } - crate::stage_split::StageConfig::Gateway(gw) => { - // Phase C: AgentRole::Gateway is now wired - // through the OpAMP role-routing path, so - // the gateway YAML is pushed to gateway-role - // collectors the same way the edge YAML is - // pushed to agent-role collectors above. - match config::emit_gateway_yaml(&gw, &st.opamp_endpoint) { - Ok(yaml) => { - let hash = short_hash(&yaml); - info!( - stage = "gateway", bytes = yaml.len(), - "[USE_TYPED_STAGE_SPLIT] pushing typed gateway YAML" - ); - st.opamp.push_to_role( - AgentRole::Gateway, - RemoteConfig { config_hash: hash, yaml }, - ).await; - } - Err(e) => warn!(error = %e, "emit_gateway_yaml failed"), - } - } - crate::stage_split::StageConfig::Backend(be) => { - // Phase C: post the typed L5 streaming-config - // JSON to ASAPQuery-backend via the shared - // BackendClient when configured. Without a - // configured endpoint this still no-ops - // silently — same fire-and-forget contract - // as the existing Replanner path. - match config::emit_backend_config_json(&be) { - Ok(json_doc) => { - info!( - stage = "backend", - aggregations = be.aggregations.len(), - readouts = be.readouts.len(), - "[USE_TYPED_STAGE_SPLIT] posting typed backend JSON" - ); - if let Some(client) = st.backend_client.as_ref() { - let body = json_doc.to_string(); - match client.post_streaming_config_json(body).await { - Ok(()) => info!( - stage = "backend", - endpoint = %client.endpoint(), - "[USE_TYPED_STAGE_SPLIT] typed backend JSON push succeeded" - ), - Err(e) => warn!( - stage = "backend", - endpoint = %client.endpoint(), - error = %e, - "[USE_TYPED_STAGE_SPLIT] typed backend JSON push failed; \ - next replan cycle will retry" - ), - } - } else { - info!( - stage = "backend", - "[USE_TYPED_STAGE_SPLIT] no backend client configured; \ - skipping JSON push (set CONTROLLER_BACKEND_ENDPOINT to enable)" - ); - } - } - Err(e) => warn!(error = %e, "emit_backend_config_json failed"), - } - - // Phase α (MVP): emit per-metric storage - // routing table from the same typed L5 - // BackendStageConfig, and POST it to the - // backend's `/api/v1/storage_routing` - // endpoint via the BackendClient sibling - // method. The classification rules live in - // `config::stage_config::emit_backend_storage_routing` - // — see that function's doc-comment for the - // sketch-family → query-shape mapping. - // - // CRITICAL: the backend's - // `POST /api/v1/storage_routing` handler is - // an atomic per-tenant SWAP — every push - // replaces the whole tenant's routing - // table. We MUST emit the cumulative - // routing JSON across every metric the - // controller has planned to date, otherwise - // each per-metric replan erases the routing - // entries for every other metric and the - // backend defaults them to - // `sketch_warm_tier` (which has nothing for - // archive-shape queries). That's the - // `archive_miss` failure mode for - // HLL/CountSketch/CountMin/KLL metrics in - // the post-#345 demo runs even though - // gorillas3 writes their TSDB blocks to - // MinIO and Thanos has them indexed. - // - // We thread the per-metric `BackendStageConfig` - // through `state.backend_routing_cache` so - // a metric replan picks up an updated entry - // for itself but preserves every previously - // planned metric's entry. - let cumulative_plans = { - let mut cache = st.backend_routing_cache.lock().await; - cache.insert(workload.metric_name.clone(), be.clone()); - cache - .iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect::>() - }; - let routing_input: Vec<(String, &BackendStageConfig)> = - cumulative_plans - .iter() - .map(|(k, v)| (k.clone(), v)) - .collect(); - match config::emit_backend_storage_routing(&routing_input) { - Ok(routing_doc) => { - info!( - stage = "backend", - metric = %workload.metric_name, - cumulative_metrics = cumulative_plans.len(), - "[USE_TYPED_STAGE_SPLIT] posting cumulative storage-routing JSON" - ); - if let Some(client) = st.backend_client.as_ref() { - let body = routing_doc.to_string(); - match client.post_storage_routing_json(body).await { - Ok(()) => info!( - stage = "backend", - metric = %workload.metric_name, - "[USE_TYPED_STAGE_SPLIT] storage-routing JSON push succeeded" - ), - Err(e) => warn!( - stage = "backend", - metric = %workload.metric_name, - error = %e, - "[USE_TYPED_STAGE_SPLIT] storage-routing JSON push failed; \ - next replan cycle will retry" - ), - } - } else { - info!( - stage = "backend", - "[USE_TYPED_STAGE_SPLIT] no backend client configured; \ - skipping storage-routing JSON push" - ); - } - } - Err(e) => warn!(error = %e, "emit_backend_storage_routing failed"), - } - - // Mention stage_id so `match` arms aren't - // collapsed into untagged log lines if the - // tracing filter drops the per-arm event. - let _ = stage_id; - } - } - } - } else { - warn!( - metric = %workload.metric_name, - "[USE_TYPED_STAGE_SPLIT] split_typed_three_stage returned None; \ - legacy plan output unaffected" - ); - } - } - } - - // ── Update scrape-endpoint sketch types and agent→metric mapping ────────── - let sketch_type = plan.agent_config.sketch_type.clone(); - for agent_id in st.opamp.connected_agents().await { - st.scraper.set_sketch_type(&agent_id, sketch_type.clone()).await; - st.replanner.register_agent(&agent_id, &workload.metric_name).await; - } - - // ── Algebra pipeline: parse → optimise → allocate ───────────────────────── - let raw_bps = plan.transmission_cost_summary.raw_bytes_per_sec; - let plan_summary = query_string.as_deref().and_then(|qs| { - match parse_query_expr(qs) { - Err(e) => { - warn!(query = qs, error = %e, "parse_query_expr failed; skipping plan_summary"); - None - } - Ok(qe) => { - let budgets = StageResourceBudgets::from_workload_chars(&wc_for_algebra); - let constraints = algebra::optimizer::DeploymentConstraints::from_budgets(&budgets); - let (opt_qe, _iters) = QueryOptimizer::with_constraints(raw_bps, constraints).optimize(qe); - let plan_node = SketchAllocator::new(budgets, raw_bps).allocate(opt_qe); - Some(plan_node.summarise(raw_bps)) - } - } - }); - - let agents = st.opamp.connected_agents().await; - let cost = &plan.transmission_cost_summary; - (StatusCode::OK, Json(json!({ - "metric": workload.metric_name, - "sketch_type": plan.agent_config.sketch_type.to_string(), - "mode": plan.agent_config.mode.to_string(), - "aggregate_by": plan.agent_config.aggregate_by, - "valid_until": plan.valid_until, - "agents_notified": agents.len(), - "precompute_jobs": plan.precompute.len(), - "delta_decision": plan.delta_decision, - "staged_plan": plan.staged_plan, - "transmission_costs": { - "raw_bytes_per_sec": cost.raw_bytes_per_sec, - "sketch_full_bytes_per_sec": cost.sketch_full_bytes_per_sec, - "sketch_delta_bytes_per_sec": cost.sketch_delta_bytes_per_sec, - "delta_cpu_overhead_micros_per_sample": cost.delta_cpu_overhead_micros_per_sample, - "delta_memory_overhead_bytes": cost.delta_memory_overhead_bytes, - "estimated_fill_rate": cost.estimated_fill_rate, - "flush_rate_hz": cost.flush_rate_hz, - }, - "plan_summary": plan_summary, - }))).into_response() -} - -/// Request body for `POST /api/v1/plan/pareto`. -#[derive(serde::Deserialize)] -struct ParetoRequest { - #[serde(flatten)] - spec: QuerySpec, - #[serde(default)] - weights: ObjectiveWeights, -} - -/// Returns the Pareto frontier of collection plans for the given workload. -/// Each point is annotated with bandwidth, CPU, memory and accuracy objectives. -/// The caller can specify `weights` to get the frontier sorted by their -/// preferred trade-off. -async fn handle_pareto( - State(st): State, - Json(req): Json, -) -> impl IntoResponse { - let wc = req.spec.workload.clone(); - let workload = match st.analyzer.analyze(req.spec) { - Ok(w) => w, - Err(e) => return (StatusCode::UNPROCESSABLE_ENTITY, e.to_string()).into_response(), - }; - - let frontier = pareto_frontier(&workload, &wc, req.weights, Some(&st.online_store)); - - if frontier.is_empty() { - return (StatusCode::UNPROCESSABLE_ENTITY, - "no sketch meets the accuracy SLA for the given workload").into_response(); - } - - let best = select_best(&frontier, req.weights) - .map(|p| p.sketch_type.to_string()); - - let points: Vec = frontier.iter().map(|p| json!({ - "sketch_type": p.sketch_type.to_string(), - "bandwidth_bytes_per_sec": p.bandwidth_bytes_per_sec, - "cpu_micros_per_sample": p.cpu_micros_per_sample, - "memory_bytes": p.memory_bytes, - "estimated_error": p.estimated_error, - })).collect(); - - (StatusCode::OK, Json(json!({ - "metric": workload.metric_name, - "frontier": points, - "best": best, - }))).into_response() -} - -async fn handle_get_plan( - State(st): State, - Path(metric): Path, -) -> impl IntoResponse { - match st.store.get(&metric) { - Ok(plan) => (StatusCode::OK, Json(json!({ - "metric": metric, - "sketch_type": plan.agent_config.sketch_type.to_string(), - "valid_until": plan.valid_until, - }))).into_response(), - Err(e) => (StatusCode::NOT_FOUND, e.to_string()).into_response(), - } -} - -async fn handle_rollback( - State(st): State, - Path(metric): Path, -) -> impl IntoResponse { - // Reset the baseline so the next POST /api/v1/plan re-runs the cost - // model and establishes a fresh baseline plan for this metric. - st.planner.reset(&metric); - match st.store.rollback(&metric) { - Ok(plan) => { - if let Ok(yaml) = generate_agent_config(&plan.agent_config, &st.opamp_endpoint) { - st.opamp.push_to_role(AgentRole::Agent, RemoteConfig { - config_hash: short_hash(&yaml), yaml, - }).await; - } - if let Ok(yaml) = generate_backend_config(&plan.backend_config, &st.opamp_endpoint) { - st.opamp.push_to_role(AgentRole::Backend, RemoteConfig { - config_hash: short_hash(&yaml), yaml, - }).await; - } - (StatusCode::OK, Json(json!({ "metric": metric, "rolled_back": true }))).into_response() - } - Err(e) => (StatusCode::BAD_REQUEST, e.to_string()).into_response(), - } -} - -async fn handle_agents(State(st): State) -> impl IntoResponse { - Json(st.opamp.connected_agents_with_roles().await) -} - -/// Returns a complete OTel collector YAML for the named metric's current plan. -/// Collectors can use this with the HTTP config provider: -/// --config=http://controller:8080/api/v1/config/ -async fn handle_get_config( - State(st): State, - Path(metric): Path, -) -> impl IntoResponse { - match st.store.get(&metric) { - Ok(plan) => match generate_agent_config(&plan.agent_config, &st.opamp_endpoint) { - Ok(yaml) => ( - StatusCode::OK, - [("content-type", "application/yaml")], - yaml, - ).into_response(), - Err(e) => (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()).into_response(), - }, - Err(e) => (StatusCode::NOT_FOUND, e.to_string()).into_response(), - } -} - -/// Bootstrap YAML config for agent collectors. -/// -/// Collectors start with: -/// `./collector --config "http://controller:8080/api/v1/collector-config/agent"` -/// -/// ## Behaviour matrix -/// -/// | `USE_TYPED_STAGE_SPLIT` | path | -/// | --- | --- | -/// | unset / `0` | **legacy** — emit a default-DDSketch [`AgentCollectorConfig`] via [`generate_agent_config`]. Backwards-compat with deployments that haven't migrated to the typed L5 emitters. | -/// | `1` / `true` / `yes` | **typed** — pick the agent's pinned workload (when `X-Agent-ID` is supplied and the replanner has a prior assignment), or fall back to the first agent-role entry in [`WorkloadRegistry`]. Run the typed L5 pipeline (`bind_workload_typed` → `split_typed_three_stage`) and emit the Edge stage config via [`emit_for_runtime`] — dispatched by the `X-Agent-Runtime` header (defaults to `AsapOtel`). When the typed path errors out (no workload, unsupported topology, no Edge stage in the per-stage map) it falls back to the legacy emitter so the bootstrap never returns a 500 just because the typed path has a gap. | -/// -/// ## Why this matters -/// -/// Without the typed path, fresh agents connecting at startup miss -/// Phase 3.2.5's `gorillas3` archive emit + warm-passthrough routing -/// processor, the per-runtime dispatch from Phase ε.1.5 (asap-otel vs -/// asap-otap vs asap-telegraf), and Phase ε.1's three operational -/// modes — they only see those once `handle_plan` is later invoked. -/// Mirroring `handle_plan`'s typed pipeline here means bootstrap and -/// plan-push converge on the same emitted YAML. -async fn handle_bootstrap_agent_config( - State(st): State, - headers: HeaderMap, -) -> impl IntoResponse { - // Phase ε.1.5 — runtime dispatch from the X-Agent-Runtime header. - // Defaults to `AsapOtel` for legacy agents that don't send - // the header so the existing OTel-collector contrib build keeps - // working with no client-side changes. - let runtime = headers - .get("X-Agent-Runtime") - .and_then(|v| v.to_str().ok()) - .map(AgentRuntime::from_header) - .unwrap_or_default(); - - // Optional X-Agent-ID — when present, look up any pinned workload - // assignment via the replanner so bootstrap returns the same plan - // a subsequent OpAMP push would pin to. Avoids drift between the - // initial fetch and the first push. - let pinned_metric: Option = headers - .get("X-Agent-ID") - .and_then(|v| v.to_str().ok()) - .map(|s| s.to_string()); - - if planner::stage_split::typed_stage_split_enabled() { - match emit_bootstrap_typed(&st, runtime, pinned_metric.as_deref()).await { - Ok(yaml) => { - info!( - runtime = ?runtime, bytes = yaml.len(), - "[USE_TYPED_STAGE_SPLIT] emitted bootstrap config from typed path" - ); - return ( - StatusCode::OK, - [("content-type", "application/yaml")], - yaml, - ).into_response(); - } - Err(e) => { - warn!( - runtime = ?runtime, error = %e, - "[USE_TYPED_STAGE_SPLIT] typed bootstrap path failed; \ - falling back to legacy generate_agent_config" - ); - // Fall through to legacy path below. - } - } - } - - // Legacy path — default DDSketch bootstrap (unchanged Phase α - // behaviour). Serves as the backwards-compat fallback when the - // typed gate is off OR when the typed path can't satisfy the - // request (no workloads registered, unsupported topology, etc.). - let cfg = AgentCollectorConfig { - output_mode: types::OutputMode::Sketch, - sketch_type: types::SketchType::DDSketch, - sketch_params: types::SketchParams::default(), - aggregate_by: vec![], - label_matchers: vec![], - window_duration: Some(std::time::Duration::from_secs(60)), - mode: types::ProcessorMode::Window, - enable_self_monitoring: true, - transmit_sketch: true, - drop_original: true, - delta_transmission: false, - delta_threshold: 0.0, - enable_series_id: false, - series_id_ttl_secs: 300, - data_sink: types::AgentDataSink::default(), - }; - match generate_agent_config(&cfg, &st.opamp_endpoint) { - Ok(yaml) => ( - StatusCode::OK, - [("content-type", "application/yaml")], - yaml, - ).into_response(), - Err(e) => (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()).into_response(), - } -} - -/// Run the typed L5 pipeline against the workload registry / pinned plan -/// and emit the Edge stage YAML for the given runtime. -/// -/// Resolution order for "which workload does this agent get": -/// 1. `X-Agent-ID` lookup → `replanner.agent_to_metric()` mapping -/// (the replanner's record of what plan the agent is currently -/// pinned to). When this hits, bootstrap == replan-push. -/// 2. First agent-role entry in [`WorkloadRegistry`] — the same -/// heuristic the OpAMP `on_connect` callback uses for unassigned -/// agents. -/// -/// Returns `Err` when none of the resolution paths land on a workload -/// the typed path can bind, when `bind_workload_typed` declines the -/// shape (multi-intent, raw-required, no aggregations), when stage -/// allocation fails, or when the per-stage map has no `Edge` entry. -/// The caller falls back to the legacy emitter on any error. -async fn emit_bootstrap_typed( - st: &AppState, - runtime: AgentRuntime, - pinned_agent_id: Option<&str>, -) -> anyhow::Result { - use anyhow::{anyhow, Context}; - - // 1. Resolve the metric this bootstrap should target. - // When the agent has a prior pinned assignment we honour it - // (pre-existing on_connect contract). The replanner's - // `agent_to_metric()` is the source of truth for this mapping. - let pinned_metric: Option = if let Some(aid) = pinned_agent_id { - st.replanner - .agent_to_metric() - .read() - .await - .get(aid) - .cloned() - } else { - None - }; - - // Candidate metric resolution. When the agent has a prior pin, we - // try it first — a pinned raw-passthrough metric (e.g. - // `http_requests_total`) declines the typed bind, but the - // bootstrap still needs to ship the 5-sketch routing-connector - // edge config so OTHER metrics in the registry get processed. - // Walk the registry until we find one the typed path accepts — - // that gives us the edge_cfg shape — then populate - // `metric_to_family` from the FULL registry (every binding - // metric, not just the chosen one) below. - // - // If pinned metric exists, it's the FIRST candidate; otherwise - // walk every agent-role entry in the registry. - let candidates: Vec = { - let mut v = Vec::new(); - if let Some(p) = pinned_metric.clone() { - v.push(p); - } - for entry in st.workload_registry.entries() { - if !entry.assign_to_role.eq_ignore_ascii_case("agent") { - continue; - } - if !v.contains(&entry.metric_name) { - v.push(entry.metric_name.clone()); - } - } - v - }; - if candidates.is_empty() { - return Err(anyhow!("no agent-role workload available for bootstrap")); - } - - // 2-3. Walk candidates: first metric that pre-populated the - // workload store AND binds via the typed path provides the - // base edge_cfg shape. - let mut chosen: Option<(String, crate::sketch_algebra::SketchExpr)> = None; - for cand in &candidates { - let Some((wl, _wc)) = st.workload_store.get(cand) else { continue }; - if let Some(expr) = planner::rules::bind_workload_typed(&wl) { - chosen = Some((cand.clone(), expr)); - break; - } - } - let (metric, sketch_expr) = chosen.ok_or_else(|| { - anyhow!( - "no registry metric binds via the typed path (all {} candidates declined)", - candidates.len() - ) - })?; - let configs = planner::stage_split::split_typed_three_stage(&sketch_expr) - .ok_or_else(|| anyhow!("split_typed_three_stage returned None for `{metric}`"))?; - - // 4. Pick the Edge stage config and emit per-runtime. The - // bootstrap caller IS the edge agent — Gateway / Backend - // configs go to other roles via OpAMP role-routing, not - // through this handler. - let mut edge_cfg = configs.into_iter().find_map(|(_, cfg)| match cfg { - crate::stage_split::StageConfig::Edge(edge) => Some(edge), - _ => None, - }).ok_or_else(|| anyhow!("typed three-stage map has no Edge entry for `{metric}`"))?; - - // 5. Bootstrap-only plumbing: extend the typed Edge config with - // metrics that the live planner doesn't see but the MVP demo - // needs the agent to handle: - // - // - Freshness probes (`http_freshness_probe_warm`, - // `http_freshness_probe_archive`): demo plumbing, not user - // metrics. The replay client polls the backend with - // `last_over_time(http_freshness_probe_warm[10s])` to gauge - // criterion ⑥. Without warm-passthrough routing the - // DDSketch processor renames them to `_quantile`; without - // gorillas3 archive write the warm engine has nothing to - // look at. - // - All non-archive workload-registry metrics: accuracy_reduce.py - // asks the archive engine for the SAME PromQL the warm sketch - // answered (criterion ④, archive-tier ground truth). If the - // under-test metric isn't in the Gorilla-S3 archive, every - // ground-truth query returns `archive_miss`. Adding the - // metrics here makes the agent's gorillas3 processor write - // them so the Thanos store-gateway can serve them later. - // - // Both extensions are bootstrap-scope only — the live planner - // stays free to plan per-metric without these defaults bleeding - // in. The actual extension lives in the shared - // [`config::extend_edge_with_demo_plumbing`] helper so the - // typed-replan push path (`replan::Replanner::push_config_to_agent`) - // can apply the same extension without duplicating the logic. - let registry_metrics = st - .workload_registry - .entries() - .iter() - .map(|e| e.metric_name.clone()); - config::extend_edge_with_demo_plumbing(&mut edge_cfg, registry_metrics); - - // 6. Stitch PR #339 (planner) → PR #340 (5-sketch routing emitter). - // - // `bind_workload_typed` is per-metric. The 5-sketch routing- - // connector edge wire shape needs every sketched metric mapped - // to its committed family up-front so the emitter can build the - // `routing` connector's per-metric OTTL condition - // statements. Walk the workload registry, classify each metric - // via the planner, and drop the resulting HashMap into the - // EdgeStageConfig before emit. Empty map ⇒ legacy single- - // pipeline emit (raw-only deployment, registry empty, etc.). - // - // Why this entry point: bootstrap is the place that already has - // all of `(WorkloadRegistry, WorkloadStore, edge_cfg)` in scope. - // Pushing the multi-metric loop down into - // `split_typed_three_stage` would change its signature for one - // caller (this one) and break the OpAMP-on-connect contract - // where the agent IS pinned to a single metric. Replan path - // (`replan::Replanner::try_emit_typed_edge_yaml_for_workload`) - // applies the same stitch via the same shared helper. - edge_cfg.metric_to_family = config::collect_metric_to_family( - &st.workload_registry, - &st.workload_store, - ); - - emit_for_runtime(runtime, &edge_cfg, &st.opamp_endpoint, None) - .with_context(|| format!("emit_for_runtime failed for `{metric}`")) -} - -/// Bootstrap YAML config for backend (merge) collectors. -async fn handle_bootstrap_backend_config( - State(st): State, -) -> impl IntoResponse { - let cfg = types::BackendCollectorConfig { - merge_sketch_type: types::SketchType::DDSketch, - group_by: vec![], - }; - match generate_backend_config(&cfg, &st.opamp_endpoint) { - Ok(yaml) => ( - StatusCode::OK, - [("content-type", "application/yaml")], - yaml, - ).into_response(), - Err(e) => (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()).into_response(), - } -} - -/// Returns the diff between the current and previous plan for `metric`. -/// 404 if the metric has no plan, 200 with `null` data if no previous plan exists. -async fn handle_plan_diff( - State(st): State, - Path(metric): Path, -) -> impl IntoResponse { - match st.store.diff(&metric) { - Ok(Some(diff)) => (StatusCode::OK, Json(json!({ - "metric": metric, - "has_diff": true, - "diff": diff, - }))).into_response(), - Ok(None) => (StatusCode::OK, Json(json!({ - "metric": metric, - "has_diff": false, - }))).into_response(), - Err(e) => (StatusCode::NOT_FOUND, e.to_string()).into_response(), - } -} - -/// Returns the current EMA cost model state — blended benchmark + observed costs -/// per sketch type. Useful for diagnosing whether the online cost model has -/// received sufficient observations to meaningfully influence plan selection. -async fn handle_cost_model(State(st): State) -> impl IntoResponse { - let table = online_cost_model::effective_table(&st.online_store); - let raw = st.online_store.try_read(); - - let entries: Vec = table.iter().map(|(sketch_type, costs)| { - let observations = raw.as_ref().ok() - .and_then(|m| m.get(sketch_type)) - .map(|o| o.observations) - .unwrap_or(0); - json!({ - "sketch_type": sketch_type.to_string(), - "bw_bytes_per_series_per_sec": costs.bytes_per_series_per_sec, - "cpu_micros_per_sample": costs.cpu_micros_per_sample, - "base_memory_bytes": costs.base_memory_bytes, - "relative_error": costs.relative_error_at_default, - "observations": observations, - }) - }).collect(); - - (StatusCode::OK, Json(json!({ "sketches": entries }))).into_response() -} - -// ── TCO endpoint ───────────────────────────────────────────────────────────── - -#[derive(serde::Deserialize)] -struct TcoRequest { - workload: tco::TcoWorkload, - pricing: Option, -} - -async fn handle_tco(Json(req): Json) -> impl IntoResponse { - let pricing = req.pricing.unwrap_or_default(); - let estimate = tco::estimate_tco(&req.workload, &pricing); - (StatusCode::OK, Json(estimate)) -} - -fn short_hash(s: &str) -> String { - use std::collections::hash_map::DefaultHasher; - use std::hash::{Hash, Hasher}; - let mut h = DefaultHasher::new(); - s.hash(&mut h); - format!("{:016x}", h.finish()) -} - -// ── Test helpers ────────────────────────────────────────────────────────────── - -/// Builds a minimal `AppState` + `Router` for integration tests. -/// No background tasks are started; OpAMP/scraper hold no real connections. -#[cfg(test)] -fn test_app() -> (AppState, axum::Router) { - test_app_with_backend(None) -} - -/// Phase C test helper: build an `AppState` whose `backend_client` is -/// optionally set to a real `BackendClient` pointed at a mock URL. The -/// `None` arm is the legacy path used by every existing test; -/// `Some(url)` is the new entry point for Phase C tests that exercise -/// the typed L5 backend-JSON push. -#[cfg(test)] -fn test_app_with_backend(backend_url: Option) -> (AppState, axum::Router) { - let online_store = init_online_store(); - let plan_store = Arc::new(PlanStore::new()); - let workload_store = Arc::new(WorkloadStore::new()); - let opamp = Arc::new(OpampServer::new()); - let scraper = Arc::new(Scraper::new( - vec![], Thresholds::default(), Arc::new(|_| {}), Duration::from_secs(60), - )); - let planner = Arc::new(BaselinePlanner::new( - CostModelPlanner::new().with_online_store(Arc::clone(&online_store)), - )); - let replanner = Arc::new(Replanner::new( - Arc::clone(&planner), - Arc::clone(&plan_store), - Arc::clone(&workload_store), - Arc::clone(&opamp), - Arc::clone(&scraper), - "ws://ctrl:4320/v1/opamp", - )); - let backend_client = backend_url - .map(|u| Arc::new(backend_client::BackendClient::new(u))); - let state = AppState { - analyzer: Arc::new(Analyzer::new()), - planner, - store: Arc::clone(&plan_store), - workload_store: Arc::clone(&workload_store), - opamp, - scraper, - replanner, - online_store, - opamp_endpoint: "ws://ctrl:4320/v1/opamp".into(), - workload_registry: Arc::new(WorkloadRegistry::empty()), - runtime_samples: runtime_samples::RuntimeSamplesStore::new(64), - backend_client, - backend_routing_cache: Arc::new(Mutex::new(HashMap::new())), - }; - let router = axum::Router::new() - .route("/api/v1/plan", axum::routing::post(handle_plan)) - .route("/api/v1/plan/pareto", axum::routing::post(handle_pareto)) - .route("/api/v1/plan/:metric", axum::routing::get(handle_get_plan)) - .route("/api/v1/plan/:metric/rollback", axum::routing::post(handle_rollback)) - .route("/api/v1/plan/:metric/diff", axum::routing::get(handle_plan_diff)) - .route("/api/v1/agents", axum::routing::get(handle_agents)) - .route("/api/v1/cost-model", axum::routing::get(handle_cost_model)) - .route("/api/v1/tco", axum::routing::post(handle_tco)) - .route("/api/v1/collector-config/agent", axum::routing::get(handle_bootstrap_agent_config)) - .with_state(state.clone()); - (state, router) -} - -#[cfg(test)] -mod api_tests { - use super::*; - use axum::body::Body; - use axum::http::{Request, StatusCode}; - use http_body_util::BodyExt; - use tower::ServiceExt; - - async fn body_json(resp: axum::response::Response) -> serde_json::Value { - let bytes = resp.into_body().collect().await.unwrap().to_bytes(); - serde_json::from_slice(&bytes).unwrap() - } - - fn plan_spec(metric: &str) -> serde_json::Value { - serde_json::json!({ - "metric_name": metric, - "aggregations": ["quantile"], - "time_window": "5m", - "accuracy_sla": 0.01 - }) - } - - // ── Phase C: AppState.backend_client wiring ─────────────────────────────── - - /// Default-constructed AppState (no `CONTROLLER_BACKEND_ENDPOINT`) - /// must leave `backend_client` as `None` so the typed L5 backend - /// JSON push silently no-ops, matching the Phase B fire-and-forget - /// contract. - #[test] - fn app_state_backend_client_none_by_default() { - let (state, _router) = test_app(); - assert!(state.backend_client.is_none(), - "backend_client should default to None when no endpoint is configured"); - } - - /// When constructed with a backend URL (the production path takes - /// it from `CONTROLLER_BACKEND_ENDPOINT`), the field is populated - /// and ready for the Phase C `handle_plan` push. - #[test] - fn app_state_backend_client_some_when_constructed_with_url() { - let (state, _router) = test_app_with_backend( - Some("http://127.0.0.1:1/api/v1/streaming-config".into()), - ); - let bc = state.backend_client.expect("backend_client must be Some"); - assert_eq!(bc.endpoint(), "http://127.0.0.1:1/api/v1/streaming-config"); - } - - // ── POST /api/v1/plan ───────────────────────────────────────────────────── - - #[tokio::test] - async fn plan_happy_path() { - let (_, app) = test_app(); - let req = Request::builder() - .method("POST") - .uri("/api/v1/plan") - .header("content-type", "application/json") - .body(Body::from(plan_spec("latency").to_string())) - .unwrap(); - let resp = app.oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let body = body_json(resp).await; - assert_eq!(body["metric"], "latency"); - assert!(body["sketch_type"].as_str().is_some()); - assert!(body["valid_until"].as_str().is_some()); - } - - #[tokio::test] - async fn plan_invalid_spec_returns_422() { - let (_, app) = test_app(); - let req = Request::builder() - .method("POST") - .uri("/api/v1/plan") - .header("content-type", "application/json") - .body(Body::from(r#"{"metric_name":"","aggregations":["quantile"],"time_window":"5m","accuracy_sla":0.01}"#)) - .unwrap(); - let resp = app.oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY); - } - - #[tokio::test] - async fn plan_invalid_aggregation_returns_422() { - let (_, app) = test_app(); - let bad = serde_json::json!({ - "metric_name": "m", "aggregations": ["histogram"], - "time_window": "5m", "accuracy_sla": 0.01 - }); - let req = Request::builder() - .method("POST").uri("/api/v1/plan") - .header("content-type", "application/json") - .body(Body::from(bad.to_string())).unwrap(); - let resp = app.oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY); - } - - // ── GET /api/v1/plan/:metric ────────────────────────────────────────────── - - #[tokio::test] - async fn get_plan_not_found_returns_404() { - let (_, app) = test_app(); - let req = Request::builder() - .uri("/api/v1/plan/nonexistent").body(Body::empty()).unwrap(); - let resp = app.oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - } - - #[tokio::test] - async fn get_plan_after_post() { - let (_, app) = test_app(); - // POST first - let post_req = Request::builder() - .method("POST").uri("/api/v1/plan") - .header("content-type", "application/json") - .body(Body::from(plan_spec("cpu").to_string())).unwrap(); - let post_resp = app.clone().oneshot(post_req).await.unwrap(); - assert_eq!(post_resp.status(), StatusCode::OK); - // Then GET - let get_req = Request::builder() - .uri("/api/v1/plan/cpu").body(Body::empty()).unwrap(); - let get_resp = app.oneshot(get_req).await.unwrap(); - assert_eq!(get_resp.status(), StatusCode::OK); - let body = body_json(get_resp).await; - assert_eq!(body["metric"], "cpu"); - } - - // ── POST /api/v1/plan/:metric/rollback ──────────────────────────────────── - - #[tokio::test] - async fn rollback_no_previous_returns_400() { - let (st, app) = test_app(); - // Seed one plan directly. - use crate::planner::rules::RulesPlanner; - let wl = crate::types::QueryWorkload { - metric_name: "m".into(), - label_filters: std::collections::HashMap::new(), - group_by_labels: vec![], - aggregations: vec![crate::types::AggType::Quantile], - time_window: std::time::Duration::from_secs(300), - repeat_every: None, accuracy_sla: 0.01, latency_sla: None, - sketch_type_override: None, exact_required: false, quantiles: vec![], - }; - st.store.set("m", RulesPlanner::new().plan(&wl)); - let req = Request::builder() - .method("POST").uri("/api/v1/plan/m/rollback") - .body(Body::empty()).unwrap(); - let resp = app.oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::BAD_REQUEST); - } - - #[tokio::test] - async fn rollback_not_found_returns_400() { - let (_, app) = test_app(); - let req = Request::builder() - .method("POST").uri("/api/v1/plan/ghost/rollback") - .body(Body::empty()).unwrap(); - let resp = app.oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::BAD_REQUEST); - } - - // ── GET /api/v1/plan/:metric/diff ───────────────────────────────────────── - - #[tokio::test] - async fn diff_no_previous_returns_has_diff_false() { - let (_, app) = test_app(); - // POST a plan once. - let req = Request::builder() - .method("POST").uri("/api/v1/plan") - .header("content-type", "application/json") - .body(Body::from(plan_spec("rtt").to_string())).unwrap(); - app.clone().oneshot(req).await.unwrap(); - // Diff should exist but has_diff=false (only one version). - let req = Request::builder() - .uri("/api/v1/plan/rtt/diff").body(Body::empty()).unwrap(); - let resp = app.oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let body = body_json(resp).await; - assert_eq!(body["has_diff"], false); - } - - #[tokio::test] - async fn diff_not_found_returns_404() { - let (_, app) = test_app(); - let req = Request::builder() - .uri("/api/v1/plan/ghost/diff").body(Body::empty()).unwrap(); - let resp = app.oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - } - - // ── GET /api/v1/cost-model ──────────────────────────────────────────────── - - #[tokio::test] - async fn cost_model_returns_all_sketch_types() { - let (_, app) = test_app(); - let req = Request::builder() - .uri("/api/v1/cost-model").body(Body::empty()).unwrap(); - let resp = app.oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let body = body_json(resp).await; - let sketches = body["sketches"].as_array().unwrap(); - assert!(sketches.len() >= 4, "expected at least 4 sketch types"); - for s in sketches { - assert!(s["sketch_type"].as_str().is_some()); - assert!(s["observations"].as_u64().is_some()); - } - } - - // ── POST /api/v1/plan/pareto ────────────────────────────────────────────── - - #[tokio::test] - async fn pareto_returns_frontier_for_quantile() { - let (_, app) = test_app(); - let body = serde_json::json!({ - "metric_name": "latency", "aggregations": ["quantile"], - "time_window": "5m", "accuracy_sla": 0.02, - "weights": { "bandwidth": 0.7, "cpu": 0.2, "memory": 0.1 } - }); - let req = Request::builder() - .method("POST").uri("/api/v1/plan/pareto") - .header("content-type", "application/json") - .body(Body::from(body.to_string())).unwrap(); - let resp = app.oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let body = body_json(resp).await; - let frontier = body["frontier"].as_array().unwrap(); - assert!(!frontier.is_empty(), "frontier should not be empty"); - assert!(body["best"].as_str().is_some(), "best sketch should be set"); - } - - // ── GET /api/v1/agents ──────────────────────────────────────────────────── - - #[tokio::test] - async fn agents_returns_empty_map_initially() { - let (_, app) = test_app(); - let req = Request::builder() - .uri("/api/v1/agents").body(Body::empty()).unwrap(); - let resp = app.oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let body = body_json(resp).await; - // No agents connected → empty object. - assert_eq!(body, serde_json::json!({})); - } - - // ── POST /api/v1/tco ───────────────────────────────────────────────────── - - #[tokio::test] - async fn tco_returns_valid_estimate() { - let (_, app) = test_app(); - let body = serde_json::json!({ - "workload": { - "series_count": 100000, - "samples_per_sec": 1.0, - "bytes_per_sample": 100, - "scrape_interval_secs": 15, - "queries_per_sec": 1.0, - "query_window_secs": 300, - "retention_days": 30, - "sketch_compression_ratio": 0.05, - "delta_compression_ratio": 0.3 - } - }); - let req = Request::builder() - .method("POST").uri("/api/v1/tco") - .header("content-type", "application/json") - .body(Body::from(body.to_string())).unwrap(); - let resp = app.oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let body = body_json(resp).await; - assert!(body["before"]["total_dollars"].as_f64().unwrap() > 0.0); - assert!(body["after"]["total_dollars"].as_f64().unwrap() > 0.0); - assert!(body["savings_percent"].as_f64().unwrap() > 0.0); - } - - // ── Integration: controller ↔ collector wiring ───────────────────────── - - /// Helper: start an OpAMP WebSocket server on a random port. - /// Returns the (server Arc, local addr string). - async fn start_opamp_server(opamp: Arc) -> String { - let router = axum::Router::new() - .route("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/v1/opamp", axum::routing::get(OpampServer::ws_handler)) - .with_state(opamp); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - format!("127.0.0.1:{}", addr.port()) - } - - /// Connect a mock agent via WebSocket, returning the stream. - async fn connect_agent( - opamp_addr: &str, - agent_id: &str, - role: &str, - ) -> tokio_tungstenite::WebSocketStream> { - use tokio_tungstenite::tungstenite::client::IntoClientRequest; - let url = format!("ws://{opamp_addr}/v1/opamp"); - let mut req = url.into_client_request().unwrap(); - req.headers_mut().insert("X-Agent-ID", agent_id.parse().unwrap()); - req.headers_mut().insert("X-Agent-Role", role.parse().unwrap()); - let (ws, _) = tokio_tungstenite::connect_async(req).await.unwrap(); - ws - } - - /// Read the next binary WebSocket frame, decode as OpAMP ServerToAgent, - /// and extract the YAML config body. - async fn recv_config_yaml( - ws: &mut tokio_tungstenite::WebSocketStream>, - ) -> String { - use tokio_tungstenite::tungstenite::Message; - let msg = tokio::time::timeout( - std::time::Duration::from_secs(5), - futures_util::StreamExt::next(ws), - ).await.expect("timeout waiting for config push") - .expect("stream ended") - .expect("ws error"); - match msg { - Message::Binary(data) => { - let payload = if !data.is_empty() && data[0] == 0 { - &data[1..] - } else { - data.as_slice() - }; - let sta = ::decode( - payload, - ).expect("decode ServerToAgent"); - let rc = sta.remote_config.expect("remote_config present"); - let cm = rc.config.expect("config present"); - let file = cm.config_map.get("").expect("empty-key config file"); - String::from_utf8(file.body.clone()).expect("yaml is utf8") - } - other => panic!("expected binary frame, got {other:?}"), - } - } - - /// Test 1: Agent connects with workloads.yaml pre-populated, receives config on connect. - #[tokio::test] - async fn agent_receives_config_on_connect_via_workload_registry() { - // Build a full AppState with a workload registry entry. - let online_store = init_online_store(); - let plan_store = Arc::new(PlanStore::new()); - let workload_store = Arc::new(WorkloadStore::new()); - let opamp = Arc::new(OpampServer::new()); - let scraper = Arc::new(Scraper::new( - vec![], Thresholds::default(), Arc::new(|_| {}), Duration::from_secs(60), - )); - let planner = Arc::new(BaselinePlanner::new( - CostModelPlanner::new().with_online_store(Arc::clone(&online_store)), - )); - - // Pre-populate plan store (simulating what main() does with workload registry). - let analyzer = Analyzer::new(); - let spec = analyzer::QuerySpec { - query_string: None, - metric_name: "http_latency".into(), - label_filters: Default::default(), - group_by_labels: vec![], - aggregations: vec!["quantile".into()], - time_window: "5m".into(), - repeat_every: None, - accuracy_sla: 0.01, - latency_sla: None, - sketch_type: None, - workload: types::WorkloadCharacteristics::default(), - id: None, - language: None, - accuracy: None, - dollars: None, - deployment_model: None, - shape: types_v2::QueryShape::default(), - data: types_v2::DataShape::default(), - }; - let wl = analyzer.analyze(spec).unwrap(); - let wc = types::WorkloadCharacteristics::default(); - let plan = planner.plan(&wl, Some(&wc)); - plan_store.set("http_latency", plan); - workload_store.set("http_latency", wl, wc); - - // Build replanner and late-binding cells. - let replanner_cell: Arc>>> = - Arc::new(tokio::sync::RwLock::new(None)); - let registry_cell: Arc>>> = - Arc::new(tokio::sync::RwLock::new(None)); - - let opamp_ep = "ws://127.0.0.1:0/v1/opamp".to_string(); - - // Wire on_connect callback — same logic as main(). - let sc = Arc::clone(&scraper); - let connect_cell = Arc::clone(&replanner_cell); - let connect_registry = Arc::clone(®istry_cell); - let opamp_srv = Arc::new( - OpampServer::new() - .with_on_connect(move |agent_id, _role| { - let url = format!("http://{agent_id}/metrics"); - let sc = Arc::clone(&sc); - let id_copy = agent_id.clone(); - let cell = Arc::clone(&connect_cell); - let reg = Arc::clone(&connect_registry); - let aid = agent_id.clone(); - tokio::spawn(async move { - sc.add_endpoint(Endpoint::new(id_copy, url)).await; - if let Some(r) = cell.read().await.as_ref() { - let pushed = r.push_config_to_agent(&aid).await; - if !pushed { - if let Some(registry) = reg.read().await.as_ref() { - if let Some(entry) = registry.first_for_role("agent") { - r.register_agent(&aid, &entry.metric_name).await; - r.push_config_to_agent(&aid).await; - } - } - } - } - }); - }), - ); - - let replanner = Arc::new(Replanner::new( - Arc::clone(&planner), - Arc::clone(&plan_store), - Arc::clone(&workload_store), - Arc::clone(&opamp_srv), - Arc::clone(&scraper), - opamp_ep, - )); - - // Build a workload registry with one entry matching the pre-populated plan. - let registry = Arc::new(WorkloadRegistry::load("/nonexistent")); // empty - // We'll create one inline with the correct metric name. - let yaml = "- metric_name: http_latency\n accuracy_sla: 0.01\n assign_to_role: agent\n"; - let entries: Vec = - serde_yaml::from_str(yaml).unwrap(); - // WorkloadRegistry doesn't have a public constructor from entries, so we - // test via the first_for_role interface that the on_connect path uses. - // Bind the cells. - *replanner_cell.write().await = Some(Arc::clone(&replanner)); - // We need a registry that returns "http_latency". Load trick: - let tmp_path = "/tmp/datacollector_test_workloads.yaml"; - std::fs::write(tmp_path, yaml).unwrap(); - let registry = Arc::new(WorkloadRegistry::load(tmp_path)); - *registry_cell.write().await = Some(Arc::clone(®istry)); - - // Start OpAMP WS server. - let addr = start_opamp_server(Arc::clone(&opamp_srv)).await; - - // Connect a mock agent. - let mut ws = connect_agent(&addr, "test-agent-1", "agent").await; - - // The on_connect callback should assign the workload and push config. - let yaml_config = recv_config_yaml(&mut ws).await; - - // Verify the config has the expected sketch processor. - assert!( - yaml_config.contains("ddsketch") - || yaml_config.contains("KLL") - || yaml_config.contains("KLL:"), - "expected a sketch processor in the pushed config:\n{yaml_config}" - ); - // Verify OpAMP extension is present. - assert!( - yaml_config.contains("opamp"), - "pushed config should include opamp extension:\n{yaml_config}" - ); - - std::fs::remove_file(tmp_path).ok(); - } - - /// Test 2: Re-plan pushes config only to agents registered for that metric. - #[tokio::test] - async fn replan_pushes_only_to_registered_agent() { - let online_store = init_online_store(); - let plan_store = Arc::new(PlanStore::new()); - let workload_store = Arc::new(WorkloadStore::new()); - let opamp_srv = Arc::new(OpampServer::new()); - let scraper = Arc::new(Scraper::new( - vec![], Thresholds::default(), Arc::new(|_| {}), Duration::from_secs(60), - )); - let planner = Arc::new(BaselinePlanner::new( - CostModelPlanner::new().with_online_store(Arc::clone(&online_store)), - )); - - // Seed workload + plan for "metric_a". - let analyzer = Analyzer::new(); - let spec = analyzer::QuerySpec { - query_string: None, - metric_name: "metric_a".into(), - label_filters: Default::default(), - group_by_labels: vec![], - aggregations: vec!["quantile".into()], - time_window: "5m".into(), - repeat_every: None, - accuracy_sla: 0.01, - latency_sla: None, - sketch_type: None, - workload: types::WorkloadCharacteristics::default(), - id: None, - language: None, - accuracy: None, - dollars: None, - deployment_model: None, - shape: types_v2::QueryShape::default(), - data: types_v2::DataShape::default(), - }; - let wl = analyzer.analyze(spec).unwrap(); - let wc = types::WorkloadCharacteristics::default(); - let plan = planner.plan(&wl, Some(&wc)); - plan_store.set("metric_a", plan); - workload_store.set("metric_a", wl, wc); - - let replanner = Arc::new(Replanner::new( - Arc::clone(&planner), - Arc::clone(&plan_store), - Arc::clone(&workload_store), - Arc::clone(&opamp_srv), - Arc::clone(&scraper), - "ws://ctrl:4320/v1/opamp", - )); - - // Start OpAMP server and connect two agents. - let addr = start_opamp_server(Arc::clone(&opamp_srv)).await; - let mut ws_a = connect_agent(&addr, "agent-a", "agent").await; - let mut ws_b = connect_agent(&addr, "agent-b", "agent").await; - // Let connections register. - tokio::time::sleep(Duration::from_millis(100)).await; - - // Register agent-a for metric_a, agent-b is NOT registered for metric_a. - replanner.register_agent("agent-a", "metric_a").await; - replanner.register_agent("agent-b", "metric_b").await; - - // Trigger replan for metric_a. - let ok = replanner.replan_metric("metric_a").await; - assert!(ok, "replan should succeed"); - - // agent-a should receive a config push. - let yaml_a = recv_config_yaml(&mut ws_a).await; - assert!(!yaml_a.is_empty(), "agent-a should have received config"); - - // agent-b should NOT receive anything (timeout). - let result_b = tokio::time::timeout( - Duration::from_millis(500), - futures_util::StreamExt::next(&mut ws_b), - ).await; - assert!( - result_b.is_err(), - "agent-b should NOT receive config for metric_a replan" - ); - } - - /// Test 3: Generated agent YAML contains extensions.opamp with correct endpoint. - #[tokio::test] - async fn generated_agent_yaml_contains_opamp_extension() { - let endpoint = "ws://my-controller:4320/v1/opamp"; - let cfg = AgentCollectorConfig { - output_mode: types::OutputMode::Sketch, - sketch_type: types::SketchType::DDSketch, - sketch_params: types::SketchParams::default(), - aggregate_by: vec![], - label_matchers: vec![], - window_duration: Some(Duration::from_secs(60)), - mode: types::ProcessorMode::Window, - enable_self_monitoring: true, - transmit_sketch: true, - drop_original: true, - delta_transmission: false, - delta_threshold: 0.0, - enable_series_id: false, - series_id_ttl_secs: 300, - // This test asserts on `doc["exporters"]["prometheus"]` - // (line ~1326). Keep the test semantics by pinning the - // sink to the legacy prometheus exporter. - data_sink: types::AgentDataSink::PrometheusScrape { - endpoint: "0.0.0.0:8889".to_string(), - }, - }; - let yaml = generate_agent_config(&cfg, endpoint).unwrap(); - - // Parse the YAML to verify structure, not just substring matches. - let doc: serde_yaml::Value = serde_yaml::from_str(&yaml).unwrap(); - - // 1. extensions.opamp.server.ws.endpoint matches the parameter. - let opamp_ext = &doc["extensions"]["opamp"]; - assert!( - !opamp_ext.is_null(), - "YAML missing extensions.opamp:\n{yaml}" - ); - let ws_endpoint = opamp_ext["server"]["ws"]["endpoint"].as_str().unwrap(); - assert_eq!( - ws_endpoint, endpoint, - "OpAMP endpoint mismatch" - ); - - // 2. service.extensions list includes "opamp". - let svc_exts = doc["service"]["extensions"].as_sequence().unwrap(); - let has_opamp = svc_exts.iter().any(|v| v.as_str() == Some("opamp")); - assert!( - has_opamp, - "service.extensions should include 'opamp':\n{yaml}" - ); - - // 3. The YAML is complete: has receivers, processors, exporters, service.pipelines. - assert!(doc["receivers"]["otlp"].is_mapping(), "missing receivers.otlp"); - assert!(doc["exporters"]["prometheus"].is_mapping(), "missing exporters.prometheus"); - let pipeline = &doc["service"]["pipelines"]["metrics"]; - assert!(pipeline["receivers"].is_sequence(), "missing pipeline receivers"); - assert!(pipeline["processors"].is_sequence(), "missing pipeline processors"); - assert!(pipeline["exporters"].is_sequence(), "missing pipeline exporters"); - } - - #[tokio::test] - async fn tco_with_custom_pricing() { - let (_, app) = test_app(); - let body = serde_json::json!({ - "workload": { - "series_count": 50000, - "samples_per_sec": 1.0, - "bytes_per_sample": 100, - "scrape_interval_secs": 15, - "queries_per_sec": 1.0, - "query_window_secs": 300, - "retention_days": 30, - "sketch_compression_ratio": 0.05, - "delta_compression_ratio": 0.3 - }, - "pricing": { - "grafana_per_1k_series_1dpm": 8.0, - "s3_storage_per_gb_month": 0.023, - "s3_put_per_1k": 0.005, - "s3_get_per_1k": 0.0004, - "s3_transfer_per_gb": 0.09, - "ec2_sketch_instance_per_hour": 0.384 - } - }); - let req = Request::builder() - .method("POST").uri("/api/v1/tco") - .header("content-type", "application/json") - .body(Body::from(body.to_string())).unwrap(); - let resp = app.oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let body = body_json(resp).await; - // With higher Grafana pricing, before cost should be higher. - assert!(body["before"]["ingestion_dollars"].as_f64().unwrap() > 0.0); - assert!(body["monthly_savings_dollars"].as_f64().unwrap() > 0.0); - } - - // ── Phase ε.1.5+ — handle_bootstrap_agent_config typed path ──────────────── - // - // These tests verify the deep fix that ports the bootstrap handler - // off `generate_agent_config` and onto the typed-stage-split emit - // pipeline that `handle_plan` already uses. See the handler's - // doc-comment for the legacy ↔ typed behaviour matrix. - - /// Serialises tests that mutate the `USE_TYPED_STAGE_SPLIT` env var - /// — `cargo test` runs tests in parallel by default and - /// `typed_stage_split_enabled()` reads the env on every call. - static ENV_GUARD: std::sync::Mutex<()> = std::sync::Mutex::new(()); - - /// RAII helper: set `USE_TYPED_STAGE_SPLIT=` for the - /// lifetime of the returned guard, restoring the prior value - /// (or unsetting) on drop. Holds the test-wide ENV_GUARD mutex - /// so concurrent tests don't trample each other. - struct EnvVarGuard { - key: &'static str, - previous: Option, - // Hold the mutex so concurrent tests serialise on env-var writes. - _lock: std::sync::MutexGuard<'static, ()>, - } - impl EnvVarGuard { - fn set(key: &'static str, value: &str) -> Self { - let lock = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner()); - let previous = std::env::var(key).ok(); - std::env::set_var(key, value); - Self { key, previous, _lock: lock } - } - fn unset(key: &'static str) -> Self { - let lock = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner()); - let previous = std::env::var(key).ok(); - std::env::remove_var(key); - Self { key, previous, _lock: lock } - } - } - impl Drop for EnvVarGuard { - fn drop(&mut self) { - match &self.previous { - Some(v) => std::env::set_var(self.key, v), - None => std::env::remove_var(self.key), - } - } - } - - /// Build an `AppState` whose `workload_registry` + `workload_store` - /// + `plan_store` are pre-populated with one agent-role workload — - /// matches what `main()` does at startup. - /// - /// Returns the (state, router, registry-tempfile-path) triple. The - /// caller is responsible for cleaning up the tempfile. - fn test_app_with_workload(metric: &str, accuracy: f64) -> (AppState, axum::Router, String) { - // 1. Write a workload registry YAML to a tempfile so - // `WorkloadRegistry::load` produces a registry with the - // metric assigned to role=agent. - let yaml = format!( - "- metric_name: {metric}\n accuracy_sla: {accuracy}\n assign_to_role: agent\n", - ); - let tmp_path = format!("/tmp/datacollector_bootstrap_test_{metric}.yaml"); - std::fs::write(&tmp_path, yaml).unwrap(); - let registry = Arc::new(WorkloadRegistry::load(&tmp_path)); - - // 2. Build a stock test_app (empty registry + empty stores). - let (mut state, _router) = test_app(); - - // 3. Pre-populate workload_store + plan_store the same way - // main()'s startup loop does. - let analyzer = Analyzer::new(); - let spec = analyzer::QuerySpec { - query_string: None, - metric_name: metric.to_string(), - label_filters: Default::default(), - group_by_labels: vec![], - aggregations: vec!["quantile".into()], - time_window: "5m".into(), - repeat_every: None, - accuracy_sla: accuracy, - latency_sla: None, - sketch_type: None, - workload: types::WorkloadCharacteristics::default(), - id: None, - language: None, - accuracy: None, - dollars: None, - deployment_model: None, - shape: types_v2::QueryShape::default(), - data: types_v2::DataShape::default(), - }; - let wl = analyzer.analyze(spec).expect("analyze"); - let wc = types::WorkloadCharacteristics::default(); - let plan = state.planner.plan(&wl, Some(&wc)); - state.store.set(metric, plan); - state.workload_store.set(metric, wl, wc); - - // 4. Swap in the populated registry. - state.workload_registry = registry; - - // 5. Rebuild the router with the updated state. - let router = axum::Router::new() - .route("/api/v1/plan", axum::routing::post(handle_plan)) - .route("/api/v1/plan/pareto", axum::routing::post(handle_pareto)) - .route("/api/v1/plan/:metric", axum::routing::get(handle_get_plan)) - .route("/api/v1/plan/:metric/rollback", axum::routing::post(handle_rollback)) - .route("/api/v1/plan/:metric/diff", axum::routing::get(handle_plan_diff)) - .route("/api/v1/agents", axum::routing::get(handle_agents)) - .route("/api/v1/cost-model", axum::routing::get(handle_cost_model)) - .route("/api/v1/tco", axum::routing::post(handle_tco)) - .route("/api/v1/collector-config/agent", - axum::routing::get(handle_bootstrap_agent_config)) - .with_state(state.clone()); - (state, router, tmp_path) - } - - /// Backwards-compat — when `USE_TYPED_STAGE_SPLIT` is unset the - /// handler must keep its legacy `generate_agent_config` shape - /// (default DDSketch, `processors.ddsketch`, `processors.batch`) - /// so deployments that haven't migrated keep working. - #[tokio::test] - async fn bootstrap_legacy_path_when_env_unset() { - let _env = EnvVarGuard::unset(planner::stage_split::ENV_USE_TYPED_STAGE_SPLIT); - - let (_, app) = test_app(); - let req = Request::builder() - .uri("/api/v1/collector-config/agent") - .body(Body::empty()).unwrap(); - let resp = app.oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let body = resp.into_body().collect().await.unwrap().to_bytes(); - let yaml = String::from_utf8(body.to_vec()).unwrap(); - - // Legacy bootstrap fingerprint: a `ddsketch:` processor block. - assert!( - yaml.contains("ddsketch:"), - "legacy bootstrap should emit ddsketch processor; got:\n{yaml}" - ); - } - - /// `USE_TYPED_STAGE_SPLIT=1` + a workload routed through the typed - /// L5 emit → the YAML is the typed Edge config (ddsketch) - /// rather than the legacy default DDSketch shape. - #[tokio::test] - async fn bootstrap_typed_path_when_env_set() { - let _env = EnvVarGuard::set(planner::stage_split::ENV_USE_TYPED_STAGE_SPLIT, "1"); - - let (_, app, tmp) = test_app_with_workload("http_latency", 0.01); - let req = Request::builder() - .uri("/api/v1/collector-config/agent") - .body(Body::empty()).unwrap(); - let resp = app.oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let body = resp.into_body().collect().await.unwrap().to_bytes(); - let yaml = String::from_utf8(body.to_vec()).unwrap(); - std::fs::remove_file(&tmp).ok(); - - // Typed Edge fingerprint: valid patched collector component id. - assert!( - yaml.contains("ddsketch:"), - "typed bootstrap should emit `ddsketch:`:\n{yaml}" - ); - } - - /// `X-Agent-Runtime: asap-otap` → emitter dispatches through - /// `emit_otap_dag_yaml` rather than the OTel-collector emit. The - /// output shape is YAML-but-not-OTel — we identify it by the - /// otap-dataflow DAG version token. - #[tokio::test] - async fn bootstrap_typed_path_asap_otap_runtime_dispatch() { - let _env = EnvVarGuard::set(planner::stage_split::ENV_USE_TYPED_STAGE_SPLIT, "1"); - - let (_, app, tmp) = test_app_with_workload("rtt_otap", 0.01); - let req = Request::builder() - .uri("/api/v1/collector-config/agent") - .header("X-Agent-Runtime", "asap-otap") - .body(Body::empty()).unwrap(); - let resp = app.oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let body = resp.into_body().collect().await.unwrap().to_bytes(); - let yaml = String::from_utf8(body.to_vec()).unwrap(); - std::fs::remove_file(&tmp).ok(); - - // Mirrors the assertion in `config::runtime_tests::emit_for_runtime_otap_yields_dag_yaml`. - assert!( - yaml.contains("otel_dataflow/v1"), - "asap-otap runtime should produce the otap-dataflow DAG YAML:\n{yaml}" - ); - } - - /// `X-Agent-Runtime: asap-telegraf` → emitter dispatches through - /// `emit_telegraf_toml` and produces TOML rather than YAML. - #[tokio::test] - async fn bootstrap_typed_path_asap_telegraf_runtime_dispatch() { - let _env = EnvVarGuard::set(planner::stage_split::ENV_USE_TYPED_STAGE_SPLIT, "1"); - - let (_, app, tmp) = test_app_with_workload("rtt_tg", 0.01); - let req = Request::builder() - .uri("/api/v1/collector-config/agent") - .header("X-Agent-Runtime", "asap-telegraf") - .body(Body::empty()).unwrap(); - let resp = app.oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let body = resp.into_body().collect().await.unwrap().to_bytes(); - let toml = String::from_utf8(body.to_vec()).unwrap(); - std::fs::remove_file(&tmp).ok(); - - // Telegraf fingerprint — see `config::runtime_tests::emit_for_runtime_telegraf_yields_toml`. - assert!( - toml.contains("[[inputs.opentelemetry]]"), - "asap-telegraf runtime should produce Telegraf TOML:\n{toml}" - ); - } - - /// `USE_TYPED_STAGE_SPLIT=1` but the registry is empty → typed - /// path fails to resolve a workload and the handler falls back - /// to the legacy `generate_agent_config` emit. Bootstrap MUST - /// NOT 500 just because the typed path hit a gap. - #[tokio::test] - async fn bootstrap_typed_path_falls_back_to_legacy_when_no_workload() { - let _env = EnvVarGuard::set(planner::stage_split::ENV_USE_TYPED_STAGE_SPLIT, "1"); - - let (_, app) = test_app(); // empty registry + empty stores - let req = Request::builder() - .uri("/api/v1/collector-config/agent") - .body(Body::empty()).unwrap(); - let resp = app.oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let body = resp.into_body().collect().await.unwrap().to_bytes(); - let yaml = String::from_utf8(body.to_vec()).unwrap(); - - // Legacy fingerprint — bare `ddsketch:` processor block. - assert!( - yaml.contains("ddsketch:"), - "fallback path should emit legacy ddsketch processor:\n{yaml}" - ); - } - - // ── MVP §46: planner ↔ 5-sketch emitter stitch (PR #339 ↔ PR #340) ───────── - // - // The acceptance contract: register the six contract metrics in the - // workload registry, hit the bootstrap GET endpoint, and verify the - // emitted YAML carries the 5-sketch routing-connector wire shape — - // every sketched metric routed to its family-specific pipeline, - // raw `http_requests_total` falling through to - // `metrics/raw_passthrough`. - // - // Without the stitch wired in `emit_bootstrap_typed`, the - // EdgeStageConfig.metric_to_family HashMap stays empty and the - // emitter falls back to single-pipeline DDSketch — none of the - // assertions below pass. - - /// Build an AppState whose workload registry carries all six MVP §46 - /// contract metrics, each pre-populated in the workload store with - /// `aggregations=["quantile"]`. The planner classifies by metric - /// name (`classify_demo_metric` wins over `aggregations[0]`) so the - /// dummy aggregation is fine. - /// - /// Returns the (state, router, registry-tempfile-path) triple. The - /// caller cleans up the tempfile. - fn test_app_with_six_contract_metrics() -> (AppState, axum::Router, String) { - // The 6 contract metrics from MVP §46. - let metrics = [ - "http_requests_total", // raw passthrough (no sketch) - "http_latency_ms", // DDSketch - "request_size_bytes", // KLL - "unique_users_per_min", // HLL - "top_endpoint_qps", // CountSketch - "endpoint_request_freq", // CountMinSketch - ]; - - // 1. Materialise a workload-registry YAML covering all six. - let mut yaml = String::new(); - for m in metrics.iter() { - yaml.push_str(&format!( - "- metric_name: {m}\n accuracy_sla: 0.01\n assign_to_role: agent\n", - )); - } - let tmp_path = "/tmp/datacollector_mvp46_six_metrics.yaml".to_string(); - std::fs::write(&tmp_path, yaml).unwrap(); - let registry = Arc::new(WorkloadRegistry::load(&tmp_path)); - - // 2. Stock test_app with empty stores, then hand-populate. - let (mut state, _router) = test_app(); - - // 3. Pre-populate workload_store + plan_store the same way - // main()'s startup loop does. - let analyzer = Analyzer::new(); - for m in metrics.iter() { - let spec = analyzer::QuerySpec { - query_string: None, - metric_name: (*m).into(), - label_filters: Default::default(), - group_by_labels: vec![], - aggregations: vec!["quantile".into()], - time_window: "5m".into(), - repeat_every: None, - accuracy_sla: 0.01, - latency_sla: None, - sketch_type: None, - workload: types::WorkloadCharacteristics::default(), - id: None, - language: None, - accuracy: None, - dollars: None, - deployment_model: None, - shape: types_v2::QueryShape::default(), - data: types_v2::DataShape::default(), - }; - let wl = analyzer.analyze(spec).expect("analyze"); - let wc = types::WorkloadCharacteristics::default(); - let plan = state.planner.plan(&wl, Some(&wc)); - state.store.set(*m, plan); - state.workload_store.set(*m, wl, wc); - } - - // 4. Swap in the populated registry. - state.workload_registry = registry; - - // 5. Rebuild router with updated state. - let router = axum::Router::new() - .route( - "/api/v1/collector-config/agent", - axum::routing::get(handle_bootstrap_agent_config), - ) - .with_state(state.clone()); - (state, router, tmp_path) - } - - /// Acceptance test: PR #339 (planner) ↔ PR #340 (emitter) stitch - /// produces the 5-sketch routing-connector wire shape when the - /// workload registry covers the six MVP §46 contract metrics. - /// - /// Asserts: - /// - All 5 sketch processors loaded under `processors:`. - /// - `routing` lives in `connectors:` (NOT `processors:`). - /// - All 6 named pipelines emitted (raw_passthrough + 5 sketches). - /// - Each metric routed to its expected pipeline via - /// `name == "..."`. - #[tokio::test] - async fn bootstrap_emits_5sketch_routing_for_six_contract_metrics() { - let _env = EnvVarGuard::set(planner::stage_split::ENV_USE_TYPED_STAGE_SPLIT, "1"); - - let (_, app, tmp) = test_app_with_six_contract_metrics(); - let req = Request::builder() - .uri("/api/v1/collector-config/agent") - .body(Body::empty()) - .unwrap(); - let resp = app.oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let body = resp.into_body().collect().await.unwrap().to_bytes(); - let yaml = String::from_utf8(body.to_vec()).unwrap(); - std::fs::remove_file(&tmp).ok(); - - // ── Contract 1: all 5 sketch processors loaded ──────────────────── - for proc in [ - "ddsketch:", - "KLL:", - "HLL:", - "countsketch:", - "countmin:", - ] { - assert!( - yaml.contains(proc), - "missing top-level sketch processor `{proc}`\n{yaml}" - ); - } - - // ── Contract 2: routing in connectors, not processors ───────────── - let connectors_idx = yaml - .find("connectors:") - .expect("missing top-level connectors block"); - let after_conn = &yaml[connectors_idx..]; - assert!( - after_conn.contains("routing:"), - "missing `routing:` under connectors:\n{yaml}" - ); - // Negative: routing is NOT under processors. - let processors_idx = yaml.find("processors:").expect("processors:"); - let proc_end = yaml[processors_idx..] - .find("\nconnectors:") - .or_else(|| yaml[processors_idx..].find("\nexporters:")) - .map(|x| processors_idx + x) - .unwrap_or(yaml.len()); - let processors_section = &yaml[processors_idx..proc_end]; - assert!( - !processors_section.contains("routing:"), - "routing must NOT live under processors: (the v0.106 bug)\n\ - processors_section:\n{processors_section}" - ); - - // ── Contract 3: all 6 named pipelines ───────────────────────────── - for pl in [ - "metrics:", // entry - "metrics/raw_passthrough:", // default for http_requests_total - "metrics/ddsketch_path:", // http_latency_ms - "metrics/kll_path:", // request_size_bytes - "metrics/hll_path:", // unique_users_per_min - "metrics/countsketch_path:", // top_endpoint_qps - "metrics/countminsketch_path:", // endpoint_request_freq - ] { - assert!( - yaml.contains(pl), - "missing pipeline `{pl}`\n{yaml}" - ); - } - - // ── Contract 4: each sketched metric carries an OTTL condition ── - // The 5 sketched metrics must each have a `name == "..."` - // rule in the routing connector. - // `http_requests_total` (raw) does NOT need a rule — it falls - // through to the default `metrics/raw_passthrough` pipeline. - for sketched in [ - "http_latency_ms", - "request_size_bytes", - "unique_users_per_min", - "top_endpoint_qps", - "endpoint_request_freq", - ] { - let needle = format!("name == \\\"{sketched}\\\""); - let alt1 = format!("name == \"{sketched}\""); - let alt2 = format!("name=='{sketched}'"); - assert!( - yaml.contains(&needle) || yaml.contains(&alt1) || yaml.contains(&alt2), - "missing routing rule for `{sketched}` — expected `name == \"{sketched}\"`\n{yaml}" - ); - } - } - - // ── Stitching-gap regression: live mvp-workload.yaml binds all 5 sketches ── - // - // Reproduces the live demo gap (3 of 6 contract metrics silently dropped - // because `WorkloadEntry` didn't carry `sketch_family_override` and - // `bind_workload_typed` early-returned on `exact_required` set by the - // bare-VectorSelector → Sum path that PromQL parsing applies inside - // `count(metric)` / `topk(K, metric)` / `rate(metric[5m])`). - // - // Loads workload entries shaped exactly like the deployed - // `deploy/configs/mvp-workload.yaml` MVP §46 rows (entries 5–8), pre-pops - // the workload store via the same code main() runs, and asserts the - // routing table emitted by the bootstrap GET endpoint covers all five - // sketched metrics. - fn test_app_with_live_mvp_workload_metrics() -> (AppState, axum::Router, String) { - // Mirror the YAML shape of `deploy/configs/mvp-workload.yaml` MVP §46 - // entries — these are the exact strings that crashed in the live demo. - let yaml = r#" -- metric_name: http_requests_total_latency_ms - query_string: "quantile_over_time(0.99, http_requests_total_latency_ms[1m])" - accuracy_sla: 0.01 - assign_to_role: agent -- metric_name: http_requests_total - query_string: "count(http_requests_total{service=\"payments\"})" - accuracy_sla: 0.0 - assign_to_role: agent -- metric_name: request_size_bytes - query_string: "quantile_over_time(0.99, request_size_bytes[1m])" - accuracy_sla: 0.05 - assign_to_role: agent - sketch_family_override: KLL -- metric_name: unique_users_per_min - query_string: "count(unique_users_per_min)" - accuracy_sla: 0.02 - assign_to_role: agent - sketch_family_override: HLL -- metric_name: top_endpoint_qps - query_string: "topk(5, top_endpoint_qps)" - accuracy_sla: 0.05 - assign_to_role: agent - sketch_family_override: CountSketch -- metric_name: endpoint_request_freq - query_string: "rate(endpoint_request_freq[5m])" - accuracy_sla: 0.05 - assign_to_role: agent - sketch_family_override: CountMinSketch -"#; - let tmp_path = "/tmp/datacollector_live_mvp46_workload.yaml".to_string(); - std::fs::write(&tmp_path, yaml).unwrap(); - let registry = Arc::new(WorkloadRegistry::load(&tmp_path)); - - let (mut state, _router) = test_app(); - - let analyzer = Analyzer::new(); - for entry in registry.entries() { - let spec = analyzer::QuerySpec { - query_string: entry.query_string.clone(), - metric_name: entry.metric_name.clone(), - label_filters: Default::default(), - group_by_labels: vec![], - aggregations: vec!["quantile".into()], - time_window: "5m".into(), - repeat_every: None, - accuracy_sla: entry.accuracy_sla, - latency_sla: None, - sketch_type: entry.sketch_family_override.clone(), - workload: types::WorkloadCharacteristics::default(), - id: None, - language: None, - accuracy: None, - dollars: None, - deployment_model: None, - shape: types_v2::QueryShape::default(), - data: types_v2::DataShape::default(), - }; - if let Ok(wl) = analyzer.analyze(spec) { - let wc = types::WorkloadCharacteristics::default(); - let plan = state.planner.plan(&wl, Some(&wc)); - let metric_name = wl.metric_name.clone(); - state.store.set(&metric_name, plan); - state.workload_store.set(&metric_name, wl, wc); - } - } - - state.workload_registry = registry; - - let router = axum::Router::new() - .route( - "/api/v1/collector-config/agent", - axum::routing::get(handle_bootstrap_agent_config), - ) - .with_state(state.clone()); - (state, router, tmp_path) - } - - /// Pinning regression: the routing table emitted by the bootstrap - /// endpoint must cover all 5 sketched contract metrics — DDSketch - /// (`http_requests_total_latency_ms`), KLL (`request_size_bytes`), - /// HLL (`unique_users_per_min`), CountSketch (`top_endpoint_qps`), - /// CountMinSketch (`endpoint_request_freq`). - /// - /// Without the fix, this test fails with only 2 sketched routes - /// (DDSketch + KLL); HLL / CountSketch / CountMinSketch silently drop. - #[tokio::test] - async fn bootstrap_routing_table_covers_all_five_sketches_for_live_mvp_yaml() { - let _env = EnvVarGuard::set(planner::stage_split::ENV_USE_TYPED_STAGE_SPLIT, "1"); - - let (state, app, tmp) = test_app_with_live_mvp_workload_metrics(); - - // ── Direct check: collect_metric_to_family produces 5 entries ──── - let map = config::collect_metric_to_family( - &state.workload_registry, - &state.workload_store, - ); - assert_eq!( - map.len(), 5, - "metric_to_family should have 5 sketched entries (raw declines), got {map:?}", - ); - for (metric, want_family) in &[ - ("http_requests_total_latency_ms", "DDSketch"), - ("request_size_bytes", "Kll"), - ("unique_users_per_min", "Hll"), - ("top_endpoint_qps", "CountSketch"), - ("endpoint_request_freq", "Cms"), - ] { - let got = map.get(*metric) - .map(|k| format!("{k:?}")) - .unwrap_or_else(|| "MISSING".into()); - assert_eq!( - got, *want_family, - "metric_to_family[{metric}] expected {want_family}, got {got}\nmap: {map:?}", - ); - } - - // ── End-to-end check: routing rules in emitted YAML ─────────────── - let req = Request::builder() - .uri("/api/v1/collector-config/agent") - .body(Body::empty()) - .unwrap(); - let resp = app.oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let body = resp.into_body().collect().await.unwrap().to_bytes(); - let yaml = String::from_utf8(body.to_vec()).unwrap(); - std::fs::remove_file(&tmp).ok(); - - for sketched in [ - "http_requests_total_latency_ms", - "request_size_bytes", - "unique_users_per_min", - "top_endpoint_qps", - "endpoint_request_freq", - ] { - let needle = format!("name == \\\"{sketched}\\\""); - let alt1 = format!("name == \"{sketched}\""); - let alt2 = format!("name=='{sketched}'"); - assert!( - yaml.contains(&needle) || yaml.contains(&alt1) || yaml.contains(&alt2), - "missing routing rule for `{sketched}`\n{yaml}" - ); - } - } - - // ── Regression: archive tier covers all 5 sketched metrics ──────────────── - // - // The backend's `POST /api/v1/storage_routing` handler is an atomic - // per-tenant SWAP — every push replaces the whole tenant's routing - // table. Pre-fix, `handle_plan` posted a single-element - // `metrics:[…]` document per call, so when the demo POSTed - // `/api/v1/plan` for each of the 5 sketched contract metrics in - // sequence, only the LAST metric's entry survived in the backend. - // The other 4 metrics defaulted to `sketch_warm_tier` (which has - // no warm-tier sketch state for archive-shape queries) → the - // demo's accuracy reducer logged `archive_miss` for those metrics - // even though gorillas3 wrote their TSDB blocks to MinIO and - // Thanos had them indexed. - // - // The fix wires `state.backend_routing_cache` so each - // `handle_plan` cycle posts the **cumulative** routing table. - // This regression test replays the demo's per-metric POST sequence - // against a mock backend, captures every body, and asserts the - // final swap covers all 5 sketched metrics simultaneously. - #[tokio::test] - async fn storage_routing_cumulative_push_covers_all_5_sketched_metrics() { - // Activate the typed-stage-split path (the only path that - // emits storage-routing JSON; the legacy path no-ops). - let _env = EnvVarGuard::set(planner::stage_split::ENV_USE_TYPED_STAGE_SPLIT, "1"); - - // Mock backend that captures every storage-routing body. - // We re-use the mock pattern from `backend_client::tests` — - // an axum router that drains the request body into a shared - // sink. Mounted at the canonical `/api/v1/storage_routing` - // path so `BackendClient`'s URL-rewrite hits it directly. - type SinkInner = std::sync::Mutex>; - let sink: Arc = Arc::new(std::sync::Mutex::new(Vec::new())); - let sink_capture = Arc::clone(&sink); - let mock_app = axum::Router::new() - .route( - "/api/v1/storage_routing", - axum::routing::post(move |body: axum::body::Bytes| { - let sink = Arc::clone(&sink_capture); - async move { - let s = String::from_utf8_lossy(&body).to_string(); - sink.lock().unwrap().push(s); - axum::http::StatusCode::OK - } - }), - ); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - tokio::spawn(async move { - axum::serve(listener, mock_app).await.unwrap(); - }); - // Brief settle so the bind is observable before the first POST. - tokio::time::sleep(Duration::from_millis(50)).await; - - // Build an AppState with the backend pointed at the mock URL. - // Use the streaming-config alias so `BackendClient` derives - // the matching `/api/v1/storage_routing` URL. - let backend_url = format!("http://{addr}/api/v1/streaming-config"); - let (state, _) = test_app_with_backend(Some(backend_url)); - - // Mount only `/api/v1/plan` — that's the path the demo - // exercises; we don't need bootstrap or other routes. - let app = axum::Router::new() - .route("/api/v1/plan", axum::routing::post(handle_plan)) - .with_state(state.clone()); - - // The 5 sketched contract metrics from MVP §46. Each gets a - // separate POST /api/v1/plan, mirroring the demo's - // per-workload plan-emit cycle. - let sketched = [ - "http_requests_total_latency_ms", // DDSketch - "request_size_bytes", // KLL - "unique_users_per_min", // HLL - "top_endpoint_qps", // CountSketch - "endpoint_request_freq", // CountMinSketch - ]; - - for m in &sketched { - let app = app.clone(); - let req = Request::builder() - .method("POST") - .uri("/api/v1/plan") - .header("content-type", "application/json") - .body(Body::from(plan_spec(m).to_string())) - .unwrap(); - let resp = app.oneshot(req).await.unwrap(); - assert_eq!( - resp.status(), - StatusCode::OK, - "POST /api/v1/plan for `{m}` must return 200", - ); - } - - // Drain the mock sink: every plan-emit must have produced - // exactly one body (5 plans → 5 bodies). - let bodies = sink.lock().unwrap().clone(); - assert_eq!( - bodies.len(), - sketched.len(), - "expected one storage-routing POST per plan; got {} bodies", - bodies.len(), - ); - - // The LAST captured body is the one the backend will leave - // installed (the swap is destructive — last write wins). It - // MUST list ALL 5 sketched metrics, otherwise the swap would - // erase the routing entries for the metrics planned earlier - // in the sequence and the backend would default them to - // `sketch_warm_tier` → archive_miss for those metrics' archive - // queries even though gorillas3's TSDB blocks are present in - // MinIO and Thanos has them indexed. - let last: serde_json::Value = - serde_json::from_str(bodies.last().unwrap()).expect("last body is valid JSON"); - let metric_names: std::collections::BTreeSet = last["metrics"] - .as_array() - .expect("metrics array") - .iter() - .map(|m| m["name"].as_str().unwrap().to_string()) - .collect(); - for m in &sketched { - assert!( - metric_names.contains(*m), - "final cumulative storage-routing table missing metric `{m}`; \ - contains only {metric_names:?}\nfull body: {}", - bodies.last().unwrap(), - ); - } - - // Each metric entry must carry a `thanos_archive` target — the - // archive-tier dispatch that lets backend forward archive-shape - // queries to Thanos. Without this target the metric falls back - // to `default_engine: sketch_warm_tier` and the archive miss - // reproduces. - for m in last["metrics"].as_array().unwrap() { - let targets = m["targets"].as_array().expect("targets array"); - let engines: Vec<&str> = targets - .iter() - .map(|t| t["engine"].as_str().unwrap()) - .collect(); - assert!( - engines.contains(&"thanos_archive"), - "metric `{}` missing `thanos_archive` target; engines={engines:?}", - m["name"].as_str().unwrap(), - ); - } - } -} diff --git a/controller/src/metrics_exposer.rs b/controller/src/metrics_exposer.rs deleted file mode 100644 index a825a231..00000000 --- a/controller/src/metrics_exposer.rs +++ /dev/null @@ -1,511 +0,0 @@ -//! `GET /metrics` — the controller as a Prometheus exporter. -//! -//! The controller already holds per-agent runtime samples in -//! [`RuntimeSamplesStore`](crate::runtime_samples::RuntimeSamplesStore) -//! (fed by the agents' `sketch-runtime` push pipeline). This -//! module renders that store's state on demand in Prometheus -//! exposition format so a bog-standard Prom `scrape_configs` -//! entry pointed at `controller:8080/metrics` picks up every -//! paper §6 signal without any extra agent-side plumbing. -//! -//! ## Rendering contract -//! -//! For each `(source, sketch, impl)` key in the store we emit -//! the **latest** sample's headline gauges. That's what a -//! scrape is — point-in-time, not history. Prom does retention; -//! we do fan-out-to-labels. -//! -//! Controller-wide counters (batches received, decode errors, -//! evictions) come from `RuntimeSamplesStats` and are pure -//! `Counter`s. -//! -//! ## What lives in the registry -//! -//! All gauges are labelled `{source, sketch, impl}`: -//! -//! | Metric | Type | Source field (in `bench`) | -//! |---|---|---| -//! | `asap_runtime_throughput_items_per_sec` | Gauge | `throughput_items_per_sec.mean` | -//! | `asap_runtime_latency_p50_ns` | Gauge | `latency_ns.p50` | -//! | `asap_runtime_latency_p99_ns` | Gauge | `latency_ns.p99` | -//! | `asap_runtime_memory_bytes` | Gauge | `memory_bytes` | -//! | `asap_runtime_last_seen_unix_seconds` | Gauge | record `timestamp` (freshness signal) | -//! -//! Controller-wide (no labels): -//! -//! | Metric | Type | -//! |---|---| -//! | `asap_runtime_samples_batches_received_total` | Counter | -//! | `asap_runtime_samples_records_stored_total` | Counter | -//! | `asap_runtime_samples_records_evicted_total` | Counter | -//! | `asap_runtime_samples_decode_errors_total` | Counter | - -use std::collections::hash_map::DefaultHasher; -use std::hash::{Hash, Hasher}; -use std::sync::atomic::Ordering; -use std::sync::Arc; - -use axum::extract::State; -use axum::http::{header, StatusCode}; -use axum::response::{IntoResponse, Response}; -use prometheus::{Encoder, GaugeVec, IntCounter, Opts, Registry, TextEncoder}; - -use crate::runtime_samples::RuntimeSamplesStats; -use crate::runtime_samples::RuntimeSamplesStore; -use crate::store::PlanStore; - -const LABELS: &[&str] = &["source", "sketch", "impl"]; -const PLAN_LABELS: &[&str] = &["metric", "plan_id"]; - -/// The Prometheus registry + pre-built metric handles. Built -/// once at startup; the `/metrics` handler pulls the latest -/// sample per key out of the store on each scrape and updates -/// the gauge vecs before encoding. -pub struct MetricsRegistry { - registry: Registry, - throughput: GaugeVec, - latency_p50: GaugeVec, - latency_p99: GaugeVec, - memory_bytes: GaugeVec, - last_seen_unix: GaugeVec, - // controller-wide counters, refreshed from atomic stats - batches_received: IntCounter, - records_stored: IntCounter, - records_evicted: IntCounter, - decode_errors: IntCounter, - /// `asap_active_plan_id{metric="...", plan_id=""} 1` — - /// rendered at scrape time from the `PlanStore` snapshot. The - /// plan_id label is a stable hash of the current plan's content, - /// so a re-plan flips the label value (and the previous time-series - /// stops being emitted on the next scrape). The replay client and - /// `plan_transition.py` look for this metric to detect plan - /// transitions. - active_plan_id: GaugeVec, -} - -impl MetricsRegistry { - pub fn new() -> Arc { - let registry = Registry::new(); - - let throughput = GaugeVec::new( - Opts::new( - "asap_runtime_throughput_items_per_sec", - "Latest runtime throughput reported by each (source, sketch, impl)", - ), - LABELS, - ) - .expect("valid gauge vec"); - let latency_p50 = GaugeVec::new( - Opts::new( - "asap_runtime_latency_p50_ns", - "Latest p50 latency ns reported by each (source, sketch, impl)", - ), - LABELS, - ) - .unwrap(); - let latency_p99 = GaugeVec::new( - Opts::new( - "asap_runtime_latency_p99_ns", - "Latest p99 latency ns reported by each (source, sketch, impl)", - ), - LABELS, - ) - .unwrap(); - let memory_bytes = GaugeVec::new( - Opts::new( - "asap_runtime_memory_bytes", - "Latest memory footprint of each (source, sketch, impl)", - ), - LABELS, - ) - .unwrap(); - let last_seen_unix = GaugeVec::new( - Opts::new( - "asap_runtime_last_seen_unix_seconds", - "Unix timestamp of the latest sample received from each (source, sketch, impl). \ - Compare against `time() - N` to alert on a stale agent.", - ), - LABELS, - ) - .unwrap(); - - let batches_received = IntCounter::new( - "asap_runtime_samples_batches_received_total", - "Count of push batches accepted by /api/v1/runtime-samples", - ) - .unwrap(); - let records_stored = IntCounter::new( - "asap_runtime_samples_records_stored_total", - "Count of individual records stored in the ring buffer", - ) - .unwrap(); - let records_evicted = IntCounter::new( - "asap_runtime_samples_records_evicted_total", - "Count of records evicted by the per-key ring-buffer FIFO", - ) - .unwrap(); - let decode_errors = IntCounter::new( - "asap_runtime_samples_decode_errors_total", - "Count of decode / parse failures on incoming batches", - ) - .unwrap(); - - let active_plan_id = GaugeVec::new( - Opts::new( - "asap_active_plan_id", - "Currently-published plan id per metric. The plan_id label is a stable \ - hash of the plan's content; a re-plan changes the label value.", - ), - PLAN_LABELS, - ) - .unwrap(); - - registry.register(Box::new(active_plan_id.clone())).unwrap(); - registry.register(Box::new(throughput.clone())).unwrap(); - registry.register(Box::new(latency_p50.clone())).unwrap(); - registry.register(Box::new(latency_p99.clone())).unwrap(); - registry.register(Box::new(memory_bytes.clone())).unwrap(); - registry.register(Box::new(last_seen_unix.clone())).unwrap(); - registry.register(Box::new(batches_received.clone())).unwrap(); - registry.register(Box::new(records_stored.clone())).unwrap(); - registry.register(Box::new(records_evicted.clone())).unwrap(); - registry.register(Box::new(decode_errors.clone())).unwrap(); - - Arc::new(Self { - registry, - throughput, - latency_p50, - latency_p99, - memory_bytes, - last_seen_unix, - batches_received, - records_stored, - records_evicted, - decode_errors, - active_plan_id, - }) - } - - /// Refresh the `asap_active_plan_id` gauge at scrape time. - /// Resets prior label sets so a re-plan stops emitting the old - /// (metric, plan_id) pair on the next scrape. - fn refresh_plan_ids(&self, plan_store: &PlanStore) { - // Reset is necessary because GaugeVec keeps every label set - // ever observed; without this a re-plan would leave the old - // plan_id label permanently emitting a stale value. - self.active_plan_id.reset(); - for metric in plan_store.metrics() { - let Ok(plan) = plan_store.get(&metric) else { continue }; - // Stable hash of the plan's debug repr — good enough for - // a label value, doesn't need to be cryptographic. - let mut hasher = DefaultHasher::new(); - // Cover the fields the planner actually changes per - // re-plan: agent sketch+mode+delta, backend merge+group_by, - // and valid_until (to catch refresh-only re-plans). - format!( - "{:?}|{:?}|{:?}|{:?}|{:?}|{:?}|{:?}", - plan.agent_config.sketch_type, - plan.agent_config.mode, - plan.agent_config.delta_transmission, - plan.agent_config.window_duration, - plan.backend_config.merge_sketch_type, - plan.backend_config.group_by, - plan.valid_until, - ) - .hash(&mut hasher); - let plan_id = format!("p{:016x}", hasher.finish()); - self.active_plan_id - .with_label_values(&[metric.as_str(), plan_id.as_str()]) - .set(1.0); - } - } - - /// Walk the store and push the latest sample per key into - /// the gauge vecs. Called at scrape time, not per-record — - /// cost scales with (# keys), not (# records). - fn refresh_gauges(&self, store: &RuntimeSamplesStore) { - for key in store.keys() { - let Some(rec) = store.latest(&key) else { - continue; - }; - let labels = &[ - key.source.as_str(), - key.sketch.as_str(), - key.impl_name.as_str(), - ]; - - // Pull the interesting numeric fields from the - // opaque JSON payload. Missing fields silently stay - // at their last value — an agent that dropped a - // metric briefly won't zap the gauge to 0. - let bench = rec.payload.get("bench"); - if let Some(v) = bench - .and_then(|b| b.get("throughput_items_per_sec")) - .and_then(|t| t.get("mean")) - .and_then(|m| m.as_f64()) - { - self.throughput.with_label_values(labels).set(v); - } - if let Some(v) = bench - .and_then(|b| b.get("latency_ns")) - .and_then(|l| l.get("p50")) - .and_then(|m| m.as_f64()) - { - self.latency_p50.with_label_values(labels).set(v); - } - if let Some(v) = bench - .and_then(|b| b.get("latency_ns")) - .and_then(|l| l.get("p99")) - .and_then(|m| m.as_f64()) - { - self.latency_p99.with_label_values(labels).set(v); - } - if let Some(v) = bench - .and_then(|b| b.get("memory_bytes")) - .and_then(|m| m.as_f64()) - { - self.memory_bytes.with_label_values(labels).set(v); - } - if let Some(ts) = rec.payload.get("timestamp").and_then(|t| t.as_str()) { - if let Ok(parsed) = chrono::DateTime::parse_from_rfc3339(ts) { - self.last_seen_unix - .with_label_values(labels) - .set(parsed.timestamp() as f64); - } - } - } - } - - /// Sync controller-wide counters from the store's atomic - /// stats. `IntCounter` only exposes `.inc()` — so we pump - /// in the delta since the last scrape rather than `.set()`. - fn refresh_counters(&self, stats: &RuntimeSamplesStats) { - fn delta(target: &IntCounter, snapshot: u64) { - let have = target.get(); - if snapshot > have { - target.inc_by(snapshot - have); - } - } - delta( - &self.batches_received, - stats.batches_received.load(Ordering::Relaxed), - ); - delta( - &self.records_stored, - stats.records_stored.load(Ordering::Relaxed), - ); - delta( - &self.records_evicted, - stats.records_evicted.load(Ordering::Relaxed), - ); - delta( - &self.decode_errors, - stats.decode_errors.load(Ordering::Relaxed), - ); - } -} - -#[derive(Clone)] -pub struct MetricsState { - pub registry: Arc, - pub store: Arc, - pub stats: Arc, - /// Optional `PlanStore` reference; when present the exposer - /// renders `asap_active_plan_id` per metric. `None` is fine for - /// unit tests that exercise only the runtime-samples path. - pub plan_store: Option>, -} - -pub async fn handle_metrics(State(state): State) -> Response { - state.registry.refresh_gauges(&state.store); - state.registry.refresh_counters(&state.stats); - if let Some(ps) = state.plan_store.as_ref() { - state.registry.refresh_plan_ids(ps); - } - - let metric_families = state.registry.registry.gather(); - let encoder = TextEncoder::new(); - let mut buf = Vec::with_capacity(1024); - if let Err(e) = encoder.encode(&metric_families, &mut buf) { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("prom encode failed: {e}"), - ) - .into_response(); - } - ( - StatusCode::OK, - [(header::CONTENT_TYPE, encoder.format_type())], - buf, - ) - .into_response() -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::runtime_samples::RuntimeRecord; - use serde_json::json; - - fn seeded_store() -> (Arc, Arc) { - let store = RuntimeSamplesStore::new(16); - let stats = store.stats(); - for (source, sketch, impl_name, tp, p99, mem) in [ - ("dc-a", "cms", "oxide", 4.2e7, 120u64, 40960u64), - ("dc-a", "hll", "lib", 7.4e7, 21, 16384), - ] { - // Build a record whose payload mirrors the v1 bench - // shape — exactly what the store would have stored - // after receiving a PushExporter batch. - let payload = json!({ - "schema_version": 1, - "mode": "runtime", - "timestamp": "2026-04-21T19:00:00Z", - "bench": { - "throughput_items_per_sec": { "mean": tp, "stddev": 0.0 }, - "latency_ns": { "p50": 10, "p99": p99 }, - "memory_bytes": mem, - } - }); - let rec = RuntimeRecord { - source: source.into(), - sketch: sketch.into(), - impl_name: impl_name.into(), - schema_version: 1, - payload, - }; - // Inject via the public append-by-record shim we - // use in tests: the store's `append` is crate-public - // from `runtime_samples::tests` only, so go through - // the handler-side entry point with a JSON body in - // a real integration; here we lean on the trait. - // Shortcut: use store's pub `stats` + manual push - // by serialising + using the handler path. Simpler - // for this unit test: add a test-only helper below. - insert_for_test(&store, rec); - } - (store, stats) - } - - // Test-only shim — uses the `append_for_test` hook the - // `runtime_samples` module exposes under `cfg(test)`. - fn insert_for_test(store: &RuntimeSamplesStore, rec: RuntimeRecord) { - store.append_for_test(rec); - } - - #[tokio::test] - async fn metrics_scrape_reflects_latest_per_key_sample() { - let (store, stats) = seeded_store(); - let registry = MetricsRegistry::new(); - let metric_state = MetricsState { - registry: Arc::clone(®istry), - store: Arc::clone(&store), - stats: Arc::clone(&stats), - plan_store: None, - }; - let resp = handle_metrics(State(metric_state)).await; - let status = resp.status(); - assert_eq!(status, StatusCode::OK); - let body = axum::body::to_bytes(resp.into_body(), 64 * 1024) - .await - .unwrap(); - let text = String::from_utf8(body.to_vec()).unwrap(); - // Sanity: exposition format + at least one of our - // gauges should be present per key. - assert!(text.contains("# TYPE asap_runtime_throughput_items_per_sec gauge")); - assert!(text.contains( - "asap_runtime_throughput_items_per_sec{impl=\"oxide\",sketch=\"cms\",source=\"dc-a\"}" - )); - assert!(text.contains( - "asap_runtime_latency_p99_ns{impl=\"lib\",sketch=\"hll\",source=\"dc-a\"}" - )); - } - - #[test] - fn plan_id_emitted_per_metric_and_changes_on_replan() { - use crate::types::*; - use chrono::Utc; - - fn make_plan(sketch: SketchType, valid_secs: i64) -> CollectionPlan { - CollectionPlan { - agent_config: AgentCollectorConfig { - output_mode: OutputMode::Sketch, - sketch_type: sketch.clone(), - sketch_params: Default::default(), - aggregate_by: vec![], - label_matchers: vec![], - window_duration: None, - mode: ProcessorMode::Window, - enable_self_monitoring: true, - transmit_sketch: true, - drop_original: true, - delta_transmission: true, - delta_threshold: 0.0, - enable_series_id: false, - series_id_ttl_secs: 300, - data_sink: AgentDataSink::default(), - }, - gateway_config: GatewayCollectorConfig { passthrough: true }, - backend_config: BackendCollectorConfig { - merge_sketch_type: sketch, - group_by: vec![], - }, - precompute: vec![], - valid_until: Utc::now() + chrono::Duration::seconds(valid_secs), - delta_decision: Default::default(), - transmission_cost_summary: Default::default(), - staged_plan: None, - } - } - - let plan_store = Arc::new(PlanStore::new()); - plan_store.set("http_requests_total", make_plan(SketchType::DDSketch, 600)); - plan_store.set("http_requests_total_latency_ms", make_plan(SketchType::HLL, 600)); - - let registry = MetricsRegistry::new(); - registry.refresh_plan_ids(&plan_store); - // Render and check exposition contains both metrics with - // distinct plan_id labels. - let mfs = registry.registry.gather(); - let encoder = TextEncoder::new(); - let mut buf = Vec::new(); - encoder.encode(&mfs, &mut buf).unwrap(); - let text = String::from_utf8(buf).unwrap(); - assert!( - text.contains("asap_active_plan_id{metric=\"http_requests_total\""), - "expected plan_id for http_requests_total in:\n{text}" - ); - assert!( - text.contains("asap_active_plan_id{metric=\"http_requests_total_latency_ms\""), - "expected plan_id for http_requests_total_latency_ms in:\n{text}" - ); - // Re-plan with a different sketch must change the plan_id label. - let before = text.clone(); - plan_store.set("http_requests_total", make_plan(SketchType::KLL, 600)); - registry.refresh_plan_ids(&plan_store); - let mfs = registry.registry.gather(); - let mut buf = Vec::new(); - encoder.encode(&mfs, &mut buf).unwrap(); - let after = String::from_utf8(buf).unwrap(); - assert_ne!(before, after, "plan_id label should change after re-plan"); - } - - #[test] - fn counters_are_monotonic_across_refreshes() { - let store = RuntimeSamplesStore::new(16); - let stats = store.stats(); - stats.batches_received.fetch_add(5, Ordering::Relaxed); - let registry = MetricsRegistry::new(); - registry.refresh_counters(&stats); - assert_eq!(registry.batches_received.get(), 5); - // Second refresh adds only the delta. - stats.batches_received.fetch_add(3, Ordering::Relaxed); - registry.refresh_counters(&stats); - assert_eq!(registry.batches_received.get(), 8); - // No regression: if the snapshot somehow went down - // (shouldn't, but guard), we hold the counter flat - // rather than decrementing. - registry.refresh_counters(&stats); - assert_eq!(registry.batches_received.get(), 8); - } -} diff --git a/controller/src/monitor/mod.rs b/controller/src/monitor/mod.rs deleted file mode 100644 index 29defa37..00000000 --- a/controller/src/monitor/mod.rs +++ /dev/null @@ -1,486 +0,0 @@ -/// Feedback loop: scrapes Prometheus /metrics from OTel collectors and fires -/// violation callbacks to trigger re-planning, and an optional metrics callback -/// to feed observed bandwidth/CPU data into the EMA cost model (SP-5/SP-8). -use std::collections::HashMap; -use std::sync::{Arc, Mutex}; -use std::time::Duration; - -use anyhow::Context; -use tokio::sync::RwLock; -use tracing::{info, warn}; - -use crate::types::SketchType; - -// ── Types ───────────────────────────────────────────────────────────────────── - -#[derive(Debug, Clone)] -pub struct CollectorMetrics { - pub agent_id: String, - pub sketch_size_bytes: f64, - pub cpu_seconds_total: f64, - pub samples_ingested: f64, - pub error_rate: f64, -} - -#[derive(Debug, Clone, PartialEq)] -pub enum ViolationKind { - Bandwidth, - Accuracy, - Cpu, -} - -impl std::fmt::Display for ViolationKind { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - ViolationKind::Bandwidth => write!(f, "bandwidth"), - ViolationKind::Accuracy => write!(f, "accuracy"), - ViolationKind::Cpu => write!(f, "cpu"), - } - } -} - -#[derive(Debug, Clone)] -pub struct Violation { - pub agent_id: String, - pub kind: ViolationKind, - pub observed: f64, - pub threshold: f64, -} - -#[derive(Debug, Clone, Copy)] -pub struct Thresholds { - pub max_sketch_size_bytes: f64, - pub max_error_rate: f64, - pub max_cpu_micros_per_sample: f64, -} - -impl Default for Thresholds { - fn default() -> Self { - Self { - max_sketch_size_bytes: 5.0 * 1024.0 * 1024.0, // 5 MB - max_error_rate: 0.02, // 2 % - max_cpu_micros_per_sample: 5.0, // 5 µs/sample - } - } -} - -#[derive(Debug, Clone)] -pub struct Endpoint { - pub agent_id: String, - pub metrics_url: String, - /// The sketch type currently deployed to this agent; used to attribute - /// scraped metrics to the right EMA bucket. - pub sketch_type: Option, -} - -impl Endpoint { - pub fn new(agent_id: impl Into, metrics_url: impl Into) -> Self { - Self { agent_id: agent_id.into(), metrics_url: metrics_url.into(), sketch_type: None } - } -} - -/// Data reported to the `on_metrics` callback after each successful scrape. -#[derive(Debug, Clone)] -pub struct ScrapedData { - pub agent_id: String, - /// The sketch type configured on this endpoint at scrape time (if known). - pub sketch_type: Option, - /// Current total sketch size in bytes at the agent. - pub sketch_size_bytes: f64, - /// Derived µs/sample over the last scrape window; `None` on the very first - /// scrape because there is no previous baseline yet. - pub cpu_micros_per_sample: Option, -} - -pub type OnViolationFn = Arc; -pub type OnMetricsFn = Arc; - -// ── Scraper ─────────────────────────────────────────────────────────────────── - -pub struct Scraper { - endpoints: Arc>>, - thresholds: Thresholds, - on_violation: OnViolationFn, - on_metrics: Option, - interval: Duration, - client: reqwest::Client, - last: Mutex>, -} - -impl Scraper { - pub fn new( - endpoints: Vec, - thresholds: Thresholds, - on_violation: OnViolationFn, - interval: Duration, - ) -> Self { - Self { - endpoints: Arc::new(RwLock::new(endpoints)), - thresholds, - on_violation, - on_metrics: None, - interval, - client: reqwest::Client::builder() - .timeout(Duration::from_secs(5)) - .build() - .expect("reqwest client"), - last: Mutex::new(HashMap::new()), - } - } - - /// Attach a callback invoked after every successful scrape with observed metrics. - pub fn with_on_metrics(mut self, cb: OnMetricsFn) -> Self { - self.on_metrics = Some(cb); - self - } - - /// Registers a new endpoint to be scraped. Safe to call from any async context. - pub async fn add_endpoint(&self, ep: Endpoint) { - info!(agent = %ep.agent_id, "adding scrape endpoint"); - self.endpoints.write().await.push(ep); - } - - /// Removes an endpoint by agent ID. No-op if not found. - pub async fn remove_endpoint(&self, agent_id: &str) { - let mut eps = self.endpoints.write().await; - eps.retain(|e| e.agent_id != agent_id); - info!(agent = %agent_id, "removed scrape endpoint"); - } - - /// Updates the sketch type recorded for an existing endpoint. - /// Called after a new plan is pushed so EMA attribution is accurate. - pub async fn set_sketch_type(&self, agent_id: &str, st: SketchType) { - let mut eps = self.endpoints.write().await; - for ep in eps.iter_mut() { - if ep.agent_id == agent_id { - ep.sketch_type = Some(st); - return; - } - } - } - - /// Starts the scrape loop; runs until the process exits. - pub async fn run(self: Arc) { - let mut ticker = tokio::time::interval(self.interval); - ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - loop { - ticker.tick().await; - self.scrape_all().await; - } - } - - /// Performs a single scrape of all endpoints. Useful for tests. - pub async fn scrape_all(&self) { - // Clone the endpoint list so we don't hold the lock across async scrapes. - let endpoints = self.endpoints.read().await.clone(); - for ep in &endpoints { - match self.scrape(ep).await { - Ok(m) => self.analyze(&m, ep.sketch_type.as_ref()), - Err(e) => warn!(agent = %ep.agent_id, "scrape failed: {e}"), - } - } - } - - async fn scrape(&self, ep: &Endpoint) -> anyhow::Result { - let text = self.client - .get(&ep.metrics_url) - .send().await - .context("GET metrics")? - .text().await - .context("read body")?; - - let mut m = CollectorMetrics { - agent_id: ep.agent_id.clone(), - sketch_size_bytes: 0.0, - cpu_seconds_total: 0.0, - samples_ingested: 0.0, - error_rate: 0.0, - }; - parse_prometheus_text(&text, &mut m); - Ok(m) - } - - fn analyze(&self, m: &CollectorMetrics, sketch_type: Option<&SketchType>) { - // Bandwidth / sketch size. - if m.sketch_size_bytes > self.thresholds.max_sketch_size_bytes { - (self.on_violation)(Violation { - agent_id: m.agent_id.clone(), - kind: ViolationKind::Bandwidth, - observed: m.sketch_size_bytes, - threshold: self.thresholds.max_sketch_size_bytes, - }); - } - - // Accuracy / error rate. - if m.error_rate > self.thresholds.max_error_rate { - (self.on_violation)(Violation { - agent_id: m.agent_id.clone(), - kind: ViolationKind::Accuracy, - observed: m.error_rate, - threshold: self.thresholds.max_error_rate, - }); - } - - // CPU: compare δCPU/δsamples with the previous scrape. - let mut last = self.last.lock().unwrap(); - let cpu_micros = if let Some(prev) = last.get(&m.agent_id) { - let delta_samples = m.samples_ingested - prev.samples_ingested; - let delta_cpu = m.cpu_seconds_total - prev.cpu_seconds_total; - if delta_samples > 0.0 { - let micros_per_sample = (delta_cpu / delta_samples) * 1e6; - if micros_per_sample > self.thresholds.max_cpu_micros_per_sample { - (self.on_violation)(Violation { - agent_id: m.agent_id.clone(), - kind: ViolationKind::Cpu, - observed: micros_per_sample, - threshold: self.thresholds.max_cpu_micros_per_sample, - }); - } - Some(micros_per_sample) - } else { - None - } - } else { - None - }; - last.insert(m.agent_id.clone(), m.clone()); - drop(last); - - // Fire on_metrics callback so callers can feed EMA / telemetry. - if let Some(cb) = &self.on_metrics { - cb(ScrapedData { - agent_id: m.agent_id.clone(), - sketch_type: sketch_type.cloned(), - sketch_size_bytes: m.sketch_size_bytes, - cpu_micros_per_sample: cpu_micros, - }); - } - } -} - -// ── Prometheus text parser ──────────────────────────────────────────────────── - -fn parse_prometheus_text(text: &str, m: &mut CollectorMetrics) { - for line in text.lines() { - let line = line.trim(); - if line.is_empty() || line.starts_with('#') { continue; } - // Handle lines with optional labels: metric_name{...} value [timestamp] - // Split on whitespace to get name and value parts. - let parts: Vec<&str> = line.splitn(2, ' ').collect(); - if parts.len() < 2 { continue; } - // Strip label block {…} from the metric name, if any. - let name = parts[0].split('{').next().unwrap_or(parts[0]); - let val_str = parts[1].split_whitespace().next().unwrap_or(""); - let Ok(val) = val_str.parse::() else { continue }; - match name { - "otelcol_sketch_size_bytes" => m.sketch_size_bytes = val, - "process_cpu_seconds_total" => m.cpu_seconds_total = val, - "otelcol_processor_accepted_metric_points" => m.samples_ingested = val, - "otelcol_sketch_error_rate" => m.error_rate = val, - _ => {} - } - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use axum::{Router, routing::get}; - use tokio::net::TcpListener; - - const NORMAL_PAYLOAD: &str = " -otelcol_sketch_size_bytes 1048576 -process_cpu_seconds_total 0.5 -otelcol_processor_accepted_metric_points 100000 -otelcol_sketch_error_rate 0.001 -"; - - const HIGH_BANDWIDTH_PAYLOAD: &str = " -otelcol_sketch_size_bytes 10485760 -process_cpu_seconds_total 1.0 -otelcol_processor_accepted_metric_points 200000 -otelcol_sketch_error_rate 0.001 -"; - - const HIGH_ERROR_RATE_PAYLOAD: &str = " -otelcol_sketch_size_bytes 512000 -process_cpu_seconds_total 1.0 -otelcol_processor_accepted_metric_points 200000 -otelcol_sketch_error_rate 0.05 -"; - - async fn serve_metrics(payload: &'static str) -> String { - let app = Router::new().route("/metrics", - get(move || async move { payload })); - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - tokio::spawn(async move { axum::serve(listener, app).await.unwrap(); }); - format!("http://{addr}/metrics") - } - - fn scraper_with_violations(url: &str) -> (Arc, Arc>>) { - let violations: Arc>> = Arc::new(Mutex::new(vec![])); - let v2 = Arc::clone(&violations); - let s = Arc::new(Scraper::new( - vec![Endpoint::new("a1", url)], - Thresholds::default(), - Arc::new(move |v| v2.lock().unwrap().push(v)), - Duration::from_secs(60), - )); - (s, violations) - } - - #[tokio::test] - async fn no_violation_on_normal_metrics() { - let url = serve_metrics(NORMAL_PAYLOAD).await; - let (s, violations) = scraper_with_violations(&url); - s.scrape_all().await; - assert!(violations.lock().unwrap().is_empty()); - } - - #[tokio::test] - async fn bandwidth_violation() { - let url = serve_metrics(HIGH_BANDWIDTH_PAYLOAD).await; // 10MB > 5MB threshold - let (s, violations) = scraper_with_violations(&url); - s.scrape_all().await; - let v = violations.lock().unwrap(); - assert_eq!(v.len(), 1); - assert_eq!(v[0].kind, ViolationKind::Bandwidth); - assert!(v[0].observed > v[0].threshold); - } - - #[tokio::test] - async fn accuracy_violation() { - let url = serve_metrics(HIGH_ERROR_RATE_PAYLOAD).await; // 5% > 2% threshold - let (s, violations) = scraper_with_violations(&url); - s.scrape_all().await; - let v = violations.lock().unwrap(); - let has_accuracy = v.iter().any(|vio| vio.kind == ViolationKind::Accuracy); - assert!(has_accuracy, "expected accuracy violation, got: {v:?}"); - } - - #[tokio::test] - async fn cpu_violation_on_delta() { - // First call: baseline (0 CPU, 0 samples). - // Second call: 5ms CPU for 100 samples → 50 µs/sample > 5 µs threshold. - let call_count = Arc::new(Mutex::new(0u32)); - let c2 = Arc::clone(&call_count); - let app = Router::new().route("/metrics", get(move || { - let count = Arc::clone(&c2); - async move { - let mut n = count.lock().unwrap(); - *n += 1; - if *n == 1 { - "process_cpu_seconds_total 0\notelcol_processor_accepted_metric_points 0\notelcol_sketch_size_bytes 0\notelcol_sketch_error_rate 0\n" - } else { - "process_cpu_seconds_total 0.005\notelcol_processor_accepted_metric_points 100\notelcol_sketch_size_bytes 0\notelcol_sketch_error_rate 0\n" - } - } - })); - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - tokio::spawn(async move { axum::serve(listener, app).await.unwrap(); }); - - let violations: Arc>> = Arc::new(Mutex::new(vec![])); - let v2 = Arc::clone(&violations); - let s = Arc::new(Scraper::new( - vec![Endpoint::new("a1", format!("http://{addr}/metrics"))], - Thresholds::default(), - Arc::new(move |v| v2.lock().unwrap().push(v)), - Duration::from_secs(60), - )); - s.scrape_all().await; // baseline - s.scrape_all().await; // delta → CPU violation - - let v = violations.lock().unwrap(); - assert!(v.iter().any(|vio| vio.kind == ViolationKind::Cpu), - "expected CPU violation, got: {v:?}"); - } - - #[tokio::test] - async fn multiple_endpoints_only_bad_violates() { - let ok_url = serve_metrics(NORMAL_PAYLOAD).await; - let bad_url = serve_metrics(HIGH_BANDWIDTH_PAYLOAD).await; - - let violations: Arc>> = Arc::new(Mutex::new(vec![])); - let v2 = Arc::clone(&violations); - let s = Arc::new(Scraper::new( - vec![ - Endpoint::new("ok", ok_url), - Endpoint::new("bad", bad_url), - ], - Thresholds::default(), - Arc::new(move |v| v2.lock().unwrap().push(v)), - Duration::from_secs(60), - )); - s.scrape_all().await; - - let v = violations.lock().unwrap(); - assert!(v.iter().all(|vio| vio.agent_id == "bad"), - "only bad agent should violate: {v:?}"); - assert!(v.iter().any(|vio| vio.agent_id == "bad")); - } - - #[tokio::test] - async fn unreachable_endpoint_no_panic() { - let (s, violations) = scraper_with_violations("http://127.0.0.1:1/metrics"); - s.scrape_all().await; // should log warning, not panic - assert!(violations.lock().unwrap().is_empty()); - } - - #[test] - fn violation_kind_display() { - assert_eq!(ViolationKind::Bandwidth.to_string(), "bandwidth"); - assert_eq!(ViolationKind::Accuracy.to_string(), "accuracy"); - assert_eq!(ViolationKind::Cpu.to_string(), "cpu"); - } - - #[tokio::test] - async fn on_metrics_callback_fires() { - let url = serve_metrics(NORMAL_PAYLOAD).await; - let scraped: Arc>> = Arc::new(Mutex::new(vec![])); - let s2 = Arc::clone(&scraped); - let s = Arc::new( - Scraper::new( - vec![Endpoint::new("a1", url)], - Thresholds::default(), - Arc::new(|_| {}), - Duration::from_secs(60), - ) - .with_on_metrics(Arc::new(move |d| s2.lock().unwrap().push(d))), - ); - s.scrape_all().await; - let got = scraped.lock().unwrap(); - assert_eq!(got.len(), 1); - assert_eq!(got[0].agent_id, "a1"); - assert!(got[0].sketch_size_bytes > 0.0); - } - - #[tokio::test] - async fn add_remove_endpoint() { - let url = serve_metrics(NORMAL_PAYLOAD).await; - let violations: Arc>> = Arc::new(Mutex::new(vec![])); - let v2 = Arc::clone(&violations); - let s = Arc::new(Scraper::new( - vec![], - Thresholds::default(), - Arc::new(move |v| v2.lock().unwrap().push(v)), - Duration::from_secs(60), - )); - // Initially no endpoints → no violations. - s.scrape_all().await; - assert!(violations.lock().unwrap().is_empty()); - - // Add endpoint and scrape. - s.add_endpoint(Endpoint::new("a1", &url)).await; - s.scrape_all().await; - // NORMAL_PAYLOAD → no violation. - assert!(violations.lock().unwrap().is_empty()); - - // Remove and verify nothing scrapes. - s.remove_endpoint("a1").await; - assert!(s.endpoints.read().await.is_empty()); - } -} diff --git a/controller/src/opamp/mod.rs b/controller/src/opamp/mod.rs deleted file mode 100644 index c3deee5c..00000000 --- a/controller/src/opamp/mod.rs +++ /dev/null @@ -1,678 +0,0 @@ -/// OpAMP server implemented over WebSocket via axum. -/// -/// Speaks the standard OpAMP protobuf protocol (ServerToAgent / AgentToServer) -/// so the `opampextension` in OTel Collectors can connect directly. -/// -/// Each OTel collector connects as an "agent" identified by the `X-Agent-ID` -/// request header. The role (`agent` vs `backend`) is determined by the -/// optional `X-Agent-Role` header (defaults to `agent`). -/// -/// The server pushes `RemoteConfig` as an OpAMP `ServerToAgent.remote_config` -/// message (protobuf binary frame) and receives `AgentToServer` status reports. -use std::collections::HashMap; -use std::sync::Arc; - -use axum::extract::ws::{Message, WebSocket, WebSocketUpgrade}; -use axum::extract::State; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::IntoResponse; -use futures_util::sink::SinkExt; -use futures_util::stream::StreamExt; -use prost::Message as ProstMessage; -use serde::{Deserialize, Serialize}; -use tokio::sync::{mpsc, RwLock}; -use tracing::{info, warn}; - -/// Generated OpAMP protobuf types (from proto/opamp.proto). -pub mod opamp_proto { - include!(concat!(env!("OUT_DIR"), "/opamp.proto.rs")); -} - -// ── Wire types ──────────────────────────────────────────────────────────────── - -/// Config message pushed to an agent collector. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RemoteConfig { - pub config_hash: String, - pub yaml: String, -} - -/// Status report sent back from an agent after applying a config. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AgentStatus { - pub agent_id: String, - pub config_hash: String, - pub healthy: bool, - pub error: Option, -} - -// ── Role ────────────────────────────────────────────────────────────────────── - -/// Role of a connected OTel collector. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum AgentRole { - /// Edge / agent collector that produces sketches. - Agent, - /// Mid-tier gateway collector that forwards / merges sketches - /// between edge agents and the backend. Phase C (MVP v6) wires - /// this role through OpAMP so the typed L5 stage_split path can - /// push the gateway YAML directly via `push_to_role`. - Gateway, - /// Backend / aggregation collector that merges sketches. - Backend, -} - -impl AgentRole { - /// Parse the `X-Agent-Role` header value into an `AgentRole`. - /// Recognises `backend`, `gateway`, and `agent` (case-insensitive); - /// any other value (including the empty string) defaults to - /// `Agent` so legacy / mis-configured collectors keep working. - pub fn from_header(value: &str) -> Self { - match value.trim().to_lowercase().as_str() { - "backend" => AgentRole::Backend, - "gateway" => AgentRole::Gateway, - _ => AgentRole::Agent, - } - } -} - -// ── Server ──────────────────────────────────────────────────────────────────── - -type AgentMap = HashMap, AgentRole)>; - -pub type OnConnectFn = Arc; -pub type OnDisconnectFn = Arc; - -pub struct OpampServer { - agents: Arc>, - on_connect: Option, - on_disconnect: Option, -} - -impl Default for OpampServer { - fn default() -> Self { - Self { - agents: Arc::new(RwLock::new(HashMap::new())), - on_connect: None, - on_disconnect: None, - } - } -} - -impl Clone for OpampServer { - fn clone(&self) -> Self { - Self { - agents: Arc::clone(&self.agents), - on_connect: self.on_connect.clone(), - on_disconnect: self.on_disconnect.clone(), - } - } -} - -impl OpampServer { - pub fn new() -> Self { Self::default() } - - /// Register a callback invoked when an agent connects. - pub fn with_on_connect(mut self, f: impl Fn(String, AgentRole) + Send + Sync + 'static) -> Self { - self.on_connect = Some(Arc::new(f)); - self - } - - /// Register a callback invoked when an agent disconnects. - pub fn with_on_disconnect(mut self, f: impl Fn(String) + Send + Sync + 'static) -> Self { - self.on_disconnect = Some(Arc::new(f)); - self - } - - /// axum handler: `GET /v1/opamp` — upgrades to WebSocket. - /// Agents must include `X-Agent-ID: ` in the upgrade request. - /// Optional `X-Agent-Role: agent|backend` (default: `agent`). - pub async fn ws_handler( - ws: WebSocketUpgrade, - headers: HeaderMap, - State(srv): State>, - ) -> impl IntoResponse { - let agent_id = headers - .get("X-Agent-ID") - .and_then(|v| v.to_str().ok()) - .unwrap_or("") - .to_string(); - - if agent_id.is_empty() { - return (StatusCode::BAD_REQUEST, "X-Agent-ID header required").into_response(); - } - - let role = headers - .get("X-Agent-Role") - .and_then(|v| v.to_str().ok()) - .map(AgentRole::from_header) - .unwrap_or(AgentRole::Agent); - - ws.on_upgrade(move |socket| handle_socket(socket, agent_id, role, srv)) - .into_response() - } - - /// Pushes a config to a specific agent. Returns false if not connected. - pub async fn push(&self, agent_id: &str, cfg: RemoteConfig) -> bool { - if let Some((tx, _)) = self.agents.read().await.get(agent_id) { - tx.send(cfg).await.is_ok() - } else { - false - } - } - - /// Broadcasts a config to every connected agent regardless of role. - pub async fn push_all(&self, cfg: RemoteConfig) { - let ids: Vec = self.agents.read().await.keys().cloned().collect(); - for id in ids { self.push(&id, cfg.clone()).await; } - } - - /// Broadcasts a config only to agents matching the given role. - pub async fn push_to_role(&self, role: AgentRole, cfg: RemoteConfig) { - let ids: Vec = self.agents.read().await - .iter() - .filter(|(_, (_, r))| *r == role) - .map(|(id, _)| id.clone()) - .collect(); - for id in ids { self.push(&id, cfg.clone()).await; } - } - - /// Returns the IDs of currently connected agents (all roles). - pub async fn connected_agents(&self) -> Vec { - self.agents.read().await.keys().cloned().collect() - } - - /// Returns a map of agent_id → role for all connected agents. - pub async fn connected_agents_with_roles(&self) -> HashMap { - self.agents.read().await - .iter() - .map(|(id, (_, role))| (id.clone(), role.clone())) - .collect() - } -} - -async fn handle_socket(socket: WebSocket, agent_id: String, role: AgentRole, srv: Arc) { - let (tx, mut rx) = mpsc::channel::(16); - srv.agents.write().await.insert(agent_id.clone(), (tx, role.clone())); - info!(agent = %agent_id, ?role, "agent connected"); - - if let Some(cb) = &srv.on_connect { - cb(agent_id.clone(), role); - } - - let (mut ws_tx, mut ws_rx) = socket.split(); - - // Forward channel messages → WebSocket as standard OpAMP protobuf. - // - // OpAMP WS wire format prepends each binary frame with a varint - // header (`uint64(0)` today). See `opamp-go/internal/wsmessage.go`. - // Without the header, the agent's `DecodeWSMessage` falls back to - // "old format" and decodes successfully — which is why pushes - // worked even before this fix. We add the header for spec - // conformance so the agent never has to take the legacy path. - let writer_id = agent_id.clone(); - let write_task = tokio::spawn(async move { - while let Some(cfg) = rx.recv().await { - // Build standard OpAMP ServerToAgent with RemoteConfig. - let server_to_agent = encode_remote_config(&cfg); - let mut payload = Vec::new(); - if server_to_agent.encode(&mut payload).is_err() { - warn!(agent = %writer_id, "failed to encode OpAMP protobuf"); - continue; - } - // Prepend the wsMsgHeader varint (zero byte today; the - // varint is `0u64`, which encodes to a single 0x00). - let mut buf = Vec::with_capacity(1 + payload.len()); - buf.push(0u8); - buf.extend_from_slice(&payload); - if ws_tx.send(Message::Binary(buf.into())).await.is_err() { break; } - info!(agent = %writer_id, hash = %cfg.config_hash, "config pushed (OpAMP protobuf)"); - } - }); - - // Receive AgentToServer protobuf messages. - // - // Strip the OpAMP wire-format header before decoding. Per - // `opamp-go/internal/wsmessage.go::DecodeWSMessage`, the spec - // header is a varint-encoded `uint64(0)` and is detected by a - // leading 0 byte. Older clients send raw protobuf with no header - // — in that case the first byte is the protobuf field tag and is - // never zero (a tag-0 wire type is illegal), so the - // "first-byte-is-zero" check is unambiguous. - while let Some(Ok(msg)) = ws_rx.next().await { - match msg { - Message::Binary(data) => { - let payload: &[u8] = if !data.is_empty() && data[0] == 0 { - // Spec format. Decode the varint header (always - // 0 today) and skip it. - match prost::encoding::decode_varint(&mut &data[..]) { - Ok(_hdr) => { - // Recompute consumed bytes = varint length. - // For the canonical zero header this is 1 - // byte; for any future non-zero header - // it's `n` bytes from the unsigned LEB128 - // encoding. - let mut tmp: &[u8] = data.as_ref(); - let _ = prost::encoding::decode_varint(&mut tmp); - let consumed = data.len() - tmp.len(); - &data[consumed..] - } - Err(_) => &data[..], - } - } else { - &data[..] - }; - match opamp_proto::AgentToServer::decode(payload) { - Ok(ats) => { - info!(agent = %agent_id, "received AgentToServer (OpAMP protobuf)"); - // Log effective config if reported. Triggered by - // the `ReportFullState` flag on our outgoing - // ServerToAgent (see `encode_remote_config`). - if let Some(ec) = &ats.effective_config { - if let Some(cm) = &ec.config_map { - for (name, file) in &cm.config_map { - let body_preview = String::from_utf8_lossy( - &file.body[..file.body.len().min(160)] - ); - info!( - agent = %agent_id, - config_name = %name, - bytes = file.body.len(), - preview = %body_preview.replace('\n', " ⏎ "), - "agent reported effective config", - ); - } - } - } - // Log remote-config apply state — this is the - // signal that "the agent received our pushed - // RemoteConfig, attempted to apply it, and ended - // up in {Applied | Failed | Applying}". - // RemoteConfigStatuses enum values: - // 0 = Unset - // 1 = Applied - // 2 = Applying - // 3 = Failed - if let Some(rcs) = &ats.remote_config_status { - let status_str = match rcs.status { - 0 => "Unset", - 1 => "Applied", - 2 => "Applying", - 3 => "Failed", - n => { - // Future spec-defined values fall through - // here; surface the raw int rather than - // claim a meaning. - return_unknown_status(n) - } - }; - let last_hash_hex = rcs - .last_remote_config_hash - .iter() - .map(|b| format!("{:02x}", b)) - .collect::(); - if rcs.status == 3 { - warn!( - agent = %agent_id, - status = status_str, - last_hash = %last_hash_hex, - error = %rcs.error_message, - "agent reported remote-config status", - ); - } else { - info!( - agent = %agent_id, - status = status_str, - last_hash = %last_hash_hex, - "agent reported remote-config status", - ); - } - } - // Log health if reported. - if let Some(health) = &ats.health { - info!(agent = %agent_id, healthy = health.healthy, "agent health"); - } - } - Err(e) => warn!(agent = %agent_id, error = %e, "failed to decode AgentToServer"), - } - } - // Also accept JSON for backward compatibility. - Message::Text(text) => match serde_json::from_str::(&text) { - Ok(s) => info!(agent = %s.agent_id, healthy = s.healthy, "agent status (legacy JSON)"), - Err(_) => warn!(agent = %agent_id, "unexpected text message"), - }, - Message::Close(_) => break, - _ => {} - } - } - - write_task.abort(); - srv.agents.write().await.remove(&agent_id); - info!(agent = %agent_id, "agent disconnected"); - - if let Some(cb) = &srv.on_disconnect { - cb(agent_id); - } -} - -/// Encode a `RemoteConfig` as a standard OpAMP `ServerToAgent` protobuf message. -/// -/// The YAML config body is wrapped in: -/// ServerToAgent.remote_config.config.config_map[""].body = yaml_bytes -/// -/// Format an unknown `RemoteConfigStatuses` int as a stable string -/// for logs. Pulled out into a helper to keep the match arm above -/// borrow-checker-friendly (returning a `&'static str`). -fn return_unknown_status(n: i32) -> &'static str { - // Leak the formatted int into a `'static str` only if needed. - // For diagnostic logs we accept the cost of a Box::leak per - // unrecognised value since this is an "out-of-spec status" - // signal that should be rare. Avoids reworking the surrounding - // match into String. - Box::leak(format!("Unknown({})", n).into_boxed_str()) -} - -/// This is the standard OpAMP way to push collector configuration. -/// The opampextension in the OTel Collector decodes this and applies the config. -/// -/// Two protocol bits the controller sets per spec: -/// -/// 1. `flags = ReportFullState` (`0x01`) asks the agent's next -/// `AgentToServer` to include the full status block — -/// `effective_config` (the YAML the agent ended up running) -/// and `remote_config_status` (Applied / Failed / Applying). -/// Without this, the agent is allowed to elide both fields as -/// an optimization once the controller has acknowledged a -/// given sequence_num, and we lose visibility into whether the -/// push actually took. -/// -/// 2. `capabilities` advertises what the controller can accept -/// back. `AcceptsStatus` is mandatory; `OffersRemoteConfig` -/// must be set whenever we send `remote_config`; -/// `AcceptsEffectiveConfig` tells the agent it's worth -/// populating the field (some agents skip it if the server -/// didn't claim it could parse it). -fn encode_remote_config(cfg: &RemoteConfig) -> opamp_proto::ServerToAgent { - let config_file = opamp_proto::AgentConfigFile { - body: cfg.yaml.as_bytes().to_vec(), - content_type: "text/yaml".to_string(), - }; - - let mut config_map = HashMap::new(); - config_map.insert(String::new(), config_file); // empty key = single config file - - let agent_config_map = opamp_proto::AgentConfigMap { config_map }; - - let remote_config = opamp_proto::AgentRemoteConfig { - config: Some(agent_config_map), - config_hash: cfg.config_hash.as_bytes().to_vec(), - }; - - // ServerToAgentFlags_ReportFullState = 0x01. - const FLAG_REPORT_FULL_STATE: u64 = 0x0000_0001; - // ServerCapabilities bitmask: - // AcceptsStatus = 0x01 - // OffersRemoteConfig = 0x02 - // AcceptsEffectiveConfig = 0x04 - const CAPS_DEFAULT: u64 = 0x01 | 0x02 | 0x04; - - opamp_proto::ServerToAgent { - remote_config: Some(remote_config), - flags: FLAG_REPORT_FULL_STATE, - capabilities: CAPS_DEFAULT, - ..Default::default() - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use axum::{Router, routing::get}; - use tokio::net::TcpListener; - - #[tokio::test] - async fn no_agents_push_returns_false() { - let srv = Arc::new(OpampServer::new()); - let sent = srv.push("unknown", RemoteConfig { - config_hash: "h".into(), yaml: "y".into() - }).await; - assert!(!sent); - } - - #[tokio::test] - async fn connected_agents_empty_initially() { - let srv = Arc::new(OpampServer::new()); - assert!(srv.connected_agents().await.is_empty()); - } - - #[test] - fn role_from_header() { - assert_eq!(AgentRole::from_header("backend"), AgentRole::Backend); - assert_eq!(AgentRole::from_header("agent"), AgentRole::Agent); - assert_eq!(AgentRole::from_header(""), AgentRole::Agent); - assert_eq!(AgentRole::from_header("BACKEND"), AgentRole::Backend); - } - - /// Phase C: `Gateway` is a recognised role and round-trips through - /// the OpAMP `X-Agent-Role` header parser. This locks in the wire - /// vocabulary that the typed L5 stage_split path relies on when it - /// calls `push_to_role(AgentRole::Gateway, ...)` and expects to - /// reach gateway-role collectors only. - #[test] - fn role_from_header_recognises_gateway() { - assert_eq!(AgentRole::from_header("gateway"), AgentRole::Gateway); - assert_eq!(AgentRole::from_header("Gateway"), AgentRole::Gateway); - assert_eq!(AgentRole::from_header("GATEWAY"), AgentRole::Gateway); - // Round-trip through serde lowercase rename. - let s = serde_json::to_string(&AgentRole::Gateway).unwrap(); - assert_eq!(s, "\"gateway\""); - let back: AgentRole = serde_json::from_str(&s).unwrap(); - assert_eq!(back, AgentRole::Gateway); - // Still distinct from the other two roles. - assert_ne!(AgentRole::Gateway, AgentRole::Agent); - assert_ne!(AgentRole::Gateway, AgentRole::Backend); - } - - /// Phase C: a gateway-role client connecting via WebSocket appears - /// in `connected_agents_with_roles` tagged as `Gateway`. Together - /// with the from_header test above this proves the role plumbs - /// through the connect path that `push_to_role` selects on. - #[tokio::test] - async fn gateway_role_round_trips_through_connection() { - let (srv, addr) = start_server().await; - let _gateway_ws = connect_ws_client(addr, "gw-1", "gateway").await; - - // Wait for server-side registration to complete. - for _ in 0..20 { - tokio::time::sleep(std::time::Duration::from_millis(25)).await; - let map = srv.connected_agents_with_roles().await; - if map.get("gw-1") == Some(&AgentRole::Gateway) { - return; - } - } - let map = srv.connected_agents_with_roles().await; - panic!("gateway role never registered; map = {:?}", map); - } - - /// Helper: start a real OpAMP server on a random port, return the server - /// Arc and the bound address. - async fn start_server() -> (Arc, std::net::SocketAddr) { - let srv = Arc::new(OpampServer::new()); - let app = Router::new() - .route("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/v1/opamp", get(OpampServer::ws_handler)) - .with_state(Arc::clone(&srv)); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); - (srv, addr) - } - - /// Connect a WebSocket client with a given agent-id and role. - async fn connect_ws_client( - addr: std::net::SocketAddr, - agent_id: &str, - role: &str, - ) -> tokio_tungstenite::WebSocketStream> - { - use tokio_tungstenite::tungstenite::http::Request; - let req = Request::builder() - .uri(format!("ws://{addr}/v1/opamp")) - .header("Host", addr.to_string()) - .header("X-Agent-ID", agent_id) - .header("X-Agent-Role", role) - .header("Upgrade", "websocket") - .header("Connection", "Upgrade") - .header("Sec-WebSocket-Key", "dGhlIHNhbXBsZSBub25jZQ==") - .header("Sec-WebSocket-Version", "13") - .body(()) - .unwrap(); - let (ws, _) = tokio_tungstenite::connect_async(req).await.unwrap(); - ws - } - - fn decode_server_to_agent_frame(data: &[u8]) -> opamp_proto::ServerToAgent { - let payload = if !data.is_empty() && data[0] == 0 { - &data[1..] - } else { - data - }; - opamp_proto::ServerToAgent::decode(payload).expect("valid protobuf") - } - - /// `push_to_role` delivers a RemoteConfig to a connected agent-role client. - #[tokio::test] - async fn push_to_role_delivers_yaml_to_agent_role() { - let (srv, addr) = start_server().await; - let mut agent_ws = connect_ws_client(addr, "agent-1", "agent").await; - - // Allow the server to register the connection. - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - - let yaml_payload = "ddsketch:\n mode: window\n"; - srv.push_to_role(AgentRole::Agent, RemoteConfig { - config_hash: "hash-1".into(), - yaml: yaml_payload.to_string(), - }).await; - - let msg = tokio::time::timeout( - std::time::Duration::from_secs(2), - futures_util::StreamExt::next(&mut agent_ws), - ) - .await - .expect("timed out waiting for message") - .unwrap() - .unwrap(); - - // Decode standard OpAMP protobuf binary frame. - let data = msg.into_data(); - let sta = decode_server_to_agent_frame(data.as_ref()); - let rc = sta.remote_config.expect("should have remote_config"); - let config = rc.config.expect("should have config"); - let file = config.config_map.get("").expect("should have empty-key entry"); - let yaml = String::from_utf8(file.body.clone()).unwrap(); - assert_eq!(yaml, yaml_payload, "delivered yaml must match"); - assert_eq!(String::from_utf8(rc.config_hash).unwrap(), "hash-1", "delivered hash must match"); - } - - /// Phase C integration test: gateway YAML emitted from the typed L5 - /// stage_split path is queued onto the gateway-role connection and - /// not onto agent-role / backend-role connections. We don't decode - /// the protobuf (an unrelated decode-tag-zero issue affects sibling - /// tests today); we only assert *delivery routing* — the gateway - /// client receives a non-empty binary frame within the timeout, the - /// other roles receive nothing. - #[tokio::test] - async fn push_to_role_gateway_routes_only_to_gateway_role() { - use futures_util::StreamExt; - let (srv, addr) = start_server().await; - let mut agent_ws = connect_ws_client(addr, "agent-1", "agent").await; - let mut gateway_ws = connect_ws_client(addr, "gateway-1", "gateway").await; - let mut backend_ws = connect_ws_client(addr, "backend-1", "backend").await; - - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - - // Mirror the call site that handle_plan now exercises: - // emit_gateway_yaml(...) → push_to_role(Gateway, ...). - // We use a stand-in YAML payload here; the emitter has its own - // tests in stage_config.rs. - let yaml = "extensions:\n opamp: {}\n".to_string(); - srv.push_to_role(AgentRole::Gateway, RemoteConfig { - config_hash: "hash-gw".into(), - yaml, - }).await; - - // Gateway must receive exactly one frame. - let msg = tokio::time::timeout( - std::time::Duration::from_secs(2), - gateway_ws.next(), - ) - .await - .expect("gateway timed out") - .unwrap() - .unwrap(); - let bytes = msg.into_data(); - assert!(!bytes.is_empty(), "gateway must receive a non-empty frame"); - - // Other roles must receive nothing within a short window. - let agent_result = tokio::time::timeout( - std::time::Duration::from_millis(200), - agent_ws.next(), - ).await; - assert!( - agent_result.is_err(), - "agent-role client must not receive gateway-role push" - ); - let backend_result = tokio::time::timeout( - std::time::Duration::from_millis(200), - backend_ws.next(), - ).await; - assert!( - backend_result.is_err(), - "backend-role client must not receive gateway-role push" - ); - } - - /// `push_to_role(Agent)` must not deliver to a backend-role client. - #[tokio::test] - async fn push_to_agent_role_does_not_reach_backend_role() { - use futures_util::StreamExt; - let (srv, addr) = start_server().await; - let mut agent_ws = connect_ws_client(addr, "agent-1", "agent").await; - let mut backend_ws = connect_ws_client(addr, "backend-1", "backend").await; - - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - - srv.push_to_role(AgentRole::Agent, RemoteConfig { - config_hash: "hash-agent".into(), - yaml: "ddsketch:\n mode: batch\n".to_string(), - }).await; - - // Agent must receive the message. - let msg = tokio::time::timeout( - std::time::Duration::from_secs(2), - agent_ws.next(), - ) - .await - .expect("agent timed out") - .unwrap() - .unwrap(); - let data = msg.into_data(); - let sta = decode_server_to_agent_frame(data.as_ref()); - let rc = sta.remote_config.expect("should have remote_config"); - assert_eq!(String::from_utf8(rc.config_hash).unwrap(), "hash-agent"); - - // Backend must receive nothing within a short window. - let backend_result = tokio::time::timeout( - std::time::Duration::from_millis(200), - backend_ws.next(), - ) - .await; - assert!( - backend_result.is_err(), - "backend-role client must not receive agent-role push" - ); - } -} diff --git a/controller/src/planner/baseline_planner.rs b/controller/src/planner/baseline_planner.rs deleted file mode 100644 index 2b56bb4c..00000000 --- a/controller/src/planner/baseline_planner.rs +++ /dev/null @@ -1,162 +0,0 @@ -/// Baseline planner. -/// -/// A *baseline* is the cost-optimised `CollectionPlan` established on the -/// **first** `POST /api/v1/plan` request for a metric. Once set, the same -/// plan is returned for every subsequent request — even if the workload -/// characteristics change — giving a stable, predictable collector -/// configuration in production. -/// -/// The baseline is intentionally static: live workload fluctuations do **not** -/// trigger re-optimisation, which prevents mid-stream sketch-type flips that -/// would break downstream aggregation pipelines. -/// -/// To replace the baseline (e.g. after an SLA violation or explicit rollback) -/// call [`BaselinePlanner::reset`] for the metric. The next plan request will -/// run the cost model afresh and lock in a new baseline. -use std::collections::HashMap; -use std::sync::{Arc, RwLock}; - -use crate::types::{CollectionPlan, QueryWorkload, WorkloadCharacteristics}; -use super::cost_model::CostModelPlanner; - -pub struct BaselinePlanner { - inner: CostModelPlanner, - cache: Arc>>, -} - -impl BaselinePlanner { - pub fn new(inner: CostModelPlanner) -> Self { - Self { - inner, - cache: Arc::new(RwLock::new(HashMap::new())), - } - } - - /// Return the baseline plan for this metric, or run the cost model and - /// establish a new baseline if this is the first request for the metric. - pub fn plan( - &self, - workload: &QueryWorkload, - wc: Option<&WorkloadCharacteristics>, - ) -> CollectionPlan { - let key = &workload.metric_name; - - // Fast path: return the cached plan if one exists. - { - let cache = self.cache.read().unwrap(); - if let Some(plan) = cache.get(key) { - return plan.clone(); - } - } - - // Slow path: first request for this metric — run cost optimisation. - let plan = self.inner.plan(workload, wc); - self.cache.write().unwrap().insert(key.clone(), plan.clone()); - plan - } - - /// Clear the baseline for a metric so the next request - /// triggers a fresh cost-model run. Called by the rollback handler or - /// any future re-plan endpoint. - pub fn reset(&self, metric: &str) { - self.cache.write().unwrap().remove(metric); - } - - /// Return the metric names that have an established baseline. - pub fn baseline_metrics(&self) -> Vec { - self.cache.read().unwrap().keys().cloned().collect() - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use std::collections::HashMap; - use std::time::Duration; - use crate::types::AggType; - - fn workload(metric: &str) -> QueryWorkload { - QueryWorkload { - metric_name: metric.into(), - label_filters: HashMap::new(), - group_by_labels: vec![], - aggregations: vec![AggType::Quantile], - time_window: Duration::from_secs(300), - repeat_every: None, - accuracy_sla: 0.01, - latency_sla: None, - sketch_type_override: None, - exact_required: false, - quantiles: vec![0.99], - } - } - - fn planner() -> BaselinePlanner { - BaselinePlanner::new(CostModelPlanner::new()) - } - - #[test] - fn first_call_produces_a_plan() { - let p = planner(); - let plan = p.plan(&workload("latency"), None); - // Cost model picks the cheapest sketch that meets the SLA; verify - // we got a valid plan. transmit_sketch defaults to false (enabled - // by CostModelPlanner when appropriate). - assert!(!plan.agent_config.transmit_sketch); - } - - #[test] - fn second_call_returns_same_plan() { - let p = planner(); - let first = p.plan(&workload("latency"), None); - // Change the workload — the baseline planner must ignore it. - let mut w2 = workload("latency"); - w2.aggregations = vec![AggType::Cardinality]; - let second = p.plan(&w2, None); - assert_eq!( - first.agent_config.sketch_type, - second.agent_config.sketch_type, - "baseline plan must not change even when workload changes" - ); - } - - #[test] - fn different_metrics_get_independent_plans() { - let p = planner(); - let a = p.plan(&workload("metric_a"), None); - let b = p.plan(&workload("metric_b"), None); - // Both plans are valid (exact sketch type may differ by cost model - // internals, but we just check they are independently produced). - let _ = (a, b); - assert_eq!(p.baseline_metrics().len(), 2); - } - - #[test] - fn reset_allows_re_plan() { - let p = planner(); - let first = p.plan(&workload("latency"), None); - p.reset("latency"); - assert!(p.baseline_metrics().is_empty()); - // After reset the planner will run the cost model again on the same - // workload and should produce an equivalent plan. - let second = p.plan(&workload("latency"), None); - assert_eq!( - first.agent_config.sketch_type, - second.agent_config.sketch_type, - "same workload after reset should produce the same sketch type" - ); - } - - #[test] - fn baseline_metrics_lists_all_seen_metrics() { - let p = planner(); - p.plan(&workload("cpu"), None); - p.plan(&workload("mem"), None); - p.plan(&workload("cpu"), None); // repeat — should not double-count - let mut metrics = p.baseline_metrics(); - metrics.sort(); - assert_eq!(metrics, vec!["cpu", "mem"]); - } -} diff --git a/controller/src/planner/cost_model.rs b/controller/src/planner/cost_model.rs deleted file mode 100644 index 980accf2..00000000 --- a/controller/src/planner/cost_model.rs +++ /dev/null @@ -1,1056 +0,0 @@ -use std::collections::HashMap; -use std::time::Duration; - -use super::delta_cost_model::decide_delta; -use super::online_cost_model; -use super::rules::{default_sketch_params, select_window_strategy, RulesPlanner}; -use crate::types::*; - -// ── Benchmark-derived cost table ────────────────────────────────────────────── -// -// Source: e2e benchmark results (2026-03-15), 1 000 series × 1 000 Hz row. -// Units: bandwidth bytes/series/sec, CPU µs/sample, memory bytes/sketch. - -#[derive(Debug, Clone, Copy)] -pub struct SketchCosts { - pub bytes_per_series_per_sec: f64, - pub cpu_micros_per_sample: f64, - pub base_memory_bytes: f64, - pub relative_error_at_default: f64, -} - -/// Public accessor used by `apply_delta_decision` to retrieve the cost table. -pub fn benchmark_table_pub() -> HashMap { - benchmark_table() -} - -fn benchmark_table() -> HashMap { - [ - ( - SketchType::DDSketch, - SketchCosts { - bytes_per_series_per_sec: 120.0, - cpu_micros_per_sample: 0.8, - base_memory_bytes: 4_096.0, - relative_error_at_default: 0.01, - }, - ), - ( - SketchType::KLL, - SketchCosts { - bytes_per_series_per_sec: 80.0, - cpu_micros_per_sample: 0.5, - base_memory_bytes: 2_048.0, - relative_error_at_default: 0.02, - }, - ), - ( - SketchType::HLL, - SketchCosts { - bytes_per_series_per_sec: 40.0, - cpu_micros_per_sample: 0.3, - base_memory_bytes: 16_384.0, // precision=14 → 16 KB - relative_error_at_default: 0.008, - }, - ), - ( - SketchType::CountSketch, - SketchCosts { - bytes_per_series_per_sec: 200.0, - cpu_micros_per_sample: 1.2, - base_memory_bytes: 40_960.0, - relative_error_at_default: 0.01, - }, - ), - ( - SketchType::CountMinSketch, - SketchCosts { - bytes_per_series_per_sec: 200.0, - cpu_micros_per_sample: 1.0, - base_memory_bytes: 40_960.0, - relative_error_at_default: 0.01, - }, - ), - ] - .into() -} - -// ── Scoring ─────────────────────────────────────────────────────────────────── - -#[derive(Debug, Clone)] -pub struct PlanScore { - pub bandwidth_bytes_per_sec: f64, - pub cpu_micros_per_sample: f64, - pub memory_bytes: f64, - pub estimated_error: f64, - pub meets_sla: bool, -} - -/// Estimates resource costs for a given plan + workload using the provided cost table. -pub fn score_with(plan: &CollectionPlan, w: &QueryWorkload, table: &HashMap) -> PlanScore { - let st = &plan.agent_config.sketch_type; - let Some(&costs) = table.get(st) else { - return PlanScore { - bandwidth_bytes_per_sec: f64::MAX, - cpu_micros_per_sample: f64::MAX, - memory_bytes: f64::MAX, - estimated_error: 1.0, - meets_sla: false, - }; - }; - - let dim_multiplier = (plan.agent_config.aggregate_by.len() + 1) as f64; - let bandwidth = costs.bytes_per_series_per_sec * dim_multiplier; - let memory = costs.base_memory_bytes * dim_multiplier; - let err = estimate_error(st, &plan.agent_config.sketch_params, costs); - let sla = if w.accuracy_sla <= 0.0 { 0.01 } else { w.accuracy_sla }; - - PlanScore { - bandwidth_bytes_per_sec: bandwidth, - cpu_micros_per_sample: costs.cpu_micros_per_sample, - memory_bytes: memory, - estimated_error: err, - meets_sla: err <= sla, - } -} - -/// Estimates resource costs for a given plan + workload. -pub fn score(plan: &CollectionPlan, w: &QueryWorkload) -> PlanScore { - let table = benchmark_table(); - let st = &plan.agent_config.sketch_type; - - let Some(&costs) = table.get(st) else { - return PlanScore { - bandwidth_bytes_per_sec: f64::MAX, - cpu_micros_per_sample: f64::MAX, - memory_bytes: f64::MAX, - estimated_error: 1.0, - meets_sla: false, - }; - }; - - // More preserved dimensions → more distinct sketches in flight. - let dim_multiplier = (plan.agent_config.aggregate_by.len() + 1) as f64; - - let bandwidth = costs.bytes_per_series_per_sec * dim_multiplier; - let memory = costs.base_memory_bytes * dim_multiplier; - let err = estimate_error(st, &plan.agent_config.sketch_params, costs); - - let sla = if w.accuracy_sla <= 0.0 { - 0.01 - } else { - w.accuracy_sla - }; - - PlanScore { - bandwidth_bytes_per_sec: bandwidth, - cpu_micros_per_sample: costs.cpu_micros_per_sample, - memory_bytes: memory, - estimated_error: err, - meets_sla: err <= sla, - } -} - -fn estimate_error(_st: &SketchType, p: &SketchParams, costs: SketchCosts) -> f64 { - match p { - SketchParams::DDSketch { relative_accuracy, .. } if *relative_accuracy > 0.0 => *relative_accuracy, - SketchParams::KLL { k, .. } if *k > 0 => 1.0 / *k as f64, - SketchParams::HLL { precision } if *precision > 0 => 1.04 / (2.0f64.powi(*precision as i32)).sqrt(), - _ => costs.relative_error_at_default, - } -} - -// ── CostModelPlanner ────────────────────────────────────────────────────────── - -/// Extends the rule-based planner by scoring all valid sketch candidates and -/// choosing the one with the lowest bandwidth that still meets the AccuracySLA. -/// -/// When an [`OnlineMetricsStore`] is attached (via [`CostModelPlanner::with_online_store`]) -/// the planner blends live EMA observations into the cost table used for scoring, -/// so that real-world behaviour gradually supersedes the static benchmark defaults. -pub struct CostModelPlanner { - inner: RulesPlanner, - online_store: Option, -} - -impl CostModelPlanner { - pub fn new() -> Self { - Self { inner: RulesPlanner::new(), online_store: None } - } - - pub fn with_sketch_defaults(mut self, defaults: SketchDefaults) -> Self { - self.inner.sketch_defaults = defaults; - self - } - - /// Attach a live EMA store so scoring uses blended benchmark + observed costs. - pub fn with_online_store(mut self, store: online_cost_model::OnlineMetricsStore) -> Self { - self.online_store = Some(store); - self - } - - /// Returns the effective cost table: online-blended when available, benchmark otherwise. - fn cost_table(&self) -> HashMap { - match &self.online_store { - Some(s) => online_cost_model::effective_table(s), - None => benchmark_table_pub(), - } - } - - /// Produces a [`CollectionPlan`] optimised for the given query workload - /// and data characteristics. - /// - /// `wc` drives the delta transmission decision: fill rate, flush rate, - /// CPU / memory overhead, and raw vs. sketch bandwidth comparison. - /// Pass `None` to use conservative defaults (1 000 series, 100 Hz, - /// 100 B/sample, Zipf distribution, no memory budget). - pub fn plan( - &self, - w: &QueryWorkload, - wc: Option<&WorkloadCharacteristics>, - ) -> CollectionPlan { - let default_wc; - let wc = match wc { - Some(c) => c, - None => { - default_wc = WorkloadCharacteristics::default(); - &default_wc - } - }; - - let table = self.cost_table(); - - // If a specific sketch type is pinned, use it directly. - if let Some(st) = &w.sketch_type_override { - let params = default_sketch_params(st, w.accuracy_sla); - let (mode, window_duration) = select_window_strategy(w); - let mut plan = self.inner.plan(w); - plan.agent_config.sketch_type = st.clone(); - plan.agent_config.sketch_params = params; - plan.agent_config.mode = mode; - plan.agent_config.window_duration = window_duration; - plan.backend_config.merge_sketch_type = st.clone(); - apply_delta_decision_with(&mut plan, w, wc, &table); - return plan; - } - - let candidates = crate::algebra::directory::candidates_for_workload(&w.aggregations); - - // Start with the rule-based plan as the baseline. - let baseline = self.inner.plan(w); - let mut best_plan = baseline; - let mut best_score = score_with(&best_plan, w, &table); - - for st in candidates { - let params = default_sketch_params(&st, w.accuracy_sla); - let (mode, window_duration) = select_window_strategy(w); - - let mut trial = self.inner.plan(w); - trial.agent_config.sketch_type = st.clone(); - trial.agent_config.sketch_params = params; - trial.agent_config.mode = mode; - trial.agent_config.window_duration = window_duration; - trial.backend_config.merge_sketch_type = st; - - let s = score_with(&trial, w, &table); - if !s.meets_sla { - continue; - } - - if s.bandwidth_bytes_per_sec < best_score.bandwidth_bytes_per_sec - || !best_score.meets_sla - { - best_plan = trial; - best_score = s; - } - } - - apply_delta_decision_with(&mut best_plan, w, wc, &table); - best_plan - } -} - -/// Runs the delta cost model and writes the decision into the plan using a provided cost table. -fn apply_delta_decision_with( - plan: &mut CollectionPlan, - w: &QueryWorkload, - wc: &WorkloadCharacteristics, - table: &HashMap, -) { - let bytes_per_series_per_sec = table - .get(&plan.agent_config.sketch_type) - .map(|c| c.bytes_per_series_per_sec) - .unwrap_or(200.0); - - let (decision, summary) = decide_delta(plan, w, wc, bytes_per_series_per_sec); - - // Propagate into agent config. - match &decision { - DeltaDecision::UseDelta { threshold, .. } => { - plan.agent_config.delta_transmission = true; - plan.agent_config.delta_threshold = *threshold; - } - _ => { - plan.agent_config.delta_transmission = false; - plan.agent_config.delta_threshold = 0.0; - } - } - - plan.delta_decision = decision; - plan.transmission_cost_summary = summary; -} - -// candidates_for_workload delegated to algebra::directory. - -// ── Workload-level cost model (DAG fan-in shared-credit) ────────────────────── -// -// Phase F per `controller/docs/design.md` §6 `core::cost`. Single-query -// scoring above (`score`, `score_with`, `CostModelPlanner`) is the L4 -// physical-plan cost; the workload-level entry point below is the L3 -// shared-producer credit. -// -// Per design.md §6 the bundled cost of N related queries is *not* a sum -// of their individual costs — when q1 and q2 share a sub-DAG via a -// `LetBinding`/`Ref` pair, the build cost of that sub-DAG is paid once, -// not twice. The function below walks the L3 IR DAG (Phase B's -// `intent_algebra::QueryExpr`) post-order, memoises the binding-name -// space, and returns the bundled total along with how much was saved. -// -// This is the proof point that `Schema::unique_keys` from Phase B is -// load-bearing — the `cse_reuse_is_legal` gatekeeper in -// `intent_algebra::schema` is what allows the lower step to emit the -// `LetBinding` in the first place; the cost model collected here is -// what makes the planner *prefer* it. -// -// Phase F has no in-tree consumer of `workload_cost` yet — the analyzer -// wiring lands in a follow-up phase. `dead_code` is suppressed on the -// new surface (every item below carries `#[allow(dead_code)]`) to keep -// the lint baseline clean — mirrors the module-wide allowance on -// `intent_algebra/mod.rs` while Phase B sat consumer-less. -#[allow(unused_imports)] -use crate::intent_algebra::{AggIntent, BindingScope, QueryExpr, QueryExprError, Schema}; -#[allow(unused_imports)] -use crate::types_v2::{BindingName, QueryId}; - -/// Bundled cost of a multi-query workload, with per-root contributions -/// and the savings unlocked by shared-producer credit. Returned by -/// [`workload_cost`]. -/// -/// `total_dollars` is the sum every consumer of `WorkloadCost` cares -/// about — the bundled plan beats N independent plans when this value -/// undercuts `Σ per_root_breakdown[i]` (no shared-producer credit). The -/// gap is exactly `reused_savings`, exposed for EXPLAIN / observability -/// per design.md §6 line ~1023. -#[allow(dead_code)] -#[derive(Debug, Clone, PartialEq)] -pub struct WorkloadCost { - /// Bundled total — each unique node costed once across all roots. - /// Units: abstract "dollars" (the same scalar L4's `dollars(plan)` - /// returns; cost-model agnostic so the same number composes with - /// the L4 cost model when the planner gains an L3+L4 stack). - pub total_dollars: f64, - /// Per-root contribution if that root were the sole consumer (i.e. - /// what the naive sum-over-roots cost model would charge it). The - /// difference between `Σ per_root_breakdown[i].1` and `total_dollars` - /// is exactly `reused_savings`. - pub per_root_breakdown: Vec<(QueryId, f64)>, - /// Savings from shared-producer credit vs. naive sum-over-roots. - /// Always `>= 0.0`; equals `0.0` when no `LetBinding` is referenced - /// from more than one consumer (the degenerate "no reuse" case). - pub reused_savings: f64, -} - -/// Multi-root container the cost model walks. Mirrors the shape of -/// `types_v2::WorkloadPlan` (`bindings + roots`) but uses the real -/// Phase B `QueryExpr` instead of the `QueryExprPlaceholder` string. -/// -/// Lives here rather than in `types_v2` because `types_v2::WorkloadPlan` -/// is the JSON wire shape (string placeholder for QueryExpr until the -/// L3 IR gets a stable serde shape downstream); the cost model needs -/// the live IR. When `types_v2::WorkloadPlan` swaps the placeholder for -/// `QueryExpr`, this struct can be replaced with that one and -/// [`workload_cost`] re-pointed without an API break. -#[allow(dead_code)] -#[derive(Debug, Clone)] -pub struct WorkloadCostPlan<'a> { - /// Named shared producers. Each binding is referenced by ≥2 roots - /// via `QueryExpr::Ref` for the cost model to credit it as reused. - pub bindings: Vec<(BindingName, &'a QueryExpr)>, - /// One root per `QuerySpec`, in input order. - pub roots: Vec<(QueryId, &'a QueryExpr)>, -} - -/// Cost a multi-query workload plan, crediting shared producers -/// (`LetBinding` referenced by ≥2 `Ref`s) ONCE across consumers. -/// -/// Per design.md §6 batched-queries example (line ~1256): when q1 + q2 -/// share a `Window` producer hoisted into a `WorkloadCostPlan::bindings` -/// entry, the Scan + Window build cost is credited once across the -/// workload, not twice. This is what makes the bundled plan beat 2 -/// independent plans on `total_dollars`. -/// -/// The walk is post-order with memoisation: -/// 1. Each `bindings[i]` entry is costed exactly once and registered -/// in a binding-name → cost map. -/// 2. Each root walks its tree; encountering `Ref(name)` adds 0.0 -/// (the binding's cost has already been paid). -/// 3. The `total_dollars` is `Σ binding_costs + Σ per_root_traversal` -/// (refs charge zero, so shared work is paid once). -/// 4. `per_root_breakdown` reports each root as if it were the sole -/// consumer (full subtree cost) so the savings are visible. -/// -/// Errors from `QueryExpr` (unresolved `Ref`, invalid by-column, -/// `Window` missing time index) propagate via [`QueryExprError`]. -#[allow(dead_code)] -pub fn workload_cost(plan: &WorkloadCostPlan<'_>) -> Result { - // 1. Cost each binding once. Bindings shadow earlier bindings in - // forward order (mirrors `LetBinding`'s lexical-scope semantics). - let mut binding_costs: HashMap = HashMap::new(); - let mut schema_scope = BindingScope::new(); - for (name, expr) in &plan.bindings { - let cost = subtree_cost(expr, &binding_costs, &schema_scope)?; - let schema = expr.output_schema_in(&schema_scope)?; - binding_costs.insert(name.as_str().to_owned(), cost); - schema_scope = schema_scope.with(name.clone(), schema); - } - - // 2. Cost each root in the bindings scope. `Ref` lookups charge 0.0 - // (the binding has already been paid for above). `LetBinding` - // nodes inside a root are within-query CTE fan-in (design.md §6 - // line ~1318) — same memoisation logic, costed once per root. - let mut bundled_root_total = 0.0; - let mut per_root_breakdown: Vec<(QueryId, f64)> = Vec::with_capacity(plan.roots.len()); - for (qid, root) in &plan.roots { - // "What the bundled plan charges this root" — refs and - // workload-level bindings are free here. - let bundled_contribution = subtree_cost_bundled(root, &binding_costs, &schema_scope)?; - bundled_root_total += bundled_contribution; - // "What this root would cost if it owned the whole sub-DAG" - // — every binding it references is paid for in full. Used - // only for breakdown reporting; the bundled total above is - // the actual cost. - let standalone = subtree_cost_standalone(root, &binding_costs, &schema_scope)?; - per_root_breakdown.push((qid.clone(), standalone)); - } - - // 3. Bundled total = sum of binding costs (each paid once) + - // bundled per-root contributions (refs charged zero above). - let bindings_total: f64 = binding_costs.values().sum(); - let total_dollars = bindings_total + bundled_root_total; - - // 4. Savings = naive-sum total − bundled total. Naive sum is - // `Σ per_root_breakdown.1` (each root pays for everything it - // transitively references, double-counting shared producers). - let naive_sum: f64 = per_root_breakdown.iter().map(|(_, c)| *c).sum(); - let reused_savings = (naive_sum - total_dollars).max(0.0); - - Ok(WorkloadCost { - total_dollars, - per_root_breakdown, - reused_savings, - }) -} - -/// Cost of a single subtree, treating `Ref(name)` as a free pointer to -/// the already-paid binding. Used for the bundled per-root contribution. -#[allow(dead_code)] -fn subtree_cost_bundled( - expr: &QueryExpr, - binding_costs: &HashMap, - schema_scope: &BindingScope, -) -> Result { - match expr { - QueryExpr::Ref { name } => { - // Ref is free — the binding has been paid for at the workload - // level. Resolve to assert the name is bound (otherwise the - // walk should error rather than silently zero-cost an - // unresolved reference). - if binding_costs.contains_key(name.as_str()) { - Ok(0.0) - } else { - Err(QueryExprError::UnresolvedRef(name.as_str().into())) - } - } - QueryExpr::Scan { schema, .. } => Ok(node_cost_scan(schema)), - QueryExpr::Window { child, .. } => { - let cs = subtree_cost_bundled(child, binding_costs, schema_scope)?; - let in_schema = child.output_schema_in(schema_scope)?; - Ok(node_cost_window(&in_schema) + cs) - } - QueryExpr::Aggregate { - by, aggs, child, .. - } => { - let cs = subtree_cost_bundled(child, binding_costs, schema_scope)?; - let in_schema = child.output_schema_in(schema_scope)?; - Ok(node_cost_aggregate(by, aggs, &in_schema) + cs) - } - QueryExpr::LetBinding { name, expr, child } => { - // Within-query LetBinding — same shared-credit logic. Cost - // `expr` once, expose it under `name`, then walk `child`. - let mut extended = binding_costs.clone(); - let bind_cost = subtree_cost(expr, binding_costs, schema_scope)?; - let bound_schema = expr.output_schema_in(schema_scope)?; - let extended_scope = schema_scope.with(name.clone(), bound_schema); - extended.insert(name.as_str().to_owned(), bind_cost); - let child_cost = subtree_cost_bundled(child, &extended, &extended_scope)?; - Ok(bind_cost + child_cost) - } - } -} - -/// Cost of a single subtree as if it owned every reference it transitively -/// makes (refs charge their full binding cost). Used to compute the -/// per-root breakdown — what each query would cost if it were the sole -/// consumer. -#[allow(dead_code)] -fn subtree_cost_standalone( - expr: &QueryExpr, - binding_costs: &HashMap, - schema_scope: &BindingScope, -) -> Result { - match expr { - QueryExpr::Ref { name } => binding_costs - .get(name.as_str()) - .copied() - .ok_or_else(|| QueryExprError::UnresolvedRef(name.as_str().into())), - QueryExpr::Scan { schema, .. } => Ok(node_cost_scan(schema)), - QueryExpr::Window { child, .. } => { - let cs = subtree_cost_standalone(child, binding_costs, schema_scope)?; - let in_schema = child.output_schema_in(schema_scope)?; - Ok(node_cost_window(&in_schema) + cs) - } - QueryExpr::Aggregate { - by, aggs, child, .. - } => { - let cs = subtree_cost_standalone(child, binding_costs, schema_scope)?; - let in_schema = child.output_schema_in(schema_scope)?; - Ok(node_cost_aggregate(by, aggs, &in_schema) + cs) - } - QueryExpr::LetBinding { name, expr, child } => { - let mut extended = binding_costs.clone(); - let bind_cost = subtree_cost(expr, binding_costs, schema_scope)?; - let bound_schema = expr.output_schema_in(schema_scope)?; - let extended_scope = schema_scope.with(name.clone(), bound_schema); - extended.insert(name.as_str().to_owned(), bind_cost); - let child_cost = subtree_cost_standalone(child, &extended, &extended_scope)?; - Ok(bind_cost + child_cost) - } - } -} - -/// Cost a sub-expression in isolation (no fan-in credit). Used by -/// `workload_cost` to price each binding's body, where there's no -/// outer `LetBinding`/`Ref` semantics to honour. -#[allow(dead_code)] -fn subtree_cost( - expr: &QueryExpr, - binding_costs: &HashMap, - schema_scope: &BindingScope, -) -> Result { - // Same shape as standalone — bindings are walked in their own - // scope where outer bindings are visible (workload-level bindings - // can reference earlier ones). - subtree_cost_standalone(expr, binding_costs, schema_scope) -} - -// ── Per-node cost primitives ───────────────────────────────────────────────── -// -// Numbers are in the same abstract-dollars unit as L4's `dollars(plan)` -// (`design.md` §6 `core::cost`). Calibration vs. real benchmarks is -// future work — what matters for Phase F is that: -// - costs are positive, -// - costs are monotonic in input size (Schema width as a proxy), -// - costs compose additively over sub-trees, -// so the test suite can assert "shared producer credited once" without -// committing to a specific magnitude. - -/// Cost of a `Scan` — proportional to the number of columns scanned. -/// The scan dominates I/O in single-query plans; making it cheap here -/// would make shared-producer credit invisible. 10.0 dollars per column -/// is the placeholder; calibration is downstream. -#[allow(dead_code)] -fn node_cost_scan(schema: &Schema) -> f64 { - 10.0 * schema.columns.len().max(1) as f64 -} - -/// Cost of a `Window` — proportional to the input row count, here proxied -/// by the input schema width. 2.0 per column reflects that windowing is -/// cheap relative to a scan but not free (state buffer per row). -#[allow(dead_code)] -fn node_cost_window(input: &Schema) -> f64 { - 2.0 * input.columns.len().max(1) as f64 -} - -/// Cost of an `Aggregate { by, aggs }` — `|by| * 1.0 + Σ cost(intent)`. -/// Each intent contributes its own cost; sketch-bound intents will be -/// re-priced in L4 via the existing `score_with` path. At L3 the only -/// signal is the intent vocabulary, which is a coarse-but-monotonic -/// proxy. -#[allow(dead_code)] -fn node_cost_aggregate(by: &[usize], aggs: &[AggIntent], _input: &Schema) -> f64 { - let by_cost = by.len() as f64; - let agg_cost: f64 = aggs.iter().map(intent_cost).sum(); - by_cost + agg_cost -} - -/// Cost of a single `AggIntent`. Sketch-amenable intents that L4 will -/// later re-price (`Quantile`, `Cardinality`, `TopK`, `Frequency`, -/// `Count{Epsilon}`) are marked more expensive so the cost model -/// agrees with L4's "sketch is cheaper than exact for these" intuition. -#[allow(dead_code)] -fn intent_cost(intent: &AggIntent) -> f64 { - match intent { - AggIntent::Sum | AggIntent::Min | AggIntent::Max | AggIntent::Avg => 5.0, - AggIntent::Count { .. } => 5.0, - AggIntent::Quantile { .. } => 20.0, - AggIntent::Cardinality { .. } => 15.0, - AggIntent::TopK { .. } => 25.0, - AggIntent::Frequency { .. } => 15.0, - AggIntent::Rate { .. } | AggIntent::Increase { .. } => 8.0, - // Phase β archive-only intents — priced as a cold-tier scan - // rather than a streaming aggregate. Higher than `Sum` (the engine - // must read the raw archive) but lower than the sketch intents - // (no per-sample sketch update on the hot path). Tightening this - // is a follow-up once real measurements land. - intent if intent.archive_only() => 12.0, - // Defensive fallback — any future intent that isn't archive-only - // and doesn't match an explicit arm prices as a generic aggregate. - _ => 5.0, - } -} - -// ── Workload-cost tests ────────────────────────────────────────────────────── - -#[cfg(test)] -mod workload_cost_tests { - use super::*; - use crate::intent_algebra::{ - AggIntent, Column, DataType, LabelFilter, QueryExpr, Schema, Source, WindowKind, - }; - use crate::types_v2::{AccuracyTarget, BindingName, QueryId}; - use std::time::Duration; - - fn col(name: &str, dtype: DataType) -> Column { - Column { - name: name.into(), - dtype, - nullable: false, - } - } - - fn ts_scan() -> QueryExpr { - QueryExpr::Scan { - source: Source::TimeSeries { - metric: "http_request_duration_seconds".into(), - }, - label_filters: vec![LabelFilter { - label: "service".into(), - equals: "api".into(), - }], - schema: Schema::with_time_index( - vec![ - col("ts", DataType::Timestamp), - col("service", DataType::Utf8), - col("value", DataType::Float64), - ], - 0, - vec![vec![0, 1]], - ), - } - } - - fn windowed_scan() -> QueryExpr { - QueryExpr::Window { - kind: WindowKind::Sliding, - size: Duration::from_secs(300), - slide: None, - child: Box::new(ts_scan()), - } - } - - /// Wrap `child` in `Aggregate { by: [], aggs: [Quantile{q}] }`. - fn quantile_root(q: f64, child: QueryExpr) -> QueryExpr { - QueryExpr::Aggregate { - by: vec![], - aggs: vec![AggIntent::Quantile { - q, - accuracy: AccuracyTarget::Epsilon(0.01), - }], - having: None, - child: Box::new(child), - } - } - - /// Wrap `child` in `Aggregate { by: [], aggs: [Max] }`. - fn max_root(child: QueryExpr) -> QueryExpr { - QueryExpr::Aggregate { - by: vec![], - aggs: vec![AggIntent::Max], - having: None, - child: Box::new(child), - } - } - - /// Single-root degenerate case — `workload_cost` for one query - /// equals the standalone cost of that query (no reuse possible). - #[test] - fn workload_cost_single_root_equals_query_cost() { - let q = quantile_root(0.99, windowed_scan()); - let plan = WorkloadCostPlan { - bindings: vec![], - roots: vec![(QueryId::new("q1"), &q)], - }; - let wc = workload_cost(&plan).unwrap(); - - // Per-root breakdown reports the standalone cost of `q`. - let standalone = subtree_cost_standalone(&q, &HashMap::new(), &BindingScope::new()).unwrap(); - assert_eq!(wc.per_root_breakdown.len(), 1); - assert_eq!(wc.per_root_breakdown[0].0, QueryId::new("q1")); - assert!((wc.per_root_breakdown[0].1 - standalone).abs() < 1e-9); - // Total equals standalone (no bindings, single root). - assert!((wc.total_dollars - standalone).abs() < 1e-9); - // No sharing → no savings. - assert!(wc.reused_savings.abs() < 1e-9); - } - - /// Two roots that share NOTHING — bundled cost equals sum of - /// standalone costs. `reused_savings` is zero. - #[test] - fn workload_cost_two_roots_no_sharing_equals_sum() { - let q1 = quantile_root(0.99, windowed_scan()); - let q2 = max_root(windowed_scan()); - let plan = WorkloadCostPlan { - bindings: vec![], - roots: vec![(QueryId::new("q1"), &q1), (QueryId::new("q2"), &q2)], - }; - let wc = workload_cost(&plan).unwrap(); - - let s1 = subtree_cost_standalone(&q1, &HashMap::new(), &BindingScope::new()).unwrap(); - let s2 = subtree_cost_standalone(&q2, &HashMap::new(), &BindingScope::new()).unwrap(); - assert!((wc.total_dollars - (s1 + s2)).abs() < 1e-9); - assert!(wc.reused_savings.abs() < 1e-9); - assert_eq!(wc.per_root_breakdown.len(), 2); - } - - /// design.md §6 batched-queries example: q1 (Quantile{0.99}) and - /// q2 (Quantile{0.95}) share a `Window` producer hoisted into a - /// workload-level binding. The Scan + Window cost is credited once, - /// not twice — `reused_savings > 0`, and `total_dollars` undercuts - /// the naive sum-over-roots. - #[test] - fn workload_cost_two_roots_shared_window_credits_once() { - let shared = windowed_scan(); - let q1 = quantile_root(0.99, QueryExpr::Ref { - name: BindingName::new("w"), - }); - let q2 = quantile_root(0.95, QueryExpr::Ref { - name: BindingName::new("w"), - }); - - let plan = WorkloadCostPlan { - bindings: vec![(BindingName::new("w"), &shared)], - roots: vec![(QueryId::new("q1"), &q1), (QueryId::new("q2"), &q2)], - }; - let wc = workload_cost(&plan).unwrap(); - - // The shared-producer cost contribution. - let shared_cost = subtree_cost(&shared, &HashMap::new(), &BindingScope::new()).unwrap(); - // Naive sum-over-roots = each root pays for its full subtree - // including the shared sub-DAG. - let naive_sum: f64 = wc.per_root_breakdown.iter().map(|(_, c)| *c).sum(); - - // Bundled total < naive sum by exactly one shared_cost (paid - // once vs. twice). - assert!( - wc.total_dollars < naive_sum, - "bundled ({}) must beat naive ({}) when a producer is shared", - wc.total_dollars, - naive_sum - ); - // Savings ≈ shared_cost (the producer paid once instead of - // twice). - assert!( - (wc.reused_savings - shared_cost).abs() < 1e-9, - "expected savings ≈ shared_cost ({}); got {}", - shared_cost, - wc.reused_savings - ); - assert_eq!(wc.per_root_breakdown.len(), 2); - } - - /// q1 + q2 share a binding; q3 is independent. Savings credit only - /// the (n_consumers − 1) × binding_cost for the shared portion. - #[test] - fn workload_cost_three_roots_two_share_partial() { - let shared = windowed_scan(); - let q1 = quantile_root(0.99, QueryExpr::Ref { - name: BindingName::new("w"), - }); - let q2 = quantile_root(0.95, QueryExpr::Ref { - name: BindingName::new("w"), - }); - // q3 builds its own scan + window — no shared producer. - let q3 = max_root(windowed_scan()); - - let plan = WorkloadCostPlan { - bindings: vec![(BindingName::new("w"), &shared)], - roots: vec![ - (QueryId::new("q1"), &q1), - (QueryId::new("q2"), &q2), - (QueryId::new("q3"), &q3), - ], - }; - let wc = workload_cost(&plan).unwrap(); - - let shared_cost = subtree_cost(&shared, &HashMap::new(), &BindingScope::new()).unwrap(); - // Two consumers of `w` ⇒ one duplicated copy avoided ⇒ - // savings ≈ shared_cost (not 2 × shared_cost). - assert!( - (wc.reused_savings - shared_cost).abs() < 1e-9, - "two consumers should save 1× shared_cost ({}); got {}", - shared_cost, - wc.reused_savings - ); - assert_eq!(wc.per_root_breakdown.len(), 3); - } - - /// All three roots share the same binding — savings = 2 × shared_cost - /// (3 consumers, 1 paid, 2 avoided). The "fan-in across {q1, q2, q3}" - /// case design.md §6 line ~1318 calls out by name. - #[test] - fn workload_cost_three_roots_all_share_one_binding() { - let shared = windowed_scan(); - let q1 = quantile_root(0.99, QueryExpr::Ref { - name: BindingName::new("w"), - }); - let q2 = quantile_root(0.95, QueryExpr::Ref { - name: BindingName::new("w"), - }); - let q3 = max_root(QueryExpr::Ref { - name: BindingName::new("w"), - }); - let plan = WorkloadCostPlan { - bindings: vec![(BindingName::new("w"), &shared)], - roots: vec![ - (QueryId::new("q1"), &q1), - (QueryId::new("q2"), &q2), - (QueryId::new("q3"), &q3), - ], - }; - let wc = workload_cost(&plan).unwrap(); - let shared_cost = subtree_cost(&shared, &HashMap::new(), &BindingScope::new()).unwrap(); - assert!( - (wc.reused_savings - 2.0 * shared_cost).abs() < 1e-9, - "three consumers should save 2× shared_cost ({}); got {}", - shared_cost, - wc.reused_savings - ); - } - - /// Savings is informational and never negative even when the - /// bindings include unused entries (defensive — a binding with no - /// `Ref` consumer means the bundled total includes the binding - /// once and the naive sum *also* counts it once, so savings is 0). - #[test] - fn workload_cost_unused_binding_is_zero_savings_not_negative() { - let unused = windowed_scan(); - let q1 = quantile_root(0.99, windowed_scan()); - let plan = WorkloadCostPlan { - bindings: vec![(BindingName::new("unused"), &unused)], - roots: vec![(QueryId::new("q1"), &q1)], - }; - let wc = workload_cost(&plan).unwrap(); - assert!( - wc.reused_savings >= 0.0, - "savings must never go negative; got {}", - wc.reused_savings - ); - } - - /// `Ref` to an undeclared binding errors rather than silently - /// zero-costing — caught at workload-cost time so the planner - /// can refuse the plan rather than under-quote it. - #[test] - fn workload_cost_unresolved_ref_errors() { - let q = quantile_root(0.99, QueryExpr::Ref { - name: BindingName::new("missing"), - }); - let plan = WorkloadCostPlan { - bindings: vec![], - roots: vec![(QueryId::new("q1"), &q)], - }; - let err = workload_cost(&plan).unwrap_err(); - assert!(matches!(err, QueryExprError::UnresolvedRef(s) if s == "missing")); - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use chrono::Utc; - use std::collections::HashMap; - - fn workload(aggs: Vec) -> QueryWorkload { - QueryWorkload { - metric_name: "test".into(), - label_filters: HashMap::new(), - group_by_labels: vec![], - aggregations: aggs, - time_window: Duration::from_secs(300), - repeat_every: None, - accuracy_sla: 0.01, - latency_sla: None, - sketch_type_override: None, - exact_required: false, - quantiles: vec![], - } - } - - fn dummy_plan(st: SketchType) -> CollectionPlan { - CollectionPlan { - agent_config: AgentCollectorConfig { - output_mode: OutputMode::Sketch, - sketch_type: st.clone(), - sketch_params: default_sketch_params(&st, 0.01), - aggregate_by: vec![], - label_matchers: vec![], - window_duration: Some(Duration::from_secs(300)), - mode: ProcessorMode::Window, - enable_self_monitoring: true, - transmit_sketch: true, - drop_original: true, - delta_transmission: false, - delta_threshold: 0.0, - enable_series_id: false, - series_id_ttl_secs: 0, - - data_sink: AgentDataSink::default(), - }, - gateway_config: GatewayCollectorConfig { passthrough: true }, - backend_config: BackendCollectorConfig { - merge_sketch_type: st, - group_by: vec![], - }, - precompute: vec![], - valid_until: Utc::now(), - delta_decision: DeltaDecision::default(), - transmission_cost_summary: TransmissionCostSummary::default(), - staged_plan: None, - } - } - - #[test] - fn ddsketch_meets_sla_at_1pct() { - let w = workload(vec![AggType::Quantile]); - let s = score(&dummy_plan(SketchType::DDSketch), &w); - assert!( - s.meets_sla, - "DDSketch at 1% should meet 1% SLA, error={}", - s.estimated_error - ); - } - - #[test] - fn ddsketch_fails_tight_sla() { - let w = QueryWorkload { - accuracy_sla: 0.001, - ..workload(vec![AggType::Quantile]) - }; - // Force 1% params despite tighter SLA. - let mut plan = dummy_plan(SketchType::DDSketch); - plan.agent_config.sketch_params = SketchParams::DDSketch { - relative_accuracy: 0.01, - quantiles: vec![0.5, 0.99], - }; - let s = score(&plan, &w); - assert!(!s.meets_sla, "DDSketch at 1% should NOT meet 0.1% SLA"); - } - - #[test] - fn hll_lower_bandwidth_than_ddsketch() { - let w = workload(vec![AggType::Quantile]); - let s_dd = score(&dummy_plan(SketchType::DDSketch), &w); - let s_hll = score(&dummy_plan(SketchType::HLL), &w); - assert!(s_hll.bandwidth_bytes_per_sec < s_dd.bandwidth_bytes_per_sec); - } - - #[test] - fn dim_multiplier_increases_bandwidth() { - let w_few = QueryWorkload { - group_by_labels: vec!["host".into()], - ..workload(vec![AggType::Quantile]) - }; - let w_many = QueryWorkload { - group_by_labels: vec![ - "host".into(), - "service".into(), - "zone".into(), - "region".into(), - ], - ..workload(vec![AggType::Quantile]) - }; - let pl = RulesPlanner::new(); - let s_few = score(&pl.plan(&w_few), &w_few); - let s_many = score(&pl.plan(&w_many), &w_many); - assert!(s_many.bandwidth_bytes_per_sec > s_few.bandwidth_bytes_per_sec); - } - - #[test] - fn kll_error_formula() { - let w = QueryWorkload { - accuracy_sla: 0.02, - ..workload(vec![AggType::Quantile]) - }; - let mut plan = dummy_plan(SketchType::KLL); - plan.agent_config.sketch_params = SketchParams::KLL { - k: 100, // error ≈ 1/100 = 1% - quantiles: vec![0.5, 0.99], - }; - let s = score(&plan, &w); - assert!(s.meets_sla, "KLL k=100 (error~1%) should meet 2% SLA"); - } - - #[test] - fn cost_model_planner_meets_sla_for_all_agg_types() { - let pl = CostModelPlanner::new(); - for (agg, sla) in [ - (AggType::Quantile, 0.01), - (AggType::Cardinality, 0.01), - (AggType::Frequency, 0.02), - ] { - let w = QueryWorkload { - accuracy_sla: sla, - ..workload(vec![agg]) - }; - let plan = pl.plan(&w, None); - let s = score(&plan, &w); - assert!( - s.meets_sla, - "agg={} sla={sla}: plan does not meet SLA (error={})", - w.aggregations[0], s.estimated_error - ); - } - } - - #[test] - fn cost_model_prefers_lower_bandwidth_for_cardinality() { - let w = QueryWorkload { - accuracy_sla: 0.02, - ..workload(vec![AggType::Cardinality]) - }; - let plan = CostModelPlanner::new().plan(&w, None); - assert_eq!( - plan.agent_config.sketch_type, - SketchType::HLL, - "HLL should win for cardinality (lowest bandwidth)" - ); - } - - #[test] - fn cost_model_valid_until_in_future() { - let plan = CostModelPlanner::new().plan(&workload(vec![AggType::Quantile]), None); - assert!(plan.valid_until > Utc::now()); - } -} diff --git a/controller/src/planner/delta_cost_model.rs b/controller/src/planner/delta_cost_model.rs deleted file mode 100644 index f08c3a1b..00000000 --- a/controller/src/planner/delta_cost_model.rs +++ /dev/null @@ -1,824 +0,0 @@ -// planner/delta_cost_model.rs -// -// Delta transmission cost model. -// -// Compares three transmission strategies for a given workload and sketch plan: -// -// 1. Raw pass-through – send every raw OTLP sample unchanged. -// 2. Sketch (full) – send a complete sketch payload each flush. -// 3. Sketch (delta) – send only the cells that changed since the last -// flush (sparse delta encoding). -// -// The two key inputs that drive the delta decision are: -// -// Fill rate – fraction of sketch cells that change per flush period. -// Lower fill rate → fewer cells in delta → better compression. -// Fill rate is a function of distinct keys per flush period, -// which itself depends on the flush rate. -// -// Flush rate – how often the sketch is transmitted (Hz). -// Window mode : 1 / window_duration_secs -// Batch mode : 1 / repeat_every_secs (or 1 Hz as fallback) -// -// Shorter flush periods mean fewer inserts per period → -// lower fill rate → better delta compression ratio, but -// also more flushes per second → higher CPU overhead per -// second (though the same CPU per sample). - -use std::collections::HashMap; -use std::time::Duration; - -use crate::types::*; - -// ── Delta benchmark table ───────────────────────────────────────────────────── -// -// Source: deltaaccbench (2026-03-15), sketch dims 5×2048, Zipf s=1.1, -// 10-second tumbling window, 2000 inserts/window. -// -// compression_at_fillX : full_bytes / delta_bytes at that estimated fill rate. -// cpu_micros_per_flush : extra CPU for snapshot-diff + sparse-encode, per -// flush, per sketch instance (µs). -// snapshot_bytes : memory for one previous-state snapshot (bytes). - -#[derive(Debug, Clone, Copy)] -pub struct DeltaCosts { - pub compression_at_fill_1pct: f64, - pub compression_at_fill_5pct: f64, - pub compression_at_fill_20pct: f64, - /// Additional CPU per flush per sketch instance (µs). - pub cpu_micros_per_flush: f64, - /// Memory for one snapshot of this sketch type (bytes). - pub snapshot_bytes_per_sketch: u64, - pub supports_delta: bool, -} - -pub fn delta_benchmark_table() -> HashMap { - [ - ( - SketchType::CountMinSketch, - DeltaCosts { - // 5 rows × 2048 cols × 3 fields (count, sum, sum²) × 8 B = 245 760 B snapshot. - compression_at_fill_1pct: 50.0, - compression_at_fill_5pct: 20.0, - compression_at_fill_20pct: 5.0, - cpu_micros_per_flush: 120.0, - snapshot_bytes_per_sketch: 245_760, - supports_delta: true, - }, - ), - ( - SketchType::CountSketch, - DeltaCosts { - // 5 rows × 2048 cols × 1 field × 8 B = 81 920 B snapshot. - compression_at_fill_1pct: 35.0, - compression_at_fill_5pct: 15.0, - compression_at_fill_20pct: 4.0, - cpu_micros_per_flush: 80.0, - snapshot_bytes_per_sketch: 81_920, - supports_delta: true, - }, - ), - ( - SketchType::HLL, - DeltaCosts { - // precision=14 → 2^14 = 16 384 uint8 registers. - compression_at_fill_1pct: 12.0, - compression_at_fill_5pct: 6.0, - compression_at_fill_20pct: 2.5, - cpu_micros_per_flush: 30.0, - snapshot_bytes_per_sketch: 16_384, - supports_delta: true, - }, - ), - ( - SketchType::DDSketch, - DeltaCosts { - // Bucket map snapshot ~8 KB (sparse, relative-accuracy-dependent). - compression_at_fill_1pct: 8.0, - compression_at_fill_5pct: 3.0, - compression_at_fill_20pct: 1.5, - cpu_micros_per_flush: 20.0, - snapshot_bytes_per_sketch: 8_192, - supports_delta: true, - }, - ), - ( - SketchType::KLL, - DeltaCosts { - // KLL uses a compactor hierarchy; no delta implementation exists. - compression_at_fill_1pct: 1.0, - compression_at_fill_5pct: 1.0, - compression_at_fill_20pct: 1.0, - cpu_micros_per_flush: 0.0, - snapshot_bytes_per_sketch: 0, - supports_delta: false, - }, - ), - ] - .into() -} - -// ── Flush rate ──────────────────────────────────────────────────────────────── - -/// Effective flush period in seconds. -/// -/// Window mode : window_duration (fixed tumbling window boundary). -/// Batch mode : repeat_every from the query workload (how often the metric -/// is re-evaluated / a new batch arrives), or 1 s if unset. -/// -/// A shorter flush period means: -/// • fewer inserts accumulate per period → lower fill rate → better delta -/// • more flushes per second → higher total CPU overhead -pub fn flush_period_secs(plan: &CollectionPlan, w: &QueryWorkload) -> f64 { - if let Some(wd) = plan.agent_config.window_duration { - return wd.as_secs_f64(); - } - // Batch mode: use repeat_every as a proxy for the batch arrival interval. - w.repeat_every - .unwrap_or(Duration::from_secs(1)) - .as_secs_f64() - .max(0.001) // guard against zero -} - -// ── Distinct key estimation ─────────────────────────────────────────────────── - -/// Estimates the number of distinct keys observed in one flush period. -/// -/// For CMS / CS this determines how many sketch cells are touched. -/// For HLL this determines how many registers receive new maximum values. -/// For DDSketch this determines what fraction of value buckets are active. -/// -/// If the caller provided `distinct_keys_per_window` we use that directly. -/// Otherwise we apply a distribution-specific analytic approximation. -fn estimate_distinct_keys(inserts_per_flush: f64, wc: &WorkloadCharacteristics) -> f64 { - if let Some(dk) = wc.distinct_keys_per_window { - return dk as f64; - } - match wc.data_distribution { - // Zipf (s ≈ 1.1): distinct count grows sub-linearly as N^(1/s) ≈ N^0.91. - // Scaling factor 0.55 calibrated against deltaaccbench at s=1.1. - DataDistribution::Zipf => 0.55 * inserts_per_flush.powf(0.91), - // Uniform: every insert is a new distinct key in the worst case. - DataDistribution::Uniform => inserts_per_flush, - // Bursty: traffic concentrates in a small key subset; use conservative - // sub-linear growth similar to Zipf but more concentrated. - DataDistribution::Bursty => 0.30 * inserts_per_flush.powf(0.85), - } -} - -// ── Fill rate estimation ────────────────────────────────────────────────────── - -/// Estimates the sketch fill rate: the fraction of cells / registers that -/// change in one flush period. -/// -/// Fill rate drives the delta compression ratio (via [`interpolate_compression`]). -/// It depends on the flush period because: -/// • longer flush period → more inserts accumulate → more cells touched -/// • shorter flush period → fewer inserts → fewer cells touched → better delta -/// -/// Per sketch type: -/// -/// CMS / CS: each distinct key touches `rows` cells (one per hash function row). -/// Fill rate ≈ min(1, distinct_keys / cols). -/// -/// HLL: each distinct key may update one of 2^precision registers. -/// Delta only sends registers that *increased* since last flush. -/// Fill rate uses the birthday-problem approximation: -/// 1 − exp(−distinct / registers) -/// This is an upper bound at steady state (many registers already -/// hold near-maximum values and are seldom updated). -/// -/// DDSketch: value range within the flush period determines which log-scale -/// buckets are touched. Empirical baseline 10 % at a 10-second -/// window, scaled by the flush period. -/// -/// KLL: no delta implementation; always returns 0. -pub fn estimate_fill_rate( - wc: &WorkloadCharacteristics, - plan: &CollectionPlan, - w: &QueryWorkload, -) -> f64 { - let flush_secs = flush_period_secs(plan, w); - let inserts_per_flush = - wc.samples_per_sec_per_series * wc.series_count as f64 * flush_secs; - let distinct = estimate_distinct_keys(inserts_per_flush, wc); - match &plan.agent_config.sketch_params { - SketchParams::CountMinSketch { cols, .. } => { - let cols = *cols as f64; - if cols > 0.0 { (distinct / cols).min(1.0) } else { 0.05 } - } - SketchParams::CountSketch { .. } => { - // CountSketch uses epsilon-based sizing; approximate cols ≈ 1/ε². - 0.05 // safe fallback - } - SketchParams::HLL { precision } => { - let registers = (1u64 << (*precision).max(1)) as f64; - 1.0_f64 - (-distinct / registers).exp() - } - SketchParams::DDSketch { .. } => { - // Scale linearly around the 10-second benchmark baseline. - let scale = (flush_secs / 10.0).clamp(0.2, 8.0); - (0.10 * scale).min(0.80) - } - SketchParams::KLL { .. } => 0.0, - } -} - -// ── Compression ratio interpolation ────────────────────────────────────────── - -/// Linearly interpolates the delta compression ratio from the three-point -/// benchmark table entries at 1 %, 5 %, and 20 % fill rate. -/// -/// Above 20 % fill the ratio decays toward 1.0 (delta payload ≈ full -/// payload), eventually crossing 1.0 when the sparse encoding overhead -/// (cell indices) exceeds the savings. -pub fn interpolate_compression(costs: &DeltaCosts, fill_rate: f64) -> f64 { - if fill_rate <= 0.01 { - costs.compression_at_fill_1pct - } else if fill_rate <= 0.05 { - let t = (fill_rate - 0.01) / (0.05 - 0.01); - lerp(costs.compression_at_fill_1pct, costs.compression_at_fill_5pct, t) - } else if fill_rate <= 0.20 { - let t = (fill_rate - 0.05) / (0.20 - 0.05); - lerp(costs.compression_at_fill_5pct, costs.compression_at_fill_20pct, t) - } else { - // Linear extrapolation toward 1.0 at 100 % fill. - let t = ((fill_rate - 0.20) / 0.80).min(1.0); - lerp(costs.compression_at_fill_20pct, 1.0, t) - } -} - -fn lerp(a: f64, b: f64, t: f64) -> f64 { - a + (b - a) * t -} - -// ── Bandwidth helpers ───────────────────────────────────────────────────────── - -/// Raw OTLP pass-through bandwidth (bytes/sec). -pub fn raw_bytes_per_sec(wc: &WorkloadCharacteristics) -> f64 { - wc.series_count as f64 * wc.samples_per_sec_per_series * wc.bytes_per_raw_sample as f64 -} - -/// Full-sketch outbound bandwidth (bytes/sec). -/// -/// Uses the sketch cost table entry `bytes_per_series_per_sec` scaled by -/// the number of sketch instances: -/// instances = series_count (each input series produces one sketch output) -/// -/// The dim_multiplier in the existing `PlanScore` captures the QUERY fanout -/// (how many group-by combinations exist); for bandwidth estimation we treat -/// `series_count` as the total sketch instances after aggregation. -pub fn sketch_full_bytes_per_sec(wc: &WorkloadCharacteristics, bytes_per_series_per_sec: f64) -> f64 { - wc.series_count as f64 * bytes_per_series_per_sec -} - -// ── Minimum delta compression ratio to enable delta ────────────────────────── - -/// Delta must offer at least this compression over full-sketch to be worth -/// the snapshot memory and diff CPU overhead. -pub const MIN_DELTA_COMPRESSION_RATIO: f64 = 2.0; - -/// Minimum total sample rate (series × Hz) below which sketches add more -/// overhead than they save; the planner falls back to raw pass-through. -pub const RAW_PASSTHROUGH_SAMPLE_RATE_THRESHOLD: f64 = 10.0; - -// ── Main decision function ──────────────────────────────────────────────────── - -/// Decides the delta transmission mode for the given plan and workload. -/// -/// Returns the resolved [`DeltaDecision`] and the full -/// [`TransmissionCostSummary`] for all three strategies. -/// -/// Decision order: -/// 1. If total sample rate < threshold → UseRaw (sketch overhead > saving). -/// 2. If sketch type has no delta support → UseFullSketch. -/// 3. Estimate fill rate from flush period and distribution. -/// 4. Estimate compression ratio from fill rate. -/// 5. If ratio < MIN_DELTA_COMPRESSION_RATIO → UseFullSketch. -/// 6. If delta snapshot memory > budget → UseFullSketch. -/// 7. Otherwise → UseDelta. -pub fn decide_delta( - plan: &CollectionPlan, - w: &QueryWorkload, - wc: &WorkloadCharacteristics, - bytes_per_series_per_sec: f64, -) -> (DeltaDecision, TransmissionCostSummary) { - let table = delta_benchmark_table(); - let st = &plan.agent_config.sketch_type; - - let raw_bw = raw_bytes_per_sec(wc); - let full_bw = sketch_full_bytes_per_sec(wc, bytes_per_series_per_sec); - let flush_secs = flush_period_secs(plan, w); - let flush_hz = if flush_secs > 0.0 { 1.0 / flush_secs } else { 1.0 }; - - // ── 1. Workload too small for sketching ────────────────────────────────── - let total_sample_rate = wc.series_count as f64 * wc.samples_per_sec_per_series; - if total_sample_rate < RAW_PASSTHROUGH_SAMPLE_RATE_THRESHOLD { - let summary = TransmissionCostSummary { - raw_bytes_per_sec: raw_bw, - sketch_full_bytes_per_sec: full_bw, - sketch_delta_bytes_per_sec: 0.0, - delta_cpu_overhead_micros_per_sample: 0.0, - delta_memory_overhead_bytes: 0.0, - estimated_fill_rate: 0.0, - flush_rate_hz: flush_hz, - }; - return ( - DeltaDecision::UseRaw { - reason: RawDataReason::WorkloadTooSmall, - estimated_raw_bytes_per_sec: raw_bw, - }, - summary, - ); - } - - // ── 2. Sketch type does not support delta ──────────────────────────────── - let Some(&costs) = table.get(st) else { - // Unknown sketch type – treat as no delta. - let summary = TransmissionCostSummary { - raw_bytes_per_sec: raw_bw, - sketch_full_bytes_per_sec: full_bw, - sketch_delta_bytes_per_sec: 0.0, - delta_cpu_overhead_micros_per_sample: 0.0, - delta_memory_overhead_bytes: 0.0, - estimated_fill_rate: 0.0, - flush_rate_hz: flush_hz, - }; - return ( - DeltaDecision::UseFullSketch { - reason: DeltaSkipReason::SketchTypeUnsupported, - estimated_full_bytes_per_sec: full_bw, - }, - summary, - ); - }; - - if !costs.supports_delta { - let summary = TransmissionCostSummary { - raw_bytes_per_sec: raw_bw, - sketch_full_bytes_per_sec: full_bw, - sketch_delta_bytes_per_sec: 0.0, - delta_cpu_overhead_micros_per_sample: 0.0, - delta_memory_overhead_bytes: 0.0, - estimated_fill_rate: 0.0, - flush_rate_hz: flush_hz, - }; - return ( - DeltaDecision::UseFullSketch { - reason: DeltaSkipReason::SketchTypeUnsupported, - estimated_full_bytes_per_sec: full_bw, - }, - summary, - ); - } - - // ── 3. Fill rate ───────────────────────────────────────────────────────── - let fill_rate = estimate_fill_rate(wc, plan, w); - - // ── 4. Delta compression ratio and bandwidth ───────────────────────────── - let compression_ratio = interpolate_compression(&costs, fill_rate); - let delta_bw = full_bw / compression_ratio; - - // ── 5. CPU overhead ────────────────────────────────────────────────────── - // Total CPU added per second by delta diff + sparse encode: - // cpu_per_sec = cpu_per_flush × flushes_per_sec × series_count × dim_mult - // Amortised per sample (what the operator cares about): - // cpu_per_sample_µs = cpu_per_flush / (samples_per_sec_per_series × flush_secs) - let dim_mult = (plan.agent_config.aggregate_by.len() + 1) as f64; - let cpu_per_sample_us = if wc.samples_per_sec_per_series > 0.0 && flush_secs > 0.0 { - costs.cpu_micros_per_flush / (wc.samples_per_sec_per_series * flush_secs) - } else { - 0.0 - }; - - // ── 6. Memory overhead ─────────────────────────────────────────────────── - // One snapshot per sketch instance; the number of sketch instances - // equals series_count × dim_mult (each group-by partition is separate). - let snapshot_mem = - wc.series_count as f64 * dim_mult * costs.snapshot_bytes_per_sketch as f64; - - let summary = TransmissionCostSummary { - raw_bytes_per_sec: raw_bw, - sketch_full_bytes_per_sec: full_bw, - sketch_delta_bytes_per_sec: delta_bw, - delta_cpu_overhead_micros_per_sample: cpu_per_sample_us, - delta_memory_overhead_bytes: snapshot_mem, - estimated_fill_rate: fill_rate, - flush_rate_hz: flush_hz, - }; - - // ── 5b. Compression ratio below minimum ────────────────────────────────── - if compression_ratio < MIN_DELTA_COMPRESSION_RATIO { - return ( - DeltaDecision::UseFullSketch { - reason: if fill_rate > 0.20 { - DeltaSkipReason::FillRateTooHigh - } else { - DeltaSkipReason::CompressionRatioBelowThreshold - }, - estimated_full_bytes_per_sec: full_bw, - }, - summary, - ); - } - - // ── 6b. Memory budget exceeded ─────────────────────────────────────────── - if let Some(budget) = wc.memory_budget_bytes { - if snapshot_mem as u64 > budget { - return ( - DeltaDecision::UseFullSketch { - reason: DeltaSkipReason::MemoryBudgetExceeded, - estimated_full_bytes_per_sec: full_bw, - }, - summary, - ); - } - } - - // ── 7. Delta is beneficial ─────────────────────────────────────────────── - ( - DeltaDecision::UseDelta { - threshold: 1.0, // lossless sparse threshold - estimated_compression_ratio: compression_ratio, - estimated_delta_bytes_per_sec: delta_bw, - delta_cpu_overhead_micros_per_sample: cpu_per_sample_us, - delta_memory_overhead_bytes: snapshot_mem, - }, - summary, - ) -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use crate::planner::rules::default_sketch_params; - use chrono::Utc; - use std::collections::HashMap; - - fn workload_for(agg: AggType) -> QueryWorkload { - QueryWorkload { - metric_name: "m".into(), - label_filters: HashMap::new(), - group_by_labels: vec![], - aggregations: vec![agg], - time_window: Duration::from_secs(300), - repeat_every: Some(Duration::from_secs(10)), - accuracy_sla: 0.01, - latency_sla: None, - sketch_type_override: None, - exact_required: false, - quantiles: vec![], - } - } - - fn make_plan(st: SketchType, window: Option) -> CollectionPlan { - let params = default_sketch_params(&st, 0.01); - CollectionPlan { - agent_config: AgentCollectorConfig { - output_mode: OutputMode::Sketch, - sketch_type: st.clone(), - sketch_params: params, - aggregate_by: vec![], - label_matchers: vec![], - window_duration: window, - mode: if window.is_some() { - ProcessorMode::Window - } else { - ProcessorMode::Batch - }, - enable_self_monitoring: true, - transmit_sketch: true, - drop_original: true, - delta_transmission: false, - delta_threshold: 0.0, - enable_series_id: false, - series_id_ttl_secs: 0, - - data_sink: AgentDataSink::default(), - }, - gateway_config: GatewayCollectorConfig { passthrough: true }, - backend_config: BackendCollectorConfig { - merge_sketch_type: st, - group_by: vec![], - }, - precompute: vec![], - valid_until: Utc::now(), - delta_decision: DeltaDecision::default(), - transmission_cost_summary: TransmissionCostSummary::default(), - staged_plan: None, - } - } - - fn default_wc() -> WorkloadCharacteristics { - // 10 series × 10 Hz keeps inserts-per-window well below cols=2048, - // so fill rates stay sub-saturated and delta decisions are meaningful. - WorkloadCharacteristics { - series_count: 10, - samples_per_sec_per_series: 10.0, - bytes_per_raw_sample: 100, - distinct_keys_per_window: None, - data_distribution: DataDistribution::Zipf, - memory_budget_bytes: None, - } - } - - // ── flush_period_secs ───────────────────────────────────────────────────── - - #[test] - fn flush_period_uses_window_duration_in_window_mode() { - let w = workload_for(AggType::Frequency); - let plan = make_plan(SketchType::CountMinSketch, Some(Duration::from_secs(30))); - assert_eq!(flush_period_secs(&plan, &w), 30.0); - } - - #[test] - fn flush_period_uses_repeat_every_in_batch_mode() { - let mut w = workload_for(AggType::Frequency); - w.repeat_every = Some(Duration::from_secs(15)); - let plan = make_plan(SketchType::CountMinSketch, None); - assert_eq!(flush_period_secs(&plan, &w), 15.0); - } - - #[test] - fn flush_period_batch_fallback_is_one_second() { - let mut w = workload_for(AggType::Frequency); - w.repeat_every = None; - let plan = make_plan(SketchType::CountMinSketch, None); - assert_eq!(flush_period_secs(&plan, &w), 1.0); - } - - // ── fill rate ───────────────────────────────────────────────────────────── - - #[test] - fn fill_rate_cms_increases_with_longer_window() { - // More inserts per flush → more cells touched → higher fill rate. - let w_short = workload_for(AggType::Frequency); - let w_long = workload_for(AggType::Frequency); - let plan_short = make_plan(SketchType::CountMinSketch, Some(Duration::from_secs(10))); - let plan_long = make_plan(SketchType::CountMinSketch, Some(Duration::from_secs(120))); - let wc = default_wc(); - let fr_short = estimate_fill_rate(&wc, &plan_short, &w_short); - let fr_long = estimate_fill_rate(&wc, &plan_long, &w_long); - assert!( - fr_long > fr_short, - "longer window should give higher fill rate: short={fr_short:.4} long={fr_long:.4}" - ); - } - - #[test] - fn fill_rate_uniform_higher_than_zipf() { - // Uniform distribution touches more unique cells than Zipf. - let w = workload_for(AggType::Frequency); - let plan = make_plan(SketchType::CountMinSketch, Some(Duration::from_secs(10))); - let wc_zipf = WorkloadCharacteristics { - data_distribution: DataDistribution::Zipf, - ..default_wc() - }; - let wc_unif = WorkloadCharacteristics { - data_distribution: DataDistribution::Uniform, - ..default_wc() - }; - let fr_zipf = estimate_fill_rate(&wc_zipf, &plan, &w); - let fr_unif = estimate_fill_rate(&wc_unif, &plan, &w); - assert!( - fr_unif > fr_zipf, - "uniform should have higher fill rate than Zipf: zipf={fr_zipf:.4} unif={fr_unif:.4}" - ); - } - - #[test] - fn fill_rate_kll_is_zero() { - let w = workload_for(AggType::Quantile); - let plan = make_plan(SketchType::KLL, Some(Duration::from_secs(30))); - let fr = estimate_fill_rate(&default_wc(), &plan, &w); - assert_eq!(fr, 0.0, "KLL has no delta; fill rate must be 0"); - } - - #[test] - fn fill_rate_hll_bounded() { - let w = workload_for(AggType::Cardinality); - let plan = make_plan(SketchType::HLL, Some(Duration::from_secs(60))); - let fr = estimate_fill_rate(&default_wc(), &plan, &w); - assert!(fr > 0.0 && fr <= 1.0, "HLL fill rate out of range: {fr}"); - } - - // ── compression interpolation ───────────────────────────────────────────── - - #[test] - fn compression_at_1pct_returns_table_entry() { - let costs = delta_benchmark_table()[&SketchType::CountMinSketch]; - let r = interpolate_compression(&costs, 0.005); - assert_eq!(r, costs.compression_at_fill_1pct); - } - - #[test] - fn compression_monotone_decreasing_with_fill_rate() { - let costs = delta_benchmark_table()[&SketchType::CountMinSketch]; - let r1 = interpolate_compression(&costs, 0.01); - let r5 = interpolate_compression(&costs, 0.05); - let r20 = interpolate_compression(&costs, 0.20); - let r80 = interpolate_compression(&costs, 0.80); - assert!(r1 >= r5, "compression should decrease as fill rises: {r1} vs {r5}"); - assert!(r5 >= r20, "{r5} vs {r20}"); - assert!(r20 >= r80, "{r20} vs {r80}"); - } - - #[test] - fn compression_at_100pct_is_near_one() { - let costs = delta_benchmark_table()[&SketchType::CountMinSketch]; - let r = interpolate_compression(&costs, 1.0); - assert!(r <= 1.05, "at 100 % fill compression ratio should be ~1: {r}"); - } - - // ── decide_delta branches ───────────────────────────────────────────────── - - #[test] - fn kll_yields_sketch_type_unsupported() { - let w = workload_for(AggType::Quantile); - let plan = make_plan(SketchType::KLL, Some(Duration::from_secs(30))); - let (decision, _) = decide_delta(&plan, &w, &default_wc(), 80.0); - assert!( - matches!( - decision, - DeltaDecision::UseFullSketch { - reason: DeltaSkipReason::SketchTypeUnsupported, - .. - } - ), - "KLL should be unsupported: {decision:?}" - ); - } - - #[test] - fn tiny_workload_yields_use_raw() { - let w = workload_for(AggType::Frequency); - let plan = make_plan(SketchType::CountMinSketch, Some(Duration::from_secs(10))); - let wc = WorkloadCharacteristics { - series_count: 1, - samples_per_sec_per_series: 0.5, // total = 0.5 Hz < threshold - ..default_wc() - }; - let (decision, _) = decide_delta(&plan, &w, &wc, 200.0); - assert!( - matches!( - decision, - DeltaDecision::UseRaw { - reason: RawDataReason::WorkloadTooSmall, - .. - } - ), - "tiny workload should fall back to raw: {decision:?}" - ); - } - - #[test] - fn cms_zipf_short_window_uses_delta() { - // 1000 series, 100 Hz, 10s window, Zipf → low fill rate → good compression. - let w = workload_for(AggType::Frequency); - let plan = make_plan(SketchType::CountMinSketch, Some(Duration::from_secs(10))); - let (decision, summary) = decide_delta(&plan, &w, &default_wc(), 200.0); - assert!( - matches!(decision, DeltaDecision::UseDelta { .. }), - "CMS Zipf 10s should use delta (fill={:.3}): {decision:?}", - summary.estimated_fill_rate - ); - } - - #[test] - fn cms_uniform_fills_fast_at_long_window() { - // Uniform distribution + very long window → high fill rate → skip delta. - let w = workload_for(AggType::Frequency); - let plan = make_plan(SketchType::CountMinSketch, Some(Duration::from_secs(3600))); - let wc = WorkloadCharacteristics { - data_distribution: DataDistribution::Uniform, - ..default_wc() - }; - let (decision, summary) = decide_delta(&plan, &w, &wc, 200.0); - assert!( - matches!( - decision, - DeltaDecision::UseFullSketch { - reason: DeltaSkipReason::FillRateTooHigh - | DeltaSkipReason::CompressionRatioBelowThreshold, - .. - } - ), - "CMS uniform 1h should skip delta (fill={:.3}): {decision:?}", - summary.estimated_fill_rate - ); - } - - #[test] - fn memory_budget_blocks_delta() { - let w = workload_for(AggType::Frequency); - let plan = make_plan(SketchType::CountMinSketch, Some(Duration::from_secs(10))); - // Budget of 1 byte — way below snapshot requirement. - let wc = WorkloadCharacteristics { - memory_budget_bytes: Some(1), - ..default_wc() - }; - let (decision, _) = decide_delta(&plan, &w, &wc, 200.0); - assert!( - matches!( - decision, - DeltaDecision::UseFullSketch { - reason: DeltaSkipReason::MemoryBudgetExceeded, - .. - } - ), - "memory budget exceeded should skip delta: {decision:?}" - ); - } - - #[test] - fn hll_short_window_uses_delta() { - let w = workload_for(AggType::Cardinality); - let plan = make_plan(SketchType::HLL, Some(Duration::from_secs(10))); - let (decision, _) = decide_delta(&plan, &w, &default_wc(), 40.0); - assert!( - matches!(decision, DeltaDecision::UseDelta { .. }), - "HLL with short window + Zipf should use delta: {decision:?}" - ); - } - - #[test] - fn delta_bw_less_than_full_bw_when_delta_used() { - let w = workload_for(AggType::Frequency); - let plan = make_plan(SketchType::CountMinSketch, Some(Duration::from_secs(10))); - let (_, summary) = decide_delta(&plan, &w, &default_wc(), 200.0); - if summary.sketch_delta_bytes_per_sec > 0.0 { - assert!( - summary.sketch_delta_bytes_per_sec < summary.sketch_full_bytes_per_sec, - "delta bw should be less than full: delta={} full={}", - summary.sketch_delta_bytes_per_sec, - summary.sketch_full_bytes_per_sec - ); - } - } - - #[test] - fn cpu_overhead_lower_with_longer_flush_period() { - // Amortised CPU per sample = cpu_per_flush / (samples_per_sec × flush_secs). - // Longer flush period → more samples to amortise over → lower per-sample cost. - let w = workload_for(AggType::Frequency); - let plan_10s = make_plan(SketchType::CountMinSketch, Some(Duration::from_secs(10))); - let plan_60s = make_plan(SketchType::CountMinSketch, Some(Duration::from_secs(60))); - let (_, s10) = decide_delta(&plan_10s, &w, &default_wc(), 200.0); - let (_, s60) = decide_delta(&plan_60s, &w, &default_wc(), 200.0); - assert!( - s60.delta_cpu_overhead_micros_per_sample - < s10.delta_cpu_overhead_micros_per_sample, - "longer flush period should lower per-sample CPU overhead: \ - 10s={:.4}µs 60s={:.4}µs", - s10.delta_cpu_overhead_micros_per_sample, - s60.delta_cpu_overhead_micros_per_sample - ); - } - - #[test] - fn memory_overhead_scales_with_series_count() { - let w = workload_for(AggType::Frequency); - let plan = make_plan(SketchType::CountMinSketch, Some(Duration::from_secs(10))); - let wc_small = WorkloadCharacteristics { - series_count: 10, - ..default_wc() - }; - let wc_large = WorkloadCharacteristics { - series_count: 10_000, - ..default_wc() - }; - let (_, s_small) = decide_delta(&plan, &w, &wc_small, 200.0); - let (_, s_large) = decide_delta(&plan, &w, &wc_large, 200.0); - assert!( - s_large.delta_memory_overhead_bytes > s_small.delta_memory_overhead_bytes, - "more series → more snapshot memory" - ); - // Ratio should be proportional to series_count ratio (10 000 / 10 = 1000). - let ratio = s_large.delta_memory_overhead_bytes / s_small.delta_memory_overhead_bytes; - assert!( - (ratio - 1000.0).abs() < 1.0, - "memory should scale linearly with series_count: ratio={ratio}" - ); - } - - #[test] - fn summary_raw_bw_matches_series_rate_size() { - let w = workload_for(AggType::Frequency); - let plan = make_plan(SketchType::CountMinSketch, Some(Duration::from_secs(10))); - let wc = WorkloadCharacteristics { - series_count: 500, - samples_per_sec_per_series: 10.0, - bytes_per_raw_sample: 120, - ..default_wc() - }; - let (_, summary) = decide_delta(&plan, &w, &wc, 200.0); - let expected = 500.0 * 10.0 * 120.0; - assert!( - (summary.raw_bytes_per_sec - expected).abs() < 0.01, - "raw_bw={} expected={expected}", - summary.raw_bytes_per_sec - ); - } -} diff --git a/controller/src/planner/mod.rs b/controller/src/planner/mod.rs deleted file mode 100644 index ef07aef7..00000000 --- a/controller/src/planner/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -pub mod rules; -pub mod cost_model; -pub mod delta_cost_model; -pub mod online_cost_model; -pub mod pareto; -pub mod baseline_planner; -pub mod stage_split; -pub mod tco; -pub mod wire_cost; - -pub use rules::RulesPlanner; -pub use cost_model::CostModelPlanner; -pub use baseline_planner::BaselinePlanner; -pub use online_cost_model::{OnlineMetricsStore, init_store as init_online_store}; -pub use pareto::{ObjectiveWeights, ParetoPoint, pareto_frontier, select_best}; -pub use wire_cost::{ - break_even_samples, est_wire_bytes_per_window_per_series, select_bind_mode, BindMode, - SketchWireCost, WireCostTable, WireWorkload, -}; diff --git a/controller/src/planner/online_cost_model.rs b/controller/src/planner/online_cost_model.rs deleted file mode 100644 index 63c4cf32..00000000 --- a/controller/src/planner/online_cost_model.rs +++ /dev/null @@ -1,207 +0,0 @@ -//! SP-5 online profiling — EMA-adjusted cost model constants. -//! -//! The static benchmark table in [`cost_model`] seeds initial estimates. -//! As [`monitor::Scraper`] collects actual observations from running agents, -//! this module blends them in via an Exponential Moving Average (EMA) so the -//! planner's cost estimates converge toward real-world behaviour. -//! -//! # Blending weight -//! -//! After `MIN_OBS` observations the online weight reaches `MAX_ONLINE_WEIGHT` -//! (70 %). Below that the benchmark retains the majority share so a single -//! noisy scrape cannot destabilise the planner. -//! -//! ```text -//! online_weight = min(observations / MIN_OBS, 1.0) × MAX_ONLINE_WEIGHT -//! effective = online_weight × observed + (1 − online_weight) × benchmark -//! ``` - -use std::collections::HashMap; -use std::sync::Arc; - -use tokio::sync::RwLock; - -use super::cost_model::{benchmark_table_pub, SketchCosts}; -use crate::types::SketchType; - -// ── Tuning constants ────────────────────────────────────────────────────────── - -/// EMA smoothing factor α. Smaller → slower adaptation, more stable. -const EMA_ALPHA: f64 = 0.15; - -/// Minimum number of observations before online data gets majority weight. -const MIN_OBS: f64 = 5.0; - -/// Maximum fraction of weight given to online (observed) data. -const MAX_ONLINE_WEIGHT: f64 = 0.70; - -// ── Types ───────────────────────────────────────────────────────────────────── - -/// EMA-smoothed cost estimates for one sketch type. -#[derive(Debug, Clone)] -pub struct OnlineSketchCosts { - /// EMA of observed bytes/series/sec transmitted by agents using this sketch. - pub bw_bytes_per_series_per_sec: f64, - /// EMA of observed CPU µs/sample at the agent collector. - pub cpu_micros_per_sample: f64, - /// Number of observations incorporated so far. - pub observations: u64, -} - -impl OnlineSketchCosts { - fn from_benchmark(base: &SketchCosts) -> Self { - Self { - bw_bytes_per_series_per_sec: base.bytes_per_series_per_sec, - cpu_micros_per_sample: base.cpu_micros_per_sample, - observations: 0, - } - } - - /// Incorporate a new observation. - pub fn update(&mut self, observed_bw: f64, observed_cpu: f64) { - self.bw_bytes_per_series_per_sec = EMA_ALPHA * observed_bw - + (1.0 - EMA_ALPHA) * self.bw_bytes_per_series_per_sec; - self.cpu_micros_per_sample = EMA_ALPHA * observed_cpu - + (1.0 - EMA_ALPHA) * self.cpu_micros_per_sample; - self.observations += 1; - } - - /// Returns a `SketchCosts` that blends the EMA observation with `base`. - /// The online weight grows with observation count, capping at 70 %. - pub fn effective_costs(&self, base: &SketchCosts) -> SketchCosts { - let online_w = (self.observations as f64 / MIN_OBS).min(1.0) * MAX_ONLINE_WEIGHT; - let bench_w = 1.0 - online_w; - SketchCosts { - bytes_per_series_per_sec: online_w * self.bw_bytes_per_series_per_sec - + bench_w * base.bytes_per_series_per_sec, - cpu_micros_per_sample: online_w * self.cpu_micros_per_sample - + bench_w * base.cpu_micros_per_sample, - base_memory_bytes: base.base_memory_bytes, - relative_error_at_default: base.relative_error_at_default, - } - } -} - -// ── Store ───────────────────────────────────────────────────────────────────── - -/// Thread-safe map of sketch type → online EMA costs. -/// Shared between the `Scraper`'s `on_metrics` callback and `CostModelPlanner`. -pub type OnlineMetricsStore = Arc>>; - -/// Initialise the store from the static benchmark table. -pub fn init_store() -> OnlineMetricsStore { - let map = benchmark_table_pub() - .iter() - .map(|(st, c)| (st.clone(), OnlineSketchCosts::from_benchmark(c))) - .collect(); - Arc::new(RwLock::new(map)) -} - -/// Update the EMA for `sketch_type` with a new bandwidth/cpu observation. -/// If `sketch_type` is not yet in the store it is inserted from the benchmark. -pub async fn update( - store: &OnlineMetricsStore, - sketch_type: &SketchType, - observed_bw: f64, - observed_cpu: f64, -) { - let mut map = store.write().await; - let entry = map.entry(sketch_type.clone()).or_insert_with(|| { - let base = benchmark_table_pub(); - let costs = base.get(sketch_type).cloned().unwrap_or(SketchCosts { - bytes_per_series_per_sec: 200.0, - cpu_micros_per_sample: 1.0, - base_memory_bytes: 4096.0, - relative_error_at_default: 0.01, - }); - OnlineSketchCosts::from_benchmark(&costs) - }); - entry.update(observed_bw, observed_cpu); -} - -/// Returns an effective cost table that blends online EMA data with benchmarks. -/// Uses `try_read()` so callers in hot paths never block. -pub fn effective_table(store: &OnlineMetricsStore) -> HashMap { - let base = benchmark_table_pub(); - match store.try_read() { - Ok(map) => base - .iter() - .map(|(st, b)| { - let eff = map - .get(st) - .map(|o| o.effective_costs(b)) - .unwrap_or_else(|| b.clone()); - (st.clone(), eff) - }) - .collect(), - Err(_) => base, // fall back to benchmark if lock is contended - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use crate::types::SketchType; - - fn base() -> SketchCosts { - SketchCosts { - bytes_per_series_per_sec: 100.0, - cpu_micros_per_sample: 1.0, - base_memory_bytes: 4096.0, - relative_error_at_default: 0.01, - } - } - - #[test] - fn no_observations_returns_benchmark() { - let o = OnlineSketchCosts::from_benchmark(&base()); - let eff = o.effective_costs(&base()); - assert_eq!(eff.bytes_per_series_per_sec, 100.0); - } - - #[test] - fn after_min_obs_online_dominates() { - let mut o = OnlineSketchCosts::from_benchmark(&base()); - for _ in 0..10 { - o.update(200.0, 2.0); // observed = 2× benchmark - } - let eff = o.effective_costs(&base()); - // Online weight = 1.0 × 0.70 = 0.70 - // effective = 0.70 × online_ema + 0.30 × benchmark - // online_ema after 10 EMA steps from 100→200: converging toward 200 - assert!(eff.bytes_per_series_per_sec > 100.0, - "effective bw should exceed benchmark after high observations: {}", - eff.bytes_per_series_per_sec); - } - - #[test] - fn single_observation_has_low_weight() { - let mut o = OnlineSketchCosts::from_benchmark(&base()); - o.update(1000.0, 10.0); // extreme spike - let eff = o.effective_costs(&base()); - // online_weight = (1/5).min(1.0) × 0.70 = 0.14 - // should only move a little from benchmark - assert!(eff.bytes_per_series_per_sec < 200.0, - "single spike should not dominate: {}", eff.bytes_per_series_per_sec); - } - - #[tokio::test] - async fn store_update_and_effective_table() { - let store = init_store(); - update(&store, &SketchType::DDSketch, 200.0, 2.0).await; - let table = effective_table(&store); - let eff = table.get(&SketchType::DDSketch).unwrap(); - assert!(eff.bytes_per_series_per_sec > 0.0); - } - - #[tokio::test] - async fn effective_table_falls_back_on_benchmark() { - // Even without any updates the store returns sensible values. - let store = init_store(); - let table = effective_table(&store); - assert!(table.contains_key(&SketchType::HLL)); - assert!(table.contains_key(&SketchType::KLL)); - } -} diff --git a/controller/src/planner/pareto.rs b/controller/src/planner/pareto.rs deleted file mode 100644 index 7dcfd6ea..00000000 --- a/controller/src/planner/pareto.rs +++ /dev/null @@ -1,340 +0,0 @@ -//! SP-6 Pareto frontier planner. -//! -//! The cost-model planner minimises bandwidth while meeting the accuracy SLA. -//! This module extends it by enumerating **all** Pareto-optimal plans across -//! three objectives — bandwidth, CPU and memory — so operators can choose the -//! trade-off that best fits their infrastructure. -//! -//! # Algorithm -//! -//! For each sketch candidate the planner generates one `ParetoPoint` (a plan -//! annotated with its scored objectives). A point is Pareto-optimal if no -//! other point is better on *every* objective simultaneously. The frontier is -//! returned sorted by weighted-sum score so that `select_best` returns the -//! single plan that best matches caller-supplied objective weights. -//! -//! # Usage -//! -//! ```ignore -//! let weights = ObjectiveWeights { bandwidth: 0.6, cpu: 0.3, memory: 0.1 }; -//! let frontier = pareto_frontier(&planner, &workload, &wc, weights); -//! let best = select_best(&frontier, weights); -//! ``` - -use std::collections::HashMap; - -use super::cost_model::{benchmark_table_pub, score_with, SketchCosts}; -use super::delta_cost_model::decide_delta; -use super::online_cost_model; -use super::rules::{default_sketch_params, select_window_strategy, RulesPlanner}; -use crate::types::*; - -// ── Types ───────────────────────────────────────────────────────────────────── - -/// Caller-supplied relative weights for the three objectives. -/// Values are normalised internally so they need not sum to 1. -#[derive(Debug, Clone, Copy, serde::Deserialize, serde::Serialize)] -pub struct ObjectiveWeights { - /// Weight given to minimising bandwidth (bytes/sec). - pub bandwidth: f64, - /// Weight given to minimising CPU (µs/sample). - pub cpu: f64, - /// Weight given to minimising memory (bytes). - pub memory: f64, -} - -impl Default for ObjectiveWeights { - fn default() -> Self { - Self { bandwidth: 1.0, cpu: 0.0, memory: 0.0 } - } -} - -impl ObjectiveWeights { - fn normalised(self) -> Self { - let total = self.bandwidth + self.cpu + self.memory; - if total <= 0.0 { - return Self { bandwidth: 1.0, cpu: 0.0, memory: 0.0 }; - } - Self { - bandwidth: self.bandwidth / total, - cpu: self.cpu / total, - memory: self.memory / total, - } - } -} - -/// A single candidate on the Pareto frontier. -#[derive(Debug, Clone)] -pub struct ParetoPoint { - pub plan: CollectionPlan, - pub sketch_type: SketchType, - pub bandwidth_bytes_per_sec: f64, - pub cpu_micros_per_sample: f64, - pub memory_bytes: f64, - pub estimated_error: f64, - pub meets_sla: bool, -} - -// ── Frontier computation ────────────────────────────────────────────────────── - -/// Enumerate all Pareto-optimal plans for the given workload. -/// -/// Only plans that meet the accuracy SLA are included. The returned slice is -/// sorted by weighted-sum score (lowest = best) according to `weights`. -/// -/// If `online_store` is `Some` the scoring uses EMA-blended costs; otherwise -/// it falls back to the static benchmark table. -pub fn pareto_frontier( - workload: &QueryWorkload, - wc: &WorkloadCharacteristics, - weights: ObjectiveWeights, - online_store: Option<&online_cost_model::OnlineMetricsStore>, -) -> Vec { - let table: HashMap = match online_store { - Some(s) => online_cost_model::effective_table(s), - None => benchmark_table_pub(), - }; - - let candidates = all_sketch_types(); - let rules = RulesPlanner::new(); - let mut points: Vec = Vec::new(); - - for st in candidates { - let params = default_sketch_params(&st, workload.accuracy_sla); - let (mode, window_duration) = select_window_strategy(workload); - - let mut plan = rules.plan(workload); - plan.agent_config.sketch_type = st.clone(); - plan.agent_config.sketch_params = params; - plan.agent_config.mode = mode; - plan.agent_config.window_duration = window_duration; - plan.backend_config.merge_sketch_type = st.clone(); - - // Apply delta decision using the cost table. - apply_delta(st.clone(), &mut plan, workload, wc, &table); - - let s = score_with(&plan, workload, &table); - if !s.meets_sla { continue; } - - points.push(ParetoPoint { - sketch_type: st, - bandwidth_bytes_per_sec: s.bandwidth_bytes_per_sec, - cpu_micros_per_sample: s.cpu_micros_per_sample, - memory_bytes: s.memory_bytes, - estimated_error: s.estimated_error, - meets_sla: s.meets_sla, - plan, - }); - } - - // Filter to Pareto-optimal subset. - let optimal = pareto_filter(&points); - - // Sort by weighted score. - let w = weights.normalised(); - let mut sorted = optimal; - sorted.sort_by(|a, b| { - weighted_score(a, w).partial_cmp(&weighted_score(b, w)) - .unwrap_or(std::cmp::Ordering::Equal) - }); - sorted -} - -/// Returns the single plan from `frontier` with the lowest weighted-sum score. -/// Returns `None` if the frontier is empty. -pub fn select_best(frontier: &[ParetoPoint], weights: ObjectiveWeights) -> Option<&ParetoPoint> { - let w = weights.normalised(); - frontier.iter().min_by(|a, b| { - weighted_score(a, w).partial_cmp(&weighted_score(b, w)) - .unwrap_or(std::cmp::Ordering::Equal) - }) -} - -// ── Private helpers ─────────────────────────────────────────────────────────── - -fn all_sketch_types() -> Vec { - vec![ - SketchType::DDSketch, - SketchType::KLL, - SketchType::HLL, - SketchType::CountSketch, - SketchType::CountMinSketch, - ] -} - -fn apply_delta( - st: SketchType, - plan: &mut CollectionPlan, - w: &QueryWorkload, - wc: &WorkloadCharacteristics, - table: &HashMap, -) { - let bytes_per_series_per_sec = table - .get(&st) - .map(|c| c.bytes_per_series_per_sec) - .unwrap_or(200.0); - - let (decision, summary) = decide_delta(plan, w, wc, bytes_per_series_per_sec); - - match &decision { - DeltaDecision::UseDelta { threshold, .. } => { - plan.agent_config.delta_transmission = true; - plan.agent_config.delta_threshold = *threshold; - } - _ => { - plan.agent_config.delta_transmission = false; - plan.agent_config.delta_threshold = 0.0; - } - } - plan.delta_decision = decision; - plan.transmission_cost_summary = summary; -} - -fn pareto_filter(points: &[ParetoPoint]) -> Vec { - points - .iter() - .filter(|p| { - !points.iter().any(|q| { - q.bandwidth_bytes_per_sec <= p.bandwidth_bytes_per_sec - && q.cpu_micros_per_sample <= p.cpu_micros_per_sample - && q.memory_bytes <= p.memory_bytes - && (q.bandwidth_bytes_per_sec < p.bandwidth_bytes_per_sec - || q.cpu_micros_per_sample < p.cpu_micros_per_sample - || q.memory_bytes < p.memory_bytes) - }) - }) - .cloned() - .collect() -} - -fn weighted_score(p: &ParetoPoint, w: ObjectiveWeights) -> f64 { - w.bandwidth * p.bandwidth_bytes_per_sec - + w.cpu * p.cpu_micros_per_sample - + w.memory * p.memory_bytes -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use std::collections::HashMap; - use std::time::Duration; - - fn quantile_workload() -> QueryWorkload { - QueryWorkload { - metric_name: "latency".into(), - label_filters: HashMap::new(), - group_by_labels: vec![], - aggregations: vec![AggType::Quantile], - time_window: Duration::from_secs(300), - repeat_every: None, - accuracy_sla: 0.02, - latency_sla: None, - sketch_type_override: None, - exact_required: false, - quantiles: vec![], - } - } - - fn default_wc() -> WorkloadCharacteristics { WorkloadCharacteristics::default() } - - #[test] - fn quantile_frontier_non_empty_and_kll_present() { - // KLL has lower BW, CPU, and memory than DDSketch, so DDSketch is - // dominated and excluded. KLL and HLL form the frontier (HLL has - // lower BW/CPU but higher memory than KLL). - let f = pareto_frontier( - &quantile_workload(), &default_wc(), - ObjectiveWeights::default(), None, - ); - assert!(!f.is_empty(), "frontier should not be empty for quantile workload"); - let types: Vec<_> = f.iter().map(|p| &p.sketch_type).collect(); - assert!(types.contains(&&SketchType::KLL), "KLL expected in frontier"); - } - - #[test] - fn all_frontier_points_meet_sla() { - let f = pareto_frontier( - &quantile_workload(), &default_wc(), - ObjectiveWeights::default(), None, - ); - for p in &f { - assert!(p.meets_sla, "{} does not meet SLA (error={})", p.sketch_type, p.estimated_error); - } - } - - #[test] - fn bandwidth_weight_selects_lowest_bw() { - let f = pareto_frontier( - &quantile_workload(), &default_wc(), - ObjectiveWeights { bandwidth: 1.0, cpu: 0.0, memory: 0.0 }, None, - ); - let best = select_best(&f, ObjectiveWeights { bandwidth: 1.0, cpu: 0.0, memory: 0.0 }).unwrap(); - for p in &f { - assert!( - p.bandwidth_bytes_per_sec >= best.bandwidth_bytes_per_sec, - "best ({}) should have minimum bandwidth", best.sketch_type - ); - } - } - - #[test] - fn memory_weight_selects_lowest_memory() { - let f = pareto_frontier( - &quantile_workload(), &default_wc(), - ObjectiveWeights { bandwidth: 0.0, cpu: 0.0, memory: 1.0 }, None, - ); - let best = select_best(&f, ObjectiveWeights { bandwidth: 0.0, cpu: 0.0, memory: 1.0 }).unwrap(); - for p in &f { - assert!( - p.memory_bytes >= best.memory_bytes, - "best ({}) should have minimum memory", best.sketch_type - ); - } - } - - #[test] - fn select_best_empty_returns_none() { - let result = select_best(&[], ObjectiveWeights::default()); - assert!(result.is_none()); - } - - #[test] - fn tight_sla_excludes_inaccurate_sketches() { - let w = QueryWorkload { - accuracy_sla: 0.001, // very tight — only DDSketch at 0.1% accuracy can meet this - ..quantile_workload() - }; - let f = pareto_frontier(&w, &default_wc(), ObjectiveWeights::default(), None); - for p in &f { - assert!(p.estimated_error <= 0.001, - "{} error {} exceeds 0.001 SLA", p.sketch_type, p.estimated_error); - } - } - - #[test] - fn pareto_no_dominated_points() { - let f = pareto_frontier( - &quantile_workload(), &default_wc(), - ObjectiveWeights::default(), None, - ); - // Verify no point in f is dominated by another. - for i in 0..f.len() { - for j in 0..f.len() { - if i == j { continue; } - let a = &f[i]; - let b = &f[j]; - let b_dominates_a = - b.bandwidth_bytes_per_sec <= a.bandwidth_bytes_per_sec - && b.cpu_micros_per_sample <= a.cpu_micros_per_sample - && b.memory_bytes <= a.memory_bytes - && (b.bandwidth_bytes_per_sec < a.bandwidth_bytes_per_sec - || b.cpu_micros_per_sample < a.cpu_micros_per_sample - || b.memory_bytes < a.memory_bytes); - assert!(!b_dominates_a, "{} dominates {} but both are in frontier", - b.sketch_type, a.sketch_type); - } - } - } -} diff --git a/controller/src/planner/rules.rs b/controller/src/planner/rules.rs deleted file mode 100644 index fa55400e..00000000 --- a/controller/src/planner/rules.rs +++ /dev/null @@ -1,842 +0,0 @@ -use chrono::Utc; -use std::time::Duration; - -use crate::types::*; - -pub const DEFAULT_VALID_FOR: Duration = Duration::from_secs(10 * 60); - -/// Env-var that opts the planner into the typed L4 binding path -/// (`sketch_algebra::bind_query_expr`). Additive — when unset, the -/// existing untyped `algebra::directory::sketch_type_for_agg` path runs -/// unchanged. Phase E (stage_split refactor) is the natural migration -/// point at which the typed path becomes the only path. -/// -/// Set `USE_TYPED_SKETCH_ALGEBRA=1` to opt in. -#[allow(dead_code)] -pub const ENV_USE_TYPED_SKETCH_ALGEBRA: &str = "USE_TYPED_SKETCH_ALGEBRA"; - -/// Whether the typed L4 binding path is enabled for this process. -/// Reads the env var once per call (cheap; called per `plan()` invocation -/// at most). Phase C is additive — both code paths produce the same -/// `CollectionPlan` shape; the typed path is a *parallel* binding that -/// the planner can compare against the legacy path during development. -#[allow(dead_code)] -pub fn typed_sketch_algebra_enabled() -> bool { - matches!( - std::env::var(ENV_USE_TYPED_SKETCH_ALGEBRA).as_deref(), - Ok("1") | Ok("true") | Ok("yes") - ) -} - -/// Bind a `QueryWorkload` into the typed L4 [`crate::sketch_algebra::SketchExpr`] -/// IR, when callers want to inspect the typed binding alongside the -/// legacy `CollectionPlan` output. -/// -/// **Family-per-metric picker** (issue #46, MVP demo). The contract pins -/// six metric→family rows (`http_latency_ms` → DDSketch, `request_size_bytes` -/// → KLL, `unique_users_per_min` → HLL, `top_endpoint_qps` → CountSketch, -/// `endpoint_request_freq` → CMS, `http_requests_total` → raw). The -/// matching is done by [`sketch_algebra::capability_matching:: -/// classify_demo_metric`] for the contract rows, and falls back to the -/// `AggType`-driven default (Quantile→DDSketch, Cardinality→HLL, -/// Frequency→CMS) for any other metric name. -/// -/// **`sketch_type_override` wins.** When the workload-spec carries a -/// `sketch_type_override`, that field bypasses the capability-matched -/// default and pins the family directly (modulo `(sketch, statistic)` -/// validity — an override that violates the catalog is rejected and the -/// fallback path runs). This is the spec's `sketch_family_override` -/// behaviour. -/// -/// **`SumRateCount` declines.** Metrics whose contract row is "raw -/// passthrough" (`http_requests_total`) return `None` — the typed path -/// has no sketch to bind, and the legacy `plan()` path produces the -/// raw-passthrough `CollectionPlan`. -/// -/// Returns `None` when the workload shape is not yet supported by the -/// typed path (multi-intent, exact-required, raw-passthrough metric, or -/// no aggregations) — the caller should then fall back to the legacy -/// `plan()` output. -/// -/// Phase B (MVP v6) wires `main::handle_plan` to call this whenever -/// the parallel `USE_TYPED_STAGE_SPLIT` gate is enabled — the bound -/// `SketchExpr` is then fed into `planner::stage_split::split_typed_three_stage` -/// + the per-stage emitters in `config::stage_config`. -pub fn bind_workload_typed( - w: &QueryWorkload, -) -> Option { - use crate::intent_algebra::{AggIntent as L3AggIntent, QueryExpr, Schema, Source, WindowKind}; - use crate::intent_algebra::schema::{Column, DataType}; - use crate::sketch_algebra::capability_matching::{ - classify_demo_metric, is_valid_pair, pick_family, AccuracyPreference, StatisticClass, - }; - use crate::sketch_algebra::params::SketchKind; - use crate::sketch_algebra::rules::{ - bind_cms_count::BindCmsOnCount, bind_cms_topk::BindCountSketchOnTopK, - bind_ddsketch_quantile::BindDDSketchOnQuantile, bind_hll_cardinality::BindHllOnCardinality, - bind_kll_quantile::BindKllOnQuantile, Rule, - }; - use crate::types_v2::AccuracyTarget; - - // Contract-row metrics (`classify_demo_metric` returns `Some`) and - // operator-supplied overrides both signal "this metric must be - // sketched". The parser's `exact_required` flag — set when a query - // bottoms out at a bare VectorSelector → `AggFunc::Sum`, or carries - // a `Sum`/`Rate`/`Increase`/`Delta` (e.g. `rate(metric[5m])`, - // `count(metric)` whose inner walk synthesizes a `Sum` over the - // VectorSelector) — must not short-circuit those signals. Without - // this carve-out, MVP §46 entries 5–8 (`unique_users_per_min` / - // `top_endpoint_qps` / `endpoint_request_freq`) parse to - // `exact_required: true` and the typed binder declines, so the - // 5-sketch routing emitter never sees them. - let metric_is_contract_row = - crate::sketch_algebra::capability_matching::classify_demo_metric(&w.metric_name).is_some(); - let operator_pinned_sketch = w.sketch_type_override.is_some(); - if w.exact_required && !metric_is_contract_row && !operator_pinned_sketch { - return None; - } - if w.aggregations.len() != 1 { - return None; - } - - // ── Pick the (statistic class, accuracy preference) ────────────── - // - // Priority: workload-spec metric-name match → AggType-driven - // default. The metric-name match owns the demo contract rows; the - // AggType fallback covers everything else. - let (statistic, accuracy_pref) = classify_demo_metric(&w.metric_name) - .unwrap_or_else(|| match w.aggregations[0] { - AggType::Quantile => (StatisticClass::Quantile, AccuracyPreference::RelativeError), - AggType::Cardinality => (StatisticClass::Cardinality, AccuracyPreference::default()), - AggType::Frequency => (StatisticClass::Frequency, AccuracyPreference::default()), - }); - - // SumRateCount → no sketch (raw passthrough). Decline the typed - // binding so the caller falls back to the legacy raw plan. - if statistic == StatisticClass::SumRateCount { - return None; - } - - // ── Resolve the SketchKind (override > capability-matched default) ─ - // - // The workload-spec's `sketch_type_override` (= the spec's - // `sketch_family_override` per orchestrator contract) wins over the - // capability-matched pick, *provided* the override is valid for the - // statistic class. An invalid override (e.g. HLL for a Quantile - // workload) is silently dropped — the catalog-default family runs - // instead so the binding never produces a nonsense (sketch, stat) - // pair. - let override_kind: Option = w - .sketch_type_override - .as_ref() - .map(|st| SketchKind::from(st.clone())); - let kind = match override_kind { - Some(k) if is_valid_pair(k.clone(), statistic) => k, - _ => pick_family(statistic, accuracy_pref)?, - }; - - // QueryWorkload::accuracy_sla in the legacy planner is interpreted - // directly as the ε bound (e.g. `0.01` ⇒ ε=0.01). The L3/L4 typed - // form is `AccuracyTarget::Epsilon(eps)` with the same semantic. - let accuracy = if w.accuracy_sla > 0.0 { - AccuracyTarget::Epsilon(w.accuracy_sla) - } else { - AccuracyTarget::Exact - }; - let intent_accuracy = accuracy.clone(); - - // Build the matching L3 `AggIntent` for the picked statistic class. - // TopK lacks an `AggType` enum entry today (the MVP-contract - // top_endpoint_qps metric is name-classified, not AggType-derived), - // so we synthesize a default k=10 — the same value the legacy - // PromQL `topk(10, …)` lowering uses. - let intent = match statistic { - StatisticClass::Quantile => L3AggIntent::Quantile { - q: w.quantiles.first().copied().unwrap_or(0.99), - accuracy: intent_accuracy, - }, - StatisticClass::Cardinality => L3AggIntent::Cardinality { - accuracy: intent_accuracy, - }, - StatisticClass::Frequency => L3AggIntent::Frequency { - accuracy: intent_accuracy, - }, - StatisticClass::TopK => L3AggIntent::TopK { - k: 10, - accuracy: intent_accuracy, - }, - // SumRateCount handled above (early return). - StatisticClass::SumRateCount => unreachable!(), - }; - - let scan = QueryExpr::Scan { - source: Source::TimeSeries { - metric: w.metric_name.clone(), - }, - label_filters: w - .label_filters - .iter() - .map(|(k, v)| crate::intent_algebra::LabelFilter { - label: k.clone(), - equals: v.clone(), - }) - .collect(), - schema: Schema::with_time_index( - vec![ - Column { - name: "ts".into(), - dtype: DataType::Timestamp, - nullable: false, - }, - Column { - name: "value".into(), - dtype: DataType::Float64, - nullable: false, - }, - ], - 0, - vec![vec![0]], - ), - }; - let windowed = QueryExpr::Window { - kind: WindowKind::Sliding, - size: w.time_window, - slide: None, - child: Box::new(scan), - }; - let aggregate = QueryExpr::Aggregate { - by: vec![], - aggs: vec![intent], - having: None, - child: Box::new(windowed), - }; - - // ── Drive the picked family-specific `Bind*` rule ───────────────── - // - // Bypass the priority-based dispatcher: we have a definitive family - // pick from the capability matrix (or the `sketch_type_override`), - // so route directly to the rule that produces that family. This - // keeps the contract-row mapping deterministic — the dispatcher's - // tie-break (DDSketch p=6 vs KLL p=5) cannot accidentally flip - // `request_size_bytes`'s KLL pick to DDSketch. - // - // CMS+TopK note: when the picker selected `SketchKind::Cms` for a - // TopK statistic (only reachable today via a `sketch_family_override: - // CountMinSketch` on a TopK metric), we emit a CMS-with-heap binding - // inline. The CMS-Heap pattern (Cormode & Muthukrishnan 2005) gives - // a valid heavy-hitter sketch; the unbiased CountSketch remains the - // canonical pick when no override is supplied. The backend's "top-K - // from CountMin state" readout path is a separate workstream — see - // `sketch_algebra::capability_matching` module docs for the gap note. - match (kind, statistic) { - (SketchKind::DDSketch, _) => BindDDSketchOnQuantile.apply(&aggregate, &accuracy), - (SketchKind::Kll, _) => BindKllOnQuantile.apply(&aggregate, &accuracy), - (SketchKind::Hll, _) => BindHllOnCardinality.apply(&aggregate, &accuracy), - (SketchKind::CountSketch, _) => BindCountSketchOnTopK.apply(&aggregate, &accuracy), - (SketchKind::Cms, StatisticClass::TopK) => { - bind_cms_with_heap_on_topk(&aggregate, &accuracy) - } - (SketchKind::Cms, _) => BindCmsOnCount.apply(&aggregate, &accuracy), - } -} - -/// Bind `Aggregate{TopK{k, accuracy}}` to a CMS-with-heap sketch — the -/// CMS-Heap pattern from Cormode & Muthukrishnan (2005). Mirrors the -/// `(eps, delta) → (w, d)` mapping used by `BindCmsOnCount` and the -/// `with_heap` flag pattern from `BindCountSketchOnTopK`. CountSketch -/// remains the canonical (unbiased) TopK pick; this binder fires only -/// when a workload override has explicitly selected `CountMinSketch` for -/// a TopK metric. -fn bind_cms_with_heap_on_topk( - expr: &crate::intent_algebra::QueryExpr, - accuracy: &crate::types_v2::AccuracyTarget, -) -> Option { - use crate::intent_algebra::{AggIntent, QueryExpr}; - use crate::sketch_algebra::params::{CmsParams, SketchKind, SketchParams}; - use crate::sketch_algebra::sketch_expr::{EstimateOp, SketchExpr}; - use crate::types_v2::AccuracyTarget; - - let (k_topk, intent_accuracy, child) = match expr { - QueryExpr::Aggregate { - aggs, child, by, .. - } if aggs.len() == 1 && by.is_empty() => match &aggs[0] { - AggIntent::TopK { k, accuracy } => (*k, accuracy.clone(), child), - _ => return None, - }, - _ => return None, - }; - - if k_topk == 0 { - return None; - } - - let (eps, delta) = match (accuracy, &intent_accuracy) { - (AccuracyTarget::Exact, _) | (_, AccuracyTarget::Exact) => return None, - (AccuracyTarget::Epsilon(a), AccuracyTarget::Epsilon(b)) => (a.min(*b), 0.01), - (AccuracyTarget::Epsilon(a), AccuracyTarget::EpsilonDelta { eps, delta }) - | (AccuracyTarget::EpsilonDelta { eps, delta }, AccuracyTarget::Epsilon(a)) => { - (a.min(*eps), *delta) - } - ( - AccuracyTarget::EpsilonDelta { eps: a, delta: da }, - AccuracyTarget::EpsilonDelta { eps: b, delta: db }, - ) => (a.min(*b), da.min(*db)), - }; - - if eps <= 0.0 || delta <= 0.0 || delta >= 1.0 { - return None; - } - - let w = (std::f64::consts::E / eps).ceil() as u32; - let d = (1.0 / delta).ln().ceil() as u32; - let w = w.max(2); - let d = d.max(1); - - Some(SketchExpr::estimate_over_agg( - EstimateOp::TopK { k: k_topk }, - SketchKind::Cms, - SketchParams::Cms(CmsParams { w, d }), - (**child).clone(), - )) -} - -pub struct RulesPlanner { - pub valid_for: Duration, - pub sketch_defaults: SketchDefaults, -} - -impl RulesPlanner { - pub fn new() -> Self { - Self { - valid_for: DEFAULT_VALID_FOR, - sketch_defaults: SketchDefaults::default(), - } - } - - pub fn with_defaults(defaults: SketchDefaults) -> Self { - Self { - valid_for: DEFAULT_VALID_FOR, - sketch_defaults: defaults, - } - } - - pub fn plan(&self, w: &QueryWorkload) -> CollectionPlan { - // When exact computation is required (RSI, MACD, stateful indicators), - // skip sketch selection and return a raw-passthrough plan. - if w.exact_required { - return self.raw_passthrough_plan(w); - } - - let sketch_type = crate::algebra::directory::sketch_type_for_agg(&w.aggregations); - let sketch_params = crate::algebra::directory::build_sketch_params(&self.sketch_defaults, &sketch_type, w.accuracy_sla, &w.quantiles); - let (mode, window_duration) = select_window_strategy(w); - - let mut aggregate_by = w.group_by_labels.clone(); - aggregate_by.sort(); - - let mut label_matchers: Vec = w - .label_filters - .iter() - .map(|(k, v)| format!("{k}={v}")) - .collect(); - label_matchers.sort(); - - let backend_sketch = sketch_type.clone(); - let group_by = aggregate_by.clone(); - - let valid_until = Utc::now() + chrono::Duration::seconds(self.valid_for.as_secs() as i64); - - CollectionPlan { - agent_config: AgentCollectorConfig { - output_mode: OutputMode::Sketch, - sketch_type, - sketch_params, - aggregate_by, - label_matchers, - window_duration, - mode, - enable_self_monitoring: true, - transmit_sketch: false, - drop_original: true, - // Delta fields are left as disabled defaults here; the - // CostModelPlanner overwrites them via decide_delta(). - delta_transmission: false, - delta_threshold: 0.0, - enable_series_id: true, - series_id_ttl_secs: 0, - - data_sink: AgentDataSink::default(), - }, - gateway_config: GatewayCollectorConfig { passthrough: true }, - backend_config: BackendCollectorConfig { - merge_sketch_type: backend_sketch, - group_by, - }, - precompute: vec![], - valid_until, - delta_decision: DeltaDecision::default(), - transmission_cost_summary: TransmissionCostSummary::default(), - staged_plan: None, - } - } - - /// Returns a raw-passthrough plan for queries that require exact per-sample - /// computation (RSI, MACD, stochastic oscillator, etc.). - fn raw_passthrough_plan(&self, w: &QueryWorkload) -> CollectionPlan { - let valid_until = Utc::now() - + chrono::Duration::seconds(self.valid_for.as_secs() as i64); - - let mut label_matchers: Vec = w - .label_filters - .iter() - .map(|(k, v)| format!("{k}={v}")) - .collect(); - label_matchers.sort(); - - CollectionPlan { - agent_config: AgentCollectorConfig { - output_mode: OutputMode::Raw, - sketch_type: SketchType::DDSketch, // unused for raw mode - sketch_params: SketchParams::default(), - aggregate_by: vec![], - label_matchers, - window_duration: None, - mode: ProcessorMode::Batch, - enable_self_monitoring: true, - transmit_sketch: false, - drop_original: false, - delta_transmission: false, - delta_threshold: 0.0, - enable_series_id: true, - series_id_ttl_secs: 0, - - data_sink: AgentDataSink::default(), - }, - gateway_config: GatewayCollectorConfig { passthrough: true }, - backend_config: BackendCollectorConfig { - merge_sketch_type: SketchType::DDSketch, - group_by: vec![], - }, - precompute: vec![], - valid_until, - delta_decision: DeltaDecision::default(), - transmission_cost_summary: TransmissionCostSummary::default(), - staged_plan: None, - } - } -} - -// ── Sketch selection (delegated to algebra::directory) ─────────────────────── - -pub use crate::algebra::directory::{default_sketch_params, build_sketch_params}; - -// ── Window strategy ─────────────────────────────────────────────────────────── - -/// Decides processor mode. -/// -/// Rule: if `latency_sla >= time_window` (or unset) → window mode. -/// otherwise → batch mode (gateway/backend merges on query). -pub fn select_window_strategy(w: &QueryWorkload) -> (ProcessorMode, Option) { - match w.latency_sla { - None => (ProcessorMode::Window, Some(w.time_window)), - Some(ls) if ls >= w.time_window => (ProcessorMode::Window, Some(w.time_window)), - _ => (ProcessorMode::Batch, None), - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use std::collections::HashMap; - - fn workload(aggs: Vec) -> QueryWorkload { - QueryWorkload { - metric_name: "test".into(), - label_filters: HashMap::new(), - group_by_labels: vec![], - aggregations: aggs, - time_window: Duration::from_secs(300), - repeat_every: None, - accuracy_sla: 0.01, - latency_sla: None, - sketch_type_override: None, - exact_required: false, - quantiles: vec![], - } - } - - #[test] - fn quantile_selects_ddsketch() { - let plan = RulesPlanner::new().plan(&workload(vec![AggType::Quantile])); - assert_eq!(plan.agent_config.sketch_type, SketchType::DDSketch); - } - - #[test] - fn cardinality_selects_hll() { - let plan = RulesPlanner::new().plan(&workload(vec![AggType::Cardinality])); - assert_eq!(plan.agent_config.sketch_type, SketchType::HLL); - } - - #[test] - fn frequency_selects_countsketch() { - let plan = RulesPlanner::new().plan(&workload(vec![AggType::Frequency])); - assert_eq!(plan.agent_config.sketch_type, SketchType::CountSketch); - } - - #[test] - fn quantile_priority_wins() { - let plan = - RulesPlanner::new().plan(&workload(vec![AggType::Quantile, AggType::Cardinality])); - assert_eq!( - plan.agent_config.sketch_type, - SketchType::DDSketch, - "quantile should take priority over cardinality" - ); - } - - #[test] - fn window_mode_when_latency_geq_time_window() { - let mut w = workload(vec![AggType::Quantile]); - w.latency_sla = Some(Duration::from_secs(600)); // 10m >= 5m - let plan = RulesPlanner::new().plan(&w); - assert_eq!(plan.agent_config.mode, ProcessorMode::Window); - assert_eq!( - plan.agent_config.window_duration, - Some(Duration::from_secs(300)) - ); - } - - #[test] - fn batch_mode_when_latency_lt_time_window() { - let mut w = workload(vec![AggType::Quantile]); - w.latency_sla = Some(Duration::from_secs(60)); // 1m < 5m - let plan = RulesPlanner::new().plan(&w); - assert_eq!(plan.agent_config.mode, ProcessorMode::Batch); - assert_eq!(plan.agent_config.window_duration, None); - } - - #[test] - fn no_latency_sla_defaults_to_window() { - let mut w = workload(vec![AggType::Quantile]); - w.latency_sla = None; - let plan = RulesPlanner::new().plan(&w); - assert_eq!(plan.agent_config.mode, ProcessorMode::Window); - } - - #[test] - fn aggregate_by_sorted() { - let mut w = workload(vec![AggType::Quantile]); - w.group_by_labels = vec!["zone".into(), "host.name".into(), "service".into()]; - let plan = RulesPlanner::new().plan(&w); - assert_eq!( - plan.agent_config.aggregate_by, - vec!["host.name", "service", "zone"] - ); - } - - #[test] - fn label_matchers_from_filters() { - let mut w = workload(vec![AggType::Quantile]); - w.label_filters = [ - ("env".into(), "prod".into()), - ("service".into(), "web".into()), - ] - .into(); - let plan = RulesPlanner::new().plan(&w); - assert_eq!(plan.agent_config.label_matchers.len(), 2); - } - - #[test] - fn ddsketch_accuracy_params() { - let mut w = workload(vec![AggType::Quantile]); - w.accuracy_sla = 0.005; - let plan = RulesPlanner::new().plan(&w); - match &plan.agent_config.sketch_params { - SketchParams::DDSketch { relative_accuracy, .. } => assert_eq!(*relative_accuracy, 0.005), - other => panic!("expected DDSketch, got {:?}", other), - } - } - - #[test] - fn hll_precision_coarse_sla() { - let mut w = workload(vec![AggType::Cardinality]); - w.accuracy_sla = 0.03; - let plan = RulesPlanner::new().plan(&w); - match &plan.agent_config.sketch_params { - SketchParams::HLL { precision } => assert_eq!(*precision, 10, "coarse SLA should use lower precision"), - other => panic!("expected HLL, got {:?}", other), - } - } - - #[test] - fn valid_until_in_future() { - let plan = RulesPlanner::new().plan(&workload(vec![AggType::Quantile])); - assert!( - plan.valid_until > Utc::now(), - "valid_until should be in the future" - ); - } - - #[test] - fn backend_config_matches_sketch_type() { - let plan = RulesPlanner::new().plan(&workload(vec![AggType::Quantile])); - assert_eq!( - plan.backend_config.merge_sketch_type, - plan.agent_config.sketch_type - ); - } - - #[test] - fn gateway_passthrough() { - let plan = RulesPlanner::new().plan(&workload(vec![AggType::Quantile])); - assert!(plan.gateway_config.passthrough); - } - - // ── Family-per-metric tests (issue #46 MVP demo contract) ───────────────── - // - // The shared MVP demo contract pins six metric→family rows. These tests - // drive each row through `bind_workload_typed` and assert the bound - // `SketchExpr` carries the expected sketch family. The contract: - // - // | metric | family | - // |-------------------------|--------------| - // | `http_requests_total` | raw (None) | - // | `http_latency_ms` | DDSketch | - // | `request_size_bytes` | KLL | - // | `unique_users_per_min` | HLL | - // | `top_endpoint_qps` | CountSketch | - // | `endpoint_request_freq` | CMS | - - use crate::sketch_algebra::params::SketchKind; - use crate::sketch_algebra::sketch_expr::SketchExpr; - - /// Walk the L4 binding output and pull out the `SketchAgg`'s family. - /// Returns `None` if no `SketchAgg` node is present (raw / pure - /// logical pass-through). - fn extract_family(expr: &SketchExpr) -> Option { - match expr { - SketchExpr::SketchAgg { sketch_type, .. } => Some(sketch_type.clone()), - SketchExpr::SketchEstimate { child, .. } => extract_family(child), - SketchExpr::SketchMerge { children, .. } => { - children.iter().find_map(extract_family) - } - SketchExpr::LetBinding { expr, child, .. } => { - extract_family(expr).or_else(|| extract_family(child)) - } - SketchExpr::Logical(_) | SketchExpr::Ref { .. } => None, - SketchExpr::RawAtEdgeSketchAtBackend { family, .. } => Some(family.clone()), - SketchExpr::RawAtEdgePrometheusArchive { .. } => None, - } - } - - /// Build a workload with the given metric name + reasonable - /// AggType-driven default for the contract row. The metric-name match - /// in `classify_demo_metric` overrides the AggType for the - /// contract rows; the AggType still has to be a valid one (the enum - /// has no `TopK` variant, so for `top_endpoint_qps` we pass - /// `Frequency` and rely on the metric-name reclassification). - fn workload_for(metric: &str, agg: AggType) -> QueryWorkload { - QueryWorkload { - metric_name: metric.into(), - label_filters: HashMap::new(), - group_by_labels: vec![], - aggregations: vec![agg], - time_window: Duration::from_secs(300), - repeat_every: None, - accuracy_sla: 0.01, - latency_sla: None, - sketch_type_override: None, - exact_required: false, - quantiles: vec![], - } - } - - #[test] - fn typed_binding_http_requests_total_is_raw_passthrough() { - // Contract: `http_requests_total` → raw passthrough (no sketch). - // The typed path declines (`bind_workload_typed` returns `None`) - // so the caller falls back to the legacy raw plan. - let w = workload_for("http_requests_total", AggType::Frequency); - let bound = bind_workload_typed(&w); - assert!( - bound.is_none(), - "http_requests_total should bind to None (raw passthrough); got {bound:?}", - ); - } - - #[test] - fn typed_binding_http_latency_ms_picks_ddsketch() { - // Contract: `http_latency_ms` → DDSketch (Quantile, rel-err). - let w = workload_for("http_latency_ms", AggType::Quantile); - let bound = bind_workload_typed(&w).expect("http_latency_ms must bind"); - assert_eq!( - extract_family(&bound), - Some(SketchKind::DDSketch), - "http_latency_ms should bind to DDSketch (Quantile, rel-err)", - ); - } - - #[test] - fn typed_binding_request_size_bytes_picks_kll() { - // Contract: `request_size_bytes` → KLL (Quantile, rank-err). - // Note: this is the rank-err preference flip — without the - // metric-name reclassification, the priority-based dispatcher - // would pick DDSketch (priority 6 > KLL priority 5). - let w = workload_for("request_size_bytes", AggType::Quantile); - let bound = bind_workload_typed(&w).expect("request_size_bytes must bind"); - assert_eq!( - extract_family(&bound), - Some(SketchKind::Kll), - "request_size_bytes should bind to KLL (Quantile, rank-err)", - ); - } - - #[test] - fn typed_binding_unique_users_per_min_picks_hll() { - // Contract: `unique_users_per_min` → HLL (Cardinality). - let w = workload_for("unique_users_per_min", AggType::Cardinality); - let bound = bind_workload_typed(&w).expect("unique_users_per_min must bind"); - assert_eq!( - extract_family(&bound), - Some(SketchKind::Hll), - "unique_users_per_min should bind to HLL (Cardinality)", - ); - } - - #[test] - fn typed_binding_top_endpoint_qps_picks_countsketch() { - // Contract: `top_endpoint_qps` → CountSketch (TopK). - // The metric-name reclassification reroutes from the AggType - // default (Frequency → CMS) to the contract row (TopK → - // CountSketch). - let w = workload_for("top_endpoint_qps", AggType::Frequency); - let bound = bind_workload_typed(&w).expect("top_endpoint_qps must bind"); - assert_eq!( - extract_family(&bound), - Some(SketchKind::CountSketch), - "top_endpoint_qps should bind to CountSketch (TopK)", - ); - } - - #[test] - fn typed_binding_endpoint_request_freq_picks_cms() { - // Contract: `endpoint_request_freq` → CMS (Frequency). - // The legacy AggType default for Frequency is *also* CMS via - // BindCmsOnCount, but the metric-name path goes through the - // capability-matching picker first — both produce CMS, the - // contract row ratifies it. - let w = workload_for("endpoint_request_freq", AggType::Frequency); - let bound = bind_workload_typed(&w).expect("endpoint_request_freq must bind"); - assert_eq!( - extract_family(&bound), - Some(SketchKind::Cms), - "endpoint_request_freq should bind to CMS (Frequency)", - ); - } - - // ── sketch_type_override (= sketch_family_override) wins ────────────────── - - #[test] - fn sketch_type_override_pins_kll_for_quantile_metric() { - // `http_latency_ms`'s contract row is DDSketch, but a workload - // override of `KLL` must win — both KLL and DDSketch are valid - // for Quantile per the capability matrix, so the override is - // honoured. - let mut w = workload_for("http_latency_ms", AggType::Quantile); - w.sketch_type_override = Some(SketchType::KLL); - let bound = bind_workload_typed(&w).expect("override should still bind"); - assert_eq!( - extract_family(&bound), - Some(SketchKind::Kll), - "sketch_type_override=KLL should pin KLL despite the contract's DDSketch default", - ); - } - - #[test] - fn sketch_type_override_pins_ddsketch_for_quantile_metric() { - // `request_size_bytes`'s contract row is KLL (rank-err); a - // workload override of `DDSketch` flips it back to DDSketch. - let mut w = workload_for("request_size_bytes", AggType::Quantile); - w.sketch_type_override = Some(SketchType::DDSketch); - let bound = bind_workload_typed(&w).expect("override should still bind"); - assert_eq!( - extract_family(&bound), - Some(SketchKind::DDSketch), - "sketch_type_override=DDSketch should pin DDSketch despite the contract's KLL default", - ); - } - - #[test] - fn planner_accepts_countmin_override_for_topk_metric() { - // CMS-Heap pattern (Cormode & Muthukrishnan 2005): when a - // workload's `sketch_family_override` (= - // `sketch_type_override`) selects CountMinSketch for a TopK - // metric, the planner should accept it instead of falling - // back to the canonical CountSketch default. - let mut w = workload_for("top_endpoint_qps", AggType::Frequency); - w.sketch_type_override = Some(SketchType::CountMinSketch); - let bound = bind_workload_typed(&w) - .expect("CountMin override on a TopK metric should still bind"); - assert_eq!( - extract_family(&bound), - Some(SketchKind::Cms), - "sketch_type_override=CountMinSketch on a TopK metric should pin CMS", - ); - } - - #[test] - fn planner_default_for_topk_remains_countsketch() { - // Without any override, the canonical pick for a TopK metric - // stays CountSketch — CMS-Heap is opt-in via override only. - let w = workload_for("top_endpoint_qps", AggType::Frequency); - let bound = bind_workload_typed(&w).expect("top_endpoint_qps must bind"); - assert_eq!( - extract_family(&bound), - Some(SketchKind::CountSketch), - "default TopK pick must remain CountSketch (unbiased estimator)", - ); - } - - #[test] - fn invalid_sketch_type_override_falls_back_to_default() { - // HLL is NOT valid for a Quantile statistic — the capability - // matrix rejects the override, and the planner falls back to - // the contract-row default (DDSketch for `http_latency_ms`). - let mut w = workload_for("http_latency_ms", AggType::Quantile); - w.sketch_type_override = Some(SketchType::HLL); - let bound = bind_workload_typed(&w).expect("fallback should bind"); - assert_eq!( - extract_family(&bound), - Some(SketchKind::DDSketch), - "invalid (HLL, Quantile) override should be rejected; planner falls back to DDSketch", - ); - } - - // ── Combined sweep: all 6 contract rows in one shot ─────────────────────── - - #[test] - fn all_six_contract_metrics_produce_expected_family() { - // Single test that drives the full contract row set through - // `bind_workload_typed` — this is the per-task acceptance test - // ("verify each produces the expected `SketchExpr` family"). - let cases: Vec<(&str, AggType, Option)> = vec![ - ("http_requests_total", AggType::Frequency, None), - ("http_latency_ms", AggType::Quantile, Some(SketchKind::DDSketch)), - ("request_size_bytes", AggType::Quantile, Some(SketchKind::Kll)), - ("unique_users_per_min", AggType::Cardinality, Some(SketchKind::Hll)), - ("top_endpoint_qps", AggType::Frequency, Some(SketchKind::CountSketch)), - ("endpoint_request_freq", AggType::Frequency, Some(SketchKind::Cms)), - ]; - for (metric, agg, expected) in cases { - let w = workload_for(metric, agg); - let bound = bind_workload_typed(&w); - let got = bound.as_ref().and_then(extract_family); - assert_eq!( - got, expected, - "metric {metric}: expected family {expected:?}, got {got:?}", - ); - } - } -} diff --git a/controller/src/planner/stage_split.rs b/controller/src/planner/stage_split.rs deleted file mode 100644 index 3a891b32..00000000 --- a/controller/src/planner/stage_split.rs +++ /dev/null @@ -1,1055 +0,0 @@ -//! SP-9: AST-aware hierarchical stage assignment. -//! -//! Splits an optimised [`QueryExpr`] tree across pipeline stages, emitting a -//! [`StagedPlan`] that carries per-stage sub-plans for: -//! -//! | Stage | Nodes | -//! |---|---| -//! | Agent OTel Collector | `Source`, `Filter`, `Window`, `SketchAgg` (sketch ops) | -//! | Backend OTel Collector | `Partition`, `Merge`, `Dedup`, `Aggregate { Exact(Sum\|Count\|Min\|Max) }` | -//! | ASAPQuery Precompute Engine | `TopK`, `HistogramQuantile`, `PromQLSubquery`, deferred sketch ops | -//! | DB-side query | `Aggregate { Avg }` (non-mergeable) | -//! -//! # ExactAgg / AggFunc deferral -//! -//! Mergeability drives `Exact` op placement: -//! - `Sum`, `Count`, `Min`, `Max` — mergeable (`agg(A∪B) = merge(agg(A), agg(B))`) → -//! **Backend** (the backend collector can combine partial results from N agents). -//! - `Avg`, `StdDev`, `Variance` — **not** mergeable → **DB-side** query. -//! -//! # Budget-driven deferral chain -//! -//! When a `SketchAgg` node's estimated memory cost exceeds the stage cap in -//! [`StageResourceBudgets`], it is deferred to the next stage: -//! -//! `Agent → Backend → Precompute` -//! -//! The degenerate-but-valid fallback (all sketch ops deferred to Precompute) -//! matches the current flat SP-3 behaviour and ensures the query is always -//! answerable. -//! -//! # PromQL serialisation -//! -//! [`expr_to_promql`] converts a [`QueryExpr`] tree to a valid PromQL expression -//! consumed by the ASAPQuery Precompute Engine's query engine. Unlike the old -//! flat-template approach, this uses a proper recursive descent so it handles -//! `histogram_quantile`, `PromQLSubquery`, and vector `BinaryOp` nodes natively. - -use std::time::Duration; - -use crate::algebra::expr::{AggFunc, BinaryOpKind, LiteralValue, QueryExpr, ScalarExpr}; -use crate::analyzer::format_duration; -use crate::algebra::expr::{AggIntent, ExactAgg}; -use crate::algebra::directory; -use crate::types::{ - AgentSubPlan, BackendSubPlan, - DbSubPlan, PrecomputeSubPlan, SketchParams, SketchType, - StagedPlan, StageResourceBudgets, -}; - -// ── Public entry point ──────────────────────────────────────────────────────── - -/// Split a [`QueryExpr`] tree across pipeline stages, respecting per-stage -/// resource budgets and the `AggFunc` mergeability rules described above. -/// -/// The returned [`StagedPlan`] is attached to -/// [`crate::types::CollectionPlan::staged_plan`] by the caller -/// (`handle_plan` in `main.rs`). -pub fn split_expr_by_stage(expr: &QueryExpr, budgets: &StageResourceBudgets) -> StagedPlan { - let mut plan = StagedPlan::default(); - walk(expr, &mut plan, budgets); - - // Build the precompute query_expr from the full tree when the precompute - // stage is active (TopK, HistogramQuantile, PromQLSubquery, or deferred ops). - if plan.precompute.active && plan.precompute.query_expr.is_empty() { - plan.precompute.query_expr = expr_to_promql(expr); - } - - // Build the DB query_expr when a non-mergeable agg was assigned to DB stage. - if plan.db.active && plan.db.query_expr.is_empty() { - plan.db.query_expr = expr_to_promql(expr); - } - - plan -} - -/// Serialise a [`QueryExpr`] tree to a valid PromQL expression string. -/// -/// The output is consumed by the ASAPQuery Precompute Engine's query engine. -/// Sketch data is already ingested from the Backend OTel Collector; the PromQL -/// describes the aggregation to apply over it. -/// -/// Uses recursive descent, so it handles `histogram_quantile`, -/// `PromQLSubquery`, and vector `BinaryOp` nodes that the old flat-template -/// could not represent. -pub fn expr_to_promql(expr: &QueryExpr) -> String { - let mut ctx = PromQLCtx::default(); - promql_from_qe(expr, &mut ctx) -} - -// ── Tree walker ─────────────────────────────────────────────────────────────── - -fn walk(expr: &QueryExpr, plan: &mut StagedPlan, budgets: &StageResourceBudgets) { - match expr { - // Source — always Agent; populate metric name. - QueryExpr::Source(_) => {} - - // Filter — push label predicates to Agent. - QueryExpr::Filter { pred, input } => { - collect_label_filters_into(pred, &mut plan.agent.label_filters); - walk(input, plan, budgets); - } - - // Window — time window lives at Agent. - QueryExpr::Window { duration, input, .. } => { - plan.agent.window_secs = Some(duration.as_secs()); - walk(input, plan, budgets); - } - - // SketchAgg — the key sketch assignment decision. - QueryExpr::SketchAgg { op, input, .. } => { - assign_sketch_agg(op, plan, budgets); - walk(input, plan, budgets); - } - - // WindowedAgg — bundles window + sketch agg intent. - QueryExpr::WindowedAgg { agg, window, input, .. } => { - if let crate::algebra::expr::WindowKind::Tumbling { size } = &window.kind { - plan.agent.window_secs = Some(size.as_secs()); - } - assign_sketch_agg(agg, plan, budgets); - walk(input, plan, budgets); - } - - // Partition — GROUP BY / `by (dims)` always assigned to Backend. - QueryExpr::Partition { keys, input } => { - for k in keys.keys() { - if !plan.backend.group_by.contains(k) { - plan.backend.group_by.push(k.clone()); - } - } - walk(input, plan, budgets); - } - - // Aggregate — SQL GROUP BY + agg functions. - // Mergeable aggs (Sum/Count/Min/Max) → Backend. - // Non-mergeable (Avg/StdDev/Variance) → Db. - // Sketchable (Quantile/CountDistinct/HeavyHitters) → treated as SketchAgg. - QueryExpr::Aggregate { keys, aggs, input, .. } => { - for k in keys { - if !plan.backend.group_by.contains(k) { - plan.backend.group_by.push(k.clone()); - } - } - for agg in aggs { - assign_agg_func(&agg.func, plan, budgets); - } - walk(input, plan, budgets); - } - - // Dedup — absorbed at Backend (HLL dedup elimination is upstream). - QueryExpr::Dedup { input, .. } => { - plan.backend.has_dedup = true; - walk(input, plan, budgets); - } - - // Merge — Backend merges N agent sketches. - QueryExpr::Merge { inputs } => { - plan.backend.has_merge = true; - for i in inputs { - walk(i, plan, budgets); - } - } - - // TopK — always at the Precompute Engine. - QueryExpr::TopK { k, input, .. } => { - plan.precompute.topk = Some(*k); - plan.precompute.active = true; - walk(input, plan, budgets); - } - - // Sort + Limit — maps to topk semantics at Precompute. - QueryExpr::Limit { n, input, .. } => { - plan.precompute.topk = Some(*n); - plan.precompute.active = true; - walk(input, plan, budgets); - } - QueryExpr::Sort { input, .. } => { - walk(input, plan, budgets); - } - - // HistogramQuantile — precompute engine applies it over ingested histograms. - QueryExpr::HistogramQuantile { input, .. } => { - plan.precompute.active = true; - walk(input, plan, budgets); - } - - // PromQLSubquery — precompute engine evaluates the sub-query. - QueryExpr::PromQLSubquery { input, .. } => { - plan.precompute.active = true; - walk(input, plan, budgets); - } - - // BinaryOp between two instant vectors — precompute evaluates. - QueryExpr::BinaryOp { lhs, rhs, .. } => { - plan.precompute.active = true; - walk(lhs, plan, budgets); - walk(rhs, plan, budgets); - } - - // JoinSketch — outer and inner both walked; join itself at Backend. - QueryExpr::JoinSketch { outer, inner, .. } => { - plan.backend.has_merge = true; - walk(outer, plan, budgets); - walk(inner, plan, budgets); - } - - // Join — Backend. - QueryExpr::Join { left, right, .. } => { - plan.backend.has_merge = true; - walk(left, plan, budgets); - walk(right, plan, budgets); - } - - // SetOp — treat as Backend merge. - QueryExpr::SetOp { left, right, .. } => { - plan.backend.has_merge = true; - walk(left, plan, budgets); - walk(right, plan, budgets); - } - - // Transparent / passthrough nodes — recurse into child. - QueryExpr::Project { input, .. } - | QueryExpr::Subquery { expr: input, .. } - | QueryExpr::WindowFunc { input, .. } => walk(input, plan, budgets), - - QueryExpr::LetBinding { expr, body, .. } => { - walk(expr, plan, budgets); - walk(body, plan, budgets); - } - - // Ref — nothing to assign (resolved externally). - QueryExpr::Ref(_) => {} - } -} - -// ── Agg assignment ──────────────────────────────────────────────────────────── - -fn assign_sketch_agg(op: &AggIntent, plan: &mut StagedPlan, budgets: &StageResourceBudgets) { - match op { - // Exact ops: mergeability decides stage. - AggIntent::Exact(ExactAgg::Sum | ExactAgg::Count | ExactAgg::Min | ExactAgg::Max) => { - plan.backend.has_merge = true; - } - AggIntent::Exact(ExactAgg::Avg) => { - plan.db.active = true; - } - // Sketch ops: resolve to physical, assign to Agent, defer if budget exceeded. - sketch_op => { - let physical = crate::algebra::physical::resolve(sketch_op); - let stage = resolve_sketch_stage(physical.estimated_memory_bytes, budgets, &mut plan.deferral_log, sketch_op); - match stage { - SketchStage::Agent => { - plan.agent.sketch_type = Some(physical.sketch_type); - plan.agent.sketch_params = physical.sketch_params; - } - SketchStage::Backend => { - plan.backend.has_merge = true; - } - SketchStage::Precompute => { - plan.precompute.active = true; - } - } - } - } -} - -fn assign_agg_func(func: &AggFunc, plan: &mut StagedPlan, budgets: &StageResourceBudgets) { - match func { - // Sketchable → synthesise the corresponding AggIntent and use existing logic. - AggFunc::Quantile(phi) => { - let op = AggIntent::default_quantile(vec![*phi]); - assign_sketch_agg(&op, plan, budgets); - } - AggFunc::CountDistinct => { - assign_sketch_agg(&AggIntent::default_cardinality(), plan, budgets); - } - AggFunc::HeavyHitters { .. } => { - assign_sketch_agg(&AggIntent::default_frequency(), plan, budgets); - } - // Mergeable exact → Backend. - AggFunc::Count | AggFunc::Sum | AggFunc::Min | AggFunc::Max - | AggFunc::Rate | AggFunc::Increase | AggFunc::Delta => { - plan.backend.has_merge = true; - } - // Non-mergeable → Db. - AggFunc::Avg | AggFunc::StdDev { .. } | AggFunc::Variance { .. } => { - plan.db.active = true; - } - AggFunc::Custom(_) => { - // Unknown; conservatively route to Precompute. - plan.precompute.active = true; - } - } -} - -// ── Budget deferral ─────────────────────────────────────────────────────────── - -#[derive(Debug, PartialEq)] -enum SketchStage { Agent, Backend, Precompute } - -fn resolve_sketch_stage( - est_mem: u64, - budgets: &StageResourceBudgets, - log: &mut Vec, - op: &AggIntent, -) -> SketchStage { - if let Some(cap) = budgets.agent_memory_bytes { - if est_mem > cap { - log.push(format!( - "deferred {op:?} Agent→Backend: est_mem={est_mem}B > agent_cap={cap}B" - )); - if let Some(be_cap) = budgets.backend_memory_bytes { - if est_mem > be_cap { - log.push(format!( - "deferred {op:?} Backend→Precompute: est_mem={est_mem}B > backend_cap={be_cap}B" - )); - return SketchStage::Precompute; - } - } - return SketchStage::Backend; - } - } - SketchStage::Agent -} - -// Sketch-type helpers delegated to algebra::directory. - -// ── PromQL serialiser — recursive descent ───────────────────────────────────── - -/// Mutable context threaded through the recursive descent. -/// Carries `group_by` and `window` that are "collected" from inner nodes -/// and applied at the enclosing aggregate. -#[derive(Default, Clone)] -struct PromQLCtx { - /// GROUP BY / `by (…)` labels gathered from `Partition` nodes above. - group_by: Vec, - /// Time window gathered from the innermost `Window` node. - window: Option, -} - -/// Recursive PromQL serialisation of a [`QueryExpr`] node. -/// -/// Returns the PromQL string fragment for this node. Inner nodes (Source, -/// Filter) return their selector string; outer nodes (SketchAgg, TopK, etc.) -/// wrap it. -fn promql_from_qe(expr: &QueryExpr, ctx: &mut PromQLCtx) -> String { - match expr { - // ── Leaf ───────────────────────────────────────────────────────────── - QueryExpr::Source(s) => s.name.clone(), - QueryExpr::Ref(name) => name.clone(), - - // ── Filter — append label matchers to the selector ─────────────────── - QueryExpr::Filter { pred, input } => { - let inner = promql_from_qe(input, ctx); - let matchers = scalar_to_label_matchers(pred); - if matchers.is_empty() { - inner - } else { - format!("{}{{{}}}", inner, matchers.join(", ")) - } - } - - // ── Window — store duration for use by enclosing aggregate ─────────── - QueryExpr::Window { duration, input, .. } => { - if ctx.window.is_none() { - ctx.window = Some(*duration); - } - promql_from_qe(input, ctx) - } - - // ── Partition — store group_by keys for enclosing aggregate ────────── - QueryExpr::Partition { keys, input } => { - for k in keys.keys() { - if !ctx.group_by.contains(k) { - ctx.group_by.push(k.clone()); - } - } - promql_from_qe(input, ctx) - } - - // ── WindowedAgg — bundled window + sketch agg ──────────────────────── - QueryExpr::WindowedAgg { agg, window, input, .. } => { - if ctx.window.is_none() { - if let crate::algebra::expr::WindowKind::Tumbling { size } = &window.kind { - ctx.window = Some(*size); - } - } - let selector = promql_from_qe(input, ctx); - let window_s = window_str(ctx.window); - let by = by_clause(&ctx.group_by); - sketch_op_to_promql(agg, &selector, &window_s, &by) - } - - // ── SketchAgg — the main aggregation node ──────────────────────────── - QueryExpr::SketchAgg { op, input, .. } => { - let selector = promql_from_qe(input, ctx); - let window = window_str(ctx.window); - let by = by_clause(&ctx.group_by); - sketch_op_to_promql(op, &selector, &window, &by) - } - - // ── Aggregate (SQL GROUP BY) ────────────────────────────────────────── - QueryExpr::Aggregate { keys, aggs, input, .. } => { - // Merge SQL GROUP BY keys into the context. - for k in keys { - if !ctx.group_by.contains(k) { - ctx.group_by.push(k.clone()); - } - } - let selector = promql_from_qe(input, ctx); - let window = window_str(ctx.window); - let by = by_clause(&ctx.group_by); - // Use the first aggregate function to drive the PromQL template. - if let Some(agg) = aggs.first() { - agg_func_to_promql(&agg.func, &selector, &window, &by) - } else { - selector - } - } - - // ── TopK ───────────────────────────────────────────────────────────── - QueryExpr::TopK { k, input, .. } => { - let inner = promql_from_qe(input, ctx); - format!("topk({k}, {inner})") - } - - // ── Sort + Limit — map to topk ─────────────────────────────────────── - QueryExpr::Limit { n, input, .. } => { - let inner = promql_from_qe(input, ctx); - format!("topk({n}, {inner})") - } - QueryExpr::Sort { input, .. } => promql_from_qe(input, ctx), - - // ── histogram_quantile(φ, rate(selector[w])) ───────────────────────── - QueryExpr::HistogramQuantile { phi, input } => { - let selector = promql_from_qe(input, ctx); - let window = window_str(ctx.window); - format!("histogram_quantile({phi}, rate({selector}{window}))") - } - - // ── PromQL subquery expr[range:step] ───────────────────────────────── - QueryExpr::PromQLSubquery { range, resolution, input } => { - let inner = promql_from_qe(input, ctx); - let step_str = resolution - .map(|r| format!(":{}", format_duration(r))) - .unwrap_or_default(); - format!("{}[{}{}]", inner, format_duration(*range), step_str) - } - - // ── Vector binary op (lhs op rhs) ─────────────────────────────────── - QueryExpr::BinaryOp { op, lhs, rhs, vector_match } => { - let lhs_str = promql_from_qe(lhs, ctx); - let rhs_str = promql_from_qe(rhs, &mut PromQLCtx::default()); - let op_str = binop_to_promql(op); - let match_str = vector_match - .as_ref() - .map(|m| { - use crate::algebra::expr::{GroupSide, VectorMatchKind}; - let kw = match m.kind { - VectorMatchKind::On => "on", - VectorMatchKind::Ignoring => "ignoring", - }; - let labels = m.labels.join(", "); - let group = m.grouping.as_ref().map(|g| { - let side = match g.side { - GroupSide::Left => "group_left", - GroupSide::Right => "group_right", - }; - if g.labels.is_empty() { - format!(" {side}") - } else { - format!(" {side}({})", g.labels.join(", ")) - } - }).unwrap_or_default(); - format!(" {kw} ({labels}){group}") - }) - .unwrap_or_default(); - format!("({lhs_str} {op_str}{match_str} {rhs_str})") - } - - // ── Merge — serialise first branch (all branches same shape) ───────── - QueryExpr::Merge { inputs } => { - inputs.first() - .map(|first| promql_from_qe(first, ctx)) - .unwrap_or_default() - } - - // ── Passthrough nodes ───────────────────────────────────────────────── - QueryExpr::Dedup { input, .. } - | QueryExpr::Project { input, .. } - | QueryExpr::WindowFunc { input, .. } => promql_from_qe(input, ctx), - - QueryExpr::Subquery { expr, .. } => promql_from_qe(expr, ctx), - - QueryExpr::LetBinding { body, .. } => promql_from_qe(body, ctx), - - // ── Join / SetOp — serialise the outer / left branch ───────────────── - QueryExpr::JoinSketch { outer, .. } => promql_from_qe(outer, ctx), - QueryExpr::Join { left, .. } => promql_from_qe(left, ctx), - QueryExpr::SetOp { left, .. } => promql_from_qe(left, ctx), - } -} - -// ── PromQL fragment helpers ─────────────────────────────────────────────────── - -fn window_str(w: Option) -> String { - w.map(|d| format!("[{}]", format_duration(d))).unwrap_or_default() -} - -fn by_clause(keys: &[String]) -> String { - if keys.is_empty() { - String::new() - } else { - format!(" by ({})", keys.join(", ")) - } -} - -fn sketch_op_to_promql(op: &AggIntent, selector: &str, window: &str, by: &str) -> String { - match op { - AggIntent::Quantile { quantiles, .. } => { - let phi = quantiles.first().copied().unwrap_or(0.99); - format!("quantile_over_time({phi}, {selector}{window}){by}") - } - AggIntent::Cardinality { .. } => { - format!("count_over_time({selector}{window}){by}") - } - AggIntent::Frequency { .. } => { - format!("count_over_time({selector}{window}){by}") - } - AggIntent::Extrema { min, max } => match (min, max) { - (true, false) => format!("min_over_time({selector}{window}){by}"), - (false, true) => format!("max_over_time({selector}{window}){by}"), - _ => format!("quantile_over_time(0.5, {selector}{window}){by}"), - }, - AggIntent::Exact(ExactAgg::Count) => format!("count_over_time({selector}{window}){by}"), - AggIntent::Exact(ExactAgg::Sum) => format!("sum_over_time({selector}{window}){by}"), - AggIntent::Exact(ExactAgg::Avg) => format!("avg_over_time({selector}{window}){by}"), - AggIntent::Exact(ExactAgg::Min) => format!("min_over_time({selector}{window}){by}"), - AggIntent::Exact(ExactAgg::Max) => format!("max_over_time({selector}{window}){by}"), - AggIntent::PerPartition { inner, .. } => sketch_op_to_promql(inner, selector, window, by), - } -} - -fn agg_func_to_promql(func: &AggFunc, selector: &str, window: &str, by: &str) -> String { - match func { - AggFunc::Quantile(phi) => format!("quantile_over_time({phi}, {selector}{window}){by}"), - AggFunc::CountDistinct => format!("count_over_time({selector}{window}){by}"), - AggFunc::HeavyHitters{k} => format!("topk({k}, count_over_time({selector}{window}){by})"), - AggFunc::Count => format!("count_over_time({selector}{window}){by}"), - AggFunc::Sum => format!("sum_over_time({selector}{window}){by}"), - AggFunc::Avg => format!("avg_over_time({selector}{window}){by}"), - AggFunc::Min => format!("min_over_time({selector}{window}){by}"), - AggFunc::Max => format!("max_over_time({selector}{window}){by}"), - AggFunc::StdDev { .. } => format!("stddev_over_time({selector}{window}){by}"), - AggFunc::Variance { .. } => format!("stdvar_over_time({selector}{window}){by}"), - AggFunc::Rate => format!("rate({selector}{window}){by}"), - AggFunc::Increase => format!("increase({selector}{window}){by}"), - AggFunc::Delta => format!("delta({selector}{window}){by}"), - AggFunc::Custom(name) => format!("{name}({selector}{window}){by}"), - } -} - -fn binop_to_promql(op: &BinaryOpKind) -> &'static str { - match op { - BinaryOpKind::Add => "+", - BinaryOpKind::Sub => "-", - BinaryOpKind::Mul => "*", - BinaryOpKind::Div => "/", - BinaryOpKind::Mod => "%", - BinaryOpKind::Pow => "^", - BinaryOpKind::Eq => "==", - BinaryOpKind::Ne => "!=", - BinaryOpKind::Lt => "<", - BinaryOpKind::Le => "<=", - BinaryOpKind::Gt => ">", - BinaryOpKind::Ge => ">=", - BinaryOpKind::And => "and", - BinaryOpKind::Or => "or", - BinaryOpKind::Unless => "unless", - BinaryOpKind::Atan2 => "atan2", - _ => "and", // bitwise/string ops not in PromQL - } -} - -// ── Label matcher extraction from ScalarExpr ───────────────────────────────── - -/// Extract PromQL-compatible label matchers from a `ScalarExpr` AND-tree. -/// -/// Only equality / inequality / regex comparisons between a `Column` and a -/// string `Literal` are extractable as label matchers. Everything else is -/// silently ignored (it won't become a label filter in the PromQL output). -fn scalar_to_label_matchers(pred: &ScalarExpr) -> Vec { - let mut out = Vec::new(); - collect_label_matchers(pred, &mut out); - out -} - -fn collect_label_matchers(pred: &ScalarExpr, out: &mut Vec) { - match pred { - // AND-tree: recurse into both sides. - ScalarExpr::BinaryOp { op: BinaryOpKind::And, lhs, rhs } => { - collect_label_matchers(lhs, out); - collect_label_matchers(rhs, out); - } - // col = "val" - ScalarExpr::BinaryOp { op: BinaryOpKind::Eq, lhs, rhs } => { - if let (ScalarExpr::Column(col), ScalarExpr::Literal(LiteralValue::Str(v))) - = (lhs.as_ref(), rhs.as_ref()) - { - out.push(format!("{}=\"{}\"", col, v)); - } - } - // col != "val" - ScalarExpr::BinaryOp { op: BinaryOpKind::Ne, lhs, rhs } => { - if let (ScalarExpr::Column(col), ScalarExpr::Literal(LiteralValue::Str(v))) - = (lhs.as_ref(), rhs.as_ref()) - { - out.push(format!("{}!=\"{}\"", col, v)); - } - } - // col =~ "regex" - ScalarExpr::BinaryOp { op: BinaryOpKind::Regex, lhs, rhs } => { - if let (ScalarExpr::Column(col), ScalarExpr::Literal(LiteralValue::Str(v))) - = (lhs.as_ref(), rhs.as_ref()) - { - out.push(format!("{}=~\"{}\"", col, v)); - } - } - // col !~ "regex" - ScalarExpr::BinaryOp { op: BinaryOpKind::NotRegex, lhs, rhs } => { - if let (ScalarExpr::Column(col), ScalarExpr::Literal(LiteralValue::Str(v))) - = (lhs.as_ref(), rhs.as_ref()) - { - out.push(format!("{}!~\"{}\"", col, v)); - } - } - _ => {} - } -} - -// ── Helper: push label filters from a ScalarExpr into a Vec ────────── - -fn collect_label_filters_into(pred: &ScalarExpr, out: &mut Vec) { - let matchers = scalar_to_label_matchers(pred); - for m in matchers { - // Store as "col=val" (without PromQL quotes) for the agent YAML. - // The YAML generator already re-quotes as needed. - if !out.contains(&m) { - out.push(m); - } - } -} - -// ── Phase E: typed L5 call path (opt-in, additive) ──────────────────────────── - -/// Env-var that opts the planner into the typed L5 stage_split path -/// (`crate::stage_split::StageAllocator` + `ThreeStageEmitter`). -/// Additive — when unset, the existing untyped `split_expr_by_stage` -/// flow runs unchanged. Mirror of `ENV_USE_TYPED_SKETCH_ALGEBRA` from -/// Phase C. -/// -/// Set `USE_TYPED_STAGE_SPLIT=1` to opt in. -#[allow(dead_code)] -pub const ENV_USE_TYPED_STAGE_SPLIT: &str = "USE_TYPED_STAGE_SPLIT"; - -/// Whether the typed L5 stage_split path is enabled for this process. -/// Reads the env var once per call (cheap; called per `plan()` -/// invocation at most). Phase E is additive — both code paths produce -/// per-stage descriptions, but the typed path's structural output is -/// `crate::stage_split::StageConfig` (sketched against design.md §6), -/// while the legacy path is the existing `StagedPlan` shape. -/// -/// Phase B (MVP v6) wires `main::handle_plan` to consult this gate. -pub fn typed_stage_split_enabled() -> bool { - matches!( - std::env::var(ENV_USE_TYPED_STAGE_SPLIT).as_deref(), - Ok("1") | Ok("true") | Ok("yes") - ) -} - -/// Run the typed L5 path on a Phase-C-bound `SketchExpr` DAG. Returns -/// the per-stage [`crate::stage_split::StageConfig`] map for the DC -/// lifecycle topology. -/// -/// Returns `None` when the typed path errors out (unsupported topology -/// shape, unresolved Ref, empty backend) — the caller should then fall -/// back to the legacy `split_expr_by_stage` output. -/// -/// Phase B (MVP v6) wires this into `main::handle_plan` behind the -/// `USE_TYPED_STAGE_SPLIT` env-var gate. Each per-stage config the -/// returned map carries is materialised into wire bytes by the -/// emitters in [`crate::config::stage_config`] — -/// [`crate::config::stage_config::emit_edge_yaml`] for `Edge`, -/// [`crate::config::stage_config::emit_gateway_yaml`] for `Gateway`, -/// [`crate::config::stage_config::emit_backend_config_json`] for -/// `Backend`. Phase C plumbs deployment-aware endpoint resolution. -pub fn split_typed_three_stage( - expr: &crate::sketch_algebra::SketchExpr, -) -> Option> -{ - use crate::stage_split::{ - Emitter, StageAllocator, ThreeStageEmitter, Topology, - }; - let dag = StageAllocator.allocate(expr, Topology::ThreeStage).ok()?; - ThreeStageEmitter.emit_per_stage(&dag).ok() -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use crate::algebra::expr::{AggItem, AggIntent, BinaryOpKind, LiteralValue, QueryExpr, ScalarExpr}; - use crate::algebra::expr::{ColumnRef, PartitionKeys, SourceSpec}; - - fn source(name: &str) -> QueryExpr { - QueryExpr::Source(SourceSpec { name: name.into() }) - } - - fn no_budget() -> StageResourceBudgets { StageResourceBudgets::default() } - - fn eq_filter(col: &str, val: &str) -> QueryExpr { - QueryExpr::Filter { - pred: ScalarExpr::BinaryOp { - op: BinaryOpKind::Eq, - lhs: Box::new(ScalarExpr::Column(col.into())), - rhs: Box::new(ScalarExpr::Literal(LiteralValue::Str(val.into()))), - }, - input: Box::new(source("latency")), - } - } - - // ── Node-to-stage assignment ────────────────────────────────────────────── - - #[test] - fn ddsketch_agg_goes_to_agent() { - let expr = QueryExpr::Window { - duration: Duration::from_secs(300), - slide: None, - input: Box::new(QueryExpr::SketchAgg { - op: AggIntent::default_quantile(vec![0.99]), - col: ColumnRef::SampleValue, - input: Box::new(source("latency")), - }), - }; - let plan = split_expr_by_stage(&expr, &no_budget()); - assert_eq!(plan.agent.sketch_type, Some(SketchType::DDSketch)); - assert_eq!(plan.agent.window_secs, Some(300)); - assert!(!plan.precompute.active); - assert!(!plan.db.active); - } - - #[test] - fn hll_stays_at_agent_by_default() { - let expr = QueryExpr::SketchAgg { - op: AggIntent::default_cardinality(), - col: ColumnRef::SampleValue, - input: Box::new(source("events")), - }; - let plan = split_expr_by_stage(&expr, &no_budget()); - assert_eq!(plan.agent.sketch_type, Some(SketchType::HLL)); - } - - #[test] - fn partition_group_by_goes_to_backend() { - let expr = QueryExpr::Partition { - keys: PartitionKeys::By(vec!["host".into(), "region".into()]), - input: Box::new(QueryExpr::SketchAgg { - op: AggIntent::default_quantile(vec![0.99]), - col: ColumnRef::SampleValue, - input: Box::new(source("latency")), - }), - }; - let plan = split_expr_by_stage(&expr, &no_budget()); - assert!(plan.backend.group_by.contains(&"host".to_string())); - assert!(plan.backend.group_by.contains(&"region".to_string())); - } - - #[test] - fn aggregate_without_group_by_avg_goes_to_db() { - let expr = QueryExpr::Aggregate { - keys: vec![], - aggs: vec![AggItem { - alias: "avg_val".into(), - func: AggFunc::Avg, - col: ColumnRef::SampleValue, - distinct: false, - }], - having: None, - input: Box::new(source("price")), - }; - let plan = split_expr_by_stage(&expr, &no_budget()); - assert!(plan.db.active); - } - - #[test] - fn aggregate_sum_goes_to_backend() { - let expr = QueryExpr::Aggregate { - keys: vec!["symbol".into()], - aggs: vec![AggItem { - alias: "total".into(), - func: AggFunc::Sum, - col: ColumnRef::SampleValue, - distinct: false, - }], - having: None, - input: Box::new(source("trades")), - }; - let plan = split_expr_by_stage(&expr, &no_budget()); - assert!(plan.backend.has_merge); - assert!(plan.backend.group_by.contains(&"symbol".to_string())); - assert!(!plan.db.active); - } - - #[test] - fn topk_goes_to_precompute() { - let expr = QueryExpr::TopK { - k: 10, - by: vec!["symbol".into()], - input: Box::new(QueryExpr::SketchAgg { - op: AggIntent::default_frequency(), - col: ColumnRef::SampleValue, - input: Box::new(source("price")), - }), - }; - let plan = split_expr_by_stage(&expr, &no_budget()); - assert!(plan.precompute.active); - assert_eq!(plan.precompute.topk, Some(10)); - } - - #[test] - fn histogram_quantile_activates_precompute() { - let expr = QueryExpr::HistogramQuantile { - phi: 0.99, - input: Box::new(QueryExpr::Window { - duration: Duration::from_secs(300), - slide: None, - input: Box::new(source("http_request_duration_seconds_bucket")), - }), - }; - let plan = split_expr_by_stage(&expr, &no_budget()); - assert!(plan.precompute.active); - assert!(!plan.precompute.query_expr.is_empty()); - assert!(plan.precompute.query_expr.contains("histogram_quantile(0.99")); - } - - #[test] - fn binary_op_activates_precompute() { - let lhs = source("metric_a"); - let rhs = source("metric_b"); - let expr = QueryExpr::BinaryOp { - op: BinaryOpKind::Div, - lhs: Box::new(lhs), - rhs: Box::new(rhs), - vector_match: None, - }; - let plan = split_expr_by_stage(&expr, &no_budget()); - assert!(plan.precompute.active); - } - - // ── Budget-driven deferral ──────────────────────────────────────────────── - - #[test] - fn ddsketch_deferred_to_backend_when_agent_budget_exceeded() { - let tiny_budget = StageResourceBudgets { - agent_memory_bytes: Some(1), // 1 byte — DDSketch (4 KiB) won't fit - ..Default::default() - }; - let expr = QueryExpr::SketchAgg { - op: AggIntent::default_quantile(vec![0.99]), - col: ColumnRef::SampleValue, - input: Box::new(source("latency")), - }; - let plan = split_expr_by_stage(&expr, &tiny_budget); - assert_eq!(plan.agent.sketch_type, None); // not at agent - assert!(plan.backend.has_merge); // deferred to backend - assert!(!plan.deferral_log.is_empty()); - } - - #[test] - fn ddsketch_deferred_to_precompute_when_both_budgets_exceeded() { - let tiny_budget = StageResourceBudgets { - agent_memory_bytes: Some(1), - backend_memory_bytes: Some(1), - ..Default::default() - }; - let expr = QueryExpr::SketchAgg { - op: AggIntent::default_quantile(vec![0.99]), - col: ColumnRef::SampleValue, - input: Box::new(source("latency")), - }; - let plan = split_expr_by_stage(&expr, &tiny_budget); - assert!(plan.precompute.active); - assert_eq!(plan.deferral_log.len(), 2); // two deferral steps logged - } - - // ── expr_to_promql ──────────────────────────────────────────────────────── - - #[test] - fn promql_ddsketch_with_filter_and_window() { - let expr = QueryExpr::Partition { - keys: PartitionKeys::By(vec!["symbol".into()]), - input: Box::new(QueryExpr::Window { - duration: Duration::from_secs(300), - slide: None, - input: Box::new(QueryExpr::SketchAgg { - op: AggIntent::default_quantile(vec![0.99]), - col: ColumnRef::SampleValue, - input: Box::new(QueryExpr::Filter { - pred: ScalarExpr::BinaryOp { - op: BinaryOpKind::Eq, - lhs: Box::new(ScalarExpr::Column("sectype".into())), - rhs: Box::new(ScalarExpr::Literal(LiteralValue::Str("E".into()))), - }, - input: Box::new(source("price")), - }), - }), - }), - }; - let ql = expr_to_promql(&expr); - assert!(ql.contains("quantile_over_time(0.99"), "expected quantile_over_time: {ql}"); - assert!(ql.contains("sectype=\"E\""), "expected label filter: {ql}"); - assert!(ql.contains("[5m]"), "expected window: {ql}"); - assert!(ql.contains("by (symbol)"), "expected group_by: {ql}"); - } - - #[test] - fn promql_topk_wraps_inner() { - let expr = QueryExpr::TopK { - k: 10, - by: vec![], - input: Box::new(QueryExpr::SketchAgg { - op: AggIntent::default_frequency(), - col: ColumnRef::SampleValue, - input: Box::new(source("events")), - }), - }; - let ql = expr_to_promql(&expr); - assert!(ql.starts_with("topk(10,"), "expected topk prefix: {ql}"); - } - - #[test] - fn promql_histogram_quantile() { - let expr = QueryExpr::HistogramQuantile { - phi: 0.95, - input: Box::new(QueryExpr::Window { - duration: Duration::from_secs(300), - slide: None, - input: Box::new(source("http_request_duration_seconds_bucket")), - }), - }; - let ql = expr_to_promql(&expr); - assert!(ql.contains("histogram_quantile(0.95"), "got: {ql}"); - assert!(ql.contains("rate("), "got: {ql}"); - assert!(ql.contains("[5m]"), "got: {ql}"); - } - - #[test] - fn promql_binary_op_renders_operator() { - let expr = QueryExpr::BinaryOp { - op: BinaryOpKind::Div, - lhs: Box::new(source("http_errors")), - rhs: Box::new(source("http_requests")), - vector_match: None, - }; - let ql = expr_to_promql(&expr); - assert!(ql.contains('/'), "expected / operator: {ql}"); - assert!(ql.contains("http_errors"), "got: {ql}"); - assert!(ql.contains("http_requests"), "got: {ql}"); - } - - #[test] - fn promql_subquery_renders_range() { - let expr = QueryExpr::PromQLSubquery { - range: Duration::from_secs(3600), - resolution: Some(Duration::from_secs(60)), - input: Box::new(source("metric")), - }; - let ql = expr_to_promql(&expr); - assert!(ql.contains("[1h:1m]") || ql.contains("[3600s:60s]"), "got: {ql}"); - } - - #[test] - fn label_matchers_and_tree() { - let pred = ScalarExpr::BinaryOp { - op: BinaryOpKind::And, - lhs: Box::new(ScalarExpr::BinaryOp { - op: BinaryOpKind::Eq, - lhs: Box::new(ScalarExpr::Column("job".into())), - rhs: Box::new(ScalarExpr::Literal(LiteralValue::Str("api".into()))), - }), - rhs: Box::new(ScalarExpr::BinaryOp { - op: BinaryOpKind::Ne, - lhs: Box::new(ScalarExpr::Column("env".into())), - rhs: Box::new(ScalarExpr::Literal(LiteralValue::Str("dev".into()))), - }), - }; - let matchers = scalar_to_label_matchers(&pred); - assert!(matchers.contains(&"job=\"api\"".to_string())); - assert!(matchers.contains(&"env!=\"dev\"".to_string())); - } - - // ── Phase E: typed L5 opt-in path ────────────────────────────────────── - - #[test] - fn typed_three_stage_path_returns_three_configs() { - use crate::intent_algebra::schema::{Column, DataType}; - use crate::intent_algebra::{ - LabelFilter, QueryExpr as L3QE, Schema, Source as L3Source, WindowKind, - }; - use crate::sketch_algebra::params::{KllParams, SketchKind, SketchParams as L4Params}; - use crate::sketch_algebra::sketch_expr::{EstimateOp, SketchExpr}; - use crate::stage_split::StageId; - let scan = L3QE::Scan { - source: L3Source::TimeSeries { - metric: "http_request_duration_seconds".into(), - }, - label_filters: vec![LabelFilter { - label: "service".into(), - equals: "api".into(), - }], - schema: Schema::with_time_index( - vec![ - Column { - name: "ts".into(), - dtype: DataType::Timestamp, - nullable: false, - }, - Column { - name: "value".into(), - dtype: DataType::Float64, - nullable: false, - }, - ], - 0, - vec![vec![0]], - ), - }; - let windowed = L3QE::Window { - kind: WindowKind::Sliding, - size: std::time::Duration::from_secs(300), - slide: None, - child: Box::new(scan), - }; - let l4 = SketchExpr::estimate_over_agg( - EstimateOp::Quantile { q: 0.99 }, - SketchKind::Kll, - L4Params::Kll(KllParams { k: 200 }), - windowed, - ); - let configs = super::split_typed_three_stage(&l4).expect("typed path produces output"); - assert!(configs.contains_key(&StageId::Edge)); - assert!(configs.contains_key(&StageId::Backend)); - } -} diff --git a/controller/src/planner/tco.rs b/controller/src/planner/tco.rs deleted file mode 100644 index ef09666c..00000000 --- a/controller/src/planner/tco.rs +++ /dev/null @@ -1,422 +0,0 @@ -use serde::{Deserialize, Serialize}; - -/// Cloud pricing configuration (loaded from YAML or defaults). -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(default)] -pub struct CloudPricing { - /// Grafana Cloud: $/1000 active series at 1 DPM (data point per minute). - pub grafana_per_1k_series_1dpm: f64, - /// AWS S3: $/GB stored per month. - pub s3_storage_per_gb_month: f64, - /// AWS S3: $/1000 PUT requests. - pub s3_put_per_1k: f64, - /// AWS S3: $/1000 GET requests. - pub s3_get_per_1k: f64, - /// AWS S3: $/GB data transfer out. - pub s3_transfer_per_gb: f64, - /// EC2 compute cost for sketch processing ($/hour for the instance type). - pub ec2_sketch_instance_per_hour: f64, -} - -impl Default for CloudPricing { - fn default() -> Self { - Self { - grafana_per_1k_series_1dpm: 6.50, - s3_storage_per_gb_month: 0.023, - s3_put_per_1k: 0.005, - s3_get_per_1k: 0.0004, - s3_transfer_per_gb: 0.09, - ec2_sketch_instance_per_hour: 0.384, // c6i.xlarge - } - } -} - -/// Workload parameters for TCO estimation. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(default)] -pub struct TcoWorkload { - /// Number of active time series. - pub series_count: u64, - /// Samples per second per series. - pub samples_per_sec: f64, - /// Bytes per raw sample (metric name + labels + value + timestamp). - pub bytes_per_sample: u64, - /// Scrape interval in seconds. - pub scrape_interval_secs: u64, - /// Average query rate (queries per second). - pub queries_per_sec: f64, - /// Average query window in seconds. - pub query_window_secs: u64, - /// Retention period in days. - pub retention_days: u64, - /// Sketch compression ratio (sketch bytes / raw bytes). Typically 0.03-0.10. - pub sketch_compression_ratio: f64, - /// Delta compression ratio (delta bytes / full sketch bytes). Typically 0.2-0.5. - pub delta_compression_ratio: f64, -} - -impl Default for TcoWorkload { - fn default() -> Self { - Self { - series_count: 100_000, - samples_per_sec: 1.0, - bytes_per_sample: 100, - scrape_interval_secs: 15, - queries_per_sec: 1.0, - query_window_secs: 300, - retention_days: 30, - sketch_compression_ratio: 0.05, - delta_compression_ratio: 0.3, - } - } -} - -/// TCO estimation result. -#[derive(Debug, Clone, Serialize)] -pub struct TcoEstimate { - /// Monthly cost breakdown for traditional pipeline. - pub before: TcoBefore, - /// Monthly cost breakdown for sketch pipeline. - pub after: TcoAfter, - /// Absolute monthly savings in dollars. - pub monthly_savings_dollars: f64, - /// Savings as a percentage of the before cost. - pub savings_percent: f64, -} - -#[derive(Debug, Clone, Serialize)] -pub struct TcoBefore { - pub ingestion_dollars: f64, - pub query_dollars: f64, - pub storage_dollars: f64, - pub total_dollars: f64, -} - -#[derive(Debug, Clone, Serialize)] -pub struct TcoAfter { - pub sketch_ingestion_dollars: f64, - pub s3_ingestion_dollars: f64, - pub sketch_query_dollars: f64, - pub s3_query_dollars: f64, - pub sketch_storage_dollars: f64, - pub s3_storage_dollars: f64, - pub compute_dollars: f64, - pub total_dollars: f64, -} - -/// Seconds in a 30-day month. -const SECS_PER_MONTH: f64 = 86_400.0 * 30.0; -/// Hours in a 30-day month. -const HOURS_PER_MONTH: f64 = 24.0 * 30.0; -/// Bytes per gigabyte. -const BYTES_PER_GB: f64 = 1_000_000_000.0; -/// Grafana bundled storage cost approximation ($/GB/month for active series). -const GRAFANA_STORAGE_PER_GB_MONTH: f64 = 0.10; -/// Approximate Grafana query cost per 1000 queries (bundled in series pricing). -const GRAFANA_QUERY_COST_PER_1K: f64 = 0.10; -/// Series capacity per EC2 instance for sketch processing. -const SERIES_PER_INSTANCE: f64 = 100_000.0; - -/// Compute a TCO estimate comparing traditional TSDB vs sketch-based pipeline. -pub fn estimate_tco(workload: &TcoWorkload, pricing: &CloudPricing) -> TcoEstimate { - let before = compute_before(workload, pricing); - let after = compute_after(workload, pricing); - - let savings = before.total_dollars - after.total_dollars; - let savings_pct = if before.total_dollars > 0.0 { - savings / before.total_dollars * 100.0 - } else { - 0.0 - }; - - TcoEstimate { - before, - after, - monthly_savings_dollars: savings, - savings_percent: savings_pct, - } -} - -fn compute_before(w: &TcoWorkload, p: &CloudPricing) -> TcoBefore { - // Data points per minute multiplier: if scrape_interval is 15s, that's 4 DPM. - let dpm_multiplier = if w.scrape_interval_secs > 0 { - 60.0 / w.scrape_interval_secs as f64 - } else { - 1.0 - }; - - // Ingestion: Grafana charges per 1000 active series at 1 DPM. - let ingestion = (w.series_count as f64 / 1000.0) - * p.grafana_per_1k_series_1dpm - * dpm_multiplier; - - // Storage: raw bytes over retention period. - let samples_per_day = w.samples_per_sec * 86_400.0; - let total_bytes = w.bytes_per_sample as f64 - * samples_per_day - * w.series_count as f64 - * w.retention_days as f64; - let storage = (total_bytes / BYTES_PER_GB) * GRAFANA_STORAGE_PER_GB_MONTH; - - // Query: approximate cost based on query rate. - let total_queries_per_month = w.queries_per_sec * SECS_PER_MONTH; - let query = (total_queries_per_month / 1000.0) * GRAFANA_QUERY_COST_PER_1K; - - let total = ingestion + storage + query; - TcoBefore { - ingestion_dollars: ingestion, - query_dollars: query, - storage_dollars: storage, - total_dollars: total, - } -} - -fn compute_after(w: &TcoWorkload, p: &CloudPricing) -> TcoAfter { - let series = w.series_count as f64; - - // Sketch bytes per second per series after both compression stages. - let sketch_bps = w.samples_per_sec - * w.bytes_per_sample as f64 - * w.sketch_compression_ratio - * w.delta_compression_ratio; - - // -- Sketch ingestion: transfer cost for sketch data -- - let sketch_bytes_per_month = series * sketch_bps * SECS_PER_MONTH; - let sketch_ingestion = (sketch_bytes_per_month / BYTES_PER_GB) * p.s3_transfer_per_gb; - - // -- S3 ingestion: sketch-compressed backup (storage + PUT costs) -- - // In the sketch pipeline, S3 stores compressed sketches, not raw data. - let sketch_gb_per_month = sketch_bytes_per_month / BYTES_PER_GB; - // PUTs are batched: one PUT per flush interval (scrape_interval) containing - // all series in a single object, not one PUT per series. - // No PUTs needed if there are no series to back up. - let puts_per_month = if w.scrape_interval_secs > 0 && w.series_count > 0 { - SECS_PER_MONTH / w.scrape_interval_secs as f64 - } else { - 0.0 - }; - let s3_ingestion = sketch_gb_per_month * p.s3_storage_per_gb_month - + (puts_per_month / 1000.0) * p.s3_put_per_1k; - - // -- Sketch query: near-zero (local sketch eval) -- - let sketch_query = 0.0; - - // -- S3 query: only for ad-hoc queries (assume 1% of total) -- - let total_queries_per_month = w.queries_per_sec * SECS_PER_MONTH; - let s3_query = 0.01 * (total_queries_per_month / 1000.0) * p.s3_get_per_1k; - - // -- Compute: EC2 for sketch processing -- - let num_instances = (series / SERIES_PER_INSTANCE).ceil().max(1.0); - let compute = p.ec2_sketch_instance_per_hour * HOURS_PER_MONTH * num_instances; - - // -- Sketch storage: in-memory sketches (small, included in compute) -- - // Approximate: sketch data retained for query_window only, not full retention. - let sketch_mem_bytes = series * sketch_bps * w.query_window_secs as f64; - let sketch_storage = (sketch_mem_bytes / BYTES_PER_GB) * p.s3_storage_per_gb_month; - - // -- S3 storage: sketch-compressed data over retention -- - let s3_storage_bytes = sketch_bytes_per_month * w.retention_days as f64 / 30.0; - let s3_storage = (s3_storage_bytes / BYTES_PER_GB) * p.s3_storage_per_gb_month; - - let total = sketch_ingestion - + s3_ingestion - + sketch_query - + s3_query - + sketch_storage - + s3_storage - + compute; - - TcoAfter { - sketch_ingestion_dollars: sketch_ingestion, - s3_ingestion_dollars: s3_ingestion, - sketch_query_dollars: sketch_query, - s3_query_dollars: s3_query, - sketch_storage_dollars: sketch_storage, - s3_storage_dollars: s3_storage, - compute_dollars: compute, - total_dollars: total, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn default_pricing_is_reasonable() { - let p = CloudPricing::default(); - assert!((p.grafana_per_1k_series_1dpm - 6.50).abs() < f64::EPSILON); - assert!((p.s3_storage_per_gb_month - 0.023).abs() < f64::EPSILON); - assert!((p.s3_put_per_1k - 0.005).abs() < f64::EPSILON); - assert!((p.s3_get_per_1k - 0.0004).abs() < f64::EPSILON); - assert!((p.s3_transfer_per_gb - 0.09).abs() < f64::EPSILON); - assert!((p.ec2_sketch_instance_per_hour - 0.384).abs() < f64::EPSILON); - } - - #[test] - fn test_100k_series_15s_scrape() { - let workload = TcoWorkload { - series_count: 100_000, - samples_per_sec: 1.0, - bytes_per_sample: 100, - scrape_interval_secs: 15, - queries_per_sec: 1.0, - query_window_secs: 300, - retention_days: 30, - sketch_compression_ratio: 0.05, - delta_compression_ratio: 0.3, - }; - let pricing = CloudPricing::default(); - let est = estimate_tco(&workload, &pricing); - - assert!(est.before.total_dollars > 0.0, "before total should be positive"); - assert!(est.after.total_dollars > 0.0, "after total should be positive"); - assert!( - est.savings_percent > 50.0, - "expected >50% savings for 100K series, got {:.1}%", - est.savings_percent - ); - assert!(est.monthly_savings_dollars > 0.0, "should save money"); - } - - #[test] - fn test_1m_series_high_cardinality() { - let workload = TcoWorkload { - series_count: 1_000_000, - samples_per_sec: 1.0, - bytes_per_sample: 150, - scrape_interval_secs: 15, - queries_per_sec: 5.0, - query_window_secs: 600, - retention_days: 90, - sketch_compression_ratio: 0.04, - delta_compression_ratio: 0.25, - }; - let pricing = CloudPricing::default(); - let est = estimate_tco(&workload, &pricing); - - assert!(est.before.total_dollars > est.after.total_dollars); - assert!(est.savings_percent > 50.0, - "expected >50% savings at 1M series, got {:.1}%", est.savings_percent); - // At 1M series we need ceil(1M/100K) = 10 instances. - let expected_compute = pricing.ec2_sketch_instance_per_hour * HOURS_PER_MONTH * 10.0; - assert!( - (est.after.compute_dollars - expected_compute).abs() < 0.01, - "compute should be ~${:.2}, got ${:.2}", - expected_compute, est.after.compute_dollars - ); - } - - #[test] - fn zero_series_returns_zero() { - let workload = TcoWorkload { - series_count: 0, - ..Default::default() - }; - let pricing = CloudPricing::default(); - let est = estimate_tco(&workload, &pricing); - - // Before costs should be zero (no series). - assert!( - est.before.ingestion_dollars.abs() < f64::EPSILON, - "ingestion should be 0" - ); - assert!( - est.before.storage_dollars.abs() < f64::EPSILON, - "storage should be 0" - ); - // After still has a minimum 1-instance compute cost. - assert!(est.after.compute_dollars > 0.0, "compute has a 1-instance minimum"); - // But sketch/s3 ingestion should be zero. - assert!( - est.after.sketch_ingestion_dollars.abs() < f64::EPSILON, - "sketch ingestion should be 0 for 0 series" - ); - assert!( - est.after.s3_ingestion_dollars.abs() < f64::EPSILON, - "s3 ingestion should be 0 for 0 series" - ); - } - - #[test] - fn sketch_compression_ratio_affects_result() { - let pricing = CloudPricing::default(); - - let low_ratio = TcoWorkload { - series_count: 100_000, - sketch_compression_ratio: 0.03, - ..Default::default() - }; - let high_ratio = TcoWorkload { - series_count: 100_000, - sketch_compression_ratio: 0.10, - ..Default::default() - }; - - let est_low = estimate_tco(&low_ratio, &pricing); - let est_high = estimate_tco(&high_ratio, &pricing); - - // Higher compression ratio means more sketch bytes, so higher after cost. - assert!( - est_high.after.sketch_ingestion_dollars > est_low.after.sketch_ingestion_dollars, - "higher sketch ratio should cost more: {:.4} vs {:.4}", - est_high.after.sketch_ingestion_dollars, - est_low.after.sketch_ingestion_dollars - ); - // Before cost should be the same (independent of sketch ratio). - assert!( - (est_high.before.total_dollars - est_low.before.total_dollars).abs() < f64::EPSILON, - "before cost should not change with sketch ratio" - ); - } - - #[test] - fn delta_compression_ratio_affects_result() { - let pricing = CloudPricing::default(); - - let low_delta = TcoWorkload { - series_count: 100_000, - delta_compression_ratio: 0.2, - ..Default::default() - }; - let high_delta = TcoWorkload { - series_count: 100_000, - delta_compression_ratio: 0.5, - ..Default::default() - }; - - let est_low = estimate_tco(&low_delta, &pricing); - let est_high = estimate_tco(&high_delta, &pricing); - - assert!( - est_high.after.sketch_ingestion_dollars > est_low.after.sketch_ingestion_dollars, - "higher delta ratio should increase sketch ingestion cost" - ); - } - - #[test] - fn retention_days_affects_storage() { - let pricing = CloudPricing::default(); - - let short = TcoWorkload { - retention_days: 7, - ..Default::default() - }; - let long = TcoWorkload { - retention_days: 90, - ..Default::default() - }; - - let est_short = estimate_tco(&short, &pricing); - let est_long = estimate_tco(&long, &pricing); - - assert!( - est_long.before.storage_dollars > est_short.before.storage_dollars, - "longer retention should cost more storage" - ); - assert!( - est_long.after.s3_storage_dollars > est_short.after.s3_storage_dollars, - "longer retention should cost more S3 storage" - ); - } -} diff --git a/controller/src/planner/wire_cost.rs b/controller/src/planner/wire_cost.rs deleted file mode 100644 index 8c91bd18..00000000 --- a/controller/src/planner/wire_cost.rs +++ /dev/null @@ -1,601 +0,0 @@ -//! Phase ε.1 — three-mode wire cost model with native OTLP everywhere. -//! -//! The Phase β / γ planner had two transmission decisions: raw vs. sketch -//! (in `delta_cost_model.rs`) and exact vs. approximate (in `cost_model.rs`). -//! Phase ε.1 collapses those two axes into a single tri-mode selector that -//! also names where the work happens: -//! -//! | Mode | Edge action | Wire | Backend role | Accuracy | -//! |-----------------------------------|----------------------------|------------------------------|------------------------------|-----------------| -//! | [`BindMode::SketchAtEdge`] | sketch processor at edge | edge → backend OTLP | SimpleEngine over sketch | bounded ε > 0 | -//! | [`BindMode::RawAtEdgeSketchAtBackend`] | no sketch processor | edge → backend OTLP (raw) | builds sketches at ingest | bounded ε > 0 | -//! | [`BindMode::RawAtEdgePrometheusArchive`] | no sketch processor | edge → Prometheus OTLP HTTP | backend HTTP-forwards queries | exact (ε = 0) | -//! -//! All three modes use OTLP on the wire (Mode 3 talks to Prometheus's -//! native OTLP receiver at `/api/v1/otlp/v1/metrics`, exposed by the -//! `--web.enable-otlp-receiver` flag in Prometheus 2.47+). This collapses -//! the agent's exporter vocabulary to a single family (`otlp` / -//! `otlphttp`) — no separate `prometheusremotewrite` exporter is needed. -//! -//! ## Cost table source -//! -//! Sketch wire-state sizes are delta-encoded where the sketch family -//! supports it. KLL has no delta variant per `Implementation.tex` -//! (randomised compaction is not additively mergeable; the wire payload -//! is always full state). Numbers are conservative for typical -//! observability workloads — a future iteration will refine via observed -//! agent telemetry once the OnlineMetricsStore feeds back into the -//! planner. - -#![allow(dead_code)] - -use crate::sketch_algebra::params::SketchKind; -use crate::types::WorkloadCharacteristics; - -// ── Wire-cost table ────────────────────────────────────────────────────────── - -/// Per-window wire footprint of one sketch family. -/// -/// `state_bytes` is the per-flush payload after delta encoding (where -/// supported); `envelope_bytes` is the OTLP `SketchEnvelope` overhead -/// (gRPC framing + resource attributes + scope info). -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct SketchWireCost { - /// Per-flush sketch state bytes (delta-encoded where supported). - pub state_bytes: u64, - /// OTLP envelope bytes per flush — gRPC framing + resource attrs + scope. - pub envelope_bytes: u64, -} - -impl SketchWireCost { - /// Total wire bytes per flush (state + envelope). - pub const fn per_flush(&self) -> u64 { - self.state_bytes + self.envelope_bytes - } -} - -/// Phase ε.1 wire-cost table — one entry per sketch family. -/// -/// All values are delta-encoded where a delta variant exists. KLL has no -/// delta variant per `Implementation.tex`; uses full state. Numbers are -/// conservative for typical observability workloads; a future iteration -/// can refine via observed agent telemetry. -#[derive(Debug, Clone, Copy)] -pub struct WireCostTable { - /// DDSketch with sparse bucket-map delta encoding. - pub ddsketch_delta: SketchWireCost, - /// KLL — no delta variant, ships full compactor hierarchy. - pub kll_full: SketchWireCost, - /// HLL register-array delta (~60% of 16 KB full). - pub hll_delta: SketchWireCost, - /// Count-Min row-by-row delta (~70% of 6 KB full). - pub count_min_delta: SketchWireCost, - /// Count-Sketch sparse-cell delta (~60% of 400 KB full). - pub count_sketch_delta: SketchWireCost, -} - -impl WireCostTable { - /// The default Phase ε.1 cost table — see module docs for the source - /// rationale. - pub const fn default_phase_eps_1() -> Self { - Self { - ddsketch_delta: SketchWireCost { - state_bytes: 600, - envelope_bytes: 200, - }, - kll_full: SketchWireCost { - state_bytes: 3_000, - envelope_bytes: 200, - }, - hll_delta: SketchWireCost { - state_bytes: 10_000, - envelope_bytes: 200, - }, - count_min_delta: SketchWireCost { - state_bytes: 4_000, - envelope_bytes: 200, - }, - count_sketch_delta: SketchWireCost { - state_bytes: 250_000, - envelope_bytes: 200, - }, - } - } - - /// Lookup the per-flush cost for a sketch family. - pub const fn for_kind(&self, kind: &SketchKind) -> SketchWireCost { - match kind { - SketchKind::DDSketch => self.ddsketch_delta, - SketchKind::Kll => self.kll_full, - SketchKind::Hll => self.hll_delta, - SketchKind::Cms => self.count_min_delta, - SketchKind::CountSketch => self.count_sketch_delta, - } - } -} - -impl Default for WireCostTable { - fn default() -> Self { - Self::default_phase_eps_1() - } -} - -/// Break-even point in samples-per-window-per-series for a sketch -/// family — the workload size at which the sketch's per-flush wire cost -/// equals the raw OTLP cost of shipping every sample. -/// -/// `break_even = ceil((state_bytes + envelope_bytes) / per_sample_bytes)`. -/// -/// At 50 B per raw sample (Phase ε.1 default), this yields the canonical -/// table: -/// -/// - DDSketch+delta: 16 samples/window -/// - KLL (full): 64 samples/window -/// - HLL+delta: 204 samples/window -/// - Count-Min+delta: 84 samples/window -/// - CountSketch+delta: 5,004 samples/window -pub fn break_even_samples(cost: SketchWireCost, per_sample_bytes: u64) -> u64 { - if per_sample_bytes == 0 { - return u64::MAX; - } - let total = cost.per_flush(); - // ceil division — at exactly the break-even sample count, sketch and - // raw are equal; one more sample tips the balance to sketch. - total.div_ceil(per_sample_bytes) -} - -// ── Workload extensions for wire cost ──────────────────────────────────────── - -/// Per-workload inputs the wire cost model needs that aren't on -/// [`WorkloadCharacteristics`] today. -/// -/// Why a parallel struct rather than fattening `WorkloadCharacteristics`: -/// the existing struct is consumed by the delta cost model (which has a -/// different feature set — fill rate, distinct keys per window, etc.). -/// Phase ε.1 adds three orthogonal fields and the wire cost model uses -/// exactly those plus `samples_per_sec_per_series` from the existing -/// struct. Folding both into one struct conflates two cost surfaces. -/// -/// Convertible from [`WorkloadCharacteristics`] via [`Self::from_chars`]. -#[derive(Debug, Clone, PartialEq)] -pub struct WireWorkload { - /// Samples emitted per window per series — derived from emission rate - /// × window. Drives the raw-vs-sketch break-even. - pub samples_per_window_per_series: u64, - /// Wire bytes of one raw OTLP metric data point after protobuf - /// encoding. Default 50 (typical observability workload — counter or - /// gauge with a small label set, after OTLP delta encoding kicks in - /// at the SDK). - pub per_sample_bytes: u64, - /// Accuracy SLA (relative error). Already present on [`crate::types::QueryWorkload`]; - /// duplicated here so the wire cost model can be exercised without a - /// full `QueryWorkload`. - pub accuracy_sla: f64, - /// Edge CPU budget in cores. `None` = unbounded. - pub edge_cpu_budget: Option, - /// Edge RAM budget in bytes. `None` = unbounded. - pub edge_ram_budget: Option, -} - -impl WireWorkload { - /// Build a [`WireWorkload`] from existing - /// [`WorkloadCharacteristics`] + a window duration. Useful for the - /// planner's wire-cost decision when the caller already has a - /// `WorkloadCharacteristics` for the delta model. - pub fn from_chars( - wc: &WorkloadCharacteristics, - window_secs: u64, - accuracy_sla: f64, - ) -> Self { - let samples_per_window = - (wc.samples_per_sec_per_series * window_secs as f64).round() as u64; - Self { - samples_per_window_per_series: samples_per_window, - per_sample_bytes: wc.bytes_per_raw_sample as u64, - accuracy_sla, - edge_cpu_budget: None, - edge_ram_budget: wc.memory_budget_bytes, - } - } - - /// Default workload — 1 Hz × 60 s × 60 samples, 50 B per sample, 1% - /// SLA, no edge budgets. Useful for tests and as a safe fallback. - pub fn default_phase_eps_1() -> Self { - Self { - samples_per_window_per_series: 60, - per_sample_bytes: 50, - accuracy_sla: 0.01, - edge_cpu_budget: None, - edge_ram_budget: None, - } - } -} - -impl Default for WireWorkload { - fn default() -> Self { - Self::default_phase_eps_1() - } -} - -// ── BindMode ───────────────────────────────────────────────────────────────── - -/// Three-mode placement of the sketch + transmission work. Returned by -/// [`select_bind_mode`]. -#[derive(Debug, Clone, PartialEq)] -pub enum BindMode { - /// Mode 1: sketch processor at the edge ships sketch state via OTLP - /// to the backend's warm tier. The default for high-sample-per-window - /// workloads where the sketch's per-flush wire cost beats raw OTLP. - SketchAtEdge { family: SketchKind }, - - /// Mode 2: no sketch processor at edge; raw OTLP forwards to the - /// gateway/backend, which builds sketches at ingest. Picked when the - /// edge is resource-constrained (CPU / RAM) but the sketch still - /// wins on backend-side bandwidth + accuracy. - RawAtEdgeSketchAtBackend { family: SketchKind }, - - /// Mode 3: no sketch processor at edge; raw OTLP ships directly to - /// Prometheus's native OTLP receiver. Backend HTTP-forwards queries - /// to Prometheus's `/api/v1/query` endpoint. Picked when no sketch - /// family beats raw on bandwidth (low cardinality OR low - /// samples_per_window). Accuracy is exact (ε = 0). - RawAtEdgePrometheusArchive, -} - -// ── Bind-mode selection ────────────────────────────────────────────────────── - -/// Selection priority (Phase ε.1): -/// -/// 1. [`BindMode::SketchAtEdge`] — sketch beats raw on bandwidth AND -/// `edge_cpu_budget` allows. -/// 2. [`BindMode::RawAtEdgeSketchAtBackend`] — sketch wins backend-side -/// but `edge_cpu_budget` tight. -/// 3. [`BindMode::RawAtEdgePrometheusArchive`] — no sketch family beats -/// raw on bandwidth (samples_per_window × per_sample_bytes < min -/// sketch state). -/// -/// `families` is the set of sketch families the L4 binding rule -/// considers viable for the workload (e.g. `[Kll, DDSketch]` for a -/// quantile intent). The planner picks the family with the lowest -/// per-flush wire cost from among the viable ones. -/// -/// `edge_cpu_tight` is a single Boolean for whether the edge can afford -/// to run a sketch processor — Phase ε.1 treats this as a binary signal -/// (either the operator declared a tight budget, or they didn't); a -/// future revision will consult observed processor CPU from the -/// OnlineMetricsStore. -pub fn select_bind_mode( - families: &[SketchKind], - workload: &WireWorkload, - table: &WireCostTable, -) -> BindMode { - let raw_bytes_per_window = workload - .samples_per_window_per_series - .saturating_mul(workload.per_sample_bytes); - - // Find the cheapest viable sketch family (the one whose per-flush - // sketch state ≤ the raw bytes). If no family wins on bandwidth, - // Mode 3 (Prometheus archive) is correct: no sketch amortises. - let cheapest_winner = families - .iter() - .map(|f| (f.clone(), table.for_kind(f))) - .filter(|(_, c)| c.per_flush() < raw_bytes_per_window) - .min_by_key(|(_, c)| c.per_flush()); - - match cheapest_winner { - Some((family, _)) => { - // Sketch beats raw — pick edge vs. backend by the CPU budget. - if edge_cpu_budget_tight(workload) { - BindMode::RawAtEdgeSketchAtBackend { family } - } else { - BindMode::SketchAtEdge { family } - } - } - None => { - // No sketch beats raw → Prometheus archive (Mode 3). Note - // this also covers the degenerate `families.is_empty()` case - // (e.g. a workload without a quantile / cardinality / topk - // intent). - BindMode::RawAtEdgePrometheusArchive - } - } -} - -/// Phase ε.1 edge-CPU heuristic. Treats anything ≤ 0.25 cores as tight — -/// running a sketch processor at-rate (>1 kHz typical observability -/// workload) needs roughly 0.5 cores headroom for a single sketch family -/// based on the cross-host-parity benchmark numbers. A future revision -/// will replace this with an OnlineMetricsStore consultation. -fn edge_cpu_budget_tight(w: &WireWorkload) -> bool { - matches!(w.edge_cpu_budget, Some(b) if b <= 0.25) -} - -// ── Wire-bytes-per-window estimator ────────────────────────────────────────── - -/// Estimated per-window per-series wire bytes for a given bind mode. -/// -/// Used by the snapshot tests to confirm the planner's mode pick is -/// monotonic in workload size. -pub fn est_wire_bytes_per_window_per_series( - mode: &BindMode, - workload: &WireWorkload, - table: &WireCostTable, -) -> u64 { - match mode { - BindMode::SketchAtEdge { family } - | BindMode::RawAtEdgeSketchAtBackend { family } => { - // Same edge → backend wire footprint either way (the sketch - // state crosses the gateway in mode 1, the raw samples then - // sketched do in mode 2; backend ingest cost is mode-2-higher - // but that's a backend concern, not a wire concern). - // - // NOTE: mode 2 actually pays the raw cost to ship to the - // gateway, then a sketch cost gateway → backend. We surface - // the **edge-egress** cost here because that's what the - // bandwidth bench measures. - match mode { - BindMode::SketchAtEdge { .. } => table.for_kind(family).per_flush(), - BindMode::RawAtEdgeSketchAtBackend { .. } => workload - .samples_per_window_per_series - .saturating_mul(workload.per_sample_bytes), - _ => unreachable!(), - } - } - BindMode::RawAtEdgePrometheusArchive => workload - .samples_per_window_per_series - .saturating_mul(workload.per_sample_bytes), - } -} - -// ── Tests ──────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - - /// The per-sketch break-even values committed in Phase ε.1. The - /// "Resulting break-evens at 50 B per raw sample" line in the design - /// doc must match these. - #[test] - fn break_even_table_at_50_bytes_per_sample() { - let t = WireCostTable::default(); - assert_eq!(break_even_samples(t.ddsketch_delta, 50), 16, "DDSketch+delta"); - assert_eq!(break_even_samples(t.kll_full, 50), 64, "KLL full"); - assert_eq!(break_even_samples(t.hll_delta, 50), 204, "HLL+delta"); - assert_eq!(break_even_samples(t.count_min_delta, 50), 84, "Count-Min+delta"); - assert_eq!( - break_even_samples(t.count_sketch_delta, 50), - 5_004, - "Count-Sketch+delta" - ); - } - - /// Per-mode wire bytes at the 1 Hz × 60 s × 60 samples baseline. - /// Mode 1 (sketch at edge) ships the sketch state; Modes 2/3 ship raw - /// samples. The planner uses these to pick the cheapest mode. - #[test] - fn est_wire_bytes_baseline() { - let table = WireCostTable::default(); - let w = WireWorkload::default_phase_eps_1(); - // 60 samples × 50 B = 3 000 B raw. - let raw = est_wire_bytes_per_window_per_series( - &BindMode::RawAtEdgePrometheusArchive, - &w, - &table, - ); - assert_eq!(raw, 3_000); - // DDSketch state — 600 + 200 = 800 B. Sketch wins. - let ddsketch = est_wire_bytes_per_window_per_series( - &BindMode::SketchAtEdge { - family: SketchKind::DDSketch, - }, - &w, - &table, - ); - assert_eq!(ddsketch, 800); - // HLL state — 10 000 + 200 = 10 200 B. HLL loses at 60 samples. - let hll = est_wire_bytes_per_window_per_series( - &BindMode::SketchAtEdge { - family: SketchKind::Hll, - }, - &w, - &table, - ); - assert_eq!(hll, 10_200); - } - - /// Bind precedence: when the cheapest sketch wins on bandwidth and - /// the edge CPU budget allows, Mode 1 is picked. - #[test] - fn bind_mode_picks_sketch_at_edge_when_sketch_wins_and_cpu_ok() { - let table = WireCostTable::default(); - // 600 samples × 50 B = 30 000 B raw — DDSketch state (800 B) wins. - let w = WireWorkload { - samples_per_window_per_series: 600, - per_sample_bytes: 50, - accuracy_sla: 0.01, - edge_cpu_budget: None, - edge_ram_budget: None, - }; - let mode = select_bind_mode(&[SketchKind::DDSketch, SketchKind::Kll], &w, &table); - assert_eq!( - mode, - BindMode::SketchAtEdge { - family: SketchKind::DDSketch - } - ); - } - - /// Bind precedence: when the cheapest sketch wins on bandwidth but - /// the edge CPU budget is tight (≤ 0.25 cores), Mode 2 is picked - /// — sketch builds at the backend instead of the edge. - #[test] - fn bind_mode_picks_raw_at_edge_sketch_at_backend_when_cpu_tight() { - let table = WireCostTable::default(); - let w = WireWorkload { - samples_per_window_per_series: 600, - per_sample_bytes: 50, - accuracy_sla: 0.01, - edge_cpu_budget: Some(0.1), - edge_ram_budget: None, - }; - let mode = select_bind_mode(&[SketchKind::DDSketch, SketchKind::Kll], &w, &table); - assert_eq!( - mode, - BindMode::RawAtEdgeSketchAtBackend { - family: SketchKind::DDSketch - } - ); - } - - /// Bind precedence: when no sketch family beats raw on bandwidth, - /// Mode 3 (Prometheus archive) is picked. Raw goes to Prometheus's - /// native OTLP receiver; query-time HTTP forwarding hits - /// `/api/v1/query`. - #[test] - fn bind_mode_picks_prometheus_archive_when_no_sketch_wins_on_bandwidth() { - let table = WireCostTable::default(); - // 60 samples × 50 B = 3 000 B raw. DDSketch is 800 B (wins) — but - // the only candidate is HLL (10 200 B, loses) and CountSketch - // (250 200 B, loses). So no sketch beats raw → Mode 3. - let w = WireWorkload { - samples_per_window_per_series: 60, - per_sample_bytes: 50, - accuracy_sla: 0.01, - edge_cpu_budget: None, - edge_ram_budget: None, - }; - let mode = select_bind_mode(&[SketchKind::Hll, SketchKind::CountSketch], &w, &table); - assert_eq!(mode, BindMode::RawAtEdgePrometheusArchive); - } - - /// Empty family list — no sketch is considered, so Mode 3 by default. - #[test] - fn bind_mode_empty_family_list_returns_prometheus_archive() { - let table = WireCostTable::default(); - let w = WireWorkload::default_phase_eps_1(); - let mode = select_bind_mode(&[], &w, &table); - assert_eq!(mode, BindMode::RawAtEdgePrometheusArchive); - } - - /// Snapshot: Phase ε.1 break-even-table values committed for the - /// report. If any of these change, the report numbers must follow. - #[test] - fn break_even_table_snapshot_phase_eps_1() { - let t = WireCostTable::default(); - // (sketch family, expected break-even at 50 B per sample) - let cases = [ - (t.ddsketch_delta, 16u64), - (t.kll_full, 64), - (t.hll_delta, 204), - (t.count_min_delta, 84), - (t.count_sketch_delta, 5_004), - ]; - for (cost, expected) in cases { - assert_eq!(break_even_samples(cost, 50), expected); - } - } - - /// Snapshot scenario from the report: 1 Hz × 60 s × 60 samples + HLL - /// → controller picks Mode 3 (Prometheus archive) because HLL - /// state (10 200 B) > raw cost (3 000 B). - #[test] - fn snapshot_workload_1hz_60s_hll_picks_prometheus_archive() { - let table = WireCostTable::default(); - let w = WireWorkload { - samples_per_window_per_series: 60, - per_sample_bytes: 50, - accuracy_sla: 0.01, - edge_cpu_budget: None, - edge_ram_budget: None, - }; - let mode = select_bind_mode(&[SketchKind::Hll], &w, &table); - assert_eq!(mode, BindMode::RawAtEdgePrometheusArchive); - } - - /// Snapshot scenario from the report: 10 Hz × 60 s × 600 samples + - /// HLL → controller picks Mode 1 (HLL at edge wins now). Raw cost - /// (30 000 B) > HLL state (10 200 B), and edge CPU budget is open. - #[test] - fn snapshot_workload_10hz_60s_hll_picks_sketch_at_edge() { - let table = WireCostTable::default(); - let w = WireWorkload { - samples_per_window_per_series: 600, - per_sample_bytes: 50, - accuracy_sla: 0.01, - edge_cpu_budget: None, - edge_ram_budget: None, - }; - let mode = select_bind_mode(&[SketchKind::Hll], &w, &table); - assert_eq!( - mode, - BindMode::SketchAtEdge { - family: SketchKind::Hll - } - ); - } - - /// Snapshot scenario from the report: 1 Hz × 60 s + edge_cpu_budget=0.1 - /// cores → Mode 2 (raw at edge → sketch at backend) to save edge CPU. - /// At 1 Hz × 60 s × 60 samples × 50 B = 3 000 B raw, only DDSketch - /// (800 B) and KLL (3 200 B) are viable; DDSketch is the cheapest - /// sketch winner. Note: in this scenario the workload size is 60 - /// samples — DDSketch still wins (800 B < 3 000 B), so the cost - /// model nominates it; the tight edge-CPU budget then routes to - /// backend-side sketching. - #[test] - fn snapshot_workload_tight_edge_cpu_picks_raw_at_edge_sketch_at_backend() { - let table = WireCostTable::default(); - let w = WireWorkload { - samples_per_window_per_series: 60, - per_sample_bytes: 50, - accuracy_sla: 0.01, - edge_cpu_budget: Some(0.1), - edge_ram_budget: None, - }; - let mode = select_bind_mode(&[SketchKind::DDSketch, SketchKind::Kll], &w, &table); - assert_eq!( - mode, - BindMode::RawAtEdgeSketchAtBackend { - family: SketchKind::DDSketch - } - ); - } - - /// Snapshot scenario at 1 Hz × 60 s × 5 K cardinality (controller - /// picks Mode 3 for every sketch except DDSketch / KLL because at 60 - /// samples/window only the two smallest sketch families amortize). - /// The report lists this matrix. - #[test] - fn snapshot_5k_cardinality_1hz_60s_per_family_mode() { - let table = WireCostTable::default(); - let w = WireWorkload { - samples_per_window_per_series: 60, - per_sample_bytes: 50, - accuracy_sla: 0.01, - edge_cpu_budget: None, - edge_ram_budget: None, - }; - // raw = 3 000 B/window/series. - // DDSketch: 800 B → wins - // KLL: 3 200 B → loses (KLL full state > raw at 60 samples) - // HLL: 10 200 B → loses - // CountMin: 4 200 B → loses - // CountSketch: 250 200 B → loses - let dd = select_bind_mode(&[SketchKind::DDSketch], &w, &table); - assert_eq!( - dd, - BindMode::SketchAtEdge { - family: SketchKind::DDSketch - } - ); - let kll = select_bind_mode(&[SketchKind::Kll], &w, &table); - assert_eq!(kll, BindMode::RawAtEdgePrometheusArchive); - let hll = select_bind_mode(&[SketchKind::Hll], &w, &table); - assert_eq!(hll, BindMode::RawAtEdgePrometheusArchive); - let cms = select_bind_mode(&[SketchKind::Cms], &w, &table); - assert_eq!(cms, BindMode::RawAtEdgePrometheusArchive); - let cs = select_bind_mode(&[SketchKind::CountSketch], &w, &table); - assert_eq!(cs, BindMode::RawAtEdgePrometheusArchive); - } -} diff --git a/controller/src/query_language/datafusion/mod.rs b/controller/src/query_language/datafusion/mod.rs deleted file mode 100644 index 94dcfc61..00000000 --- a/controller/src/query_language/datafusion/mod.rs +++ /dev/null @@ -1,27 +0,0 @@ -//! DataFusion L1 backend — stub. -//! -//! The asap-fusion deployment model consumes a pre-built DataFusion -//! `LogicalPlan` upstream (its L1 happens in the caller's -//! `SessionContext`); see `controller/docs/design.md` §3 row 1. This -//! stub exists so a future asap-fusion adapter only has to fill in the -//! `parse` body. - -use super::language::{Language, ParseError}; -use super::language_ast::LanguageAst; -use crate::types_v2::QueryLanguage; - -/// DataFusion implementation of the [`Language`] trait. Currently stubbed. -#[derive(Debug, Default, Clone, Copy)] -pub struct DataFusionLanguage; - -impl Language for DataFusionLanguage { - fn id(&self) -> QueryLanguage { - QueryLanguage::DataFusion - } - - fn parse(&self, _source: &str) -> Result { - Err(ParseError::Unimplemented( - "DataFusion parser not implemented in DC deployment mode", - )) - } -} diff --git a/controller/src/query_language/elastic_dsl/mod.rs b/controller/src/query_language/elastic_dsl/mod.rs deleted file mode 100644 index 836224c9..00000000 --- a/controller/src/query_language/elastic_dsl/mod.rs +++ /dev/null @@ -1,25 +0,0 @@ -//! ElasticDSL L1 backend — stub. -//! -//! Reserved for the future ElasticDSL deployment model -//! (`controller/docs/design.md` §3 row 1). No L1 parser is shipped in -//! the DC deployment build. - -use super::language::{Language, ParseError}; -use super::language_ast::LanguageAst; -use crate::types_v2::QueryLanguage; - -/// ElasticDSL implementation of the [`Language`] trait. Currently stubbed. -#[derive(Debug, Default, Clone, Copy)] -pub struct ElasticDslLanguage; - -impl Language for ElasticDslLanguage { - fn id(&self) -> QueryLanguage { - QueryLanguage::ElasticDsl - } - - fn parse(&self, _source: &str) -> Result { - Err(ParseError::Unimplemented( - "ElasticDSL parser not implemented in DC deployment mode", - )) - } -} diff --git a/controller/src/query_language/language.rs b/controller/src/query_language/language.rs deleted file mode 100644 index 86ac1993..00000000 --- a/controller/src/query_language/language.rs +++ /dev/null @@ -1,56 +0,0 @@ -//! L1 `Language` trait — abstract over the per-language parser. -//! -//! See `controller/docs/design.md` §6 `core::query_language`. -//! -//! Every backend (PromQL, SQL, DataFusion, ElasticDSL) implements -//! [`Language`] and returns a [`LanguageAst`] variant tagged with the -//! same [`QueryLanguage`] enum the public `QuerySpec` carries. - -use crate::types_v2::QueryLanguage; - -use super::language_ast::LanguageAst; - -// ── Errors ──────────────────────────────────────────────────────────────────── - -/// Errors a [`Language`] backend can return from `parse`. -/// -/// `Unimplemented` is used by the stubbed-out non-PromQL backends so the -/// type system stays uniform without committing to a parser implementation. -#[derive(Debug, thiserror::Error)] -pub enum ParseError { - /// The underlying language parser rejected the source string. - #[error("parse failed for {language:?}: {source_err}")] - Backend { - /// Which backend rejected the source. - language: QueryLanguage, - /// Free-form parser-specific message. - source_err: String, - }, - - /// The backend is registered but not implemented in this build (DC - /// deployment mode ships PromQL only). - #[error("{0}")] - Unimplemented(&'static str), -} - -impl ParseError { - /// Convenience: wrap any `Display` parser error into [`ParseError::Backend`]. - pub fn backend(language: QueryLanguage, e: impl std::fmt::Display) -> Self { - ParseError::Backend { language, source_err: e.to_string() } - } -} - -// ── Trait ───────────────────────────────────────────────────────────────────── - -/// Per-language L1 parser façade. One impl per [`QueryLanguage`] variant. -/// -/// `parse` is the only required method; `id` is a static tag so callers -/// can cross-check the variant returned by [`LanguageAst`] without -/// downcasting. -pub trait Language: Send + Sync { - /// The [`QueryLanguage`] variant this backend implements. - fn id(&self) -> QueryLanguage; - - /// Parse `source` into a language-flavored AST. - fn parse(&self, source: &str) -> Result; -} diff --git a/controller/src/query_language/language_ast.rs b/controller/src/query_language/language_ast.rs deleted file mode 100644 index 86e4f806..00000000 --- a/controller/src/query_language/language_ast.rs +++ /dev/null @@ -1,36 +0,0 @@ -//! [`LanguageAst`] — the discriminated union over per-language ASTs. -//! -//! One variant per [`crate::types_v2::QueryLanguage`]. The PromQL variant -//! is the only one carrying real data today; the rest are reserved for -//! future deployment models (asap-fusion, ElasticDSL). - -use super::promql::ast::PromQLAst; - -/// Tagged union of language-flavored ASTs returned by [`super::Language::parse`]. -/// -/// Adding a new language is mechanical: add a new variant here and wire -/// up a backend in `query_language::::`. -#[derive(Debug, Clone)] -pub enum LanguageAst { - /// PromQL AST — wraps the existing `query_parser::ParsedQuery` plus - /// the full `QueryExpr` from the legacy parser. - PromQL(PromQLAst), - // Future backends (kept commented to surface intent): - // Sql(SqlAst), - // DataFusion(DfAst), - // ElasticDsl(EsAst), -} - -impl LanguageAst { - /// Return `true` when this AST belongs to the PromQL backend. - pub fn is_promql(&self) -> bool { - matches!(self, LanguageAst::PromQL(_)) - } - - /// Borrow the inner `PromQLAst` if the variant is PromQL. - pub fn as_promql(&self) -> Option<&PromQLAst> { - match self { - LanguageAst::PromQL(a) => Some(a), - } - } -} diff --git a/controller/src/query_language/mod.rs b/controller/src/query_language/mod.rs deleted file mode 100644 index b14a92a3..00000000 --- a/controller/src/query_language/mod.rs +++ /dev/null @@ -1,48 +0,0 @@ -// Layer 1 scaffolding ships ahead of any in-tree call site (the -// downstream `pipeline` driver lands in a later phase). Dead-code -// warnings are silenced here, not on individual items, so the public -// surface is uncluttered. -#![allow(dead_code, unused_imports)] - -//! Layer 1 — `query_language`. -//! -//! Per-language parser façade. See `controller/docs/design.md` §6 -//! `core::query_language` for the design contract. -//! -//! # Module layout -//! -//! ```text -//! query_language/ -//! ├── language.rs — Language trait + ParseError -//! ├── language_ast.rs — LanguageAst sum type -//! ├── promql/ — PromQL backend (active; wraps query_parser::promql) -//! ├── sql/ — SQL backend (stub; Unimplemented) -//! ├── datafusion/ — DataFusion backend (stub; Unimplemented) -//! └── elastic_dsl/ — ElasticDSL backend (stub; Unimplemented) -//! ``` -//! -//! # DC deployment scope -//! -//! Only [`promql::PromQLLanguage`] is implemented. The other backends -//! return [`language::ParseError::Unimplemented`] so the type system is -//! uniform; adding a real backend later is a localised change to the -//! relevant sub-module. - -pub mod language; -pub mod language_ast; - -pub mod promql; -pub mod sql; -pub mod datafusion; -pub mod elastic_dsl; - -pub use language::{Language, ParseError}; -pub use language_ast::LanguageAst; - -pub use promql::PromQLLanguage; -pub use sql::SqlLanguage; -pub use datafusion::DataFusionLanguage; -pub use elastic_dsl::ElasticDslLanguage; - -#[cfg(test)] -mod tests; diff --git a/controller/src/query_language/promql/ast.rs b/controller/src/query_language/promql/ast.rs deleted file mode 100644 index 86ccfe19..00000000 --- a/controller/src/query_language/promql/ast.rs +++ /dev/null @@ -1,38 +0,0 @@ -//! `PromQLAst` — the L1 output of the PromQL backend. -//! -//! Thin wrapper that bundles the legacy parser's two outputs (the rich -//! `QueryExpr` algebra tree and the flat `ParsedQuery` summary) so -//! downstream L2 lowering can pick whichever shape it needs without -//! re-parsing. - -use crate::algebra::expr::QueryExpr; -use crate::query_parser::ParsedQuery; - -/// PromQL parse output. Carries both the algebra tree and the flat -/// summary the legacy analyzer consumes. -/// -/// We bundle both because the existing `query_parser::parse_query` -/// already produces them, and L2 lowering wants the structured tree -/// while back-compat callers (`Analyzer`) still want the flat summary. -#[derive(Debug, Clone)] -pub struct PromQLAst { - /// Original PromQL source string (preserved for debugging / errors). - pub source: String, - /// Algebra tree — the existing `parse_query_expr` output. - pub expr: QueryExpr, - /// Flat summary — the existing `parse_query` output. - pub summary: ParsedQuery, -} - -impl PromQLAst { - /// Construct from the legacy parser's two outputs. - pub fn new(source: String, expr: QueryExpr, summary: ParsedQuery) -> Self { - Self { source, expr, summary } - } - - /// Borrow the algebra tree. - pub fn expr(&self) -> &QueryExpr { &self.expr } - - /// Borrow the flat summary. - pub fn summary(&self) -> &ParsedQuery { &self.summary } -} diff --git a/controller/src/query_language/promql/mod.rs b/controller/src/query_language/promql/mod.rs deleted file mode 100644 index 9f099317..00000000 --- a/controller/src/query_language/promql/mod.rs +++ /dev/null @@ -1,39 +0,0 @@ -//! PromQL L1 backend — wraps the existing `query_parser::promql` parser -//! behind the [`super::Language`] trait. -//! -//! No new parsing logic lives here; this is purely an adapter that -//! collects the legacy parser's two outputs into a [`PromQLAst`] and -//! tags it as [`LanguageAst::PromQL`]. - -pub mod ast; - -use super::language::{Language, ParseError}; -use super::language_ast::LanguageAst; -use crate::query_parser::{parse_query, parse_query_expr}; -use crate::types_v2::QueryLanguage; - -pub use ast::PromQLAst; - -/// PromQL implementation of the [`Language`] trait. -/// -/// Construct with `PromQLLanguage::default()`; the type is unit so it -/// is cheap to clone / store in a registry. -#[derive(Debug, Default, Clone, Copy)] -pub struct PromQLLanguage; - -impl Language for PromQLLanguage { - fn id(&self) -> QueryLanguage { - QueryLanguage::PromQL - } - - fn parse(&self, source: &str) -> Result { - // Delegate to the existing parser — both entry points re-parse the - // same string today; the cost is negligible (microseconds) and we - // get the legacy `ParsedQuery` for free for back-compat callers. - let expr = parse_query_expr(source) - .map_err(|e| ParseError::backend(QueryLanguage::PromQL, e))?; - let summary = parse_query(source) - .map_err(|e| ParseError::backend(QueryLanguage::PromQL, e))?; - Ok(LanguageAst::PromQL(PromQLAst::new(source.to_string(), expr, summary))) - } -} diff --git a/controller/src/query_language/sql/mod.rs b/controller/src/query_language/sql/mod.rs deleted file mode 100644 index ac7ee4d1..00000000 --- a/controller/src/query_language/sql/mod.rs +++ /dev/null @@ -1,26 +0,0 @@ -//! SQL L1 backend — stub for the DC deployment mode. -//! -//! The DC deployment ships PromQL only; the SQL parser path remains -//! addressable through the legacy `query_parser::sql` entry point but -//! is not exposed via the new [`Language`] trait yet. A real impl wraps -//! `sqlparser` and emits a `SqlAst` analogous to [`super::promql::PromQLAst`]. - -use super::language::{Language, ParseError}; -use super::language_ast::LanguageAst; -use crate::types_v2::QueryLanguage; - -/// SQL implementation of the [`Language`] trait. Currently stubbed. -#[derive(Debug, Default, Clone, Copy)] -pub struct SqlLanguage; - -impl Language for SqlLanguage { - fn id(&self) -> QueryLanguage { - QueryLanguage::Sql - } - - fn parse(&self, _source: &str) -> Result { - Err(ParseError::Unimplemented( - "SQL parser not implemented in DC deployment mode", - )) - } -} diff --git a/controller/src/query_language/tests.rs b/controller/src/query_language/tests.rs deleted file mode 100644 index 1100e462..00000000 --- a/controller/src/query_language/tests.rs +++ /dev/null @@ -1,69 +0,0 @@ -//! Tests for the L1 `query_language` façade. - -use super::*; -use crate::types_v2::QueryLanguage; - -#[test] -fn promql_language_id_is_promql() { - let l = PromQLLanguage; - assert_eq!(l.id(), QueryLanguage::PromQL); -} - -#[test] -fn promql_language_parses_basic() { - let ast = PromQLLanguage.parse("up").expect("PromQL parse should succeed"); - assert!(ast.is_promql(), "expected PromQL variant"); - let promql = ast.as_promql().unwrap(); - assert_eq!(promql.summary().metric_name, "up"); - assert_eq!(promql.source, "up"); -} - -#[test] -fn promql_language_parses_quantile_over_time() { - let ast = PromQLLanguage - .parse("sum by (host) (quantile_over_time(0.99, latency[5m]))") - .expect("PromQL quantile parse should succeed"); - let promql = ast.as_promql().unwrap(); - assert!(promql.summary().quantiles.contains(&0.99)); - assert_eq!(promql.summary().metric_name, "latency"); -} - -#[test] -fn promql_language_surfaces_parse_error_as_backend_error() { - let err = PromQLLanguage.parse("@!not promql@!").unwrap_err(); - match err { - ParseError::Backend { language, .. } => { - assert_eq!(language, QueryLanguage::PromQL); - } - other => panic!("expected ParseError::Backend, got {other:?}"), - } -} - -#[test] -fn sql_language_returns_unimplemented() { - let err = SqlLanguage.parse("SELECT 1").unwrap_err(); - assert!(matches!(err, ParseError::Unimplemented(_))); - assert_eq!(SqlLanguage.id(), QueryLanguage::Sql); -} - -#[test] -fn datafusion_language_returns_unimplemented() { - let err = DataFusionLanguage.parse("SELECT 1").unwrap_err(); - assert!(matches!(err, ParseError::Unimplemented(_))); - assert_eq!(DataFusionLanguage.id(), QueryLanguage::DataFusion); -} - -#[test] -fn elastic_dsl_language_returns_unimplemented() { - let err = ElasticDslLanguage.parse("{}").unwrap_err(); - assert!(matches!(err, ParseError::Unimplemented(_))); - assert_eq!(ElasticDslLanguage.id(), QueryLanguage::ElasticDsl); -} - -#[test] -fn language_trait_object_is_object_safe() { - // Compile-time check: the trait can be used as `dyn Language` so - // future code can store a `Vec>` registry. - let l: Box = Box::new(PromQLLanguage); - assert_eq!(l.id(), QueryLanguage::PromQL); -} diff --git a/controller/src/query_parser/mod.rs b/controller/src/query_parser/mod.rs deleted file mode 100644 index fe4acdc3..00000000 --- a/controller/src/query_parser/mod.rs +++ /dev/null @@ -1,529 +0,0 @@ -//! SP-1 query workload extraction — PromQL and SQL parsers. -//! -//! # Entry points -//! -//! | Function | Returns | Use | -//! |---|---|---| -//! | [`parse_query_expr`] | `QueryExpr` | Full algebra IR | -//! | [`parse_query`] | `ParsedQuery` | Backward compat with existing analyzer | -//! -//! # Supported PromQL patterns (via `promql-parser` AST) -//! - `quantile_over_time(φ, m{f}[w]) by (dims)` -//! - `histogram_quantile(φ, rate(m{f}[w])) by (le)` -//! - `avg/min/max/stddev/stdvar_over_time(m{f}[w]) by (dims)` -//! - `sum/count_over_time(m{f}[w]) by (dims)` -//! - `topk(k, *_over_time(…) by (dims))` -//! - `count(*_over_time(…) by (dims))` — cardinality -//! - `changes/resets(m{f}[w])` -//! - Bare metric selector / binary op → `exact_required` -//! -//! # Supported SQL patterns (doc §SQL Operators) -//! - `COUNT(*)` with/without GROUP BY → frequency / exact -//! - `COUNT(DISTINCT col)` ± GROUP BY → cardinality / Hydra -//! - `AVG/MIN/MAX(col)` ± GROUP BY → quantile / exact extrema -//! - `SUM(col)` → exact -//! - ORDER BY … DESC LIMIT k → heavy-hitter CountSketch -//! - Multiple aggs in one SELECT → all ops collected (Merge) -//! - JOIN … ON key → JoinSketch push-down -//! - UNION ALL → Merge (sketch linearity) - -pub mod promql; -pub mod sql; - -use std::collections::HashMap; -use std::time::Duration; - -use crate::algebra::expr::{AggIntent, QueryExpr}; -use crate::types::AggType; - -// ── Output types (legacy — consumed by analyzer and planner) ────────────────── - -/// Flat intermediate representation consumed by [`crate::analyzer::Analyzer`]. -/// -/// Produced by [`parse_query`] via [`QueryExpr`] tree walking. -#[derive(Debug, Clone)] -pub struct ParsedQuery { - /// Metric name (PromQL: from selector; SQL: FROM clause table). - pub metric_name: String, - /// Aggregation types inferred from the query. - pub aggregations: Vec, - /// Dimensions that must be preserved for GROUP BY / `by (dims)`. - pub group_by_labels: Vec, - /// Equality label filters extracted from the query. - pub label_filters: HashMap, - /// Time window extracted from the range vector or query context. - pub time_window: Duration, - /// True when the query requires per-sample exact values. - pub exact_required: bool, - /// Quantile φ values implied by the query. - pub quantiles: Vec, - /// Named pattern hint for domain-specific planner defaults. - pub hint: Option, -} - -/// Named query pattern recognised by the DEBS-aware planner. -#[derive(Debug, Clone)] -pub enum QueryHint { - // ── DEBS 2022 financial queries ─────────────────────────────────────────── - /// Q1 – per-symbol EMA via quantile proxy (DDSketch / KLL). - DebsEma, - /// Q3 – top-K symbols by event count or price move (CountSketch). - DebsTopK { k: u64 }, - /// Q4 – per-symbol high / low / last / range (extreme-quantile DDSketch). - DebsPriceStats, - /// Q5 / Q9 – realized volatility / Bollinger bands via IQR proxy. - DebsVolatility, - /// Q6 – distinct active symbols per window (HLL). - DebsCardinality, - /// Q7 – TWAP as median / p50 (DDSketch). - DebsTwap, - /// Q8 – price anomaly detection via IQR (DDSketch). - DebsAnomaly, - // ── Exact-only patterns ─────────────────────────────────────────────────── - /// Query requires stateful per-sample computation; no sketch benefit. - ExactRequired { reason: String }, -} - -// ── Public entry points ─────────────────────────────────────────────────────── - -/// Parse a raw query string (PromQL or SQL) into the general [`QueryExpr`] IR. -/// -/// Both parsers emit Layer 2 relational operators (`Aggregate { AggFunc }`). -/// The shared lowering pass converts `Aggregate` → `SketchAgg { AggIntent }` -/// where applicable. -pub fn parse_query_expr(query: &str) -> anyhow::Result { - let q = query.trim(); - let upper = q.to_ascii_uppercase(); - let layer2 = if upper.starts_with("SELECT") || upper.starts_with("WITH") { - sql::parse_sql_expr(q)? - } else { - promql::parse_promql_expr(q)? - }; - // Layer 2 → Layer 3 lowering (shared by both languages). - Ok(crate::algebra::lower::lower_to_sketch_algebra(layer2)) -} - -/// Parse a raw query string (PromQL or SQL) into a [`ParsedQuery`]. -/// -/// This is the backward-compatible entry point for the existing -/// [`crate::analyzer::Analyzer`]. Internally it parses via [`parse_query_expr`] -/// and extracts the flat summary by walking the [`QueryExpr`] tree. -pub fn parse_query(query: &str) -> anyhow::Result { - let qe = parse_query_expr(query)?; - Ok(qe_to_parsed_query(&qe)) -} - -/// Extract a flat [`ParsedQuery`] by walking a [`QueryExpr`] tree. -fn qe_to_parsed_query(qe: &QueryExpr) -> ParsedQuery { - let mut c = QeCollector::default(); - c.visit(qe); - c.build() -} - -#[derive(Default)] -struct QeCollector { - metric_name: Option, - agg_types: Vec, - group_by_labels: Vec, - label_filters: HashMap, - time_window: Option, - exact_required: bool, - quantiles: Vec, - topk: Option, - /// True when currently visiting inside a TopK node (affects Count handling). - inside_topk: bool, -} - -impl QeCollector { - fn visit(&mut self, expr: &QueryExpr) { - use crate::algebra::expr::{FilterOp, FilterVal, LiteralValue, ScalarExpr}; - match expr { - QueryExpr::Source(s) => { - if self.metric_name.is_none() { - self.metric_name = Some(s.name.clone()); - } - } - QueryExpr::Filter { pred, input } => { - // Extract equality label filters from the predicate tree. - collect_filters_from_scalar(pred, &mut self.label_filters); - self.visit(input); - } - QueryExpr::Window { duration, input, .. } => { - if self.time_window.is_none() { - self.time_window = Some(*duration); - } - self.visit(input); - } - QueryExpr::Partition { keys, input } => { - for k in keys.keys() { - if !self.group_by_labels.contains(k) { - self.group_by_labels.push(k.clone()); - } - } - self.visit(input); - } - QueryExpr::SketchAgg { op, input, .. } => { - self.collect_op(op); - self.visit(input); - } - QueryExpr::WindowedAgg { agg, window, input, .. } => { - if self.time_window.is_none() { - if let crate::algebra::expr::WindowKind::Tumbling { size } = &window.kind { - self.time_window = Some(*size); - } - } - self.collect_op(agg); - self.visit(input); - } - QueryExpr::TopK { k, input, .. } => { - self.topk = Some(*k); - let prev = self.inside_topk; - self.inside_topk = true; - self.visit(input); - self.inside_topk = prev; - } - QueryExpr::Dedup { input, .. } => self.visit(input), - QueryExpr::Merge { inputs } => { - for i in inputs { self.visit(i); } - } - QueryExpr::JoinSketch { outer, inner, .. } => { - self.visit(outer); - self.visit(inner); - } - QueryExpr::Aggregate { keys, aggs, input, .. } => { - for k in keys { - if !self.group_by_labels.contains(k) { - self.group_by_labels.push(k.clone()); - } - } - let has_group_by = !keys.is_empty(); - for agg in aggs { - self.collect_agg_func_with_group(&agg.func, has_group_by); - } - self.visit(input); - } - QueryExpr::Project { input, .. } - | QueryExpr::Sort { input, .. } - | QueryExpr::Limit { input, .. } - | QueryExpr::HistogramQuantile { input, .. } - | QueryExpr::PromQLSubquery { input, .. } - | QueryExpr::WindowFunc { input, .. } => self.visit(input), - QueryExpr::Join { left, right, .. } - | QueryExpr::SetOp { left, right, .. } - | QueryExpr::BinaryOp { lhs: left, rhs: right, .. } => { - self.visit(left); - self.visit(right); - } - QueryExpr::Subquery { expr, .. } => self.visit(expr), - QueryExpr::LetBinding { expr, body, .. } => { - self.visit(expr); - self.visit(body); - } - QueryExpr::Ref(_) => {} - } - } - - fn collect_agg_func_with_group(&mut self, func: &crate::algebra::expr::AggFunc, has_group_by: bool) { - use crate::algebra::expr::AggFunc; - // COUNT(*) without GROUP BY → exact (no sketch benefit), unless inside topk - // where Count means frequency counting. - if matches!(func, AggFunc::Count) && !has_group_by && !self.inside_topk { - self.exact_required = true; - return; - } - self.collect_agg_func(func); - } - - fn collect_agg_func(&mut self, func: &crate::algebra::expr::AggFunc) { - use crate::algebra::expr::AggFunc; - match func { - AggFunc::CountDistinct => { - if !self.agg_types.contains(&AggType::Cardinality) { - self.agg_types.push(AggType::Cardinality); - } - } - AggFunc::Count => { - if !self.agg_types.contains(&AggType::Frequency) { - self.agg_types.push(AggType::Frequency); - } - } - AggFunc::HeavyHitters { .. } => { - if !self.agg_types.contains(&AggType::Frequency) { - self.agg_types.push(AggType::Frequency); - } - } - AggFunc::Quantile(phi) => { - if !self.agg_types.contains(&AggType::Quantile) { - self.agg_types.push(AggType::Quantile); - } - if !self.quantiles.contains(phi) { - self.quantiles.push(*phi); - } - } - AggFunc::Avg => { - if !self.agg_types.contains(&AggType::Quantile) { - self.agg_types.push(AggType::Quantile); - } - // AVG maps to p50 (median) sketch - if !self.quantiles.contains(&0.5) { self.quantiles.push(0.5); } - } - AggFunc::Min => { - if !self.agg_types.contains(&AggType::Quantile) { - self.agg_types.push(AggType::Quantile); - } - if !self.quantiles.contains(&0.0) { self.quantiles.push(0.0); } - } - AggFunc::Max => { - if !self.agg_types.contains(&AggType::Quantile) { - self.agg_types.push(AggType::Quantile); - } - if !self.quantiles.contains(&1.0) { self.quantiles.push(1.0); } - } - AggFunc::StdDev { .. } | AggFunc::Variance { .. } => { - if !self.agg_types.contains(&AggType::Quantile) { - self.agg_types.push(AggType::Quantile); - } - // StdDev/Variance use IQR proxy via p25/p75. - if !self.quantiles.contains(&0.25) { self.quantiles.push(0.25); } - if !self.quantiles.contains(&0.75) { self.quantiles.push(0.75); } - } - AggFunc::Sum | AggFunc::Rate | AggFunc::Increase | AggFunc::Delta - | AggFunc::Custom(_) => { - self.exact_required = true; - } - } - } - - fn collect_op(&mut self, op: &AggIntent) { - match op { - AggIntent::Cardinality { .. } => { - if !self.agg_types.contains(&AggType::Cardinality) { - self.agg_types.push(AggType::Cardinality); - } - } - AggIntent::Frequency { .. } => { - if !self.agg_types.contains(&AggType::Frequency) { - self.agg_types.push(AggType::Frequency); - } - } - AggIntent::Quantile { quantiles, .. } => { - if !self.agg_types.contains(&AggType::Quantile) { - self.agg_types.push(AggType::Quantile); - } - for &q in quantiles { - if !self.quantiles.contains(&q) { self.quantiles.push(q); } - } - } - AggIntent::Extrema { min, max } => { - if !self.agg_types.contains(&AggType::Quantile) { - self.agg_types.push(AggType::Quantile); - } - // Extrema map to boundary quantiles for legacy compat. - if *min && !self.quantiles.contains(&0.0) { self.quantiles.push(0.0); } - if *max && !self.quantiles.contains(&1.0) { self.quantiles.push(1.0); } - } - AggIntent::Exact(_) => { self.exact_required = true; } - AggIntent::PerPartition { inner, .. } => self.collect_op(inner), - } - } - - fn build(self) -> ParsedQuery { - let metric_name = self.metric_name.unwrap_or_default(); - let mut qs = self.quantiles; - qs.sort_by(|a, b| a.partial_cmp(b).unwrap()); - qs.dedup(); - let hint = debs_hint( - &metric_name, - &self.agg_types, - &qs, - self.exact_required, - self.topk, - ); - ParsedQuery { - metric_name, - aggregations: self.agg_types, - group_by_labels: self.group_by_labels, - label_filters: self.label_filters, - time_window: self.time_window.unwrap_or(Duration::from_secs(300)), - exact_required: self.exact_required, - quantiles: qs, - hint, - } - } -} - -fn collect_filters_from_scalar( - pred: &crate::algebra::expr::ScalarExpr, - out: &mut HashMap, -) { - use crate::algebra::expr::{BinaryOpKind, LiteralValue, ScalarExpr}; - match pred { - ScalarExpr::BinaryOp { op: BinaryOpKind::Eq, lhs, rhs } => { - if let (ScalarExpr::Column(col), ScalarExpr::Literal(LiteralValue::Str(v))) = - (lhs.as_ref(), rhs.as_ref()) - { - out.insert(col.clone(), v.clone()); - } - } - ScalarExpr::BinaryOp { op: BinaryOpKind::And, lhs, rhs } => { - collect_filters_from_scalar(lhs, out); - collect_filters_from_scalar(rhs, out); - } - _ => {} - } -} - -// ── DEBS hint classifier (shared by both parsers via to_parsed_query) ───────── - -/// Returns the DEBS-specific hint for `financial.last_trade_price` queries. -pub(super) fn debs_hint( - metric: &str, - aggs: &[AggType], - quantiles: &[f64], - exact_required: bool, - topk: Option, -) -> Option { - let is_debs = metric == "financial.last_trade_price" - || metric == "financial_last_trade_price"; - if !is_debs { return None; } - - if exact_required { - return Some(QueryHint::ExactRequired { - reason: "query requires per-sample stateful computation".into(), - }); - } - if let Some(k) = topk { - return Some(QueryHint::DebsTopK { k }); - } - let primary = aggs.first()?; - match primary { - AggType::Cardinality => Some(QueryHint::DebsCardinality), - AggType::Frequency => Some(QueryHint::DebsTopK { k: 10 }), - AggType::Quantile => { - let qs: std::collections::HashSet = quantiles - .iter() - .map(|&q| (q * 100.0).round() as i32) - .collect(); - if qs.contains(&50) && qs.len() == 1 { - Some(QueryHint::DebsTwap) - } else if qs.contains(&0) || qs.contains(&100) { - Some(QueryHint::DebsPriceStats) - } else if qs.contains(&25) && qs.contains(&75) && qs.contains(&50) { - Some(QueryHint::DebsAnomaly) - } else if qs.contains(&25) && qs.contains(&75) { - Some(QueryHint::DebsVolatility) - } else { - Some(QueryHint::DebsEma) - } - } - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - - // Smoke tests for the unified entry point. - - #[test] - fn sql_dispatched_correctly() { - let pq = parse_query("SELECT COUNT(*) FROM hits GROUP BY AdvEngineID").unwrap(); - assert!(pq.aggregations.contains(&AggType::Frequency)); - } - - #[test] - fn promql_dispatched_correctly() { - // `by` belongs to the aggregate operator, not the function call. - let pq = parse_query( - "sum by (host) (quantile_over_time(0.99, latency[5m]))" - ).unwrap(); - assert!(pq.aggregations.contains(&AggType::Quantile)); - assert_eq!(pq.quantiles, vec![0.99]); - } - - #[test] - fn parse_query_expr_returns_expr() { - let pq = parse_query( - "topk by (symbol) (10, count_over_time(financial_last_trade_price[5m]))" - ).unwrap(); - // Should parse without error and extract the metric name. - assert_eq!(pq.metric_name, "financial_last_trade_price"); - } -} - -#[cfg(test)] -mod doc_verify_all { - use super::*; - use crate::algebra::expr::*; - - #[test] - fn example4_promql_quantile() { - let expr = parse_query_expr( - "quantile_over_time(0.99, http_request_duration{env=\"prod\"}[5m])" - ).unwrap(); - // Doc: WindowedAgg { Quantile([0.99]), Tumbling(5m), Filter(Source) } - assert!(matches!(&expr, QueryExpr::WindowedAgg { agg: AggIntent::Quantile { .. }, .. })); - } - - #[test] - fn example5_promql_topk() { - let expr = parse_query_expr( - "topk by (service) (10, count_over_time(requests{env=\"prod\"}[1m]))" - ).unwrap(); - // Doc: TopK { 10, Partition { ["service"], WindowedAgg { Frequency } } } - match &expr { - QueryExpr::TopK { k: 10, input, .. } => { - match input.as_ref() { - QueryExpr::Partition { keys, input: inner } => { - assert_eq!(keys.keys(), &["service".to_string()]); - assert!(matches!(inner.as_ref(), QueryExpr::WindowedAgg { agg: AggIntent::Frequency { .. }, .. })); - } - other => panic!("expected Partition, got {other:?}"), - } - } - other => panic!("expected TopK, got {other:?}"), - } - } - - #[test] - fn example6_sql_avg() { - let expr = parse_query_expr( - "SELECT symbol, AVG(price) FROM trades GROUP BY symbol" - ).unwrap(); - // Doc: Partition { ["symbol"], SketchAgg { Quantile([0.5]), Source } } - match &expr { - QueryExpr::Partition { keys, input } => { - assert_eq!(keys.keys(), &["symbol".to_string()]); - assert!(matches!(input.as_ref(), QueryExpr::SketchAgg { op: AggIntent::Quantile { .. }, .. })); - } - other => panic!("expected Partition, got {other:?}"), - } - } - - #[test] - fn example7_sql_tumble() { - let expr = parse_query_expr( - "SELECT region, COUNT(DISTINCT user_id) AS cnt FROM sessions GROUP BY region, TUMBLE(ts, INTERVAL '5' MINUTE) ORDER BY cnt DESC LIMIT 10" - ).unwrap(); - // Doc: Limit { 10, Sort { Partition { ["region"], WindowedAgg { Cardinality } } } } - match &expr { - QueryExpr::Limit { n: 10, input, .. } => { - match input.as_ref() { - QueryExpr::Sort { input: sort_inner, .. } => { - match sort_inner.as_ref() { - QueryExpr::Partition { keys, input: part_inner } => { - assert_eq!(keys.keys(), &["region".to_string()]); - assert!(matches!(part_inner.as_ref(), QueryExpr::WindowedAgg { agg: AggIntent::Cardinality { .. }, .. })); - } - other => panic!("expected Partition, got {other:?}"), - } - } - other => panic!("expected Sort, got {other:?}"), - } - } - other => panic!("expected Limit, got {other:?}"), - } - } -} diff --git a/controller/src/query_parser/promql.rs b/controller/src/query_parser/promql.rs deleted file mode 100644 index 2c61bdae..00000000 --- a/controller/src/query_parser/promql.rs +++ /dev/null @@ -1,715 +0,0 @@ -//! Layers 1→2 lowering: PromQL string → QueryExpr (relational plan). -//! -//! - **Layer 1**: the `promql-parser` crate parses the PromQL string into a -//! language-specific AST (`promql_parser::parser::Expr`). -//! - **Layer 2**: the walk functions (`walk_qe`, `walk_call_qe`, `walk_aggregate_qe`) -//! interpret PromQL semantics (range vectors, aggregation operators, label matchers) -//! and emit relational operators (`Aggregate { AggFunc }` + `Window`). -//! -//! The output is a Layer 2 `QueryExpr` tree — the same relational operators that -//! the SQL parser emits. A shared lowering pass (`algebra::lower`) converts -//! `Aggregate { AggFunc }` → `SketchAgg { AggIntent }` for both languages. -//! -//! # PromQL → AggFunc mapping (summary) -//! -//! | Expression | AggFunc | -//! |---|---| -//! | `quantile_over_time(φ, m[w])` | Quantile(φ) | -//! | `histogram_quantile(φ, rate(m[w]))` | (HistogramQuantile node — not an Aggregate) | -//! | `avg_over_time(m[w])` | Avg | -//! | `min_over_time(m[w])` | Min | -//! | `max_over_time(m[w])` | Max | -//! | `stddev/stdvar_over_time(m[w])` | StdDev / Variance | -//! | `count_over_time(m[w])` | Count / CountDistinct (context) | -//! | `sum_over_time(m[w])` | Sum | -//! | `last_over_time / delta / deriv / predict_linear` | Delta / Sum (exact) | -//! | `changes / resets` | Count | -//! | `rate / irate / increase` | Rate / Increase | -//! | `topk(k, …)` outer | TopK (structural — not AggFunc) | -//! | `count(…over_time… by (d))` outer | CountDistinct | -//! | `m{filters}` bare | Sum (exact) | -//! | `m_a op m_b` binary | (BinaryOp — not an Aggregate) | - -use std::time::Duration; - -use anyhow::anyhow; -use promql_parser::parser::{self, AggregateExpr, Call, Expr, LabelModifier, MatrixSelector, VectorSelector}; - -use crate::algebra::expr::{FilterOp, FilterVal, PartitionKeys, Predicate}; - -// ── Walk context ────────────────────────────────────────────────────────────── - -/// Context accumulated as we descend the AST. -#[derive(Default, Clone)] -struct WalkCtx { - /// GROUP BY / `without` clause from an outer Aggregate node. - partition: Option, - /// Top-K k from an outer `topk` / `bottomk` operator. - topk: Option, - /// Whether the outer context is a `count()` aggregate (→ CountDistinct). - outer_count: bool, -} - -// ── Helpers: MatrixSelector extraction ─────────────────────────────────────── - -/// Extract `(metric_name, filters, window)` from a MatrixSelector argument at -/// position `arg_idx` of a Call. -fn extract_matrix_arg( - call: &Call, - arg_idx: usize, -) -> anyhow::Result<(String, Vec, Duration)> { - let arg = call.args.args.get(arg_idx) - .map(|b| b.as_ref()) - .ok_or_else(|| anyhow!("missing arg {} in call to {}", arg_idx, call.func.name))?; - extract_inner_matrix(arg) -} - -/// Walk into an expression until we find a MatrixSelector, then extract its info. -fn extract_inner_matrix(expr: &Expr) -> anyhow::Result<(String, Vec, Duration)> { - match expr { - Expr::MatrixSelector(ms) => { - let (name, filters) = extract_vs_info(&ms.vs); - Ok((name, filters, ms.range)) - } - Expr::Paren(p) => extract_inner_matrix(p.expr.as_ref()), - Expr::Call(c) => { - // rate/irate wraps a MatrixSelector. - extract_inner_matrix(c.args.args[0].as_ref()) - } - other => Err(anyhow!("expected MatrixSelector, got {:?}", std::mem::discriminant(other))), - } -} - -// ── Helpers: VectorSelector info ───────────────────────────────────────────── - -fn extract_vs_info(vs: &VectorSelector) -> (String, Vec) { - // Metric name: prefer the explicit name field, fall back to __name__ matcher. - let name = vs.name.clone().unwrap_or_else(|| { - vs.matchers.matchers.iter() - .find(|m| m.name == "__name__") - .map(|m| m.value.clone()) - .unwrap_or_default() - }); - - let filters = vs.matchers.matchers.iter() - .filter(|m| m.name != "__name__") - .filter_map(matcher_to_predicate) - .collect(); - - (name, filters) -} - -fn matcher_to_predicate(m: &promql_parser::label::Matcher) -> Option { - use promql_parser::label::MatchOp; - let (op, val) = match &m.op { - MatchOp::Equal => (FilterOp::Eq, FilterVal::Str(m.value.clone())), - MatchOp::NotEqual => (FilterOp::Ne, FilterVal::Str(m.value.clone())), - MatchOp::Re(re) => (FilterOp::Regex(re.to_string()), FilterVal::Str(m.value.clone())), - MatchOp::NotRe(re)=> (FilterOp::NotRegex(re.to_string()), FilterVal::Str(m.value.clone())), - }; - Some(Predicate { col: m.name.clone(), op, val }) -} - -// ── Helpers: number extraction ──────────────────────────────────────────────── - -fn extract_call_num_arg(call: &Call, idx: usize) -> anyhow::Result { - match call.args.args.get(idx).map(|b| b.as_ref()) { - Some(Expr::NumberLiteral(n)) => Ok(n.val), - Some(other) => Err(anyhow!( - "expected number at arg {} of {}, got {:?}", - idx, call.func.name, std::mem::discriminant(other) - )), - None => Err(anyhow!("missing arg {} in {}", idx, call.func.name)), - } -} - -fn extract_number_param(param: &Option>) -> anyhow::Result { - match param { - Some(e) => match e.as_ref() { - Expr::NumberLiteral(n) => Ok(n.val), - other => Err(anyhow!("expected number param, got {:?}", std::mem::discriminant(other))), - }, - None => Err(anyhow!("missing required numeric parameter")), - } -} - -// ── Helpers: PartitionKeys from LabelModifier ───────────────────────────────── - -fn modifier_to_partition(modifier: &LabelModifier) -> PartitionKeys { - match modifier { - LabelModifier::Include(labels) => PartitionKeys::By(labels.labels.clone()), - LabelModifier::Exclude(labels) => PartitionKeys::Without(labels.labels.clone()), - } -} - -// ── Direct QueryExpr emission ───────────────────────────────────────────────── -// -// `parse_promql_expr` walks the same PromQL AST but emits [`QueryExpr`] nodes -// natively, preserving semantic nodes for the algebra optimizer: -// -// | PromQL pattern | QueryExpr node | -// |--------------------------|---------------------------------------| -// | `histogram_quantile(φ…)` | HistogramQuantile { phi } | -// | `m[5m:1m]` subquery | PromQLSubquery { 5m, Some(1m) } | -// | `a op b` binary | BinaryOp { VectorMatch } | - -use crate::algebra::expr::{ - AggFunc, AggItem, - BinaryOpKind, ColumnRef as QeColumnRef, GroupSide, - PartitionKeys as QePartitionKeys, QueryExpr, - SourceSpec as QeSourceSpec, VectorGrouping, VectorMatch, VectorMatchKind, -}; -use promql_parser::parser::{token::TokenType, BinaryExpr, VectorMatchCardinality}; - -/// Parse a PromQL expression string directly into an optimised [`QueryExpr`]. -/// -/// This preserves -/// `HistogramQuantile`, `PromQLSubquery`, and `BinaryOp` nodes natively. -pub fn parse_promql_expr(query: &str) -> anyhow::Result { - let expr = parser::parse(query) - .map_err(|e| anyhow!("PromQL parse error: {e}"))?; - walk_qe(&expr, WalkCtx::default()) -} - -fn walk_qe(expr: &Expr, ctx: WalkCtx) -> anyhow::Result { - match expr { - Expr::Aggregate(agg) => walk_aggregate_qe(agg, ctx), - Expr::Call(call) => walk_call_qe(call, ctx), - - // Binary op: map to QueryExpr::BinaryOp with VectorMatch. - Expr::Binary(bin) => walk_binary_qe(bin), - - Expr::Paren(p) => walk_qe(p.expr.as_ref(), ctx), - - // Subquery `expr[range:resolution]` → PromQLSubquery. - Expr::Subquery(sq) => { - let inner = walk_qe(sq.expr.as_ref(), ctx.clone())?; - Ok(QueryExpr::PromQLSubquery { - range: sq.range, - resolution: sq.step, - input: Box::new(inner), - }) - } - - // Bare vector selector → Source + Filter + Aggregate(Sum). - Expr::VectorSelector(vs) => { - let (name, filters) = extract_vs_info(vs); - let source = QueryExpr::Source(QeSourceSpec { name }); - let filtered = apply_qe_filters(source, filters); - Ok(QueryExpr::Aggregate { - keys: vec![], - aggs: vec![AggItem { - alias: "value".into(), - func: AggFunc::Sum, - col: QeColumnRef::SampleValue, - distinct: false, - }], - having: None, - input: Box::new(filtered), - }) - } - - Expr::NumberLiteral(_) | Expr::StringLiteral(_) => - Err(anyhow!("unexpected literal at top level of PromQL expression")), - - #[allow(unreachable_patterns)] - _ => Err(anyhow!("unsupported PromQL expression type")), - } -} - -fn walk_aggregate_qe(agg: &AggregateExpr, ctx: WalkCtx) -> anyhow::Result { - let partition = agg.modifier.as_ref().map(modifier_to_partition); - let op_name = format!("{}", agg.op); - - match op_name.as_str() { - "topk" | "bottomk" => { - let k = extract_number_param(&agg.param)? as u64; - let inner_ctx = WalkCtx { partition: partition.clone(), topk: Some(k), outer_count: false }; - let inner = walk_qe(agg.expr.as_ref(), inner_ctx)?; - // Don't wrap with Partition here — inner Aggregate already has the keys, - // and the lowering pass will create the Partition when it lowers the Aggregate. - let result = QueryExpr::TopK { k, by: partition.as_ref().map(|p| p.keys().to_vec()).unwrap_or_default(), input: Box::new(inner) }; - Ok(result) - } - "count" => { - let inner_ctx = WalkCtx { partition: partition.clone(), topk: None, outer_count: true }; - let inner = walk_qe(agg.expr.as_ref(), inner_ctx)?; - let result = QueryExpr::Aggregate { - keys: vec![], - aggs: vec![AggItem { - alias: "count".into(), - func: AggFunc::CountDistinct, - col: QeColumnRef::SampleValue, - distinct: false, - }], - having: None, - input: Box::new(inner), - }; - Ok(apply_qe_partition(result, partition)) - } - "sum" | "avg" | "min" | "max" | "group" => { - let inner_ctx = WalkCtx { partition: partition.clone(), topk: ctx.topk, outer_count: false }; - let inner = walk_qe(agg.expr.as_ref(), inner_ctx)?; - Ok(apply_qe_partition(inner, partition)) - } - "stddev" => { - let inner_ctx = WalkCtx { partition: partition.clone(), topk: None, outer_count: false }; - let inner = walk_qe(agg.expr.as_ref(), inner_ctx)?; - let result = QueryExpr::Aggregate { - keys: vec![], - aggs: vec![AggItem { - alias: "stddev".into(), - func: AggFunc::StdDev { population: false }, - col: QeColumnRef::SampleValue, - distinct: false, - }], - having: None, - input: Box::new(inner), - }; - Ok(apply_qe_partition(result, partition)) - } - "stdvar" => { - let inner_ctx = WalkCtx { partition: partition.clone(), topk: None, outer_count: false }; - let inner = walk_qe(agg.expr.as_ref(), inner_ctx)?; - let result = QueryExpr::Aggregate { - keys: vec![], - aggs: vec![AggItem { - alias: "stdvar".into(), - func: AggFunc::Variance { population: false }, - col: QeColumnRef::SampleValue, - distinct: false, - }], - having: None, - input: Box::new(inner), - }; - Ok(apply_qe_partition(result, partition)) - } - "quantile" => { - let phi = extract_number_param(&agg.param)?; - let inner_ctx = WalkCtx { partition: partition.clone(), topk: None, outer_count: false }; - let inner = walk_qe(agg.expr.as_ref(), inner_ctx)?; - let result = QueryExpr::Aggregate { - keys: vec![], - aggs: vec![AggItem { - alias: "quantile".into(), - func: AggFunc::Quantile(phi), - col: QeColumnRef::SampleValue, - distinct: false, - }], - having: None, - input: Box::new(inner), - }; - Ok(apply_qe_partition(result, partition)) - } - other => Err(anyhow!("unsupported PromQL aggregate operator: {other}")), - } -} - -fn walk_call_qe(call: &Call, ctx: WalkCtx) -> anyhow::Result { - let name = call.func.name; - match name { - // histogram_quantile → native HistogramQuantile node (PromQL-specific). - "histogram_quantile" => { - let phi = extract_call_num_arg(call, 0)?; - let rate_expr = call.args.args[1].as_ref(); - let (source, filters, window) = extract_inner_matrix(rate_expr)?; - let inner = build_qe_aggregate(source, filters, window, - AggFunc::Quantile(phi), - WalkCtx::default()); - Ok(QueryExpr::HistogramQuantile { phi, input: Box::new(inner) }) - } - // All other function calls: map to AggFunc (Layer 2). - "quantile_over_time" => { - let phi = extract_call_num_arg(call, 0)?; - let (source, filters, window) = extract_matrix_arg(call, 1)?; - let func = AggFunc::Quantile(phi); - Ok(build_qe_aggregate(source, filters, window, func, ctx)) - } - _ => { - let func = walk_call_to_op(call, &ctx)?; - let (source, filters, window) = if call.func.name == "rate" - || call.func.name == "irate" - || call.func.name == "increase" - { - let arg = call.args.args.first() - .map(|b| b.as_ref()) - .ok_or_else(|| anyhow!("rate/irate/increase requires a matrix arg"))?; - extract_inner_matrix(arg)? - } else { - extract_matrix_arg(call, 0)? - }; - Ok(build_qe_aggregate(source, filters, window, func, ctx)) - } - } -} - -fn walk_binary_qe(bin: &BinaryExpr) -> anyhow::Result { - let lhs = walk_qe(bin.lhs.as_ref(), WalkCtx::default())?; - let rhs = walk_qe(bin.rhs.as_ref(), WalkCtx::default())?; - - let op = promql_token_to_binop(bin.op); - - let vector_match = bin.modifier.as_ref().map(|m| { - let (kind, labels) = match &m.matching { - Some(LabelModifier::Include(ls)) => (VectorMatchKind::On, ls.labels.clone()), - Some(LabelModifier::Exclude(ls)) => (VectorMatchKind::Ignoring, ls.labels.clone()), - None => (VectorMatchKind::On, vec![]), - }; - let grouping = match &m.card { - VectorMatchCardinality::ManyToOne(ls) => Some(VectorGrouping { - side: GroupSide::Left, - labels: ls.labels.clone(), - }), - VectorMatchCardinality::OneToMany(ls) => Some(VectorGrouping { - side: GroupSide::Right, - labels: ls.labels.clone(), - }), - _ => None, - }; - VectorMatch { kind, labels, grouping } - }); - - Ok(QueryExpr::BinaryOp { - op, - lhs: Box::new(lhs), - rhs: Box::new(rhs), - vector_match, - }) -} - -fn promql_token_to_binop(tok: TokenType) -> BinaryOpKind { - use promql_parser::parser::token; - // token::T_* are u8 constants; TokenType wraps them as TokenType(u8). - let id = tok.id(); - match id { - token::T_ADD => BinaryOpKind::Add, - token::T_SUB => BinaryOpKind::Sub, - token::T_MUL => BinaryOpKind::Mul, - token::T_DIV => BinaryOpKind::Div, - token::T_MOD => BinaryOpKind::Mod, - token::T_POW => BinaryOpKind::Pow, - token::T_EQLC => BinaryOpKind::Eq, - token::T_NEQ => BinaryOpKind::Ne, - token::T_LSS => BinaryOpKind::Lt, - token::T_LTE => BinaryOpKind::Le, - token::T_GTR => BinaryOpKind::Gt, - token::T_GTE => BinaryOpKind::Ge, - token::T_LAND => BinaryOpKind::And, - token::T_LOR => BinaryOpKind::Or, - token::T_LUNLESS => BinaryOpKind::Unless, - token::T_ATAN2 => BinaryOpKind::Atan2, - _ => BinaryOpKind::Add, // unknown — default to add - } -} - -/// Map a PromQL function call to an [`AggFunc`] (Layer 2 relational operator). -fn walk_call_to_op(call: &Call, ctx: &WalkCtx) -> anyhow::Result { - let name = call.func.name; - match name { - "quantile_over_time" => { - let phi = extract_call_num_arg(call, 0)?; - Ok(AggFunc::Quantile(phi)) - } - "avg_over_time" => Ok(AggFunc::Avg), - "min_over_time" => Ok(AggFunc::Min), - "max_over_time" => Ok(AggFunc::Max), - "stddev_over_time" => Ok(AggFunc::StdDev { population: false }), - "stdvar_over_time" => Ok(AggFunc::Variance { population: false }), - "count_over_time" => { - Ok(if ctx.outer_count { AggFunc::CountDistinct } else { AggFunc::Count }) - } - "sum_over_time" | "last_over_time" | "present_over_time" | "absent_over_time" => - Ok(AggFunc::Sum), - "delta" | "idelta" | "deriv" | "predict_linear" => - Ok(AggFunc::Delta), - "changes" | "resets" => Ok(AggFunc::Count), - "rate" | "irate" => Ok(AggFunc::Rate), - "increase" => Ok(AggFunc::Increase), - other => Err(anyhow!("unsupported PromQL function: {other}")), - } -} - -/// Build a Layer 2 `QueryExpr`: `Aggregate { AggFunc, input: Window { ... } }`. -fn build_qe_aggregate( - metric: String, - filters: Vec, - window: std::time::Duration, - func: AggFunc, - ctx: WalkCtx, -) -> QueryExpr { - let source = QueryExpr::Source(QeSourceSpec { name: metric }); - let filtered = apply_qe_filters(source, filters); - let windowed = QueryExpr::Window { - duration: window, - slide: None, - input: Box::new(filtered), - }; - let actual_func = if ctx.topk.is_some() { - // Inside topk context, the aggregation is frequency-based. - AggFunc::Count - } else { - func - }; - // Propagate partition keys into the Aggregate's GROUP BY so the lowering - // pass sees Count-with-GROUP-BY → Frequency (not bare Count → no sketch). - let group_keys: Vec = ctx.partition.as_ref() - .map(|p| p.keys().to_vec()) - .unwrap_or_default(); - let alias = format!("{}", actual_func).to_lowercase(); - let agg = QueryExpr::Aggregate { - keys: group_keys, - aggs: vec![AggItem { - alias, - func: actual_func, - col: QeColumnRef::SampleValue, - distinct: false, - }], - having: None, - input: Box::new(windowed), - }; - // Don't wrap with Partition separately — keys are already in the Aggregate. - // The lowering pass will create the Partition node when it lowers the Aggregate. - agg -} - -fn apply_qe_filters( - input: QueryExpr, - filters: Vec, -) -> QueryExpr { - if filters.is_empty() { - input - } else { - use crate::algebra::expr::{BinaryOpKind, LiteralValue, ScalarExpr}; - let pred = filters.iter().fold( - ScalarExpr::Literal(LiteralValue::Bool(true)), - |acc, p| { - let col = ScalarExpr::Column(p.col.clone()); - let val = match &p.val { - FilterVal::Str(s) => ScalarExpr::Literal(LiteralValue::Str(s.clone())), - FilterVal::Num(n) => ScalarExpr::Literal(LiteralValue::Float(*n)), - FilterVal::Int(i) => ScalarExpr::Literal(LiteralValue::Int(*i)), - FilterVal::Null => ScalarExpr::Literal(LiteralValue::Null), - }; - let this = match &p.op { - FilterOp::Eq => ScalarExpr::BinaryOp { op: BinaryOpKind::Eq, lhs: Box::new(col), rhs: Box::new(val) }, - FilterOp::Ne => ScalarExpr::BinaryOp { op: BinaryOpKind::Ne, lhs: Box::new(col), rhs: Box::new(val) }, - FilterOp::Lt => ScalarExpr::BinaryOp { op: BinaryOpKind::Lt, lhs: Box::new(col), rhs: Box::new(val) }, - FilterOp::Le => ScalarExpr::BinaryOp { op: BinaryOpKind::Le, lhs: Box::new(col), rhs: Box::new(val) }, - FilterOp::Gt => ScalarExpr::BinaryOp { op: BinaryOpKind::Gt, lhs: Box::new(col), rhs: Box::new(val) }, - FilterOp::Ge => ScalarExpr::BinaryOp { op: BinaryOpKind::Ge, lhs: Box::new(col), rhs: Box::new(val) }, - FilterOp::Regex(r) => ScalarExpr::BinaryOp { op: BinaryOpKind::Regex, lhs: Box::new(col), rhs: Box::new(ScalarExpr::Literal(LiteralValue::Str(r.clone()))) }, - FilterOp::NotRegex(r) => ScalarExpr::BinaryOp { op: BinaryOpKind::NotRegex, lhs: Box::new(col), rhs: Box::new(ScalarExpr::Literal(LiteralValue::Str(r.clone()))) }, - FilterOp::Like => ScalarExpr::BinaryOp { op: BinaryOpKind::Like, lhs: Box::new(col), rhs: Box::new(val) }, - FilterOp::NotLike => ScalarExpr::BinaryOp { op: BinaryOpKind::NotLike, lhs: Box::new(col), rhs: Box::new(val) }, - FilterOp::IsNull => ScalarExpr::IsNull { expr: Box::new(col), negated: false }, - FilterOp::IsNotNull => ScalarExpr::IsNull { expr: Box::new(col), negated: true }, - }; - ScalarExpr::BinaryOp { - op: BinaryOpKind::And, - lhs: Box::new(acc), - rhs: Box::new(this), - } - }, - ); - QueryExpr::Filter { pred, input: Box::new(input) } - } -} - -fn apply_qe_partition( - input: QueryExpr, - partition: Option, -) -> QueryExpr { - match partition { - None => input, - Some(p) if p.is_empty() => input, - Some(keys) => { - // Convert PromQL by/without → PartitionKeys. - let qe_keys = match keys { - PartitionKeys::By(k) => QePartitionKeys::By(k), - PartitionKeys::Without(k) => QePartitionKeys::Without(k), - }; - QueryExpr::Partition { keys: qe_keys, input: Box::new(input) } - } - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use crate::types::AggType; - use std::time::Duration; - - fn pq(q: &str) -> super::super::ParsedQuery { - super::super::parse_query(q) - .unwrap_or_else(|e| panic!("parse_query failed: {e}\nquery={q:?}")) - } - - // ── quantile_over_time ──────────────────────────────────────────────────── - - #[test] - fn quantile_over_time_basic() { - // PromQL: `by` is part of the aggregate operator, not the function call. - let pq = pq("sum by (host) (quantile_over_time(0.99, latency{service=\"web\"}[5m]))"); - assert_eq!(pq.aggregations, vec![AggType::Quantile]); - assert_eq!(pq.quantiles, vec![0.99]); - assert_eq!(pq.group_by_labels, vec!["host"]); - assert_eq!(pq.label_filters.get("service").map(String::as_str), Some("web")); - assert_eq!(pq.time_window, Duration::from_secs(300)); - } - - #[test] - fn quantile_over_time_debs_ema() { - // Dotted names are invalid PromQL; use underscores. - let pq = pq("sum by (symbol) (quantile_over_time(0.5, financial_last_trade_price[5m]))"); - assert_eq!(pq.aggregations, vec![AggType::Quantile]); - assert_eq!(pq.quantiles, vec![0.5]); - assert_eq!(pq.group_by_labels, vec!["symbol"]); - } - - // ── histogram_quantile ──────────────────────────────────────────────────── - - #[test] - fn histogram_quantile_via_rate() { - let pq = pq("histogram_quantile(0.95, rate(http_duration_seconds_bucket[5m]))"); - assert_eq!(pq.aggregations, vec![AggType::Quantile]); - assert_eq!(pq.quantiles, vec![0.95]); - } - - // ── avg_over_time ───────────────────────────────────────────────────────── - - #[test] - fn avg_over_time_maps_to_p50() { - let pq = pq("avg by (symbol) (avg_over_time(financial_last_trade_price[5m]))"); - assert_eq!(pq.aggregations, vec![AggType::Quantile]); - assert_eq!(pq.quantiles, vec![0.5]); - } - - // ── min/max_over_time ───────────────────────────────────────────────────── - - #[test] - fn min_over_time_with_by_is_ddsketch() { - let pq = pq("min by (symbol) (min_over_time(financial_last_trade_price[5m]))"); - assert_eq!(pq.aggregations, vec![AggType::Quantile]); - assert_eq!(pq.quantiles, vec![0.0]); - } - - #[test] - fn max_over_time_with_by_is_ddsketch() { - let pq = pq("max by (symbol) (max_over_time(financial_last_trade_price[5m]))"); - assert_eq!(pq.aggregations, vec![AggType::Quantile]); - assert_eq!(pq.quantiles, vec![1.0]); - } - - // ── topk ────────────────────────────────────────────────────────────────── - - #[test] - fn topk_count_over_time() { - let pq = pq("topk by (symbol) (10, count_over_time(financial_last_trade_price[5m]))"); - assert_eq!(pq.aggregations, vec![AggType::Frequency]); - assert_eq!(pq.group_by_labels, vec!["symbol"]); - } - - #[test] - fn topk_avg_over_time() { - let pq = pq("topk by (host) (5, avg_over_time(cpu[5m]))"); - assert_eq!(pq.aggregations, vec![AggType::Frequency]); - } - - // ── count cardinality ───────────────────────────────────────────────────── - - #[test] - fn count_count_over_time_is_hll() { - let pq = pq("count by (symbol) (count_over_time(financial_last_trade_price[5m]))"); - assert_eq!(pq.aggregations, vec![AggType::Cardinality]); - } - - // ── stddev_over_time ────────────────────────────────────────────────────── - - #[test] - fn stddev_over_time_iqr_proxy() { - let pq = pq("avg by (host) (stddev_over_time(cpu[5m]))"); - assert_eq!(pq.aggregations, vec![AggType::Quantile]); - assert!(pq.quantiles.contains(&0.25) && pq.quantiles.contains(&0.75)); - } - - // ── sum_over_time → exact ───────────────────────────────────────────────── - - #[test] - fn sum_over_time_exact() { - let pq = pq("sum by (service) (sum_over_time(request_bytes[1h]))"); - assert!(pq.exact_required); - } - - // ── label filters ───────────────────────────────────────────────────────── - - #[test] - fn label_eq_filter() { - let pq = pq(r#"sum by (service) (count_over_time(hits{env="prod"}[5m]))"#); - assert_eq!(pq.label_filters.get("env").map(String::as_str), Some("prod")); - } - - // ── duration parsing ────────────────────────────────────────────────────── - - #[test] - fn duration_1h() { - let pq = pq("avg by (host) (avg_over_time(cpu[1h]))"); - assert_eq!(pq.time_window, Duration::from_secs(3600)); - } - - // ── DEBS hints ──────────────────────────────────────────────────────────── - - #[test] - fn debs_price_stats_min() { - use super::super::QueryHint; - let pq = pq("min by (symbol) (min_over_time(financial_last_trade_price[5m]))"); - assert!(matches!(pq.hint, Some(QueryHint::DebsPriceStats))); - } - - #[test] - fn debs_cardinality() { - use super::super::QueryHint; - let pq = pq("count by (symbol) (count_over_time(financial_last_trade_price[5m]))"); - assert!(matches!(pq.hint, Some(QueryHint::DebsCardinality))); - } - - // ── Complex queries ─────────────────────────────────────────────────────── - - #[test] - fn complex_topk_count_over_time_multi_label() { - // topk absorbs CountSketch (R8); multiple label filters extracted - let pq = pq( - r#"topk by (service) (10, count_over_time(http_requests_total{status="500",env="prod"}[5m]))"#, - ); - assert_eq!(pq.aggregations, vec![AggType::Frequency]); - assert_eq!(pq.group_by_labels, vec!["service"]); - assert_eq!( - pq.label_filters.get("status").map(String::as_str), - Some("500") - ); - assert_eq!( - pq.label_filters.get("env").map(String::as_str), - Some("prod") - ); - assert_eq!(pq.time_window, Duration::from_secs(300)); - } - - #[test] - fn complex_histogram_quantile_multi_label() { - // histogram_quantile wraps rate → DDSketch; two label selectors - let pq = pq( - r#"histogram_quantile(0.99, rate(request_duration_seconds_bucket{service="checkout",region="us-east"}[10m]))"#, - ); - assert_eq!(pq.aggregations, vec![AggType::Quantile]); - assert_eq!(pq.quantiles, vec![0.99]); - assert_eq!( - pq.label_filters.get("service").map(String::as_str), - Some("checkout") - ); - assert_eq!( - pq.label_filters.get("region").map(String::as_str), - Some("us-east") - ); - assert_eq!(pq.time_window, Duration::from_secs(600)); - } -} diff --git a/controller/src/query_parser/sql.rs b/controller/src/query_parser/sql.rs deleted file mode 100644 index 3ef1440d..00000000 --- a/controller/src/query_parser/sql.rs +++ /dev/null @@ -1,890 +0,0 @@ -//! Layers 1→3 lowering: SQL string → QueryExpr (sketch logical plan). -//! -//! - **Layer 1**: the `sqlparser` crate parses the SQL string into a -//! language-specific AST (`sqlparser::ast::Statement`). -//! - **Layer 2**: the extraction functions (`extract_query_expr`, `extract_select_qe`) -//! interpret SQL semantics (SELECT projection, GROUP BY, WHERE, JOIN, ORDER BY, -//! LIMIT, UNION ALL) and lower them to the sketch algebra. -//! - **Layer 3**: the output is a `QueryExpr` tree with **relational operators only** -//! (`Source`, `Filter`, `Aggregate`, `Join`, `Sort`, `Limit`, `SetOp`). -//! -//! # Key difference from the PromQL parser -//! -//! The SQL parser does **not** emit `SketchAgg` or `WindowedAgg` nodes. It emits -//! generic `Aggregate { func: Avg/Count/CountDistinct/... }` nodes. Sketch assignment -//! happens later: -//! - **Layer 4 (optimizer)**: R5 TopKFusion rewrites `Limit(Sort(Aggregate))` → `TopK`; -//! R9 HydraConversion rewrites multi-key `CountDistinct` → `PerPartition`. -//! - **Layer 5 (physical planner / stage-split)**: `assign_agg_func` maps each `AggFunc` -//! to an `AggIntent` (e.g., `CountDistinct` → `Cardinality`, `Quantile(φ)` → `Quantile`). -//! -//! This means the SQL path goes: relational plan → optimizer rewrites → physical -//! sketch assignment, whereas PromQL goes: sketch plan directly → optimizer → physical. -//! -//! # Algorithm -//! -//! For each FUNCTION edge in the SELECT projection (leaf → root): -//! 1. Collect context: GROUP BY, WHERE, HAVING, JOIN, DISTINCT, UNION ALL -//! 2. Emit the corresponding `QueryExpr` node: -//! - WHERE predicates → `Filter { ScalarExpr }` -//! - GROUP BY + aggs → `Aggregate { keys, aggs }` -//! - ORDER BY + LIMIT → `Sort` + `Limit` (→ `TopK` via optimizer R5) -//! - JOIN … ON → `Join { kind, pred }` -//! - UNION ALL → `SetOp { Union, all: true }` -//! -//! Multiple aggregations in one SELECT each emit their own `AggItem`, -//! collected inside a single `Aggregate` node. - -use std::collections::HashMap; -use std::time::Duration; - -use anyhow::{anyhow, Context}; -use sqlparser::ast::{ - BinaryOperator, DuplicateTreatment, Expr, FunctionArg, FunctionArgExpr, - FunctionArgumentList, FunctionArguments, GroupByExpr, Join, JoinConstraint, - JoinOperator, LimitClause, ObjectName, OrderBy, OrderByExpr, OrderByKind, - Query, Select, SelectItem, SetExpr, SetOperator, Statement, TableFactor, - Value, ValueWithSpan, -}; -use sqlparser::dialect::GenericDialect; - -use crate::algebra::expr::{ - AggFunc, AggItem as AlgAggItem, BinaryOpKind, ColumnRef, JoinKind, LiteralValue, - ProjectItem, QueryExpr, ScalarExpr, SetOpKind, SortKey, SourceSpec, -}; - -// ── Public entry point ──────────────────────────────────────────────────────── - -/// Parse a SQL SELECT statement into a [`QueryExpr`] tree. -/// -/// Preserves `Sort`, `Limit`, `Join`, and `SetOp` nodes natively so the -/// [`crate::algebra`] optimizer and allocator can reason about them. -pub fn parse_sql_expr(sql: &str) -> anyhow::Result { - let dialect = GenericDialect {}; - let mut stmts = sqlparser::parser::Parser::parse_sql(&dialect, sql) - .with_context(|| format!("SQL parse error: {sql:?}"))?; - let stmt = stmts.pop().ok_or_else(|| anyhow!("no SQL statement found"))?; - let query = match stmt { - Statement::Query(q) => *q, - other => return Err(anyhow!("expected SELECT, got {:?}", other)), - }; - extract_query_expr(&query) -} - -// ── Query-level dispatch ────────────────────────────────────────────────────── - -fn extract_query_expr(query: &Query) -> anyhow::Result { - let order_by: Vec = match &query.order_by { - Some(OrderBy { kind: OrderByKind::Expressions(exprs), .. }) => exprs.clone(), - _ => vec![], - }; - let (limit_n, offset_n) = match &query.limit_clause { - Some(LimitClause::LimitOffset { limit: Some(e), offset, .. }) => { - (Some(e.clone()), offset.as_ref().and_then(|o| expr_to_u64(&o.value))) - } - Some(LimitClause::OffsetCommaLimit { limit: e, offset, .. }) => { - (Some(e.clone()), Some(expr_to_u64(offset).unwrap_or(0))) - } - _ => (None, None), - }; - let limit_val = limit_n.as_ref().and_then(|e| expr_to_u64(e)); - let offset_val = offset_n.unwrap_or(0); - - let body = extract_set_expr_qe(query.body.as_ref(), &order_by, limit_val, offset_val)?; - Ok(body) -} - -fn extract_set_expr_qe( - set_expr: &SetExpr, - order_by: &[OrderByExpr], - limit_n: Option, - offset_n: u64, -) -> anyhow::Result { - match set_expr { - SetExpr::Select(sel) => extract_select_qe(sel, order_by, limit_n, offset_n), - SetExpr::Query(inner) => extract_query_expr(inner), - - // UNION / INTERSECT / EXCEPT - SetExpr::SetOperation { left, right, op, set_quantifier } => { - use sqlparser::ast::{SetOperator, SetQuantifier}; - let left_qe = extract_set_expr_qe(left, &[], None, 0)?; - let right_qe = extract_set_expr_qe(right, &[], None, 0)?; - let kind = match op { - SetOperator::Union => SetOpKind::Union, - SetOperator::Intersect => SetOpKind::Intersect, - SetOperator::Except | SetOperator::Minus => SetOpKind::Except, - }; - let all = matches!(set_quantifier, SetQuantifier::All | SetQuantifier::ByName); - Ok(QueryExpr::SetOp { - kind, - all, - left: Box::new(left_qe), - right: Box::new(right_qe), - }) - } - other => Err(anyhow!("unsupported query body: {:?}", other)), - } -} - -// ── SELECT-level extraction ─────────────────────────────────────────────────── - -fn extract_select_qe( - sel: &Select, - order_by: &[OrderByExpr], - limit_n: Option, - offset_n: u64, -) -> anyhow::Result { - let metric_name = extract_table_name(sel)?; - let where_scalar = sel.selection.as_ref().map(sql_expr_to_scalar); - let group_keys = extract_group_by(&sel.group_by); - let having_scalar = sel.having.as_ref().map(sql_expr_to_scalar); - let agg_items = collect_agg_items_qe(&sel.projection); - let join_qe = extract_join_qe(sel); - let window_spec = extract_group_by_window(&sel.group_by); - - let source = QueryExpr::Source(SourceSpec { - name: metric_name.clone(), - }); - - // WHERE → Filter - let after_where = match where_scalar { - Some(pred) => QueryExpr::Filter { pred, input: Box::new(source) }, - None => source, - }; - - // JOIN - let after_join = if let Some((inner_table, join_kind, join_pred)) = join_qe { - let inner_source = QueryExpr::Source(SourceSpec { name: inner_table }); - QueryExpr::Join { - kind: join_kind, - pred: join_pred, - left: Box::new(after_where), - right: Box::new(inner_source), - } - } else { - after_where - }; - - // TUMBLE / HOP → Window node wrapping the source - let after_window = if let Some(ws) = window_spec { - QueryExpr::Window { - duration: ws.size, - slide: ws.slide, - input: Box::new(after_join), - } - } else { - after_join - }; - - // GROUP BY + aggs OR bare projection - let after_agg = if agg_items.is_empty() { - // No aggregation — bare projection with possible DISTINCT. - let cols = collect_project_items(&sel.projection); - QueryExpr::Project { cols, input: Box::new(after_window) } - } else { - let having = having_scalar; - QueryExpr::Aggregate { - keys: group_keys, - aggs: agg_items, - having, - input: Box::new(after_window), - } - }; - - // ORDER BY → Sort - let after_sort = if order_by.is_empty() { - after_agg - } else { - let keys: Vec = order_by.iter().map(|o| SortKey { - col: expr_to_col_name(&o.expr).unwrap_or_else(|| "?".into()), - desc: matches!(o.options.asc, Some(false) | None), - nulls_first: None, - }).collect(); - QueryExpr::Sort { keys, input: Box::new(after_agg) } - }; - - // LIMIT / OFFSET - let result = match limit_n { - Some(n) => QueryExpr::Limit { n, offset: offset_n, input: Box::new(after_sort) }, - None => after_sort, - }; - - Ok(result) -} - -// ── Aggregation item collection ────────────────────────────────────────────── - -fn collect_agg_items_qe(projection: &[SelectItem]) -> Vec { - let mut out = Vec::new(); - for item in projection { - let (expr, alias) = match item { - SelectItem::UnnamedExpr(e) => (e, None), - SelectItem::ExprWithAlias { expr, alias } => (expr, Some(alias.value.clone())), - _ => continue, - }; - collect_agg_from_expr_qe(expr, alias, &mut out); - } - out -} - -fn collect_agg_from_expr_qe(expr: &Expr, alias: Option, out: &mut Vec) { - match expr { - Expr::Function(f) => { - let fn_name = f.name.0.last() - .and_then(|i| i.as_ident()) - .map(|id| id.value.to_uppercase()) - .unwrap_or_default(); - - let (distinct, args) = match &f.args { - FunctionArguments::List(FunctionArgumentList { duplicate_treatment, args, .. }) => { - let is_distinct = matches!(duplicate_treatment, Some(DuplicateTreatment::Distinct)); - (is_distinct, args.as_slice()) - } - _ => (false, &[][..]), - }; - - let col = first_col_from_args(args); - - let func = match fn_name.as_str() { - "COUNT" if distinct => AggFunc::CountDistinct, - "COUNT" => AggFunc::Count, - "SUM" => AggFunc::Sum, - "AVG" => AggFunc::Avg, - "MIN" => AggFunc::Min, - "MAX" => AggFunc::Max, - _ => return, - }; - - out.push(AlgAggItem { - alias: alias.unwrap_or_else(|| fn_name.to_lowercase()), - func, - col, - distinct, - }); - } - Expr::BinaryOp { left, right, .. } => { - collect_agg_from_expr_qe(left, None, out); - collect_agg_from_expr_qe(right, None, out); - } - Expr::Nested(inner) => collect_agg_from_expr_qe(inner, alias, out), - _ => {} - } -} - -fn collect_project_items(projection: &[SelectItem]) -> Vec { - projection.iter().filter_map(|item| match item { - SelectItem::UnnamedExpr(e) => Some(ProjectItem { - alias: None, - expr: sql_expr_to_scalar(e), - }), - SelectItem::ExprWithAlias { expr, alias } => Some(ProjectItem { - alias: Some(alias.value.clone()), - expr: sql_expr_to_scalar(expr), - }), - SelectItem::Wildcard(_) => Some(ProjectItem { - alias: None, - expr: ScalarExpr::Column("*".into()), - }), - _ => None, - }).collect() -} - -// ── AST helpers: aggregation arguments ─────────────────────────────────────── - -fn first_col_from_args(args: &[FunctionArg]) -> ColumnRef { - for arg in args { - match arg { - FunctionArg::Unnamed(FunctionArgExpr::Wildcard) => return ColumnRef::Wildcard, - FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Identifier(id))) => { - return ColumnRef::Named(id.value.clone()); - } - FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::CompoundIdentifier(parts))) => { - if let Some(last) = parts.last() { - return ColumnRef::Named(last.value.clone()); - } - } - _ => {} - } - } - ColumnRef::Wildcard -} - -// ── AST helpers: GROUP BY ────────────────────────────────────────────────────── - -/// Window spec extracted from a TUMBLE() or HOP() call in GROUP BY. -struct SqlWindowSpec { - size: Duration, - slide: Option, - time_col: Option, -} - -fn extract_group_by(group_by: &GroupByExpr) -> Vec { - let exprs = match group_by { - GroupByExpr::All(_) => return vec![], - GroupByExpr::Expressions(e, _) => e, - }; - exprs.iter().filter_map(|e| match e { - Expr::Identifier(id) => Some(id.value.clone()), - Expr::CompoundIdentifier(parts) => parts.last().map(|i| i.value.clone()), - // Skip TUMBLE/HOP function calls — extracted separately. - Expr::Function(f) => { - let name = f.name.0.last() - .and_then(|i| i.as_ident()) - .map(|id| id.value.to_uppercase()) - .unwrap_or_default(); - if name == "TUMBLE" || name == "HOP" || name == "TIME_BUCKET" { - None - } else { - None // unknown function in GROUP BY — skip - } - } - _ => None, - }).collect() -} - -/// Extract a TUMBLE / HOP / time_bucket window from the GROUP BY clause. -/// -/// Supported forms: -/// - `TUMBLE(ts, INTERVAL '5' MINUTE)` → Tumbling { size: 5m } -/// - `HOP(ts, INTERVAL '1' MINUTE, INTERVAL '5' MINUTE)` → Sliding { slide: 1m, size: 5m } -/// - `time_bucket('5 minutes', ts)` → Tumbling { size: 5m } -fn extract_group_by_window(group_by: &GroupByExpr) -> Option { - let exprs = match group_by { - GroupByExpr::Expressions(e, _) => e, - _ => return None, - }; - for expr in exprs { - if let Expr::Function(f) = expr { - let name = f.name.0.last() - .and_then(|i| i.as_ident()) - .map(|id| id.value.to_uppercase()) - .unwrap_or_default(); - - let args = match &f.args { - FunctionArguments::List(FunctionArgumentList { args, .. }) => args, - _ => continue, - }; - - match name.as_str() { - "TUMBLE" if args.len() >= 2 => { - // TUMBLE(ts_col, interval) - let time_col = func_arg_to_col_name(&args[0]); - let size = func_arg_to_duration(&args[1])?; - return Some(SqlWindowSpec { size, slide: None, time_col }); - } - "HOP" if args.len() >= 3 => { - // HOP(ts_col, slide_interval, size_interval) - let time_col = func_arg_to_col_name(&args[0]); - let slide = func_arg_to_duration(&args[1])?; - let size = func_arg_to_duration(&args[2])?; - return Some(SqlWindowSpec { size, slide: Some(slide), time_col }); - } - "TIME_BUCKET" if args.len() >= 2 => { - // time_bucket('5 minutes', ts_col) — first arg is interval string - let size = func_arg_to_duration(&args[0])?; - let time_col = func_arg_to_col_name(&args[1]); - return Some(SqlWindowSpec { size, slide: None, time_col }); - } - _ => {} - } - } - } - None -} - -fn func_arg_to_col_name(arg: &FunctionArg) -> Option { - match arg { - FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Identifier(id))) => - Some(id.value.clone()), - FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::CompoundIdentifier(parts))) => - parts.last().map(|i| i.value.clone()), - _ => None, - } -} - -fn func_arg_to_duration(arg: &FunctionArg) -> Option { - match arg { - FunctionArg::Unnamed(FunctionArgExpr::Expr(expr)) => expr_to_duration(expr), - _ => None, - } -} - -fn expr_to_duration(expr: &Expr) -> Option { - match expr { - // INTERVAL '5' MINUTE - Expr::Interval(iv) => { - let val_str = match iv.value.as_ref() { - Expr::Value(vws) => match &vws.value { - Value::SingleQuotedString(s) | Value::DoubleQuotedString(s) => s.clone(), - Value::Number(n, _) => n.clone(), - _ => return None, - }, - _ => return None, - }; - let val: u64 = val_str.trim().parse().ok()?; - let unit = iv.leading_field.as_ref()?; - let secs = match unit { - sqlparser::ast::DateTimeField::Second => val, - sqlparser::ast::DateTimeField::Minute => val * 60, - sqlparser::ast::DateTimeField::Hour => val * 3600, - sqlparser::ast::DateTimeField::Day => val * 86400, - _ => return None, - }; - Some(Duration::from_secs(secs)) - } - // '5 minutes' string (time_bucket style) - Expr::Value(vws) => match &vws.value { - Value::SingleQuotedString(s) | Value::DoubleQuotedString(s) => { - parse_duration_string(s) - } - _ => None, - }, - _ => None, - } -} - -fn parse_duration_string(s: &str) -> Option { - let s = s.trim().to_lowercase(); - // Try "Nm", "Ns", "Nmin", "N minutes", "N seconds", "N hours" - let (num_str, unit) = if let Some(n) = s.strip_suffix("minutes") { - (n.trim(), 60u64) - } else if let Some(n) = s.strip_suffix("minute") { - (n.trim(), 60) - } else if let Some(n) = s.strip_suffix("min") { - (n.trim(), 60) - } else if let Some(n) = s.strip_suffix("hours") { - (n.trim(), 3600) - } else if let Some(n) = s.strip_suffix("hour") { - (n.trim(), 3600) - } else if let Some(n) = s.strip_suffix('h') { - (n.trim(), 3600) - } else if let Some(n) = s.strip_suffix("seconds") { - (n.trim(), 1) - } else if let Some(n) = s.strip_suffix("second") { - (n.trim(), 1) - } else if let Some(n) = s.strip_suffix('s') { - (n.trim(), 1) - } else if let Some(n) = s.strip_suffix('m') { - (n.trim(), 60) - } else { - return None; - }; - let n: u64 = num_str.parse().ok()?; - Some(Duration::from_secs(n * unit)) -} - -// ── AST helpers: table name ─────────────────────────────────────────────────── - -fn extract_table_name(sel: &Select) -> anyhow::Result { - sel.from.first() - .and_then(|t| match &t.relation { - TableFactor::Table { name, .. } => Some(object_name_str(name)), - _ => None, - }) - .ok_or_else(|| anyhow!("could not determine table name from FROM clause")) -} - -fn object_name_str(name: &ObjectName) -> String { - name.0.iter() - .map(|i| i.as_ident().map(|id| id.value.as_str()).unwrap_or("")) - .collect::>() - .join(".") -} - -// ── SQL Expr → ScalarExpr ───────────────────────────────────────────────────── - -fn sql_expr_to_scalar(expr: &Expr) -> ScalarExpr { - match expr { - Expr::Identifier(id) => ScalarExpr::Column(id.value.clone()), - Expr::CompoundIdentifier(parts) => { - ScalarExpr::Column(parts.iter().map(|i| i.value.as_str()).collect::>().join(".")) - } - Expr::Value(vws) => sql_value_to_scalar(&vws.value), - Expr::BinaryOp { left, op, right } => { - let lhs = sql_expr_to_scalar(left); - let rhs = sql_expr_to_scalar(right); - let bop = sql_binop_to_algebra(op); - ScalarExpr::BinaryOp { op: bop, lhs: Box::new(lhs), rhs: Box::new(rhs) } - } - Expr::IsNull(inner) => ScalarExpr::IsNull { - expr: Box::new(sql_expr_to_scalar(inner)), - negated: false, - }, - Expr::IsNotNull(inner) => ScalarExpr::IsNull { - expr: Box::new(sql_expr_to_scalar(inner)), - negated: true, - }, - Expr::Between { expr, negated, low, high } => ScalarExpr::Between { - expr: Box::new(sql_expr_to_scalar(expr)), - low: Box::new(sql_expr_to_scalar(low)), - high: Box::new(sql_expr_to_scalar(high)), - negated: *negated, - }, - Expr::InList { expr, list, negated } => ScalarExpr::InList { - expr: Box::new(sql_expr_to_scalar(expr)), - list: list.iter().map(sql_expr_to_scalar).collect(), - negated: *negated, - }, - Expr::Like { expr, pattern, negated, .. } => { - let op = if *negated { BinaryOpKind::NotLike } else { BinaryOpKind::Like }; - ScalarExpr::BinaryOp { - op, - lhs: Box::new(sql_expr_to_scalar(expr)), - rhs: Box::new(sql_expr_to_scalar(pattern)), - } - } - Expr::Nested(inner) => sql_expr_to_scalar(inner), - Expr::Function(f) => { - let name = f.name.0.last() - .and_then(|i| i.as_ident()) - .map(|id| id.value.clone()) - .unwrap_or_default(); - ScalarExpr::FunctionCall { name, args: vec![] } - } - _ => ScalarExpr::Column("?".into()), // unknown expr → opaque column ref - } -} - -fn sql_value_to_scalar(v: &Value) -> ScalarExpr { - match v { - Value::SingleQuotedString(s) | Value::DoubleQuotedString(s) => - ScalarExpr::Literal(LiteralValue::Str(s.clone())), - Value::Number(n, _) => { - if let Ok(i) = n.parse::() { - ScalarExpr::Literal(LiteralValue::Int(i)) - } else if let Ok(f) = n.parse::() { - ScalarExpr::Literal(LiteralValue::Float(f)) - } else { - ScalarExpr::Literal(LiteralValue::Null) - } - } - Value::Boolean(b) => ScalarExpr::Literal(LiteralValue::Bool(*b)), - Value::Null => ScalarExpr::Literal(LiteralValue::Null), - _ => ScalarExpr::Literal(LiteralValue::Null), - } -} - -fn sql_binop_to_algebra(op: &BinaryOperator) -> BinaryOpKind { - match op { - BinaryOperator::Plus => BinaryOpKind::Add, - BinaryOperator::Minus => BinaryOpKind::Sub, - BinaryOperator::Multiply => BinaryOpKind::Mul, - BinaryOperator::Divide => BinaryOpKind::Div, - BinaryOperator::Modulo => BinaryOpKind::Mod, - BinaryOperator::Eq => BinaryOpKind::Eq, - BinaryOperator::NotEq => BinaryOpKind::Ne, - BinaryOperator::Lt => BinaryOpKind::Lt, - BinaryOperator::LtEq => BinaryOpKind::Le, - BinaryOperator::Gt => BinaryOpKind::Gt, - BinaryOperator::GtEq => BinaryOpKind::Ge, - BinaryOperator::And => BinaryOpKind::And, - BinaryOperator::Or => BinaryOpKind::Or, - BinaryOperator::BitwiseAnd => BinaryOpKind::BitAnd, - BinaryOperator::BitwiseOr => BinaryOpKind::BitOr, - BinaryOperator::BitwiseXor => BinaryOpKind::BitXor, - BinaryOperator::StringConcat => BinaryOpKind::Concat, - _ => BinaryOpKind::Eq, // unknown → eq - } -} - -// ── JOIN → QueryExpr::Join ──────────────────────────────────────────────────── - -fn extract_join_qe(sel: &Select) -> Option<(String, JoinKind, Option)> { - let table_with_joins = sel.from.first()?; - let join = table_with_joins.joins.first()?; - let inner_table = match &join.relation { - TableFactor::Table { name, .. } => object_name_str(name), - _ => return None, - }; - let (kind, pred) = match &join.join_operator { - JoinOperator::Inner(c) => - (JoinKind::Inner, join_constraint_to_scalar(c)), - JoinOperator::LeftOuter(c) => - (JoinKind::LeftOuter, join_constraint_to_scalar(c)), - JoinOperator::RightOuter(c) => - (JoinKind::RightOuter, join_constraint_to_scalar(c)), - JoinOperator::FullOuter(c) => - (JoinKind::FullOuter, join_constraint_to_scalar(c)), - JoinOperator::CrossJoin(_) => - (JoinKind::Cross, None), - _ => return None, - }; - Some((inner_table, kind, pred)) -} - -fn join_constraint_to_scalar(c: &JoinConstraint) -> Option { - match c { - JoinConstraint::On(e) => Some(sql_expr_to_scalar(e)), - _ => None, - } -} - -// ── Misc helpers ────────────────────────────────────────────────────────────── - -fn expr_to_u64(expr: &Expr) -> Option { - match expr { - Expr::Value(vws) => match &vws.value { - Value::Number(n, _) => n.parse::().ok(), - _ => None, - }, - _ => None, - } -} - -fn expr_to_col_name(expr: &Expr) -> Option { - match expr { - Expr::Identifier(id) => Some(id.value.clone()), - Expr::CompoundIdentifier(parts) => parts.last().map(|i| i.value.clone()), - _ => None, - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::parse_sql_expr; - use crate::algebra::expr::QueryExpr; - use crate::types::AggType; - - fn parse(sql: &str) -> QueryExpr { - parse_sql_expr(sql).unwrap_or_else(|e| panic!("parse_sql_expr failed: {e}\nSQL: {sql}")) - } - - fn pq(sql: &str) -> super::super::ParsedQuery { - super::super::parse_query(sql) - .unwrap_or_else(|e| panic!("parse_query failed: {e}\nSQL: {sql}")) - } - - // ── Basic aggregations ──────────────────────────────────────────────────── - - #[test] - fn count_star_no_group_is_exact() { - let pq = pq("SELECT COUNT(*) FROM hits"); - assert!(pq.exact_required); - assert!(pq.aggregations.is_empty()); - } - - #[test] - fn count_star_group_by_is_frequency() { - let pq = pq("SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID"); - assert!(pq.aggregations.contains(&AggType::Frequency)); - assert!(pq.group_by_labels.contains(&"AdvEngineID".to_string())); - } - - #[test] - fn count_distinct_is_cardinality() { - let pq = pq("SELECT COUNT(DISTINCT UserID) FROM hits"); - assert!(pq.aggregations.contains(&AggType::Cardinality)); - assert!(!pq.exact_required); - } - - #[test] - fn count_star_order_by_desc_limit_is_topk() { - let pq = pq( - "SELECT SearchPhrase, COUNT(*) AS c FROM hits \ - WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10", - ); - assert!(pq.aggregations.contains(&AggType::Frequency)); - } - - #[test] - fn avg_with_group_by_is_quantile_p50() { - let pq = pq("SELECT symbol, AVG(last) FROM hits GROUP BY symbol"); - assert!(pq.aggregations.contains(&AggType::Quantile)); - assert!(pq.quantiles.contains(&0.5)); - } - - #[test] - fn min_max_with_group_by_are_extremes() { - let pq = pq("SELECT symbol, MIN(last), MAX(last) FROM hits GROUP BY symbol"); - assert!(pq.aggregations.contains(&AggType::Quantile)); - assert!(pq.quantiles.contains(&0.0)); - assert!(pq.quantiles.contains(&1.0)); - } - - #[test] - fn min_max_no_group_by_is_exact_minmax() { - let pq = pq("SELECT MIN(EventDate), MAX(EventDate) FROM hits"); - // MIN/MAX map to Quantile in legacy AggType - assert!(pq.aggregations.contains(&AggType::Quantile)); - } - - #[test] - fn sum_is_always_exact() { - let pq = pq("SELECT SUM(AdvEngineID) FROM hits"); - assert!(pq.exact_required); - } - - // ── WHERE predicates ────────────────────────────────────────────────────── - - #[test] - fn where_equality_captured() { - let pq = pq("SELECT COUNT(*) FROM hits WHERE sectype = 'E' GROUP BY symbol"); - assert_eq!(pq.label_filters.get("sectype").map(String::as_str), Some("E")); - } - - #[test] - fn where_inequality_captured() { - let pq = pq("SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID"); - assert!(pq.aggregations.contains(&AggType::Frequency)); - } - - // ── Multi-aggregation ───────────────────────────────────────────────────── - - #[test] - fn multi_agg_collects_all() { - let pq = pq( - "SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) \ - FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10", - ); - assert!(pq.aggregations.contains(&AggType::Cardinality), "missing cardinality"); - assert!(pq.aggregations.contains(&AggType::Frequency), "missing frequency"); - assert!(pq.aggregations.contains(&AggType::Quantile), "missing quantile"); - // SUM adds exact_required alongside sketch ops - assert!(pq.exact_required, "SUM should set exact_required"); - } - - // ── Table / metric name ─────────────────────────────────────────────────── - - #[test] - fn dotted_table_name() { - let pq = pq("SELECT COUNT(*) FROM financial.last_trade_price GROUP BY symbol"); - assert_eq!(pq.metric_name, "financial.last_trade_price"); - } - - // ── DEBS SQL variants ───────────────────────────────────────────────────── - - #[test] - fn debs_q6_cardinality() { - use super::super::QueryHint; - let pq = pq("SELECT COUNT(DISTINCT symbol) FROM financial.last_trade_price"); - assert!(pq.aggregations.contains(&AggType::Cardinality)); - assert!(matches!(pq.hint, Some(QueryHint::DebsCardinality))); - } - - #[test] - fn debs_q3_topk() { - let pq = pq( - "SELECT symbol, COUNT(*) AS c FROM financial.last_trade_price \ - GROUP BY symbol ORDER BY c DESC LIMIT 10", - ); - assert!(pq.aggregations.contains(&AggType::Frequency)); - } - - // ── COUNT(DISTINCT) with GROUP BY ──────────────────────────────────────── - - #[test] - fn count_distinct_with_group_by() { - let pq = pq( - "SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10", - ); - assert!(pq.aggregations.contains(&AggType::Cardinality)); - assert!(pq.group_by_labels.contains(&"RegionID".to_string())); - } - - // ── TUMBLE / HOP windows ──────────────────────────────────────────────── - - /// Helper that runs the full pipeline (parse + lower), not just Layer 2. - fn parse_full(sql: &str) -> QueryExpr { - super::super::parse_query_expr(sql) - .unwrap_or_else(|e| panic!("parse_query_expr failed: {e}\nSQL: {sql}")) - } - - fn has_windowed_agg(e: &QueryExpr) -> bool { - match e { - QueryExpr::WindowedAgg { .. } => true, - QueryExpr::Partition { input, .. } - | QueryExpr::TopK { input, .. } - | QueryExpr::Sort { input, .. } - | QueryExpr::Limit { input, .. } => has_windowed_agg(input), - _ => false, - } - } - - #[test] - fn tumble_in_group_by_produces_windowed_agg() { - let expr = parse_full( - "SELECT symbol, AVG(price) FROM trades \ - GROUP BY symbol, TUMBLE(ts, INTERVAL '5' MINUTE)", - ); - assert!(has_windowed_agg(&expr), "expected WindowedAgg in tree, got {expr:?}"); - } - - #[test] - fn hop_in_group_by_produces_windowed_agg() { - let expr = parse_full( - "SELECT symbol, COUNT(*) FROM trades \ - GROUP BY symbol, HOP(ts, INTERVAL '1' MINUTE, INTERVAL '5' MINUTE)", - ); - assert!(has_windowed_agg(&expr), "expected WindowedAgg in tree, got {expr:?}"); - } - - #[test] - fn time_bucket_in_group_by_produces_windowed_agg() { - let expr = parse_full( - "SELECT symbol, AVG(price) FROM trades \ - GROUP BY symbol, time_bucket('5 minutes', ts)", - ); - assert!(has_windowed_agg(&expr), "expected WindowedAgg in tree, got {expr:?}"); - } - - #[test] - fn tumble_layer2_emits_window_node() { - // Layer 2 only (no lowering): should be Aggregate { input: Window { Source } } - let expr = parse( - "SELECT symbol, AVG(price) FROM trades \ - GROUP BY symbol, TUMBLE(ts, INTERVAL '5' MINUTE)", - ); - match &expr { - QueryExpr::Aggregate { input, .. } => { - assert!(matches!(input.as_ref(), QueryExpr::Window { .. }), - "expected Window inside Aggregate, got {input:?}"); - } - other => panic!("expected Aggregate, got {other:?}"), - } - } - - // ── UNION ALL → SetOp ───────────────────────────────────────────────────── - - #[test] - fn union_all_produces_set_op() { - let expr = parse( - "SELECT COUNT(DISTINCT UserID) FROM R \ - UNION ALL \ - SELECT COUNT(DISTINCT UserID) FROM S", - ); - assert!(matches!(expr, QueryExpr::SetOp { .. })); - } - - // ── Complex queries ─────────────────────────────────────────────────────── - - #[test] - fn complex_multi_agg_multi_dim_group_by_topk() { - let pq = pq( - "SELECT region, dc, COUNT(*) AS c, COUNT(DISTINCT UserID), AVG(ResponseTime) \ - FROM hits WHERE env = 'prod' GROUP BY region, dc ORDER BY c DESC LIMIT 5", - ); - assert!(pq.aggregations.contains(&AggType::Frequency)); - assert!(pq.aggregations.contains(&AggType::Cardinality)); - assert!(pq.aggregations.contains(&AggType::Quantile)); - assert!(pq.group_by_labels.contains(&"region".to_string())); - assert!(pq.group_by_labels.contains(&"dc".to_string())); - assert_eq!( - pq.label_filters.get("env").map(String::as_str), - Some("prod") - ); - assert!(pq.quantiles.contains(&0.5)); - } - - #[test] - fn complex_union_all_hll_with_where_on_each_branch() { - let pq = pq( - "SELECT region, COUNT(DISTINCT UserID) FROM sessions WHERE status = 'active' GROUP BY region \ - UNION ALL \ - SELECT region, COUNT(DISTINCT UserID) FROM sessions WHERE status = 'expired' GROUP BY region", - ); - assert!(pq.aggregations.contains(&AggType::Cardinality)); - assert!(pq.group_by_labels.contains(&"region".to_string())); - } -} diff --git a/controller/src/replan.rs b/controller/src/replan.rs deleted file mode 100644 index 1ebb55d0..00000000 --- a/controller/src/replan.rs +++ /dev/null @@ -1,685 +0,0 @@ -//! SP-8 re-planning automation. -//! -//! [`Replanner`] closes the feedback loop from the [`monitor::Scraper`] back -//! to the planner. Two triggers drive re-planning: -//! -//! 1. **Violation-triggered**: when the scraper fires an SLA violation callback -//! the replanner looks up which metric the violating agent is serving and -//! immediately requests a fresh plan. -//! -//! 2. **Expiry-triggered**: a periodic ticker calls [`Replanner::replan_expired`] -//! to re-plan any metric whose `CollectionPlan::valid_until` has passed. -//! -//! After a plan is updated the replanner pushes role-appropriate OTel YAML to -//! all connected collectors via OpAMP and updates scraper endpoint sketch-type -//! bookkeeping so the EMA cost model receives correctly attributed updates. -use std::collections::HashMap; -use std::sync::Arc; -use std::time::Duration; - -use tokio::sync::RwLock; -use tracing::{info, warn}; - -use crate::backend_client::{push_or_log, BackendClient}; -use crate::config::{ - build_precompute_jobs, collect_metric_to_family, emit_for_runtime, - extend_edge_with_demo_plumbing, generate_agent_config, generate_backend_config, - generate_streaming_config_yaml, AgentRuntime, WorkloadRegistry, -}; -use crate::monitor::Scraper; -use crate::opamp::{AgentRole, OpampServer, RemoteConfig}; -use crate::planner::{self, BaselinePlanner}; -use crate::store::{PlanStore, WorkloadStore}; -use crate::types::QueryWorkload; - -fn short_hash(s: &str) -> String { - use std::collections::hash_map::DefaultHasher; - use std::hash::{Hash, Hasher}; - let mut h = DefaultHasher::new(); - s.hash(&mut h); - format!("{:016x}", h.finish()) -} - -// ── Replanner ───────────────────────────────────────────────────────────────── - -pub struct Replanner { - planner: Arc, - plan_store: Arc, - workload_store: Arc, - opamp: Arc, - scraper: Arc, - opamp_endpoint: String, - /// Optional client for pushing newly-generated `StreamingConfig` - /// YAML to the ASAPQuery-backend's `/api/v1/streaming-config` - /// endpoint. When present, every successful replan POSTs the new - /// plan to the backend in addition to the existing OpAMP pushes - /// to agent-role and backend-role collectors. Configured via the - /// `CONTROLLER_BACKEND_ENDPOINT` env var; defaults to `None` so - /// existing deployments that don't yet run ASAPQuery-backend - /// behave exactly as before. - backend_client: Option>, - /// Optional handle to the controller-wide [`WorkloadRegistry`]. - /// Used only by the typed-emit path - /// ([`Replanner::try_emit_typed_edge_yaml`]) to extend the edge - /// stage config with the bootstrap-scope archive-tier metrics so - /// the OpAMP-pushed YAML matches what the bootstrap GET path - /// emits via `main::emit_bootstrap_typed`. When unset the typed - /// path still works — it just skips the workload-registry archive - /// extension and only adds the freshness probes. - workload_registry: Option>, - /// Maps agent_id → metric_name so violation callbacks can look up which - /// metric a particular agent is serving. - agent_to_metric: Arc>>, -} - -impl Replanner { - pub fn new( - planner: Arc, - plan_store: Arc, - workload_store: Arc, - opamp: Arc, - scraper: Arc, - opamp_endpoint: impl Into, - ) -> Self { - Self { - planner, - plan_store, - workload_store, - opamp, - scraper, - opamp_endpoint: opamp_endpoint.into(), - backend_client: None, - workload_registry: None, - agent_to_metric: Arc::new(RwLock::new(HashMap::new())), - } - } - - /// Attach a [`BackendClient`] so every replan also pushes the new - /// `StreamingConfig` YAML to the ASAPQuery-backend via HTTP. - /// Builder-style — call during controller startup in `main.rs`. - /// Without this call, replans continue to push only via OpAMP and - /// the ASAPQuery-backend (if running) keeps its startup config. - pub fn with_backend_client(mut self, client: Arc) -> Self { - self.backend_client = Some(client); - self - } - - /// Attach the controller-wide [`WorkloadRegistry`] so the typed - /// emit path (gated by `USE_TYPED_STAGE_SPLIT`) can extend the - /// edge stage config with the workload-registry archive metrics - /// — same demo-scope plumbing the bootstrap GET path applies in - /// `main::emit_bootstrap_typed`. Builder-style; safe to omit - /// (the typed path falls back to freshness-probe-only extension). - pub fn with_workload_registry(mut self, registry: Arc) -> Self { - self.workload_registry = Some(registry); - self - } - - // ── Agent registry ──────────────────────────────────────────────────────── - - /// Record that `agent_id` is serving `metric`. Called from `handle_plan` - /// after pushing configs so violations can be mapped back to a metric. - pub async fn register_agent(&self, agent_id: impl Into, metric: impl Into) { - self.agent_to_metric.write().await - .insert(agent_id.into(), metric.into()); - } - - /// Remove the mapping for a disconnected agent. - pub async fn unregister_agent(&self, agent_id: &str) { - self.agent_to_metric.write().await.remove(agent_id); - } - - /// Returns a read-only reference to the agent→metric mapping so that - /// callers (e.g. the on_connect callback) can check if an agent has a - /// prior assignment. - pub fn agent_to_metric(&self) -> &Arc>> { - &self.agent_to_metric - } - - // ── Config push helpers ────────────────────────────────────────────────── - - /// Try to emit edge YAML via the typed L5 pipeline for `metric`, - /// matching `main::emit_bootstrap_typed`'s flow. - /// - /// Steps: - /// 1. `bind_workload_typed(&workload)` → SketchExpr - /// 2. `split_typed_three_stage(&sketch_expr)` → per-stage configs - /// 3. Pick the `Edge` stage config - /// 4. Apply `extend_edge_with_demo_plumbing` (freshness probes + - /// workload-registry archive metrics) so the OpAMP-pushed YAML - /// matches what the bootstrap GET path emits - /// 5. `emit_for_runtime(AsapOtel, &edge, …)` → YAML string - /// - /// Defaults the runtime to `AgentRuntime::AsapOtel` (mirrors the - /// bootstrap default when no `X-Agent-Runtime` header is present; - /// the OpAMP `on_connect` payload doesn't surface a per-agent - /// runtime today). If a future commit threads runtime info through - /// OpAMP, swap the default for a per-agent lookup. - /// - /// Returns `None` whenever the typed path can't satisfy the - /// request (workload missing from store, `bind_workload_typed` - /// declines the shape, no `Edge` entry, emit failure) — caller - /// then falls back to the legacy emitter. - fn try_emit_typed_edge_yaml(&self, metric: &str) -> Option { - let (workload, _wc) = self.workload_store.get(metric)?; - self.try_emit_typed_edge_yaml_for_workload(&workload) - } - - /// Same as [`try_emit_typed_edge_yaml`] but takes the - /// `QueryWorkload` directly. Used by `replan_metric` which already - /// has the workload in scope. - fn try_emit_typed_edge_yaml_for_workload(&self, workload: &QueryWorkload) -> Option { - let sketch_expr = planner::rules::bind_workload_typed(workload)?; - let configs = planner::stage_split::split_typed_three_stage(&sketch_expr)?; - let mut edge_cfg = configs.into_iter().find_map(|(_, cfg)| match cfg { - crate::stage_split::StageConfig::Edge(edge) => Some(edge), - _ => None, - })?; - - // Apply the bootstrap-scope demo plumbing — freshness probes - // + workload-registry archive metrics — so the OpAMP-pushed - // YAML carries the SAME `gorillas3` + `routing` + - // `metrics/warm_passthrough` blocks the bootstrap GET path - // emits. Without this, an agent that reconnects gets edge - // YAML missing freshness-probe routing → criterion ⑥ fails - // for any plan-pinned agent. - let registry_metrics: Vec = self - .workload_registry - .as_ref() - .map(|r| r.entries().iter().map(|e| e.metric_name.clone()).collect()) - .unwrap_or_default(); - extend_edge_with_demo_plumbing(&mut edge_cfg, registry_metrics); - - // MVP §46 — stitch planner per-metric output into the emitter's - // `metric_to_family` map so the 5-sketch routing-connector wire - // shape activates on the OpAMP-pushed YAML too. Mirror of the - // bootstrap path's stitch in `main::emit_bootstrap_typed` — - // without this an agent that reconnects (or a metric that - // replans) gets a single-pipeline YAML, even though the - // bootstrap GET path it received first carried the routing - // connector. When `workload_registry` is None (test fixture), - // skip the stitch — the legacy single-pipeline emit still - // covers correctness for the metric being replanned. - if let Some(registry) = self.workload_registry.as_ref() { - edge_cfg.metric_to_family = - collect_metric_to_family(registry, &self.workload_store); - } - - // OpAMP `on_connect` doesn't expose the agent's runtime - // header, so default to `AsapOtel` — matches the bootstrap - // default for legacy / unspecified clients. This is the same - // assumption `main::handle_plan`'s typed push path makes - // (`push_to_role(Agent, …)` with edge YAML, no runtime - // dispatch). - emit_for_runtime(AgentRuntime::AsapOtel, &edge_cfg, &self.opamp_endpoint, None).ok() - } - - /// Push the current plan config to a specific agent. - /// - /// Looks up the metric assigned to this agent, retrieves the plan from - /// `plan_store`, generates agent YAML, and pushes via OpAMP. - /// Returns `true` if config was pushed, `false` if the agent has no - /// metric assignment or no plan exists for that metric. - /// - /// ## Behaviour matrix - /// - /// | `USE_TYPED_STAGE_SPLIT` | path | - /// | --- | --- | - /// | unset / `0` | **legacy** — emit a single-pipeline DDSketch YAML via [`generate_agent_config`]. No routing, no `gorillas3`, no warm-passthrough. Backwards-compat for deployments that haven't migrated. | - /// | `1` / `true` / `yes` | **typed** — run [`try_emit_typed_edge_yaml`] (mirror of `main::emit_bootstrap_typed`). On error, fall back to the legacy emitter so the push never silently drops. | - /// - /// Together with the bootstrap GET path (PR #333) this finishes - /// the OpAMP-on-connect side of the typed emit so reconnecting - /// agents receive the same routed YAML as fresh-connect agents. - pub async fn push_config_to_agent(&self, agent_id: &str) -> bool { - let metric = self.agent_to_metric.read().await.get(agent_id).cloned(); - let Some(metric) = metric else { return false }; - - let Ok(plan) = self.plan_store.get(&metric) else { return false }; - - let yaml = if planner::stage_split::typed_stage_split_enabled() { - match self.try_emit_typed_edge_yaml(&metric) { - Some(y) => { - info!( - agent = agent_id, metric = %metric, bytes = y.len(), - "[USE_TYPED_STAGE_SPLIT] pushed typed edge YAML on connect" - ); - y - } - None => { - warn!( - agent = agent_id, metric = %metric, - "[USE_TYPED_STAGE_SPLIT] typed emit failed on connect; \ - falling back to legacy generate_agent_config" - ); - match generate_agent_config(&plan.agent_config, &self.opamp_endpoint) { - Ok(y) => y, - Err(_) => { - warn!(agent = agent_id, metric = %metric, "failed to generate agent config on connect"); - return false; - } - } - } - } - } else { - match generate_agent_config(&plan.agent_config, &self.opamp_endpoint) { - Ok(y) => y, - Err(_) => { - warn!(agent = agent_id, metric = %metric, "failed to generate agent config on connect"); - return false; - } - } - }; - - self.opamp.push(agent_id, RemoteConfig { - config_hash: short_hash(&yaml), - yaml, - }).await; - info!(agent = agent_id, metric = %metric, "pushed config to reconnecting agent"); - true - } - - // ── Re-plan helpers ─────────────────────────────────────────────────────── - - /// Re-plans a single metric and pushes updated configs. - /// Returns `true` if re-planning succeeded, `false` if the metric is unknown. - pub async fn replan_metric(&self, metric: &str) -> bool { - let Some((workload, wc)) = self.workload_store.get(metric) else { - warn!(metric, "replan requested but workload not found in store"); - return false; - }; - - info!(metric, "re-planning metric"); - - // Reset the baseline so the cost model runs fresh rather than returning the - // previously established baseline — the whole point of a re-plan is to - // re-optimise with current EMA data. - self.planner.reset(metric); - let mut plan = self.planner.plan(&workload, Some(&wc)); - plan.precompute = build_precompute_jobs(&workload, &plan, "backend:4317"); - self.plan_store.set(metric, plan.clone()); - - // Push agent config only to agents registered for this specific metric, - // rather than broadcasting to all agent-role collectors. Same gate - // as `push_config_to_agent` — typed path on, legacy fallback on - // emit failure or when the gate is off. - let agent_yaml: Option = if planner::stage_split::typed_stage_split_enabled() { - match self.try_emit_typed_edge_yaml_for_workload(&workload) { - Some(y) => { - info!( - metric, bytes = y.len(), - "[USE_TYPED_STAGE_SPLIT] re-plan emitted typed edge YAML" - ); - Some(y) - } - None => { - warn!( - metric, - "[USE_TYPED_STAGE_SPLIT] re-plan typed emit failed; \ - falling back to legacy generate_agent_config" - ); - generate_agent_config(&plan.agent_config, &self.opamp_endpoint).ok() - } - } - } else { - generate_agent_config(&plan.agent_config, &self.opamp_endpoint).ok() - }; - if let Some(yaml) = agent_yaml { - let cfg = RemoteConfig { config_hash: short_hash(&yaml), yaml }; - let agents = self.agent_to_metric.read().await; - let target_agents: Vec = agents.iter() - .filter(|(_, m)| m.as_str() == metric) - .map(|(id, _)| id.clone()) - .collect(); - drop(agents); - for agent_id in target_agents { - self.opamp.push(&agent_id, cfg.clone()).await; - } - } - if let Ok(yaml) = generate_backend_config(&plan.backend_config, &self.opamp_endpoint) { - self.opamp.push_to_role( - AgentRole::Backend, - RemoteConfig { config_hash: short_hash(&yaml), yaml }, - ).await; - } - - // Push the ASAPQuery-backend StreamingConfig YAML via HTTP if a - // backend client is configured. This is the producer side of the - // ASAPQuery PR E hot-reload contract: the backend receives the - // new plan on its /api/v1/streaming-config endpoint and makes it - // visible to the next query without restarting. - if let Some(backend_client) = self.backend_client.as_ref() { - match generate_streaming_config_yaml(metric, &plan) { - Ok(yaml) => { - push_or_log(backend_client, metric, yaml).await; - } - Err(e) => { - warn!( - metric, - error = %e, - "failed to build ASAPQuery streaming-config YAML — \ - skipping backend HTTP push for this replan cycle" - ); - } - } - } - - // Update scraper endpoint sketch types for correct EMA attribution. - let sketch_type = plan.agent_config.sketch_type; - for agent_id in self.opamp.connected_agents().await { - self.scraper.set_sketch_type(&agent_id, sketch_type.clone()).await; - } - - info!(metric, sketch_type = %sketch_type, "re-plan complete"); - true - } - - /// Re-plans all metrics whose `valid_until` has already passed. - pub async fn replan_expired(&self) { - let expired = self.plan_store.expired(chrono::Utc::now()); - if expired.is_empty() { return; } - info!(count = expired.len(), "re-planning expired metrics"); - for metric in expired { - self.replan_metric(&metric).await; - } - } - - /// Called from the violation callback. Looks up the metric served by - /// `agent_id` and triggers an immediate re-plan. - pub async fn handle_violation(&self, agent_id: &str) { - let metric = self.agent_to_metric.read().await.get(agent_id).cloned(); - match metric { - Some(m) => { - info!(agent = agent_id, metric = %m, "SLA violation → triggering re-plan"); - self.replan_metric(&m).await; - } - None => { - warn!(agent = agent_id, "SLA violation but no metric mapping found; re-planning all expired"); - self.replan_expired().await; - } - } - } - - /// Starts a background loop that re-plans expired metrics every `interval`. - pub async fn run_expiry_ticker(self: Arc, interval: Duration) { - let mut ticker = tokio::time::interval(interval); - ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - loop { - ticker.tick().await; - self.replan_expired().await; - } - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use std::collections::HashMap; - use std::time::Duration; - - use chrono::Utc; - - use crate::planner::{CostModelPlanner, BaselinePlanner}; - use crate::store::{PlanStore, WorkloadStore}; - use crate::types::*; - - fn make_replanner() -> Arc { - let plan_store = Arc::new(PlanStore::new()); - let workload_store = Arc::new(WorkloadStore::new()); - let planner = Arc::new(BaselinePlanner::new(CostModelPlanner::new())); - let opamp = Arc::new(crate::opamp::OpampServer::new()); - let scraper = Arc::new(crate::monitor::Scraper::new( - vec![], crate::monitor::Thresholds::default(), - Arc::new(|_| {}), Duration::from_secs(60), - )); - Arc::new(Replanner::new( - planner, plan_store, workload_store, opamp, scraper, - "ws://ctrl:4320/v1/opamp", - )) - } - - fn test_workload(metric: &str) -> (QueryWorkload, WorkloadCharacteristics) { - let wl = QueryWorkload { - metric_name: metric.into(), - label_filters: HashMap::new(), - group_by_labels: vec![], - aggregations: vec![AggType::Quantile], - time_window: Duration::from_secs(300), - repeat_every: None, - accuracy_sla: 0.01, - latency_sla: None, - sketch_type_override: None, - exact_required: false, - quantiles: vec![], - }; - (wl, WorkloadCharacteristics::default()) - } - - fn make_plan() -> CollectionPlan { - CollectionPlan { - agent_config: AgentCollectorConfig { - output_mode: OutputMode::Sketch, - sketch_type: SketchType::DDSketch, - sketch_params: SketchParams::DDSketch { relative_accuracy: 0.01, quantiles: vec![0.5, 0.99] }, - aggregate_by: vec![], - label_matchers: vec![], - window_duration: None, - mode: ProcessorMode::Batch, - enable_self_monitoring: true, - transmit_sketch: true, - drop_original: true, - delta_transmission: false, - delta_threshold: 0.0, - enable_series_id: false, - series_id_ttl_secs: 300, - - data_sink: AgentDataSink::default(), - }, - gateway_config: GatewayCollectorConfig { passthrough: true }, - backend_config: BackendCollectorConfig { - merge_sketch_type: SketchType::DDSketch, - group_by: vec![], - }, - precompute: vec![], - valid_until: Utc::now() + chrono::Duration::seconds(3600), - delta_decision: DeltaDecision::default(), - transmission_cost_summary: TransmissionCostSummary::default(), - staged_plan: None, - } - } - - #[tokio::test] - async fn replan_unknown_metric_returns_false() { - let r = make_replanner(); - assert!(!r.replan_metric("unknown").await); - } - - #[tokio::test] - async fn replan_known_metric_updates_plan_store() { - let r = make_replanner(); - let (wl, wc) = test_workload("latency"); - r.workload_store.set("latency", wl, wc); - r.plan_store.set("latency", make_plan()); - - let ok = r.replan_metric("latency").await; - assert!(ok); - // Plan store should now have a new entry (valid_until in the future). - let updated = r.plan_store.get("latency").unwrap(); - assert!(updated.valid_until > Utc::now()); - } - - #[tokio::test] - async fn replan_expired_replans_only_expired() { - let r = make_replanner(); - let (wl, wc) = test_workload("old"); - r.workload_store.set("old", wl, wc); - - // Insert an already-expired plan. - let mut expired_plan = make_plan(); - expired_plan.valid_until = Utc::now() - chrono::Duration::seconds(60); - r.plan_store.set("old", expired_plan); - - // Insert a still-active plan for "active". - let (awl, awc) = test_workload("active"); - r.workload_store.set("active", awl, awc); - r.plan_store.set("active", make_plan()); - - r.replan_expired().await; - - // "old" should now have a freshly computed plan. - let old_plan = r.plan_store.get("old").unwrap(); - assert!(old_plan.valid_until > Utc::now()); - } - - #[tokio::test] - async fn register_then_violation_replans_correct_metric() { - let r = make_replanner(); - let (wl, wc) = test_workload("req_rate"); - r.workload_store.set("req_rate", wl, wc); - r.plan_store.set("req_rate", make_plan()); - - r.register_agent("agent-1", "req_rate").await; - r.handle_violation("agent-1").await; - - // Plan should have been refreshed. - assert!(r.plan_store.get("req_rate").is_ok()); - } - - #[tokio::test] - async fn unregister_removes_mapping() { - let r = make_replanner(); - r.register_agent("a1", "m").await; - r.unregister_agent("a1").await; - // After unregister, handle_violation falls back to replan_expired (no-op). - r.handle_violation("a1").await; // should not panic - } - - // ── Typed-emit path tests ───────────────────────────────────────────────── - // - // These tests exercise the `USE_TYPED_STAGE_SPLIT`-gated emit path - // ported from `main::emit_bootstrap_typed` so the OpAMP-pushed YAML - // matches what the bootstrap GET path returns. The acceptance bar - // is criterion ⑥ on issue #46: the agent's edge pipeline must - // include `gorillas3` (archive write), `routing` (warm-passthrough - // dispatch), and the `metrics/warm_passthrough` pipeline. - - /// Serialises tests that mutate `USE_TYPED_STAGE_SPLIT`. Mirror of - /// the guard in `main::tests` — `cargo test` runs tests in - /// parallel by default and `typed_stage_split_enabled()` reads the - /// env var on every call. - static TYPED_ENV_GUARD: std::sync::Mutex<()> = std::sync::Mutex::new(()); - - struct TypedEnvGuard { - previous: Option, - _lock: std::sync::MutexGuard<'static, ()>, - } - impl TypedEnvGuard { - fn enable() -> Self { - let lock = TYPED_ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner()); - let previous = std::env::var("USE_TYPED_STAGE_SPLIT").ok(); - std::env::set_var("USE_TYPED_STAGE_SPLIT", "1"); - Self { previous, _lock: lock } - } - } - impl Drop for TypedEnvGuard { - fn drop(&mut self) { - match &self.previous { - Some(v) => std::env::set_var("USE_TYPED_STAGE_SPLIT", v), - None => std::env::remove_var("USE_TYPED_STAGE_SPLIT"), - } - } - } - - /// Given an agent runtime + a quantile workload registered in the - /// workload store, the typed emit path produces YAML containing - /// `gorillas3`, `routing`, and `metrics/warm_passthrough`. - /// - /// Proves freshness-probe routing reaches OpAMP-pushed agents — - /// the legacy `generate_agent_config` path emits NONE of these - /// (it builds a single-pipeline DDSketch YAML with no routing). - #[tokio::test] - async fn typed_replan_emit_includes_freshness_probe_routing() { - let _env = TypedEnvGuard::enable(); - - let r = make_replanner(); - let (wl, wc) = test_workload("latency"); - r.workload_store.set("latency", wl, wc); - r.plan_store.set("latency", make_plan()); - - let yaml = r - .try_emit_typed_edge_yaml("latency") - .expect("typed emit should succeed for a quantile workload"); - - // gorillas3 — archive-tier write to MinIO. Without this the - // warm-tier query engine has nothing to read for criterion ⑥. - assert!( - yaml.contains("gorillas3"), - "typed emit must include the gorillas3 processor block:\n{yaml}" - ); - - // routing — OTTL routing processor that dispatches the - // freshness probes to `metrics/warm_passthrough`. Without this - // the DDSketch processor renames them to `_quantile`. - assert!( - yaml.contains("routing"), - "typed emit must include the routing processor block:\n{yaml}" - ); - - // metrics/warm_passthrough — the bypass pipeline that carries - // the freshness probe samples through gorillas3 + exporter - // WITHOUT the sketch processor. - assert!( - yaml.contains("metrics/warm_passthrough"), - "typed emit must include the metrics/warm_passthrough pipeline:\n{yaml}" - ); - - // Sanity: the freshness probe metric names appear in the YAML - // (in the warm-passthrough route + the gorillas3 archive - // metric list). - assert!( - yaml.contains("http_freshness_probe_warm"), - "typed emit must reference the warm freshness probe metric:\n{yaml}" - ); - } - - /// With `USE_TYPED_STAGE_SPLIT` unset, `push_config_to_agent` - /// falls back to the legacy single-pipeline DDSketch emitter and - /// produces YAML WITHOUT `gorillas3` / `routing` / warm-passthrough. - /// This pins the gate semantics — without it a regression that - /// always-on'd the typed path would silently break agents that - /// can't yet handle the new processors. - #[tokio::test] - async fn legacy_path_omits_typed_processors_when_gate_off() { - // Hold the env-guard lock so a parallel typed test can't - // flip the var underneath us, and explicitly unset. - let _lock = TYPED_ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner()); - let prior = std::env::var("USE_TYPED_STAGE_SPLIT").ok(); - std::env::remove_var("USE_TYPED_STAGE_SPLIT"); - - let r = make_replanner(); - let (wl, wc) = test_workload("latency"); - r.workload_store.set("latency", wl, wc); - r.plan_store.set("latency", make_plan()); - - // Drive the legacy emitter directly — same code - // `push_config_to_agent` runs when the gate is off. - let plan = r.plan_store.get("latency").unwrap(); - let yaml = generate_agent_config(&plan.agent_config, &r.opamp_endpoint) - .expect("legacy emit should succeed"); - - // Legacy single-pipeline DDSketch output has NONE of the - // typed-path processors. - assert!(!yaml.contains("gorillas3"), - "legacy path must not emit gorillas3 processor:\n{yaml}"); - assert!(!yaml.contains("metrics/warm_passthrough"), - "legacy path must not emit warm-passthrough pipeline:\n{yaml}"); - - // Restore. - match prior { - Some(v) => std::env::set_var("USE_TYPED_STAGE_SPLIT", v), - None => std::env::remove_var("USE_TYPED_STAGE_SPLIT"), - } - } -} diff --git a/controller/src/runtime_samples.rs b/controller/src/runtime_samples.rs deleted file mode 100644 index 09fb198e..00000000 --- a/controller/src/runtime_samples.rs +++ /dev/null @@ -1,329 +0,0 @@ -//! Receiver for agents' [`sketch-runtime::GrpcExporter`] -//! batches — the **push side** of the controller's real-time -//! decision loop. -//! -//! Agents running an embedded `sketch-runtime` Sampler call the -//! `asap.runtime.v1.RuntimeSamples.Push` RPC with batched -//! records. The handler appends each record to a bounded ring -//! buffer keyed by `(source, sketch, impl)`. Decision loops -//! peek the tail of that buffer to see the freshest throughput -//! / latency / accuracy signal. -//! -//! ## Why gRPC (vs the earlier HTTP+JSONL) -//! -//! HTTP/2 flow control surfaces controller back-pressure to -//! the agent — critical when the decision loop is real-time. -//! See the design discussion thread for the full trade-off; -//! the summary is in -//! [`sketch-runtime::exporter::grpc`](https://github.com/ProjectASAP/sketch-bench/blob/main/sketch-runtime/src/exporter/grpc.rs). -//! -//! ## Why a ring buffer, not a stream -//! -//! Real-time decisions want the freshest N records, not a full -//! replay. Bounded memory, O(1) append + peek, FIFO eviction. -//! Longer windows live in the agent's `FileExporter` artifact. - -use std::collections::{HashMap, VecDeque}; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Arc; - -use parking_lot::RwLock; -use serde::{Deserialize, Serialize}; -use serde_json::Value; - -// Generated from proto/feedback.proto by build.rs. -pub mod feedback { - tonic::include_proto!("asap.runtime.v1"); -} -use feedback::runtime_samples_server::{RuntimeSamples, RuntimeSamplesServer}; -use feedback::{PushAck, PushBatch}; - -/// One record as stored in the ring buffer. Kept as -/// `serde_json::Value` payload for schema forward-compat — new -/// fields on `sketch-core::report::Record` flow through without -/// a controller rebump. Decision loops extract numeric fields -/// on demand. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RuntimeRecord { - pub source: String, - pub sketch: String, - #[serde(rename = "impl")] - pub impl_name: String, - #[serde(default)] - pub schema_version: u32, - /// The full `sketch-core::Record` payload, minus the - /// labelling fields already in this struct. Parsed from the - /// `RuntimeRecord.payload_json` field of the proto. - #[serde(flatten)] - pub payload: Value, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct SampleKey { - pub source: String, - pub sketch: String, - pub impl_name: String, -} - -#[derive(Debug, Default)] -pub struct RuntimeSamplesStats { - pub batches_received: AtomicU64, - pub records_stored: AtomicU64, - pub records_evicted: AtomicU64, - pub decode_errors: AtomicU64, -} - -impl RuntimeSamplesStats { - pub fn snapshot(&self) -> RuntimeSamplesStatsSnapshot { - RuntimeSamplesStatsSnapshot { - batches_received: self.batches_received.load(Ordering::Relaxed), - records_stored: self.records_stored.load(Ordering::Relaxed), - records_evicted: self.records_evicted.load(Ordering::Relaxed), - decode_errors: self.decode_errors.load(Ordering::Relaxed), - } - } -} - -#[derive(Debug, Clone, Copy, Serialize)] -pub struct RuntimeSamplesStatsSnapshot { - pub batches_received: u64, - pub records_stored: u64, - pub records_evicted: u64, - pub decode_errors: u64, -} - -/// Bounded FIFO ring buffer of runtime records, keyed by -/// `(source, sketch, impl)`. Each key gets its own buffer so a -/// chatty source can't starve a quiet one. -pub struct RuntimeSamplesStore { - buffers: RwLock>>, - per_key_capacity: usize, - stats: Arc, -} - -impl RuntimeSamplesStore { - pub fn new(per_key_capacity: usize) -> Arc { - Arc::new(Self { - buffers: RwLock::new(HashMap::new()), - per_key_capacity, - stats: Arc::new(RuntimeSamplesStats::default()), - }) - } - - pub fn stats(&self) -> Arc { - Arc::clone(&self.stats) - } - - pub fn stats_handle(&self) -> Arc { - Arc::clone(&self.stats) - } - - #[cfg(test)] - pub(crate) fn append_for_test(&self, rec: RuntimeRecord) { - self.append(rec) - } - - fn append(&self, rec: RuntimeRecord) { - let key = SampleKey { - source: rec.source.clone(), - sketch: rec.sketch.clone(), - impl_name: rec.impl_name.clone(), - }; - let mut map = self.buffers.write(); - let buf = map - .entry(key) - .or_insert_with(|| VecDeque::with_capacity(self.per_key_capacity)); - if buf.len() >= self.per_key_capacity { - buf.pop_front(); - self.stats.records_evicted.fetch_add(1, Ordering::Relaxed); - } - buf.push_back(rec); - self.stats.records_stored.fetch_add(1, Ordering::Relaxed); - } - - /// Peek the latest record for a given key, or `None` if the - /// key has never been seen. Used by the replanner / - /// decision loop to read freshness signals. - pub fn latest(&self, key: &SampleKey) -> Option { - self.buffers.read().get(key).and_then(|b| b.back().cloned()) - } - - /// Snapshot the full ring for a key. O(n) clone; non-hot-path only. - pub fn snapshot(&self, key: &SampleKey) -> Vec { - self.buffers - .read() - .get(key) - .map(|b| b.iter().cloned().collect()) - .unwrap_or_default() - } - - pub fn keys(&self) -> Vec { - self.buffers.read().keys().cloned().collect() - } -} - -/// tonic service impl. One instance wraps the shared store and -/// is added to a tonic `Server` listening on the runtime-samples -/// port. -pub struct RuntimeSamplesService { - store: Arc, -} - -impl RuntimeSamplesService { - pub fn new(store: Arc) -> Self { - Self { store } - } - - /// Return the server as a tonic-routed service with gzip - /// compression negotiated on both sides. - pub fn into_server(self) -> RuntimeSamplesServer { - RuntimeSamplesServer::new(self) - .accept_compressed(tonic::codec::CompressionEncoding::Gzip) - .send_compressed(tonic::codec::CompressionEncoding::Gzip) - } -} - -#[tonic::async_trait] -impl RuntimeSamples for RuntimeSamplesService { - async fn push( - &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - self.store - .stats - .batches_received - .fetch_add(1, Ordering::Relaxed); - let batch = request.into_inner(); - let mut accepted = 0_u64; - for pb in batch.records { - // The agent's RuntimeRecord carries the full v1 - // `sketch-core::Record` as JSON in `payload_json`. - // Parse it into our `Value`-flattened store struct; - // reject individual malformed records rather than - // failing the whole batch. - let mut payload: Value = match serde_json::from_str(&pb.payload_json) { - Ok(v) => v, - Err(e) => { - self.store - .stats - .decode_errors - .fetch_add(1, Ordering::Relaxed); - tracing::warn!(error = %e, "runtime-samples: malformed payload_json, skipping"); - continue; - } - }; - // Strip the labelling fields out of `payload` — our - // RuntimeRecord carries them as typed fields, so - // duplicates inside payload would confuse downstream - // consumers. - if let Some(obj) = payload.as_object_mut() { - obj.remove("source"); - obj.remove("sketch"); - obj.remove("impl"); - } - let rec = RuntimeRecord { - source: pb.source, - sketch: pb.sketch, - impl_name: pb.impl_name, - schema_version: pb.schema_version, - payload, - }; - self.store.append(rec); - accepted += 1; - } - Ok(tonic::Response::new(PushAck { accepted })) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use feedback::RuntimeRecord as PbRecord; - - fn make_pb_record(source: &str, sketch: &str, impl_name: &str, tp: f64) -> PbRecord { - let payload = serde_json::json!({ - "schema_version": 1, - "mode": "runtime", - "timestamp": "2026-04-21T19:00:00Z", - "bench": { - "throughput_items_per_sec": { "mean": tp, "stddev": 0.0 }, - "latency_ns": { "p50": 10, "p99": 100 }, - "memory_bytes": 2048 - } - }); - PbRecord { - source: source.into(), - sketch: sketch.into(), - impl_name: impl_name.into(), - schema_version: 1, - payload_json: payload.to_string(), - } - } - - #[tokio::test] - async fn push_stores_records_under_per_key_rings() { - let store = RuntimeSamplesStore::new(16); - let svc = RuntimeSamplesService::new(Arc::clone(&store)); - let batch = PushBatch { - records: vec![ - make_pb_record("dc-a", "cms", "oxide", 1e6), - make_pb_record("dc-b", "hll", "lib", 2e6), - make_pb_record("dc-a", "cms", "oxide", 1.5e6), - ], - }; - let resp = svc.push(tonic::Request::new(batch)).await.expect("ok"); - assert_eq!(resp.into_inner().accepted, 3); - assert_eq!(store.keys().len(), 2); - let key = SampleKey { - source: "dc-a".into(), - sketch: "cms".into(), - impl_name: "oxide".into(), - }; - assert_eq!(store.snapshot(&key).len(), 2); - } - - #[tokio::test] - async fn push_rejects_malformed_payload_individually_but_accepts_rest() { - let store = RuntimeSamplesStore::new(16); - let svc = RuntimeSamplesService::new(Arc::clone(&store)); - let good1 = make_pb_record("dc-a", "cms", "oxide", 1e6); - let bad = PbRecord { - source: "dc-a".into(), - sketch: "cms".into(), - impl_name: "oxide".into(), - schema_version: 1, - payload_json: "{not json".into(), - }; - let good2 = make_pb_record("dc-a", "cms", "oxide", 2e6); - let batch = PushBatch { - records: vec![good1, bad, good2], - }; - let resp = svc.push(tonic::Request::new(batch)).await.expect("ok"); - assert_eq!(resp.into_inner().accepted, 2); - let snap = store.stats.snapshot(); - assert_eq!(snap.records_stored, 2); - assert_eq!(snap.decode_errors, 1); - } - - #[tokio::test] - async fn ring_evicts_oldest_past_capacity() { - let store = RuntimeSamplesStore::new(3); - let svc = RuntimeSamplesService::new(Arc::clone(&store)); - for _ in 0..5 { - svc.push(tonic::Request::new(PushBatch { - records: vec![make_pb_record("dc-a", "cms", "oxide", 1e6)], - })) - .await - .unwrap(); - } - let snap = store.stats.snapshot(); - assert_eq!(snap.records_stored, 5); - assert_eq!(snap.records_evicted, 2); - let key = SampleKey { - source: "dc-a".into(), - sketch: "cms".into(), - impl_name: "oxide".into(), - }; - assert_eq!(store.snapshot(&key).len(), 3); - } -} diff --git a/controller/src/sketch_algebra/capability_matching.rs b/controller/src/sketch_algebra/capability_matching.rs deleted file mode 100644 index f0b90e98..00000000 --- a/controller/src/sketch_algebra/capability_matching.rs +++ /dev/null @@ -1,414 +0,0 @@ -//! Capability matrix — which (sketch, statistic) pairs are valid. -//! -//! The shared MVP demo contract pins a per-metric family-per-statistic -//! mapping (issue #46): -//! -//! | metric | family | query class | -//! |-------------------------|--------------|-----------------------| -//! | `http_requests_total` | raw | sum / rate / count | -//! | `http_latency_ms` | DDSketch | quantile (rel-err) | -//! | `request_size_bytes` | KLL | quantile (rank-err) | -//! | `unique_users_per_min` | HLL | cardinality | -//! | `top_endpoint_qps` | CountSketch | top-K | -//! | `endpoint_request_freq` | CountMinSketch (CMS) | frequency | -//! -//! Note that CountMinSketch ALSO supports top-K via the CMS-Heap pattern -//! (Cormode & Muthukrishnan, 2005 — "An Improved Data Stream Summary: The -//! Count-Min Sketch and its Applications"). The capability matrix below -//! reflects this: CMS validly answers Frequency *and* TopK. CountSketch -//! remains the canonical (unbiased) TopK pick — `pick_family` prefers it -//! when no override is supplied — but a workload's -//! `sketch_family_override` may still pin CountMin for a TopK metric. -//! -//! Backend gap: declaring CMS-supports-TopK here is a planner-side concern. -//! The backend's actual "top-K from CountMin state" query path (the heap -//! readout) is a separate workstream and is not yet implemented in -//! `ASAPQuery-backend`. Until that lands, a planner-pinned CMS-for-TopK -//! binding will produce a sketch the backend cannot extract heavy hitters -//! from. Keep this caveat in mind when reviewing override-driven plans. -//! -//! The asap-common docstring (`asap-common/dependencies/rs/asap_types/src/ -//! capability_matching.rs` per the orchestrator spec) keeps this matrix -//! as the single source of truth so the planner's `bind_workload_typed` -//! and the L4 `Bind*` rule dispatcher agree on which family to pick. -//! -//! There are two perspectives the matrix has to satisfy: -//! -//! 1. **Static.** `(sketch, statistic)` is a *valid* binding — a CMS can -//! answer Frequency, but an HLL can not. Used by the Phase β catalog -//! check (`is_valid_pair`) and the `Bind*` rule guards. -//! 2. **Dynamic.** Given a `StatisticClass` + `AccuracyPreference`, pick -//! the *preferred* sketch family. The MVP picks DDSketch when the -//! preference is relative-error and KLL when it's rank-error; the rest -//! of the catalog is one-to-one. -//! -//! The asap-common path referenced in the orchestrator spec doesn't exist -//! in this monorepo today (the controller crate is the only Rust consumer -//! of the catalog), so the canonical home is `controller/src/sketch_algebra/ -//! capability_matching.rs`. When asap-common ships as a separate Cargo -//! crate, this module lifts there verbatim. - -#![allow(dead_code)] - -use crate::sketch_algebra::params::SketchKind; - -/// Query intent the user is expressing — abstracted away from the L1 -/// language (PromQL `quantile_over_time`, SQL `PERCENTILE_CONT`, etc.) and -/// from the L4 sketch family. Each variant enumerates one entry in the -/// MVP-contract row. -/// -/// Phase β scope: the variants below are the six classes the demo -/// workload exercises. Adding a class (e.g. `Histogram`) is purely -/// additive: extend [`StatisticClass`], extend [`valid_pair`] / -/// [`pick_family`], add a row to the matrix tests. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum StatisticClass { - /// `quantile(φ, x)` — DDSketch (relative-error) and KLL (rank-error) - /// are both valid; the [`AccuracyPreference`] picks which one. - Quantile, - /// `count_distinct(x)` — only HLL is valid. - Cardinality, - /// `topk(k, x)` — CountSketch (with-heap) is the canonical pick, but - /// CountMinSketch (with-heap) is *also* valid via the CMS-Heap pattern - /// (Cormode & Muthukrishnan 2005). Misra-Gries / SpaceSaving would - /// extend the matrix but are not part of Phase β. - TopK, - /// `freq(x = k)` — only CMS (CountMinSketch) is valid in the MVP - /// catalog. CountSketch could in principle answer it but the - /// MVP-contract row pins CMS. - Frequency, - /// `sum(x) / rate(x[w]) / count(x)` — no sketch needed; the agent - /// emits raw OTLP and the backend / Prometheus computes the answer - /// directly. Maps to "raw passthrough" in the contract. - SumRateCount, -} - -impl StatisticClass { - /// Stable, human-readable identifier (`"quantile"`, …) — used for - /// diagnostics + the `cargo test` matrix. - pub fn as_str(&self) -> &'static str { - match self { - StatisticClass::Quantile => "quantile", - StatisticClass::Cardinality => "cardinality", - StatisticClass::TopK => "topk", - StatisticClass::Frequency => "frequency", - StatisticClass::SumRateCount => "sum_rate_count", - } - } -} - -/// Tie-breaker preference for [`StatisticClass::Quantile`] — relative-error -/// (DDSketch) vs rank-error (KLL). Other statistic classes ignore this -/// field today (each maps to exactly one family). -/// -/// Default is [`AccuracyPreference::RelativeError`] — matches the legacy -/// `algebra::directory::sketch_type_for_agg` default of DDSketch for -/// Quantile workloads. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] -pub enum AccuracyPreference { - /// DDSketch — `|estimate − true| ≤ alpha · true`. The right pick when - /// the user cares about tail-relative error (e.g. p99 latency in - /// milliseconds, where 1ms vs 1s should be 1% off, not "rank 99%"). - #[default] - RelativeError, - /// KLL — rank-error bound (φ̂ within ε of φ in CDF space). The right - /// pick when the user cares about rank-stability (e.g. byte-size - /// distribution where the absolute value range is many orders of - /// magnitude and rank-distance is the meaningful metric). - RankError, -} - -/// Whether `(sketch, statistic)` is a valid pair in the MVP catalog. -/// -/// The capability matrix: -/// -/// | sketch | Quantile | Cardinality | TopK | Frequency | SumRateCount | -/// |--------------|----------|-------------|------|-----------|--------------| -/// | DDSketch | yes | no | no | no | no | -/// | KLL | yes | no | no | no | no | -/// | HLL | no | yes | no | no | no | -/// | CountSketch | no | no | yes | no | no | -/// | CMS | no | no | yes | yes | no | -/// -/// `SumRateCount` has no valid sketch — the agent emits raw OTLP for -/// those statistic classes (see [`pick_family`]). -/// -/// CMS gains TopK validity via the CMS-Heap pattern (Cormode & -/// Muthukrishnan 2005). NB: this is the planner-side capability -/// declaration; the backend's "top-K from CountMin state" readout path -/// is a separate workstream — see the module-level docs. -pub fn is_valid_pair(sketch: SketchKind, statistic: StatisticClass) -> bool { - use SketchKind::*; - use StatisticClass::*; - match (sketch, statistic) { - (DDSketch, Quantile) - | (Kll, Quantile) - | (Hll, Cardinality) - | (CountSketch, TopK) - | (Cms, TopK) - | (Cms, Frequency) => true, - _ => false, - } -} - -/// Pick the preferred sketch family for a `(StatisticClass, AccuracyPreference)` -/// pair. Returns `None` for [`StatisticClass::SumRateCount`] — that class -/// uses raw passthrough, no sketch needed. -/// -/// The mapping: -/// -/// - Quantile + RelativeError → DDSketch -/// - Quantile + RankError → KLL -/// - Cardinality → HLL -/// - TopK → CountSketch -/// - Frequency → CMS (CountMinSketch) -/// - SumRateCount → None (raw passthrough) -/// -/// `sketch_family_override` (treated as `QueryWorkload::sketch_type_override` -/// at the planner-rules layer) wins over the capability-matched default — -/// see `planner::rules::bind_workload_typed`. -pub fn pick_family( - statistic: StatisticClass, - accuracy: AccuracyPreference, -) -> Option { - use AccuracyPreference::*; - use SketchKind::*; - use StatisticClass::*; - let kind = match (statistic, accuracy) { - (Quantile, RelativeError) => DDSketch, - (Quantile, RankError) => Kll, - (Cardinality, _) => Hll, - (TopK, _) => CountSketch, - (Frequency, _) => Cms, - (SumRateCount, _) => return None, - }; - debug_assert!( - is_valid_pair(kind.clone(), statistic), - "pick_family produced an invalid (sketch={kind:?}, stat={statistic:?}) pair" - ); - Some(kind) -} - -/// Heuristic mapping from the MVP-demo metric names in the shared contract -/// (issue #46) to a `(StatisticClass, AccuracyPreference)` pair. When the -/// metric name matches one of the six contract rows, this is the canonical -/// classification; otherwise [`None`] is returned and the caller falls -/// back to its `AggType`-driven default. -/// -/// The name-matching is exact (case-sensitive) to keep the contract row -/// the single source of truth — a typo'd metric name should fall through -/// to the AggType default rather than silently bind to the wrong family. -pub fn classify_demo_metric( - metric_name: &str, -) -> Option<(StatisticClass, AccuracyPreference)> { - use AccuracyPreference::*; - use StatisticClass::*; - Some(match metric_name { - "http_requests_total" => (SumRateCount, RelativeError), - "http_latency_ms" => (Quantile, RelativeError), - "request_size_bytes" => (Quantile, RankError), - "unique_users_per_min" => (Cardinality, RelativeError), - "top_endpoint_qps" => (TopK, RelativeError), - "endpoint_request_freq" => (Frequency, RelativeError), - _ => return None, - }) -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - - // ── (sketch, statistic) capability matrix ───────────────────────────────── - - #[test] - fn ddsketch_is_quantile_only() { - assert!(is_valid_pair(SketchKind::DDSketch, StatisticClass::Quantile)); - assert!(!is_valid_pair(SketchKind::DDSketch, StatisticClass::Cardinality)); - assert!(!is_valid_pair(SketchKind::DDSketch, StatisticClass::TopK)); - assert!(!is_valid_pair(SketchKind::DDSketch, StatisticClass::Frequency)); - assert!(!is_valid_pair(SketchKind::DDSketch, StatisticClass::SumRateCount)); - } - - #[test] - fn kll_is_quantile_only() { - assert!(is_valid_pair(SketchKind::Kll, StatisticClass::Quantile)); - assert!(!is_valid_pair(SketchKind::Kll, StatisticClass::Cardinality)); - assert!(!is_valid_pair(SketchKind::Kll, StatisticClass::TopK)); - assert!(!is_valid_pair(SketchKind::Kll, StatisticClass::Frequency)); - assert!(!is_valid_pair(SketchKind::Kll, StatisticClass::SumRateCount)); - } - - #[test] - fn hll_is_cardinality_only() { - assert!(is_valid_pair(SketchKind::Hll, StatisticClass::Cardinality)); - assert!(!is_valid_pair(SketchKind::Hll, StatisticClass::Quantile)); - assert!(!is_valid_pair(SketchKind::Hll, StatisticClass::TopK)); - assert!(!is_valid_pair(SketchKind::Hll, StatisticClass::Frequency)); - assert!(!is_valid_pair(SketchKind::Hll, StatisticClass::SumRateCount)); - } - - #[test] - fn countsketch_is_topk_only() { - assert!(is_valid_pair(SketchKind::CountSketch, StatisticClass::TopK)); - assert!(!is_valid_pair(SketchKind::CountSketch, StatisticClass::Quantile)); - assert!(!is_valid_pair(SketchKind::CountSketch, StatisticClass::Cardinality)); - assert!(!is_valid_pair(SketchKind::CountSketch, StatisticClass::Frequency)); - assert!(!is_valid_pair(SketchKind::CountSketch, StatisticClass::SumRateCount)); - } - - #[test] - fn cms_supports_frequency_and_topk() { - // CMS validly answers Frequency (point-frequency, additive bound) - // AND TopK via the CMS-Heap pattern (Cormode & Muthukrishnan 2005). - assert!(is_valid_pair(SketchKind::Cms, StatisticClass::Frequency)); - assert!(is_valid_pair(SketchKind::Cms, StatisticClass::TopK)); - assert!(!is_valid_pair(SketchKind::Cms, StatisticClass::Quantile)); - assert!(!is_valid_pair(SketchKind::Cms, StatisticClass::Cardinality)); - assert!(!is_valid_pair(SketchKind::Cms, StatisticClass::SumRateCount)); - } - - #[test] - fn countmin_supports_topk_capability() { - // Pin the new matrix entry: CMS validly answers TopK. The - // canonical pick remains CountSketch — see - // `pick_family_topk_picks_countsketch` — but the catalog now - // accepts a `sketch_family_override: CountMinSketch` for a - // TopK-shaped workload (CMS-Heap pattern, Cormode & - // Muthukrishnan 2005). - assert!( - is_valid_pair(SketchKind::Cms, StatisticClass::TopK), - "CMS should support TopK via the CMS-Heap pattern", - ); - } - - // ── pick_family — capability-matched defaults ───────────────────────────── - - #[test] - fn pick_family_quantile_relative_picks_ddsketch() { - assert_eq!( - pick_family(StatisticClass::Quantile, AccuracyPreference::RelativeError), - Some(SketchKind::DDSketch), - ); - } - - #[test] - fn pick_family_quantile_rank_picks_kll() { - assert_eq!( - pick_family(StatisticClass::Quantile, AccuracyPreference::RankError), - Some(SketchKind::Kll), - ); - } - - #[test] - fn pick_family_cardinality_picks_hll() { - for pref in [AccuracyPreference::RelativeError, AccuracyPreference::RankError] { - assert_eq!( - pick_family(StatisticClass::Cardinality, pref), - Some(SketchKind::Hll), - ); - } - } - - #[test] - fn pick_family_topk_picks_countsketch() { - assert_eq!( - pick_family(StatisticClass::TopK, AccuracyPreference::default()), - Some(SketchKind::CountSketch), - ); - } - - #[test] - fn pick_family_frequency_picks_cms() { - assert_eq!( - pick_family(StatisticClass::Frequency, AccuracyPreference::default()), - Some(SketchKind::Cms), - ); - } - - #[test] - fn pick_family_sum_rate_count_is_raw_passthrough() { - assert_eq!( - pick_family(StatisticClass::SumRateCount, AccuracyPreference::default()), - None, - "SumRateCount must produce no sketch (raw passthrough)", - ); - } - - // ── classify_demo_metric — every contract row ───────────────────────────── - - #[test] - fn classify_demo_metric_six_contract_rows() { - let cases = [ - ( - "http_requests_total", - StatisticClass::SumRateCount, - AccuracyPreference::RelativeError, - ), - ( - "http_latency_ms", - StatisticClass::Quantile, - AccuracyPreference::RelativeError, - ), - ( - "request_size_bytes", - StatisticClass::Quantile, - AccuracyPreference::RankError, - ), - ( - "unique_users_per_min", - StatisticClass::Cardinality, - AccuracyPreference::RelativeError, - ), - ( - "top_endpoint_qps", - StatisticClass::TopK, - AccuracyPreference::RelativeError, - ), - ( - "endpoint_request_freq", - StatisticClass::Frequency, - AccuracyPreference::RelativeError, - ), - ]; - for (metric, want_class, want_pref) in cases { - assert_eq!( - classify_demo_metric(metric), - Some((want_class, want_pref)), - "metric {metric} should classify to ({want_class:?}, {want_pref:?})", - ); - } - } - - #[test] - fn classify_demo_metric_unknown_returns_none() { - assert_eq!(classify_demo_metric("foo_bar_baz"), None); - // Case-sensitive — typo'd capitalisation must NOT silently match. - assert_eq!(classify_demo_metric("HTTP_LATENCY_MS"), None); - assert_eq!(classify_demo_metric(""), None); - } - - // ── End-to-end: every contract row maps to its expected SketchKind ──────── - - #[test] - fn every_contract_metric_picks_its_contract_family() { - let cases = [ - ("http_requests_total", None), - ("http_latency_ms", Some(SketchKind::DDSketch)), - ("request_size_bytes", Some(SketchKind::Kll)), - ("unique_users_per_min", Some(SketchKind::Hll)), - ("top_endpoint_qps", Some(SketchKind::CountSketch)), - ("endpoint_request_freq", Some(SketchKind::Cms)), - ]; - for (metric, want_kind) in cases { - let (stat, pref) = classify_demo_metric(metric) - .unwrap_or_else(|| panic!("contract metric {metric} must classify")); - let got = pick_family(stat, pref); - assert_eq!( - got, want_kind, - "metric {metric}: expected family {want_kind:?}, got {got:?}", - ); - } - } -} diff --git a/controller/src/sketch_algebra/lower.rs b/controller/src/sketch_algebra/lower.rs deleted file mode 100644 index 1874c97f..00000000 --- a/controller/src/sketch_algebra/lower.rs +++ /dev/null @@ -1,83 +0,0 @@ -//! L3 → L4 lowering — `QueryExpr` walk that fires `Bind*` rules. -//! -//! Per `controller/docs/design.md` §6 sketch_algebra (line ~616): "the -//! optimizer's job is to selectively replace logical aggregates / joins -//! with their sketch-bound variants when a binding rule fires; everything -//! else stays inside `SketchExpr::Logical(…)`." -//! -//! Phase C ships the bottom-up walk — every `QueryExpr` sub-tree is -//! offered to the rule dispatcher; if a rule fires, its output replaces -//! the sub-tree; otherwise we recurse into the children and wrap the -//! result in `SketchExpr::Logical`. -//! -//! `LetBinding` / `Ref` survive the lowering: the bound expression is -//! lowered to L4, the child is lowered against the same workload-level -//! accuracy target, and the result is a `SketchExpr::LetBinding` / -//! `SketchExpr::Ref` with the L4-bound payload. - -#![allow(dead_code)] - -use thiserror::Error; - -use crate::intent_algebra::QueryExpr; -use crate::sketch_algebra::rules::dispatch; -use crate::sketch_algebra::sketch_expr::SketchExpr; -use crate::types_v2::AccuracyTarget; - -/// Errors surfaced by the `bind_query_expr` lowering. Reserved — Phase C -/// has no bind-time errors that aren't expressible as "no rule fires" -/// (the dispatcher returns `None` and the caller wraps the input in -/// `SketchExpr::Logical`). Defined now so future rules that *can* fail -/// at bind time (catalog mismatch, parameter overflow) plug in without -/// an API break. -#[derive(Debug, Error)] -pub enum BindingError { - /// Carried for downstream consumers — Phase C has no producers yet. - #[error("binding failed: {0}")] - Other(String), -} - -/// Lower an L3 `QueryExpr` to an L4 [`SketchExpr`] under the supplied -/// workload-level accuracy target. Bottom-up walk; `Bind*` rules consult -/// the `accuracy` param + the per-intent `accuracy` field on each -/// `Aggregate` and pick the tighter of the two. -/// -/// Return value: `Ok(SketchExpr)` always — Phase C never errors. The -/// caller observes "no binding" via the returned `SketchExpr::Logical` -/// at the matched sub-tree position. -pub fn bind_query_expr( - expr: &QueryExpr, - accuracy: AccuracyTarget, -) -> Result { - Ok(bind_recursive(expr, &accuracy)) -} - -fn bind_recursive(expr: &QueryExpr, accuracy: &AccuracyTarget) -> SketchExpr { - // Try the rule dispatcher first — if a `Bind*` rule fires, its - // output replaces the matched sub-tree wholesale. The rule's output - // already wraps the L3 child in `SketchExpr::Logical(...)` per the - // `estimate_over_agg` constructor. - if let Some(bound) = dispatch(expr, accuracy) { - return bound; - } - - // No rule matched — recurse into the children to find sub-trees that - // bind. For pass-through nodes (`Scan`, `Window`, `LetBinding`, - // `Ref`) we surface the recursive structure in `SketchExpr` directly - // when relevant, otherwise we wrap the L3 sub-tree in `Logical`. - match expr { - QueryExpr::LetBinding { name, expr, child } => SketchExpr::LetBinding { - name: name.clone(), - expr: Box::new(bind_recursive(expr, accuracy)), - child: Box::new(bind_recursive(child, accuracy)), - }, - QueryExpr::Ref { name } => SketchExpr::Ref { name: name.clone() }, - // For `Aggregate`, the rule dispatcher already had a chance and - // declined. For `Scan` and `Window`, the recursive walk is a - // no-op (no binding rule applies to these shapes today). In all - // cases, wrap the L3 sub-tree as a logical pass-through. - QueryExpr::Aggregate { .. } | QueryExpr::Scan { .. } | QueryExpr::Window { .. } => { - SketchExpr::Logical(expr.clone()) - } - } -} diff --git a/controller/src/sketch_algebra/mod.rs b/controller/src/sketch_algebra/mod.rs deleted file mode 100644 index 2907bb2b..00000000 --- a/controller/src/sketch_algebra/mod.rs +++ /dev/null @@ -1,53 +0,0 @@ -//! Layer 4 IR — `core::sketch_algebra` per `controller/docs/design.md` §6. -//! -//! Phase C introduces the L4 vocabulary the planner pivots on: -//! -//! - [`SketchExpr`] — the L4 algebra DAG (sketch-bound, language-orthogonal, -//! deployment-independent). Single-rooted per query; multi-root -//! workload-level fan-in lives one layer up in -//! `types_v2::WorkloadPlan`. -//! - [`SketchKind`] / [`SketchParams`] — typed sketch-family selector + -//! parameter payload. Convertible to the legacy `crate::types` -//! shape via [`SketchParams::to_legacy`] for the L5 emitter side. -//! - [`SketchStateSchema`] — the L4 type-system primitive that mirrors -//! the §6.4 input/output spec table (`Sketch(SketchKind, SketchParams)` -//! field type + catalog-capability flags). -//! - [`bind_query_expr`] — the L3→L4 lowering driver: bottom-up walk -//! that fires `Bind*` rules. -//! - [`rules`] — the `Bind*` rule family. Each rule pattern-matches on a -//! `QueryExpr::Aggregate` shape, reads the `AccuracyTarget`, and emits -//! a typed `SketchExpr` with the sketch family + parameters committed. -//! -//! Scope reduction (per orchestrator spec): the variant set ships the -//! subset DC + PromQL needs. `SketchJoin`, `SketchSubtract`, -//! `SketchDelete` from design.md §6 are intentionally *not* surfaced -//! yet — they're gated on rules that haven't landed. Adding them is -//! purely additive. -//! -//! Wire-up state. The typed path is opt-in via the -//! `USE_TYPED_SKETCH_ALGEBRA` env var consulted by `planner::rules`; -//! existing call sites continue to use the legacy untyped binding path. -//! Phase E (stage_split) is the natural migration point. - -#![allow(dead_code, unused_imports)] - -pub mod capability_matching; -pub mod lower; -pub mod params; -pub mod rules; -pub mod schema; -pub mod sketch_expr; - -#[cfg(test)] -mod tests; - -// Re-exports — `crate::sketch_algebra::*` for downstream callers. -pub use capability_matching::{ - classify_demo_metric, is_valid_pair, pick_family, AccuracyPreference, StatisticClass, -}; -pub use lower::{bind_query_expr, BindingError}; -pub use params::{ - CmsParams, CountSketchParams, DDSketchParams, HllParams, KllParams, SketchKind, SketchParams, -}; -pub use schema::{SketchCapabilities, SketchStateSchema}; -pub use sketch_expr::{EstimateOp, MergeAlgebra, SketchExpr}; diff --git a/controller/src/sketch_algebra/params.rs b/controller/src/sketch_algebra/params.rs deleted file mode 100644 index 0f1946db..00000000 --- a/controller/src/sketch_algebra/params.rs +++ /dev/null @@ -1,299 +0,0 @@ -//! Layer 4 sketch parameters — typed per family. -//! -//! Mirrors `controller/docs/design.md` §6 `core::sketch_algebra` (around -//! line ~580). The L3 IR carries an `AggIntent` + `AccuracyTarget`; L4 -//! `Bind*` rules read those, consult the catalog, and emit a typed -//! [`SketchParams`] payload alongside the chosen [`SketchType`]. -//! -//! Why a parallel typed enum rather than reusing `crate::types::SketchParams`? -//! The legacy `SketchParams` (in `crate::types`) is shaped for the wire -//! format the OTel-collector consumes: it carries pre-baked `quantiles` -//! grids, a `metric_name` field for the CMS partition operator, etc. — -//! all of which are L5 emitter concerns. L4 needs only the parameters -//! that affect cost / accuracy: the KLL sketch's `k`, DDSketch's `alpha`, -//! HLL's `precision`, CMS's `(w, d)`. Keeping the L4 parameter shape -//! minimal makes the `Bind*` rule signatures narrow and the typed -//! `SketchExpr` IR independent of wire-format drift. -//! -//! Convertibility — see [`SketchParams::to_legacy`]. The legacy form is -//! what the existing `algebra::directory::build_sketch_params` produces; -//! this `to_legacy` adapter is what lets the typed-path rewrite of -//! `planner::rules` (Phase E) land additively. - -#![allow(dead_code)] - -use serde::{Deserialize, Serialize}; - -use crate::types::{SketchDefaults, SketchParams as LegacySketchParams, SketchType}; - -/// L4 sketch family selector. Mirrors `crate::types::SketchType` but with -/// the variants L4 sketch-binding rules emit. Naming kept distinct from -/// the legacy enum so the typed path is unambiguous in error messages / -/// diagnostics. -/// -/// The values are 1:1 convertible to the legacy `SketchType` via -/// [`SketchType::from`]; all rule outputs are round-trippable. -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum SketchKind { - /// Karnin-Lang-Liberty quantile sketch. Mergeable, exact rank-error - /// bound. Picked for `Quantile` intents when relative-error budget is - /// not specified or when the quantile target is in the tail. - Kll, - /// DDSketch — log-bucketed quantile sketch with a relative-error - /// guarantee. Picked for `Quantile` intents when an explicit ε is - /// supplied and tail-relative error matters more than rank-error. - DDSketch, - /// HyperLogLog — approximate cardinality. Picked for `Cardinality`. - Hll, - /// Count-Min sketch. Picked for `Count` and `Frequency` intents when - /// approximation is allowed. - Cms, - /// Count-Sketch (with optional heavy-hitter heap). Picked for `TopK` - /// when paired with a Misra-Gries / heap-of-counters extractor; also - /// the substrate for general `Frequency` sketching when balanced - /// (zero-mean) error is preferable to CMS's one-sided bias. - CountSketch, -} - -impl From for SketchType { - fn from(k: SketchKind) -> Self { - match k { - SketchKind::Kll => SketchType::KLL, - SketchKind::DDSketch => SketchType::DDSketch, - SketchKind::Hll => SketchType::HLL, - SketchKind::Cms => SketchType::CountMinSketch, - SketchKind::CountSketch => SketchType::CountSketch, - } - } -} - -/// Inverse of `From for SketchType`. Round-trippable: -/// `SketchKind::from(SketchType::from(k)) == k` for every variant. Used -/// by `planner::rules::bind_workload_typed` to translate the legacy -/// `QueryWorkload::sketch_type_override` field into the typed `SketchKind` -/// the capability matrix consumes. -impl From for SketchKind { - fn from(t: SketchType) -> Self { - match t { - SketchType::KLL => SketchKind::Kll, - SketchType::DDSketch => SketchKind::DDSketch, - SketchType::HLL => SketchKind::Hll, - SketchType::CountMinSketch => SketchKind::Cms, - SketchType::CountSketch => SketchKind::CountSketch, - } - } -} - -/// L4 sketch parameters per family. Parameter names match the canonical -/// sketch literature: `k` for KLL stream size, `alpha` for DDSketch -/// relative-accuracy bound, `precision` for HLL register width, `(w, d)` -/// for CMS / CountSketch dimensions. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(tag = "family", rename_all = "snake_case")] -pub enum SketchParams { - /// KLL stream size. `k=200` ≈ ε≈0.01 rank error, `k=2048` ≈ ε≈0.0035 - /// — see `accuracy_profile.rs` (ASAPQuery-backend) for the formal - /// rank-error bound. - Kll(KllParams), - /// DDSketch relative-accuracy. `alpha=0.01` is the catalog default - /// for `Epsilon(0.01)` quantile budgets. - DDSketch(DDSketchParams), - /// HLL register width. `precision=14` ≈ ε≈0.81%/√m; `precision=10` is - /// the coarse default. - Hll(HllParams), - /// Count-Min: `w` columns × `d` rows. Error bound: ε ≤ e/w with prob. - /// 1 − 2^(−d) (Cormode-Muthukrishnan). - Cms(CmsParams), - /// Count-Sketch: `w` columns × `d` rows; balanced (zero-mean) error. - /// When `with_heap` is set, pairs with a heavy-hitter heap to extract - /// `TopK`. - CountSketch(CountSketchParams), -} - -/// KLL parameters. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub struct KllParams { - /// Stream-size parameter. Higher k → tighter rank-error / more memory. - pub k: u32, -} - -/// DDSketch parameters. -#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] -pub struct DDSketchParams { - /// Relative-error bound — DDSketch guarantees |estimate − true| ≤ - /// alpha · true on every quantile. - pub alpha: f64, -} - -/// HLL parameters. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub struct HllParams { - /// log2 of the register count. `precision=p` → 2^p registers, error - /// ≈ 1.04 / √(2^p). - pub precision: u32, -} - -/// Count-Min parameters. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub struct CmsParams { - /// Number of columns (width). Drives the additive ε bound (≤ e/w). - pub w: u32, - /// Number of rows (depth). Drives the failure probability (≤ 2^−d). - pub d: u32, -} - -/// Count-Sketch parameters. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub struct CountSketchParams { - /// Number of columns. - pub w: u32, - /// Number of rows. - pub d: u32, - /// Whether to pair with a heavy-hitter heap (for TopK extraction). - pub with_heap: bool, -} - -impl SketchParams { - /// The matching [`SketchKind`] for this parameter payload. - pub fn kind(&self) -> SketchKind { - match self { - SketchParams::Kll(_) => SketchKind::Kll, - SketchParams::DDSketch(_) => SketchKind::DDSketch, - SketchParams::Hll(_) => SketchKind::Hll, - SketchParams::Cms(_) => SketchKind::Cms, - SketchParams::CountSketch(_) => SketchKind::CountSketch, - } - } - - /// Convert to the legacy wire-shaped `crate::types::SketchParams`. - /// `quantile_grid` is read from defaults — Phase C does not yet plumb - /// query-specific quantile grids into the typed path; that's an L5 - /// emitter concern picked up in Phase E (stage_split + emitter - /// migration). - pub fn to_legacy(&self, defaults: &SketchDefaults) -> LegacySketchParams { - match self { - SketchParams::Kll(p) => LegacySketchParams::KLL { - k: p.k, - quantiles: defaults.quantile_grid.clone(), - }, - SketchParams::DDSketch(p) => LegacySketchParams::DDSketch { - relative_accuracy: p.alpha, - quantiles: defaults.quantile_grid.clone(), - }, - SketchParams::Hll(p) => LegacySketchParams::HLL { - precision: p.precision, - }, - SketchParams::Cms(p) => LegacySketchParams::CountMinSketch { - rows: p.d, - cols: p.w, - metric_name: defaults.count_min_sketch.metric_name.clone(), - }, - SketchParams::CountSketch(p) => LegacySketchParams::CountSketch { - // Translate (w, d) back into the legacy (epsilon, delta) - // surface — `epsilon ≈ e/w`, `delta ≈ 2^−d`. The legacy - // CountSketch processor expects these directly. - epsilon: std::f64::consts::E / (p.w as f64), - delta: 2f64.powi(-(p.d as i32)), - }, - } - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn sketch_kind_to_legacy_roundtrip() { - let cases = [ - (SketchKind::Kll, SketchType::KLL), - (SketchKind::DDSketch, SketchType::DDSketch), - (SketchKind::Hll, SketchType::HLL), - (SketchKind::Cms, SketchType::CountMinSketch), - (SketchKind::CountSketch, SketchType::CountSketch), - ]; - for (k, expected) in cases { - let legacy: SketchType = k.into(); - assert_eq!(legacy, expected); - } - } - - #[test] - fn sketch_params_kind_matches() { - assert_eq!( - SketchParams::Kll(KllParams { k: 200 }).kind(), - SketchKind::Kll - ); - assert_eq!( - SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }).kind(), - SketchKind::DDSketch - ); - assert_eq!( - SketchParams::Hll(HllParams { precision: 14 }).kind(), - SketchKind::Hll - ); - assert_eq!( - SketchParams::Cms(CmsParams { w: 2048, d: 5 }).kind(), - SketchKind::Cms - ); - assert_eq!( - SketchParams::CountSketch(CountSketchParams { - w: 2048, - d: 5, - with_heap: true, - }) - .kind(), - SketchKind::CountSketch - ); - } - - #[test] - fn to_legacy_kll_carries_k_and_grid() { - let defaults = SketchDefaults::default(); - let p = SketchParams::Kll(KllParams { k: 200 }).to_legacy(&defaults); - match p { - LegacySketchParams::KLL { k, quantiles } => { - assert_eq!(k, 200); - assert_eq!(quantiles, defaults.quantile_grid); - } - other => panic!("expected KLL legacy, got {other:?}"), - } - } - - #[test] - fn to_legacy_ddsketch_carries_alpha() { - let defaults = SketchDefaults::default(); - let p = SketchParams::DDSketch(DDSketchParams { alpha: 0.005 }).to_legacy(&defaults); - match p { - LegacySketchParams::DDSketch { - relative_accuracy, .. - } => { - assert!((relative_accuracy - 0.005).abs() < 1e-12); - } - other => panic!("expected DDSketch legacy, got {other:?}"), - } - } - - #[test] - fn params_serde_roundtrip() { - let cases = [ - SketchParams::Kll(KllParams { k: 200 }), - SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), - SketchParams::Hll(HllParams { precision: 14 }), - SketchParams::Cms(CmsParams { w: 2048, d: 5 }), - SketchParams::CountSketch(CountSketchParams { - w: 2048, - d: 5, - with_heap: true, - }), - ]; - for c in cases { - let json = serde_json::to_string(&c).unwrap(); - let back: SketchParams = serde_json::from_str(&json).unwrap(); - assert_eq!(c, back); - } - } -} diff --git a/controller/src/sketch_algebra/rules/bind_archive_only.rs b/controller/src/sketch_algebra/rules/bind_archive_only.rs deleted file mode 100644 index 44c70d4e..00000000 --- a/controller/src/sketch_algebra/rules/bind_archive_only.rs +++ /dev/null @@ -1,229 +0,0 @@ -//! `BindArchiveOnly` — route Phase β archive-only `AggIntent`s to the -//! cold tier. -//! -//! This rule is the L4 catch for [`AggIntent`]s that don't have a warm- -//! tier streaming sketch family today (`HistogramQuantile`, `Absent`, -//! `Delta`, `Deriv`, `PredictLinear`, `HoltWinters`, `Idelta`, `Irate`, -//! `Resets`, `Changes`, `Present`). It matches a single-intent -//! `Aggregate` carrying any of those, and emits an -//! [`SketchExpr::Logical`] pass-through. The L5 emitter looks at the -//! enclosed [`AggIntent::archive_only`] flag and routes the corresponding -//! StreamingConfig entry to the archive (Gorilla / Thanos) tier rather -//! than the warm sketch tier. -//! -//! Why a rule rather than the recursive walker's default? -//! `lower::bind_recursive` already wraps unmatched `Aggregate` in -//! `SketchExpr::Logical`, but that branch fires on EVERY unmatched -//! aggregate — including intents the planner is still trying to bind -//! (an `Aggregate{Sum}` over a tabular leaf, etc.). Surfacing the -//! archive-only cases through an explicit named rule lets the -//! StreamingConfig emitter and Phase α routing emit distinguish "no -//! warm-tier rule fired but the intent IS warm-eligible" from "this -//! intent is intentionally archive-only, route it cold". -//! -//! Reference: `controller/docs/design.md` §6 line ~689 ("the optimizer -//! framework") and Phase β orchestrator scope. - -#![allow(dead_code)] - -use crate::intent_algebra::{AggIntent, QueryExpr}; -use crate::sketch_algebra::rules::Rule; -use crate::sketch_algebra::sketch_expr::SketchExpr; -use crate::types_v2::AccuracyTarget; - -/// Route `Aggregate{}` to a `Logical` pass-through. -/// Phase α's emitter consults `AggIntent::archive_only()` to flag the -/// resulting StreamingConfig entry for the cold tier. -pub struct BindArchiveOnly; - -impl Rule for BindArchiveOnly { - fn name(&self) -> &'static str { - "bind_archive_only" - } - - fn priority(&self) -> u16 { - // Lowest priority — every warm-tier rule should out-rank this - // one so the only path to BindArchiveOnly is "no warm rule - // fired AND the intent is archive-only". - 1 - } - - fn apply(&self, expr: &QueryExpr, _accuracy: &AccuracyTarget) -> Option { - match expr { - QueryExpr::Aggregate { aggs, .. } => { - // Single-intent Aggregate is the canonical Phase β shape; - // multi-intent fans out to per-intent rules elsewhere. - if aggs.len() == 1 && aggs[0].archive_only() { - Some(SketchExpr::Logical(expr.clone())) - } else { - None - } - } - _ => None, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::intent_algebra::{LabelFilter, Schema, Source, WindowKind}; - use crate::intent_algebra::schema::{Column, DataType}; - use std::time::Duration; - - fn ts_scan() -> QueryExpr { - QueryExpr::Scan { - source: Source::TimeSeries { - metric: "http_request_duration_seconds_bucket".into(), - }, - label_filters: vec![LabelFilter { - label: "service".into(), - equals: "api".into(), - }], - schema: Schema::with_time_index( - vec![ - Column { - name: "ts".into(), - dtype: DataType::Timestamp, - nullable: false, - }, - Column { - name: "service".into(), - dtype: DataType::Utf8, - nullable: false, - }, - Column { - name: "value".into(), - dtype: DataType::Float64, - nullable: false, - }, - ], - 0, - vec![vec![0, 1]], - ), - } - } - - fn windowed_scan() -> QueryExpr { - QueryExpr::Window { - kind: WindowKind::Sliding, - size: Duration::from_secs(300), - slide: None, - child: Box::new(ts_scan()), - } - } - - fn agg_with(intent: AggIntent) -> QueryExpr { - QueryExpr::Aggregate { - by: vec![], - aggs: vec![intent], - having: None, - child: Box::new(windowed_scan()), - } - } - - #[test] - fn binds_histogram_quantile() { - let expr = agg_with(AggIntent::HistogramQuantile { q: 0.99 }); - let out = BindArchiveOnly - .apply(&expr, &AccuracyTarget::Epsilon(0.01)) - .expect("rule should match histogram_quantile"); - match out { - SketchExpr::Logical(inner) => assert_eq!(inner, expr), - other => panic!("expected Logical pass-through, got {other:?}"), - } - } - - #[test] - fn binds_each_archive_only_intent() { - let intents = vec![ - AggIntent::HistogramQuantile { q: 0.5 }, - AggIntent::Absent, - AggIntent::Present, - AggIntent::Delta { - window: Duration::from_secs(60), - }, - AggIntent::Deriv { - window: Duration::from_secs(60), - }, - AggIntent::PredictLinear { - window: Duration::from_secs(300), - ahead: Duration::from_secs(60), - }, - AggIntent::HoltWinters { - window: Duration::from_secs(300), - smoothing_factor: 0.3, - trend_factor: 0.3, - }, - AggIntent::Idelta { - window: Duration::from_secs(60), - }, - AggIntent::Irate { - window: Duration::from_secs(60), - }, - AggIntent::Resets { - window: Duration::from_secs(300), - }, - AggIntent::Changes { - window: Duration::from_secs(300), - }, - ]; - for intent in intents { - let expr = agg_with(intent.clone()); - let out = BindArchiveOnly.apply(&expr, &AccuracyTarget::Epsilon(0.01)); - assert!( - out.is_some(), - "BindArchiveOnly should bind {intent:?} (archive-only Phase β intent)" - ); - } - } - - #[test] - fn does_not_bind_warm_tier_intents() { - // Sum / Quantile / Cardinality / TopK are NOT archive-only — they - // must NOT trigger BindArchiveOnly (the warm-tier rules own them). - for intent in [ - AggIntent::Sum, - AggIntent::Quantile { - q: 0.99, - accuracy: AccuracyTarget::Epsilon(0.01), - }, - AggIntent::Cardinality { - accuracy: AccuracyTarget::Epsilon(0.01), - }, - AggIntent::TopK { - k: 10, - accuracy: AccuracyTarget::Epsilon(0.05), - }, - AggIntent::Frequency { - accuracy: AccuracyTarget::Epsilon(0.01), - }, - AggIntent::Rate { - window: Duration::from_secs(60), - }, - AggIntent::Increase { - window: Duration::from_secs(60), - }, - ] { - let expr = agg_with(intent.clone()); - assert!( - BindArchiveOnly - .apply(&expr, &AccuracyTarget::Epsilon(0.01)) - .is_none(), - "BindArchiveOnly must not bind warm-tier intent {intent:?}" - ); - } - } - - #[test] - fn does_not_bind_non_aggregate_shapes() { - let scan = ts_scan(); - assert!(BindArchiveOnly - .apply(&scan, &AccuracyTarget::Epsilon(0.01)) - .is_none()); - let window = windowed_scan(); - assert!(BindArchiveOnly - .apply(&window, &AccuracyTarget::Epsilon(0.01)) - .is_none()); - } -} diff --git a/controller/src/sketch_algebra/rules/bind_cms_count.rs b/controller/src/sketch_algebra/rules/bind_cms_count.rs deleted file mode 100644 index e808604e..00000000 --- a/controller/src/sketch_algebra/rules/bind_cms_count.rs +++ /dev/null @@ -1,106 +0,0 @@ -//! `BindCmsOnCount` — `Aggregate{Count{accuracy}}` and `Aggregate{Frequency{accuracy}}` → CountMin. -//! -//! Reference: `controller/docs/design.md` §6 line ~713 lists `BindCmsOnCount` -//! in the shared rule library. CMS is the canonical sketch for both: -//! -//! - `AggIntent::Count{accuracy}` — when the user wants COUNT(*) per -//! group with relaxed accuracy. The L4 readout is `EstimateOp::PointCount` -//! over each group key (the L5 emitter materialises the per-key reads). -//! - `AggIntent::Frequency{accuracy}` — `count(*) WHERE key = k`. CMS is -//! the textbook fit (Cormode-Muthukrishnan). -//! -//! Accuracy mapping: `AccuracyTarget::EpsilonDelta { eps, delta }` → -//! `(w, d) = (⌈e/eps⌉, ⌈ln(1/delta)⌉)`. CMS guarantees additive error -//! `≤ eps · ‖f‖₁` with probability `≥ 1 − delta` (see -//! `accuracy_profile.rs` in ASAPQuery-backend for the formal bound). -//! -//! `AccuracyTarget::Epsilon(eps)` (no delta) defaults to `delta = 0.01` -//! per the in-tree default (`SketchDefaults::count_min_sketch.delta`). -//! `Exact` does not bind. - -#![allow(dead_code)] - -use crate::intent_algebra::{AggIntent, QueryExpr}; -use crate::sketch_algebra::params::{CmsParams, SketchKind, SketchParams}; -use crate::sketch_algebra::rules::Rule; -use crate::sketch_algebra::sketch_expr::{EstimateOp, SketchExpr}; -use crate::types_v2::AccuracyTarget; - -/// Bind `Aggregate{Count}` / `Aggregate{Frequency}` to CMS. -pub struct BindCmsOnCount; - -impl Rule for BindCmsOnCount { - fn name(&self) -> &'static str { - "bind_cms_count" - } - - fn priority(&self) -> u16 { - 5 - } - - fn apply(&self, expr: &QueryExpr, accuracy: &AccuracyTarget) -> Option { - let (intent_accuracy, readout, child) = match expr { - QueryExpr::Aggregate { - aggs, child, by, .. - } if aggs.len() == 1 && by.is_empty() => match &aggs[0] { - AggIntent::Count { accuracy } => { - // Generic Count without a per-key readout falls back - // to the legacy logical aggregate; the only sketch- - // bound shape we emit here is a per-key PointCount. - // Use a sentinel "*" key meaning "all rows"; the L5 - // emitter resolves it against the stage allocator's - // per-group output. - ( - accuracy.clone(), - EstimateOp::PointCount { key: "*".into() }, - child, - ) - } - AggIntent::Frequency { accuracy } => ( - accuracy.clone(), - EstimateOp::PointCount { key: "*".into() }, - child, - ), - _ => return None, - }, - _ => return None, - }; - - // Read `(eps, delta)` from the tighter of the workload-level - // and per-intent accuracy targets. - let (eps, delta) = match (accuracy, &intent_accuracy) { - (AccuracyTarget::Exact, _) | (_, AccuracyTarget::Exact) => return None, - (AccuracyTarget::Epsilon(a), AccuracyTarget::Epsilon(b)) => (a.min(*b), 0.01), - (AccuracyTarget::Epsilon(a), AccuracyTarget::EpsilonDelta { eps, delta }) - | (AccuracyTarget::EpsilonDelta { eps, delta }, AccuracyTarget::Epsilon(a)) => { - (a.min(*eps), *delta) - } - ( - AccuracyTarget::EpsilonDelta { - eps: a, - delta: da, - }, - AccuracyTarget::EpsilonDelta { - eps: b, - delta: db, - }, - ) => (a.min(*b), da.min(*db)), - }; - - if eps <= 0.0 || delta <= 0.0 || delta >= 1.0 { - return None; - } - - let w = (std::f64::consts::E / eps).ceil() as u32; - let d = (1.0 / delta).ln().ceil() as u32; - let w = w.max(2); - let d = d.max(1); - - Some(SketchExpr::estimate_over_agg( - readout, - SketchKind::Cms, - SketchParams::Cms(CmsParams { w, d }), - (**child).clone(), - )) - } -} diff --git a/controller/src/sketch_algebra/rules/bind_cms_topk.rs b/controller/src/sketch_algebra/rules/bind_cms_topk.rs deleted file mode 100644 index 4d9b465d..00000000 --- a/controller/src/sketch_algebra/rules/bind_cms_topk.rs +++ /dev/null @@ -1,96 +0,0 @@ -//! `BindCountSketchOnTopK` — `Aggregate{TopK{k, accuracy}}` → CountSketch-with-heap. -//! -//! Reference: `controller/docs/design.md` §6 line ~419 — "`SketchAgg -//! { intent, col }` … L4 emits `SketchExpr::SketchAgg`" — and -//! `intent_algebra::AggIntent::TopK` (heavy-hitter intent) maps directly -//! to a heavy-hitter sketch primitive. CountSketch with a heap of size -//! `k` is the textbook fit (Charikar-Chen-Farach-Colton); CMS with a -//! heap is an alternative the cost model can pick instead. -//! -//! Phase C ships the CountSketch-with-heap variant only — it pairs -//! cleanly with the existing `algebra::directory` defaults -//! (CountSketch is what the legacy in-tree planner already emits for -//! Frequency-shaped workloads) and the heap is the part that turns it -//! into a TopK primitive. -//! -//! Accuracy mapping: `AccuracyTarget::EpsilonDelta { eps, delta }` → -//! `(w, d) = (⌈e/eps⌉, ⌈ln(1/delta)⌉)`, identical to CMS. The heap size -//! is fixed at the requested `k`. See `accuracy_profile.rs` -//! (ASAPQuery-backend) for the formal heavy-hitter recall guarantee -//! (CountSketch + size-k heap recovers all heavy hitters with -//! frequency `≥ ‖f‖₁ / k` w.h.p.). - -#![allow(dead_code)] - -use crate::intent_algebra::{AggIntent, QueryExpr}; -use crate::sketch_algebra::params::{CountSketchParams, SketchKind, SketchParams}; -use crate::sketch_algebra::rules::Rule; -use crate::sketch_algebra::sketch_expr::{EstimateOp, SketchExpr}; -use crate::types_v2::AccuracyTarget; - -pub struct BindCountSketchOnTopK; - -impl Rule for BindCountSketchOnTopK { - fn name(&self) -> &'static str { - "bind_cms_topk" - } - - fn priority(&self) -> u16 { - 5 - } - - fn apply(&self, expr: &QueryExpr, accuracy: &AccuracyTarget) -> Option { - let (k_topk, intent_accuracy, child) = match expr { - QueryExpr::Aggregate { - aggs, child, by, .. - } if aggs.len() == 1 && by.is_empty() => match &aggs[0] { - AggIntent::TopK { k, accuracy } => (*k, accuracy.clone(), child), - _ => return None, - }, - _ => return None, - }; - - if k_topk == 0 { - return None; - } - - let (eps, delta) = match (accuracy, &intent_accuracy) { - (AccuracyTarget::Exact, _) | (_, AccuracyTarget::Exact) => return None, - (AccuracyTarget::Epsilon(a), AccuracyTarget::Epsilon(b)) => (a.min(*b), 0.01), - (AccuracyTarget::Epsilon(a), AccuracyTarget::EpsilonDelta { eps, delta }) - | (AccuracyTarget::EpsilonDelta { eps, delta }, AccuracyTarget::Epsilon(a)) => { - (a.min(*eps), *delta) - } - ( - AccuracyTarget::EpsilonDelta { - eps: a, - delta: da, - }, - AccuracyTarget::EpsilonDelta { - eps: b, - delta: db, - }, - ) => (a.min(*b), da.min(*db)), - }; - - if eps <= 0.0 || delta <= 0.0 || delta >= 1.0 { - return None; - } - - let w = (std::f64::consts::E / eps).ceil() as u32; - let d = (1.0 / delta).ln().ceil() as u32; - let w = w.max(2); - let d = d.max(1); - - Some(SketchExpr::estimate_over_agg( - EstimateOp::TopK { k: k_topk }, - SketchKind::CountSketch, - SketchParams::CountSketch(CountSketchParams { - w, - d, - with_heap: true, - }), - (**child).clone(), - )) - } -} diff --git a/controller/src/sketch_algebra/rules/bind_ddsketch_quantile.rs b/controller/src/sketch_algebra/rules/bind_ddsketch_quantile.rs deleted file mode 100644 index f4ac5a09..00000000 --- a/controller/src/sketch_algebra/rules/bind_ddsketch_quantile.rs +++ /dev/null @@ -1,84 +0,0 @@ -//! `BindDDSketchOnQuantile` — `Aggregate{Quantile{q, accuracy}}` → DDSketch. -//! -//! Reference: `controller/docs/design.md` §6 sketch_algebra (line ~565) -//! lists DDSketch alongside KLL as a quantile family. DDSketch has a -//! tail-relative-error guarantee — `|estimate − true| ≤ alpha · true` — -//! which makes it the preferred choice when the user asks for relative -//! tail-error rather than rank-error. -//! -//! Accuracy mapping: `AccuracyTarget::Epsilon(eps)` → `alpha = eps`. -//! DDSketch's parameter *is* the relative-error bound, so the mapping is -//! the identity. See `accuracy_profile.rs` (ASAPQuery-backend) for the -//! formal proof. -//! -//! Rule selection: this rule has priority 6 (just above -//! `BindKllOnQuantile`'s priority 5). The dispatcher's tie-break gives -//! DDSketch precedence whenever both rules fire on the same intent — -//! that matches the legacy `algebra::directory::sketch_type_for_agg` -//! behaviour, which already picks DDSketch as the default Quantile -//! sketch (see `controller/src/algebra/directory.rs` line ~59). - -#![allow(dead_code)] - -use crate::intent_algebra::{AggIntent, QueryExpr}; -use crate::sketch_algebra::params::{DDSketchParams, SketchKind, SketchParams}; -use crate::sketch_algebra::rules::Rule; -use crate::sketch_algebra::sketch_expr::{EstimateOp, SketchExpr}; -use crate::types_v2::AccuracyTarget; - -/// Bind a single-intent `Aggregate{Quantile{q, accuracy}}` to DDSketch. -pub struct BindDDSketchOnQuantile; - -impl Rule for BindDDSketchOnQuantile { - fn name(&self) -> &'static str { - "bind_ddsketch_quantile" - } - - fn priority(&self) -> u16 { - // DDSketch wins the tie-break when both rules fire — see module - // docstring for the rationale (matches the legacy directory - // default for SP-2/SP-4). - 6 - } - - fn apply(&self, expr: &QueryExpr, accuracy: &AccuracyTarget) -> Option { - let (q, intent_accuracy, child) = match expr { - QueryExpr::Aggregate { - aggs, child, by, .. - } if aggs.len() == 1 && by.is_empty() => match &aggs[0] { - AggIntent::Quantile { q, accuracy } => (*q, accuracy.clone(), child), - _ => return None, - }, - _ => return None, - }; - - if !(0.0..=1.0).contains(&q) { - return None; - } - - // DDSketch needs an explicit relative-error budget. `Exact` - // disables the rule; the dispatcher then picks KLL (or falls - // through to logical pass-through). - let alpha = match (accuracy, &intent_accuracy) { - (AccuracyTarget::Exact, _) | (_, AccuracyTarget::Exact) => return None, - (AccuracyTarget::Epsilon(a), AccuracyTarget::Epsilon(b)) => a.min(*b), - (AccuracyTarget::Epsilon(a), AccuracyTarget::EpsilonDelta { eps, .. }) - | (AccuracyTarget::EpsilonDelta { eps, .. }, AccuracyTarget::Epsilon(a)) => a.min(*eps), - ( - AccuracyTarget::EpsilonDelta { eps: a, .. }, - AccuracyTarget::EpsilonDelta { eps: b, .. }, - ) => a.min(*b), - }; - - if alpha <= 0.0 || alpha >= 1.0 { - return None; - } - - Some(SketchExpr::estimate_over_agg( - EstimateOp::Quantile { q }, - SketchKind::DDSketch, - SketchParams::DDSketch(DDSketchParams { alpha }), - (**child).clone(), - )) - } -} diff --git a/controller/src/sketch_algebra/rules/bind_hll_cardinality.rs b/controller/src/sketch_algebra/rules/bind_hll_cardinality.rs deleted file mode 100644 index 296735c6..00000000 --- a/controller/src/sketch_algebra/rules/bind_hll_cardinality.rs +++ /dev/null @@ -1,101 +0,0 @@ -//! `BindHllOnCardinality` — `Aggregate{Cardinality{accuracy}}` → HLL. -//! -//! Reference: `controller/docs/design.md` §6 line ~714 lists -//! `BindHllOnCardinality` in the shared rule library. HLL is the -//! catalog-default sketch family for COUNT DISTINCT — see -//! `algebra::directory::sketch_type_for_agg` line ~59 for the existing -//! in-tree binding. -//! -//! Accuracy mapping: HLL standard error is `≈ 1.04 / √m` where -//! `m = 2^precision`. Inverting: `precision ≈ 2 · log2(1.04 / eps)`. -//! See `accuracy_profile.rs` (ASAPQuery-backend) for the formal bound -//! and the in-tree default rungs (precision 10 / 12 / 14 / 16 covering -//! ε ≈ 3% / 1.5% / 0.8% / 0.4%). - -#![allow(dead_code)] - -use crate::intent_algebra::{AggIntent, QueryExpr}; -use crate::sketch_algebra::params::{HllParams, SketchKind, SketchParams}; -use crate::sketch_algebra::rules::Rule; -use crate::sketch_algebra::sketch_expr::{EstimateOp, SketchExpr}; -use crate::types_v2::AccuracyTarget; - -pub struct BindHllOnCardinality; - -impl Rule for BindHllOnCardinality { - fn name(&self) -> &'static str { - "bind_hll_cardinality" - } - - fn priority(&self) -> u16 { - 5 - } - - fn apply(&self, expr: &QueryExpr, accuracy: &AccuracyTarget) -> Option { - let (intent_accuracy, child) = match expr { - QueryExpr::Aggregate { - aggs, child, by, .. - } if aggs.len() == 1 && by.is_empty() => match &aggs[0] { - AggIntent::Cardinality { accuracy } => (accuracy.clone(), child), - _ => return None, - }, - _ => return None, - }; - - // Read the tighter of the workload-level and per-intent budgets. - let eps = match (accuracy, &intent_accuracy) { - (AccuracyTarget::Exact, _) | (_, AccuracyTarget::Exact) => return None, - (AccuracyTarget::Epsilon(a), AccuracyTarget::Epsilon(b)) => a.min(*b), - (AccuracyTarget::Epsilon(a), AccuracyTarget::EpsilonDelta { eps, .. }) - | (AccuracyTarget::EpsilonDelta { eps, .. }, AccuracyTarget::Epsilon(a)) => a.min(*eps), - ( - AccuracyTarget::EpsilonDelta { eps: a, .. }, - AccuracyTarget::EpsilonDelta { eps: b, .. }, - ) => a.min(*b), - }; - - if eps <= 0.0 { - return None; - } - - let precision = hll_precision_for_eps(eps); - - Some(SketchExpr::estimate_over_agg( - EstimateOp::Cardinality, - SketchKind::Hll, - SketchParams::Hll(HllParams { precision }), - (**child).clone(), - )) - } -} - -/// Map an ε standard-error budget to the HLL `precision` (log2 register -/// count). Mirrors the in-tree default rungs in `algebra::directory` — -/// precision 10 (ε≈3.25%) / 12 (ε≈1.6%) / 14 (ε≈0.81%) / 16 (ε≈0.41%). -fn hll_precision_for_eps(eps: f64) -> u32 { - if eps <= 0.0 { - return 16; - } - if eps >= 0.03 { - 10 - } else if eps >= 0.015 { - 12 - } else if eps >= 0.008 { - 14 - } else { - 16 - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn precision_at_canonical_rungs() { - assert_eq!(hll_precision_for_eps(0.03), 10); - assert_eq!(hll_precision_for_eps(0.015), 12); - assert_eq!(hll_precision_for_eps(0.008), 14); - assert_eq!(hll_precision_for_eps(0.001), 16); - } -} diff --git a/controller/src/sketch_algebra/rules/bind_kll_quantile.rs b/controller/src/sketch_algebra/rules/bind_kll_quantile.rs deleted file mode 100644 index 8111f145..00000000 --- a/controller/src/sketch_algebra/rules/bind_kll_quantile.rs +++ /dev/null @@ -1,121 +0,0 @@ -//! `BindKllOnQuantile` — `Aggregate{Quantile{q, accuracy}}` → `SketchEstimate{Quantile{q}}` over `SketchAgg{KLL}`. -//! -//! Reference: `controller/docs/design.md` §6 line ~1157 — "the shared rule -//! `core::optimizer::rules::BindKllOnQuantile` matches `Aggregate { aggs: -//! [Quantile{q, accuracy}] }`, consults the sketch catalog (KLL has -//! `supported_intents: [Quantile]`, is mergeable, satisfies `ε=0.01` at -//! `k=200`), and rewrites the matched sub-DAG into a `SketchAgg` wrapped -//! in a `SketchEstimate`". -//! -//! Accuracy mapping: `AccuracyTarget::Epsilon(eps)` → KLL stream-size `k`. -//! KLL's rank-error bound is `≈ 2.6 / (eps · √π)` per Karnin-Lang-Liberty; -//! see `accuracy_profile.rs` (ASAPQuery-backend) for the formal mapping -//! table — `eps=0.01 → k=200`, `eps=0.005 → k=400`, `eps=0.001 → k=2048`, -//! `eps=0.0001 → k=8192`. `Exact` does not bind. - -#![allow(dead_code)] - -use crate::intent_algebra::{AggIntent, QueryExpr}; -use crate::sketch_algebra::params::{KllParams, SketchKind, SketchParams}; -use crate::sketch_algebra::rules::Rule; -use crate::sketch_algebra::sketch_expr::{EstimateOp, SketchExpr}; -use crate::types_v2::AccuracyTarget; - -/// Bind a single-intent `Aggregate{Quantile{q, accuracy}}` to KLL. -pub struct BindKllOnQuantile; - -impl Rule for BindKllOnQuantile { - fn name(&self) -> &'static str { - "bind_kll_quantile" - } - - fn priority(&self) -> u16 { - // KLL is the planner's preferred quantile sketch when the - // accuracy budget is unspecified or when an `Exact`-by-default - // SLA is in play (see `algebra::directory::sketch_type_for_agg`, - // which already picks KLL as the in-tree default for SP-2/SP-4). - // It edges out DDSketch on rank-error tightness when the budget - // is rank-driven; DDSketch (priority 6) wins when the user - // supplied a tail-relative epsilon. - 5 - } - - fn apply(&self, expr: &QueryExpr, accuracy: &AccuracyTarget) -> Option { - // Only the single-intent Aggregate{Quantile{...}} shape binds — - // multi-intent Aggregates and TopK-shaped quantiles take other - // rules. - let (q, intent_accuracy, child) = match expr { - QueryExpr::Aggregate { - aggs, child, by, .. - } if aggs.len() == 1 && by.is_empty() => match &aggs[0] { - AggIntent::Quantile { q, accuracy } => (*q, accuracy.clone(), child), - _ => return None, - }, - _ => return None, - }; - - // Quantile φ must be in [0, 1]. - if !(0.0..=1.0).contains(&q) { - return None; - } - - // Pick the binding budget. Per the orchestrator-level convention, - // the per-intent `accuracy` field overrides the workload-level - // `AccuracyTarget` when both are present; here we use the more - // restrictive (lower-eps) of the two when both are `Epsilon`. If - // either side is `Exact`, the rule does not bind. - let eps = match (accuracy, &intent_accuracy) { - (AccuracyTarget::Exact, _) | (_, AccuracyTarget::Exact) => return None, - (AccuracyTarget::Epsilon(a), AccuracyTarget::Epsilon(b)) => a.min(*b), - (AccuracyTarget::Epsilon(a), AccuracyTarget::EpsilonDelta { eps, .. }) - | (AccuracyTarget::EpsilonDelta { eps, .. }, AccuracyTarget::Epsilon(a)) => a.min(*eps), - ( - AccuracyTarget::EpsilonDelta { eps: a, .. }, - AccuracyTarget::EpsilonDelta { eps: b, .. }, - ) => a.min(*b), - }; - - let k = kll_k_for_eps(eps); - - Some(SketchExpr::estimate_over_agg( - EstimateOp::Quantile { q }, - SketchKind::Kll, - SketchParams::Kll(KllParams { k }), - (**child).clone(), - )) - } -} - -/// Map an ε rank-error budget to a KLL stream-size `k`. Mirrors the -/// `accuracy_profile.rs` table referenced in the module docstring. -/// Bumped to power-of-two rungs (200, 400, 800, 2048, 8192) so the -/// in-tree `algebra::directory` continues to recognise the parameter. -fn kll_k_for_eps(eps: f64) -> u32 { - if eps <= 0.0 { - return 8192; - } - if eps >= 0.01 { - 200 - } else if eps >= 0.005 { - 400 - } else if eps >= 0.0025 { - 800 - } else if eps >= 0.001 { - 2048 - } else { - 8192 - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn k_at_canonical_rungs() { - assert_eq!(kll_k_for_eps(0.01), 200); - assert_eq!(kll_k_for_eps(0.005), 400); - assert_eq!(kll_k_for_eps(0.001), 2048); - assert_eq!(kll_k_for_eps(0.0001), 8192); - } -} diff --git a/controller/src/sketch_algebra/rules/mod.rs b/controller/src/sketch_algebra/rules/mod.rs deleted file mode 100644 index 0fb425a0..00000000 --- a/controller/src/sketch_algebra/rules/mod.rs +++ /dev/null @@ -1,83 +0,0 @@ -//! Layer 4 `Bind*` rules — transform L3 [`QueryExpr`] sub-trees into L4 -//! [`SketchExpr`] sub-trees. -//! -//! Per `controller/docs/design.md` §6 ("`core::optimizer` — Layer 4 -//! framework", around line ~689) and §6 sketch_algebra (line ~565). A -//! `Bind*` rule: -//! -//! 1. Pattern-matches on a `QueryExpr::Aggregate` shape. -//! 2. Reads the [`AccuracyTarget`] off the matched intent. -//! 3. Consults the catalog (here: the family-default capability flags in -//! [`crate::sketch_algebra::schema::SketchStateSchema::for_kind`]). -//! 4. Returns `Some(SketchExpr)` if it can bind, `None` otherwise. -//! -//! Rule selection is cost-aware: when multiple rules match (e.g. KLL vs -//! DDSketch on a `Quantile` intent), the dispatcher picks one by -//! consulting per-rule [`Rule::priority`] + the accuracy-driven hints -//! returned by [`Rule::cost_hint`]. The `cost_hint` is intentionally -//! coarse for Phase C — the cost-model integration is Phase F's domain. -//! -//! Each rule documents the binding with a comment referencing -//! `accuracy_profile.rs` (in ASAPQuery-backend) for the formal accuracy -//! bound that justifies the chosen parameter mapping. - -#![allow(dead_code)] - -pub mod bind_archive_only; -pub mod bind_cms_count; -pub mod bind_cms_topk; -pub mod bind_ddsketch_quantile; -pub mod bind_hll_cardinality; -pub mod bind_kll_quantile; - -use crate::intent_algebra::QueryExpr; -use crate::sketch_algebra::sketch_expr::SketchExpr; -use crate::types_v2::AccuracyTarget; - -/// Bind-rule trait. Phase C keeps the trait minimal — `apply` + a -/// `priority` for the dispatcher's tie-break + a `cost_hint` (intent + -/// accuracy → relative cost). Phase F wires `cost_hint` into the -/// per-deployment cost model. -pub trait Rule: Sync { - /// Stable, human-readable identifier (`"bind_kll_quantile"`, …) — - /// used for diagnostics + the `cargo test` matrix. - fn name(&self) -> &'static str; - - /// Lower the matched sub-tree under the given accuracy target. Return - /// `None` if this rule does not apply to the supplied `expr`. - fn apply(&self, expr: &QueryExpr, accuracy: &AccuracyTarget) -> Option; - - /// Coarse priority used for tie-break when multiple rules match. - /// Higher = preferred. Default 0. - fn priority(&self) -> u16 { - 0 - } -} - -/// Dispatch a `QueryExpr` sub-tree against the full Phase C rule set. -/// Returns the highest-priority binding that fires, or `None` if no rule -/// matches (caller wraps the input in `SketchExpr::Logical`). -pub fn dispatch(expr: &QueryExpr, accuracy: &AccuracyTarget) -> Option { - let rules: Vec> = vec![ - Box::new(bind_kll_quantile::BindKllOnQuantile), - Box::new(bind_ddsketch_quantile::BindDDSketchOnQuantile), - Box::new(bind_cms_count::BindCmsOnCount), - Box::new(bind_cms_topk::BindCountSketchOnTopK), - Box::new(bind_hll_cardinality::BindHllOnCardinality), - // Phase β: archive-only catch-all. Lowest priority — fires only - // when no warm-tier rule matches AND the intent is archive-only. - Box::new(bind_archive_only::BindArchiveOnly), - ]; - - let mut best: Option<(u16, SketchExpr)> = None; - for r in &rules { - if let Some(out) = r.apply(expr, accuracy) { - let p = r.priority(); - best = match best { - Some((bp, _)) if bp >= p => best, - _ => Some((p, out)), - }; - } - } - best.map(|(_, e)| e) -} diff --git a/controller/src/sketch_algebra/schema.rs b/controller/src/sketch_algebra/schema.rs deleted file mode 100644 index dd5235ca..00000000 --- a/controller/src/sketch_algebra/schema.rs +++ /dev/null @@ -1,174 +0,0 @@ -//! Layer 4 sketch-state schema. -//! -//! Per `controller/docs/design.md` §6.4 ("Per-node input/output spec for -//! `SketchExpr`", around line ~618). L4 introduces a new field type into -//! `Schema`: `DataType::Sketch(SketchKind, SketchParams)`. This module -//! defines that extension as a parallel typed layer that the L4 type -//! checker consults. -//! -//! Keeping this in a parallel struct (rather than mutating the L3 -//! `intent_algebra::DataType`) avoids touching the recently-shipped L3 -//! IR. The L4 type checker asks: "for this `SketchExpr` node, what is -//! the input sketch-state schema, and what is the output?" Each `Bind*` -//! rule populates an [`SketchStateSchema`] when it produces a -//! sketch-state-bearing node. -//! -//! Local-checkable invariants (per design.md §6.4 line ~643): -//! 1. **Sketch-family mismatch is a plan-time error.** A `SketchMerge` -//! over inputs with mismatched `(kind, params)` fails at L4. -//! 2. **Catalog capability flags gate which nodes can fire.** The -//! capability flags live on the [`SketchStateSchema`] so the type -//! checker has them locally without re-consulting the catalog. - -#![allow(dead_code)] - -use serde::{Deserialize, Serialize}; - -use crate::sketch_algebra::params::{SketchKind, SketchParams}; - -/// Sketch-state schema annotation on a [`crate::sketch_algebra::SketchExpr`] -/// edge. Carries the family + params + the catalog-capability flags so -/// the L4 type checker can decide locally whether a downstream node may -/// merge / subtract / delete from the state. -/// -/// Mirrors the per-node input/output spec table in design.md §6.4: a -/// `SketchAgg` produces a sketch-state schema; a `SketchEstimate` -/// consumes one and emits a regular row schema; a `SketchMerge` consumes -/// N matching sketch-state schemas and emits one of the same family. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct SketchStateSchema { - /// Sketch family of the state. - pub kind: SketchKind, - /// Parameter payload — must match across all inputs to a `SketchMerge`. - pub params: SketchParams, - /// Capability flags from the sketch catalog. - pub caps: SketchCapabilities, -} - -/// Catalog capability flags. Populated from the sketch catalog at -/// `Bind*`-rule time. `mergeable` gates `SketchMerge`; `subtractable` -/// gates `SketchSubtract`; `deletable` gates `SketchDelete`. See design.md -/// §6 line ~646 ("catalog is the single source of truth for these flags; -/// binding rules consult it before producing the node"). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub struct SketchCapabilities { - /// Whether two states of this family + params can be unioned — - /// catalog default for KLL / HLL / DDSketch / CMS / CountSketch. - pub mergeable: bool, - /// Whether `SketchSubtract` is meaningful for this family — true for - /// CMS / count-based sketches, false for KLL / HLL / DDSketch. - pub subtractable: bool, - /// Whether `SketchDelete` is meaningful — true for deletable Bloom - /// filters and CMS (with -1 update); false for KLL / HLL / DDSketch. - pub deletable: bool, -} - -impl SketchStateSchema { - /// Construct the catalog-default schema for a given family + params. - /// Capability flags follow the design.md §6 catalog defaults; if a - /// future catalog reorder changes them, this is the single point of - /// truth that downstream rules must consult. - pub fn for_kind(kind: SketchKind, params: SketchParams) -> Self { - let caps = match kind { - SketchKind::Kll => SketchCapabilities { - mergeable: true, - subtractable: false, - deletable: false, - }, - SketchKind::DDSketch => SketchCapabilities { - mergeable: true, - subtractable: false, - deletable: false, - }, - SketchKind::Hll => SketchCapabilities { - mergeable: true, - subtractable: false, - deletable: false, - }, - SketchKind::Cms => SketchCapabilities { - mergeable: true, - subtractable: true, - deletable: true, - }, - SketchKind::CountSketch => SketchCapabilities { - mergeable: true, - subtractable: true, - deletable: false, - }, - }; - SketchStateSchema { kind, params, caps } - } - - /// Whether two sketch-state schemas can be `SketchMerge`-d. Per - /// design.md §6.4 invariant 1: the family + params must match - /// exactly, and the family must be `mergeable`. - pub fn is_compatible_for_merge(&self, other: &Self) -> bool { - self.kind == other.kind && self.params == other.params && self.caps.mergeable - } - - /// Whether two sketch-state schemas can be `SketchSubtract`-ed. - pub fn is_compatible_for_subtract(&self, other: &Self) -> bool { - self.kind == other.kind && self.params == other.params && self.caps.subtractable - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use crate::sketch_algebra::params::{CmsParams, KllParams}; - - #[test] - fn kll_default_caps() { - let s = SketchStateSchema::for_kind( - SketchKind::Kll, - SketchParams::Kll(KllParams { k: 200 }), - ); - assert!(s.caps.mergeable); - assert!(!s.caps.subtractable); - assert!(!s.caps.deletable); - } - - #[test] - fn cms_supports_subtract_and_delete() { - let s = SketchStateSchema::for_kind( - SketchKind::Cms, - SketchParams::Cms(CmsParams { w: 2048, d: 5 }), - ); - assert!(s.caps.mergeable); - assert!(s.caps.subtractable); - assert!(s.caps.deletable); - } - - #[test] - fn merge_compatibility_requires_matching_params() { - let a = SketchStateSchema::for_kind( - SketchKind::Kll, - SketchParams::Kll(KllParams { k: 200 }), - ); - let b = SketchStateSchema::for_kind( - SketchKind::Kll, - SketchParams::Kll(KllParams { k: 200 }), - ); - let c = SketchStateSchema::for_kind( - SketchKind::Kll, - SketchParams::Kll(KllParams { k: 400 }), - ); - assert!(a.is_compatible_for_merge(&b)); - assert!(!a.is_compatible_for_merge(&c)); // different k - } - - #[test] - fn merge_compatibility_rejects_family_mismatch() { - let kll = SketchStateSchema::for_kind( - SketchKind::Kll, - SketchParams::Kll(KllParams { k: 200 }), - ); - let cms = SketchStateSchema::for_kind( - SketchKind::Cms, - SketchParams::Cms(CmsParams { w: 2048, d: 5 }), - ); - assert!(!kll.is_compatible_for_merge(&cms)); - } -} diff --git a/controller/src/sketch_algebra/sketch_expr.rs b/controller/src/sketch_algebra/sketch_expr.rs deleted file mode 100644 index 480953a8..00000000 --- a/controller/src/sketch_algebra/sketch_expr.rs +++ /dev/null @@ -1,373 +0,0 @@ -//! Layer 4 IR — `SketchExpr` DAG. -//! -//! Per `controller/docs/design.md` §6 "`core::sketch_algebra` — Layer 4 IR -//! (`SketchExpr`)" (around line ~565). -//! -//! Two-IR split: L3 [`crate::intent_algebra::QueryExpr`] is intent-only; -//! L4 [`SketchExpr`] is sketch-bound. `Bind*` rules consume the L3 IR -//! and produce the L4 IR with the sketch family + parameters committed. -//! -//! The variant set ships the subset DC + PromQL needs (the orchestrator's -//! current scope-reduction). `SketchJoin`, `SketchSubtract`, `SketchDelete` -//! from design.md §6 are intentionally *not* surfaced yet — they're -//! gated on rules that haven't landed (no `Bind*OnJoin`, no subtract / -//! delete consumer) and the orchestrator's spec restricts Phase C to -//! `SketchAgg / SketchEstimate / SketchMerge / Logical`. Adding them -//! later is a purely additive enum extension. - -#![allow(dead_code)] - -use serde::{Deserialize, Serialize}; - -use crate::intent_algebra::QueryExpr; -use crate::sketch_algebra::params::{SketchKind, SketchParams}; -use crate::types_v2::BindingName; - -/// Readout operation extracted from a built sketch state. Inverse of -/// `SketchAgg`. Mirrors design.md §6 line ~607 — `SketchEstimate` plus -/// the readout `query` that says what to extract from the state. -/// -/// PromQL convention: a `Quantile` op carries the φ; a `PointCount` op -/// carries the key; `Cardinality` and `TopK` need no payload beyond the -/// ones already on the producing `SketchAgg` (cardinality ops are -/// parameter-free; the TopK `k` rides on the `SketchAgg` for -/// CountSketch-with-heap). -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(tag = "op", rename_all = "snake_case")] -pub enum EstimateOp { - /// φ-th quantile readout — KLL / DDSketch / t-digest input. - Quantile { q: f64 }, - /// Approximate cardinality (count-distinct) — HLL / theta-sketch input. - Cardinality, - /// Approximate point count for a key — CMS input. - PointCount { key: String }, - /// Heavy-hitter top-k extraction — CountSketch-with-heap / Misra-Gries. - TopK { k: usize }, -} - -/// Algebra of a `SketchMerge` node — at L4 today this is always a union -/// of mergeable sketch states. The enum-shape is forward-compatible with -/// future merge algebras (weighted union for sampling sketches, etc.). -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum MergeAlgebra { - /// Set-union of two or more sketch states — `KLL ∪ KLL`, `HLL ∪ HLL`, - /// etc. The catalog `mergeable` flag must be true on all inputs. - Union, -} - -/// L4 algebra node. See module doc for the variant subset rationale. -/// -/// Serde tag is `"sketch_node"` (not `"node"`) so it doesn't collide with -/// the L3 `QueryExpr`'s `"node"` tag — `SketchExpr::Logical(QueryExpr)` -/// nests a JSON-tagged enum inside an internally-tagged outer enum, and -/// reusing the same tag would surface as a `duplicate field "node"` -/// deserialization error. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(tag = "sketch_node", rename_all = "snake_case")] -pub enum SketchExpr { - /// Logical pass-through: an L3 node that no L4 rule rewrote. A - /// `Filter`, a row-shaped `Aggregate{Sum}`, or any other operator - /// whose semantics are unchanged by the sketch-binding pass lives - /// here unchanged. - Logical(QueryExpr), - - /// Sketch aggregation — produces a sketch state on its output edge. - /// The L3 `AggIntent` was lowered by a `Bind*` rule into the - /// committed `(sketch_type, params)` pair. - SketchAgg { - /// Sketch family (KLL / DDSketch / HLL / CMS / CountSketch). - sketch_type: SketchKind, - /// Sketch parameters (validated by the catalog at bind time). - params: SketchParams, - /// Input sub-tree — typically `Logical(Window{...})` or - /// `Logical(Scan{...})`. - child: Box, - }, - - /// Read out a query result from a built sketch state. Inverse of - /// `SketchAgg`. The `op` says what to extract — a quantile φ, the - /// approximate cardinality, the top-k heavy hitters. - SketchEstimate { - /// Readout operation (see [`EstimateOp`]). - op: EstimateOp, - /// Sketch-state-bearing sub-tree (a `SketchAgg`, a `SketchMerge`, - /// or a `Ref` to one). - child: Box, - }, - - /// Merge multiple sketches into one — set-union under - /// [`MergeAlgebra::Union`]. The L4 type checker rejects mismatched - /// families / params at plan time (design.md §6.4 invariant 1). - SketchMerge { - /// Merge algebra — currently always `Union`. - algebra: MergeAlgebra, - /// Sketch-state-bearing inputs. All must agree on `(kind, params)`. - children: Vec, - }, - - /// SQL `WITH name AS (expr) SELECT ... FROM name` / sketch-state - /// fan-in: name a sub-expression so multiple parents can reference - /// it. `SketchExpr::LetBinding` carries the two-tier fan-in described - /// in design.md §1339 — outer let names a `Window` output, inner let - /// names a `SketchAgg{KLL}` shared by two `SketchEstimate` parents - /// reading different quantiles. - LetBinding { - /// Binding name; must be unique within the surrounding scope. - name: BindingName, - /// Bound sub-expression. - expr: Box, - /// In-scope sub-tree — references the binding via `Ref`. - child: Box, - }, - - /// Reference a `LetBinding` by name. Resolution is lexical (scope - /// follows the surrounding `LetBinding` chain). - Ref { - /// Bound name. - name: BindingName, - }, - - // ── Phase ε.1: three-mode placement variants ──────────────────────── - // - // Phase ε.1 collapses the planner's raw-vs-sketch + edge-vs-backend - // axes into a single tri-mode selector. The two new variants name - // the two new placements; the existing `SketchAgg` corresponds to - // Mode 1 (sketch at edge). See `planner::wire_cost::BindMode`. - /// Mode 2 (Phase ε.1): no sketch processor at the edge — raw OTLP - /// forwards to the backend, which builds the sketch at ingest. The - /// `family` and `params` are the sketch the backend will build, so - /// the backend's `StreamingConfig` `aggregation_input` is `raw` for - /// this metric (Phase ε.2 implements the raw-input ingest path). - RawAtEdgeSketchAtBackend { - /// Sketch family the backend will build at ingest. - family: SketchKind, - /// Sketch parameters (validated by the catalog at bind time). - params: SketchParams, - /// Input sub-tree — typically `Logical(Window{...})` or - /// `Logical(Scan{...})`. Mirrors `SketchAgg`'s child field so the - /// L5 emitter's walk uniform. - child: Box, - }, - - /// Mode 3 (Phase ε.1): no sketch processor at the edge — raw OTLP - /// ships directly to Prometheus's native OTLP receiver at - /// `/api/v1/otlp/v1/metrics`. The backend HTTP-forwards queries to - /// Prometheus's `/api/v1/query` endpoint (the `prometheus_remote` - /// engine). Accuracy is exact (ε = 0) — Prometheus owns the raw - /// samples; no sketch math is involved. - /// - /// The variant carries enough projection info for the edge agent's - /// pipeline to ship the right metric with the right labels, and for - /// the backend's storage routing to claim the metric. - RawAtEdgePrometheusArchive { - /// Metric name as it appears at the edge (and in - /// `BackendStorageRouting`). - metric: String, - /// Optional window — when present, the planner pre-bucketed the - /// metric into windowed scrape data. Prometheus stores the raw - /// stream regardless; the field is informational for the L5 - /// emitter so it can size scrape intervals consistently. - window: Option, - /// Label projection — labels promoted from OTLP resource - /// attributes by Prometheus's - /// `otlp.promote_resource_attributes` config. Default - /// `["service.name", "service.namespace", "service.instance.id"]` - /// — see `deploy/configs/prometheus-otlp-receiver.yml`. - label_proj: Vec, - }, -} - -impl SketchExpr { - /// Convenience constructor for the canonical - /// `SketchEstimate{SketchAgg{Logical(qe)}}` shape produced by every - /// `Bind*` rule. Keeps rule call sites short. - pub fn estimate_over_agg( - op: EstimateOp, - sketch_type: SketchKind, - params: SketchParams, - logical: QueryExpr, - ) -> Self { - SketchExpr::SketchEstimate { - op, - child: Box::new(SketchExpr::SketchAgg { - sketch_type, - params, - child: Box::new(SketchExpr::Logical(logical)), - }), - } - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use crate::intent_algebra::{ - AggIntent, LabelFilter, QueryExpr, Schema, Source, WindowKind, - }; - use crate::intent_algebra::schema::{Column, DataType}; - use crate::sketch_algebra::params::{ - CountSketchParams, DDSketchParams, HllParams, KllParams, - }; - use crate::types_v2::AccuracyTarget; - use std::time::Duration; - - fn ts_scan() -> QueryExpr { - QueryExpr::Scan { - source: Source::TimeSeries { - metric: "http_request_duration_seconds".into(), - }, - label_filters: vec![LabelFilter { - label: "service".into(), - equals: "api".into(), - }], - schema: Schema::with_time_index( - vec![ - Column { - name: "ts".into(), - dtype: DataType::Timestamp, - nullable: false, - }, - Column { - name: "service".into(), - dtype: DataType::Utf8, - nullable: false, - }, - Column { - name: "value".into(), - dtype: DataType::Float64, - nullable: false, - }, - ], - 0, - vec![vec![0, 1]], - ), - } - } - - fn windowed_scan() -> QueryExpr { - QueryExpr::Window { - kind: WindowKind::Sliding, - size: Duration::from_secs(300), - slide: None, - child: Box::new(ts_scan()), - } - } - - #[test] - fn estimate_over_agg_ctor_shape() { - let e = SketchExpr::estimate_over_agg( - EstimateOp::Quantile { q: 0.99 }, - SketchKind::Kll, - SketchParams::Kll(KllParams { k: 200 }), - windowed_scan(), - ); - match e { - SketchExpr::SketchEstimate { op, child } => { - assert_eq!(op, EstimateOp::Quantile { q: 0.99 }); - match *child { - SketchExpr::SketchAgg { - sketch_type, - params, - child, - } => { - assert_eq!(sketch_type, SketchKind::Kll); - assert_eq!(params, SketchParams::Kll(KllParams { k: 200 })); - assert!(matches!(*child, SketchExpr::Logical(_))); - } - other => panic!("expected SketchAgg, got {other:?}"), - } - } - other => panic!("expected SketchEstimate, got {other:?}"), - } - } - - fn agg_quantile() -> QueryExpr { - QueryExpr::Aggregate { - by: vec![], - aggs: vec![AggIntent::Quantile { - q: 0.99, - accuracy: AccuracyTarget::Epsilon(0.01), - }], - having: None, - child: Box::new(windowed_scan()), - } - } - - #[test] - fn sketch_expr_serde_roundtrip_logical() { - let e = SketchExpr::Logical(agg_quantile()); - let json = serde_json::to_string(&e).unwrap(); - let back: SketchExpr = serde_json::from_str(&json).unwrap(); - assert_eq!(e, back); - } - - #[test] - fn sketch_expr_serde_roundtrip_sketch_agg() { - let e = SketchExpr::SketchAgg { - sketch_type: SketchKind::Kll, - params: SketchParams::Kll(KllParams { k: 200 }), - child: Box::new(SketchExpr::Logical(windowed_scan())), - }; - let json = serde_json::to_string(&e).unwrap(); - let back: SketchExpr = serde_json::from_str(&json).unwrap(); - assert_eq!(e, back); - } - - #[test] - fn sketch_expr_serde_roundtrip_estimate() { - let e = SketchExpr::estimate_over_agg( - EstimateOp::Quantile { q: 0.95 }, - SketchKind::DDSketch, - SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), - windowed_scan(), - ); - let json = serde_json::to_string(&e).unwrap(); - let back: SketchExpr = serde_json::from_str(&json).unwrap(); - assert_eq!(e, back); - } - - #[test] - fn sketch_expr_serde_roundtrip_merge() { - let leaf = SketchExpr::SketchAgg { - sketch_type: SketchKind::Hll, - params: SketchParams::Hll(HllParams { precision: 14 }), - child: Box::new(SketchExpr::Logical(windowed_scan())), - }; - let e = SketchExpr::SketchMerge { - algebra: MergeAlgebra::Union, - children: vec![leaf.clone(), leaf], - }; - let json = serde_json::to_string(&e).unwrap(); - let back: SketchExpr = serde_json::from_str(&json).unwrap(); - assert_eq!(e, back); - } - - #[test] - fn sketch_expr_serde_roundtrip_let_ref() { - let inner_agg = SketchExpr::SketchAgg { - sketch_type: SketchKind::CountSketch, - params: SketchParams::CountSketch(CountSketchParams { - w: 2048, - d: 5, - with_heap: true, - }), - child: Box::new(SketchExpr::Logical(windowed_scan())), - }; - let e = SketchExpr::LetBinding { - name: BindingName::new("kll_state"), - expr: Box::new(inner_agg), - child: Box::new(SketchExpr::SketchEstimate { - op: EstimateOp::TopK { k: 10 }, - child: Box::new(SketchExpr::Ref { - name: BindingName::new("kll_state"), - }), - }), - }; - let json = serde_json::to_string(&e).unwrap(); - let back: SketchExpr = serde_json::from_str(&json).unwrap(); - assert_eq!(e, back); - } -} diff --git a/controller/src/sketch_algebra/tests.rs b/controller/src/sketch_algebra/tests.rs deleted file mode 100644 index e28d01b7..00000000 --- a/controller/src/sketch_algebra/tests.rs +++ /dev/null @@ -1,793 +0,0 @@ -//! Integration tests for the L4 IR + `Bind*` rules. - -#![cfg(test)] - -use std::time::Duration; - -use crate::intent_algebra::{ - AggIntent, LabelFilter, QueryExpr, Schema, Source, WindowKind, -}; -use crate::intent_algebra::schema::{Column, DataType}; -use crate::sketch_algebra::lower::bind_query_expr; -use crate::sketch_algebra::params::{KllParams, SketchKind, SketchParams}; -use crate::sketch_algebra::rules::{ - bind_ddsketch_quantile::BindDDSketchOnQuantile, bind_kll_quantile::BindKllOnQuantile, Rule, -}; -use crate::sketch_algebra::sketch_expr::{EstimateOp, MergeAlgebra, SketchExpr}; -use crate::types_v2::{AccuracyTarget, BindingName}; - -// ── Test fixtures ───────────────────────────────────────────────────────────── - -fn col(name: &str, dtype: DataType) -> Column { - Column { - name: name.into(), - dtype, - nullable: false, - } -} - -fn ts_scan() -> QueryExpr { - QueryExpr::Scan { - source: Source::TimeSeries { - metric: "http_request_duration_seconds".into(), - }, - label_filters: vec![LabelFilter { - label: "service".into(), - equals: "api".into(), - }], - schema: Schema::with_time_index( - vec![ - col("ts", DataType::Timestamp), - col("service", DataType::Utf8), - col("value", DataType::Float64), - ], - 0, - vec![vec![0, 1]], - ), - } -} - -fn windowed_scan() -> QueryExpr { - QueryExpr::Window { - kind: WindowKind::Sliding, - size: Duration::from_secs(300), - slide: None, - child: Box::new(ts_scan()), - } -} - -fn agg_quantile(q: f64, accuracy: AccuracyTarget) -> QueryExpr { - QueryExpr::Aggregate { - by: vec![], - aggs: vec![AggIntent::Quantile { q, accuracy }], - having: None, - child: Box::new(windowed_scan()), - } -} - -// ── Serde round-trip across all variants ────────────────────────────────────── - -#[test] -fn sketch_expr_serde_roundtrip() { - use crate::sketch_algebra::params::{ - CmsParams, CountSketchParams, DDSketchParams, HllParams, - }; - let cases = vec![ - SketchExpr::Logical(windowed_scan()), - SketchExpr::SketchAgg { - sketch_type: SketchKind::Kll, - params: SketchParams::Kll(KllParams { k: 200 }), - child: Box::new(SketchExpr::Logical(windowed_scan())), - }, - SketchExpr::SketchEstimate { - op: EstimateOp::Quantile { q: 0.5 }, - child: Box::new(SketchExpr::SketchAgg { - sketch_type: SketchKind::DDSketch, - params: SketchParams::DDSketch(DDSketchParams { alpha: 0.005 }), - child: Box::new(SketchExpr::Logical(windowed_scan())), - }), - }, - SketchExpr::SketchMerge { - algebra: MergeAlgebra::Union, - children: vec![ - SketchExpr::SketchAgg { - sketch_type: SketchKind::Hll, - params: SketchParams::Hll(HllParams { precision: 14 }), - child: Box::new(SketchExpr::Logical(windowed_scan())), - }, - SketchExpr::SketchAgg { - sketch_type: SketchKind::Hll, - params: SketchParams::Hll(HllParams { precision: 14 }), - child: Box::new(SketchExpr::Logical(windowed_scan())), - }, - ], - }, - SketchExpr::LetBinding { - name: BindingName::new("kll_state"), - expr: Box::new(SketchExpr::SketchAgg { - sketch_type: SketchKind::CountSketch, - params: SketchParams::CountSketch(CountSketchParams { - w: 2048, - d: 5, - with_heap: false, - }), - child: Box::new(SketchExpr::Logical(windowed_scan())), - }), - child: Box::new(SketchExpr::Ref { - name: BindingName::new("kll_state"), - }), - }, - SketchExpr::Ref { - name: BindingName::new("alone"), - }, - SketchExpr::SketchAgg { - sketch_type: SketchKind::Cms, - params: SketchParams::Cms(CmsParams { w: 2048, d: 5 }), - child: Box::new(SketchExpr::Logical(windowed_scan())), - }, - ]; - for c in cases { - let json = serde_json::to_string(&c).unwrap(); - let back: SketchExpr = serde_json::from_str(&json).unwrap(); - assert_eq!(c, back); - } -} - -// ── Bind rule tests ─────────────────────────────────────────────────────────── - -#[test] -fn bind_kll_quantile_basic() { - // The KLL rule on its own (priority 5) — DDSketch (priority 6) wins - // the dispatcher tie-break, so test the KLL rule's `apply` directly. - let expr = agg_quantile(0.99, AccuracyTarget::Epsilon(0.01)); - let bound = BindKllOnQuantile - .apply(&expr, &AccuracyTarget::Epsilon(0.01)) - .expect("KLL rule should bind a Quantile{0.99, ε=0.01}"); - match bound { - SketchExpr::SketchEstimate { op, child } => { - assert_eq!(op, EstimateOp::Quantile { q: 0.99 }); - match *child { - SketchExpr::SketchAgg { - sketch_type, - params, - child, - } => { - assert_eq!(sketch_type, SketchKind::Kll); - assert_eq!(params, SketchParams::Kll(KllParams { k: 200 })); - assert!(matches!(*child, SketchExpr::Logical(QueryExpr::Window { .. }))); - } - other => panic!("expected SketchAgg, got {other:?}"), - } - } - other => panic!("expected SketchEstimate, got {other:?}"), - } -} - -#[test] -fn bind_ddsketch_quantile_basic() { - let expr = agg_quantile(0.99, AccuracyTarget::Epsilon(0.01)); - let bound = BindDDSketchOnQuantile - .apply(&expr, &AccuracyTarget::Epsilon(0.01)) - .expect("DDSketch rule should bind a Quantile{0.99, ε=0.01}"); - match bound { - SketchExpr::SketchEstimate { op, child } => { - assert_eq!(op, EstimateOp::Quantile { q: 0.99 }); - match *child { - SketchExpr::SketchAgg { - sketch_type, - params, - .. - } => { - assert_eq!(sketch_type, SketchKind::DDSketch); - match params { - SketchParams::DDSketch(p) => assert!((p.alpha - 0.01).abs() < 1e-12), - other => panic!("expected DDSketchParams, got {other:?}"), - } - } - other => panic!("expected SketchAgg, got {other:?}"), - } - } - other => panic!("expected SketchEstimate, got {other:?}"), - } -} - -/// Cost-aware rule selection: the dispatcher should pick DDSketch over -/// KLL for an explicit ε-driven Quantile because DDSketch has higher -/// `priority()` (6 vs 5) — that matches the legacy -/// `algebra::directory::sketch_type_for_agg` default for SP-2/SP-4. -#[test] -fn bind_picks_ddsketch_over_kll_when_eps_explicit() { - let expr = agg_quantile(0.99, AccuracyTarget::Epsilon(0.01)); - let bound = - bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)).expect("bind_query_expr should not error"); - match bound { - SketchExpr::SketchEstimate { child, .. } => match *child { - SketchExpr::SketchAgg { sketch_type, .. } => { - assert_eq!( - sketch_type, - SketchKind::DDSketch, - "dispatcher should pick DDSketch (priority 6) over KLL (priority 5) on ε-driven Quantile" - ); - } - other => panic!("expected SketchAgg, got {other:?}"), - }, - other => panic!("expected SketchEstimate, got {other:?}"), - } -} - -#[test] -fn bind_cms_topk_basic() { - let expr = QueryExpr::Aggregate { - by: vec![], - aggs: vec![AggIntent::TopK { - k: 10, - accuracy: AccuracyTarget::EpsilonDelta { - eps: 0.01, - delta: 0.001, - }, - }], - having: None, - child: Box::new(windowed_scan()), - }; - let bound = bind_query_expr( - &expr, - AccuracyTarget::EpsilonDelta { - eps: 0.01, - delta: 0.001, - }, - ) - .expect("bind_query_expr should not error"); - match bound { - SketchExpr::SketchEstimate { op, child } => { - assert_eq!(op, EstimateOp::TopK { k: 10 }); - match *child { - SketchExpr::SketchAgg { - sketch_type, - params, - .. - } => { - assert_eq!(sketch_type, SketchKind::CountSketch); - match params { - SketchParams::CountSketch(p) => { - assert!(p.with_heap, "TopK binding must enable the heavy-hitter heap"); - assert!(p.w >= 2); - assert!(p.d >= 1); - } - other => panic!("expected CountSketchParams, got {other:?}"), - } - } - other => panic!("expected SketchAgg, got {other:?}"), - } - } - other => panic!("expected SketchEstimate, got {other:?}"), - } -} - -#[test] -fn bind_hll_cardinality_basic() { - let expr = QueryExpr::Aggregate { - by: vec![], - aggs: vec![AggIntent::Cardinality { - accuracy: AccuracyTarget::Epsilon(0.01), - }], - having: None, - child: Box::new(windowed_scan()), - }; - let bound = bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)).expect("no error"); - match bound { - SketchExpr::SketchEstimate { op, child } => { - assert_eq!(op, EstimateOp::Cardinality); - match *child { - SketchExpr::SketchAgg { - sketch_type, - params, - .. - } => { - assert_eq!(sketch_type, SketchKind::Hll); - match params { - SketchParams::Hll(p) => { - assert!( - p.precision >= 12, - "ε=0.01 should land on at least precision 12 (~1.6%) per the rung table" - ); - } - other => panic!("expected HllParams, got {other:?}"), - } - } - other => panic!("expected SketchAgg, got {other:?}"), - } - } - other => panic!("expected SketchEstimate, got {other:?}"), - } -} - -#[test] -fn bind_no_match_passes_through_logical() { - // Sum is exact at L3 — no `Bind*` rule covers it. Should pass - // through unchanged in `SketchExpr::Logical`. - let expr = QueryExpr::Aggregate { - by: vec![], - aggs: vec![AggIntent::Sum], - having: None, - child: Box::new(windowed_scan()), - }; - let bound = bind_query_expr(&expr, AccuracyTarget::Exact).expect("no error"); - assert!( - matches!(bound, SketchExpr::Logical(QueryExpr::Aggregate { .. })), - "Sum should pass through as Logical(Aggregate{{Sum}})" - ); -} - -#[test] -fn bind_exact_accuracy_disables_quantile_binding() { - // Quantile under `AccuracyTarget::Exact` should NOT bind — the - // optimizer falls back to an exact path. (Per design.md §6 line - // ~1254 — "the sketch path is selected, not mandated".) - let expr = agg_quantile(0.99, AccuracyTarget::Exact); - let bound = bind_query_expr(&expr, AccuracyTarget::Exact).expect("no error"); - assert!( - matches!(bound, SketchExpr::Logical(QueryExpr::Aggregate { .. })), - "Exact accuracy should disable sketch binding and pass through as Logical" - ); -} - -/// Two `SketchEstimate` parents reading different quantiles can share -/// one underlying `SketchAgg{KLL}` via `LetBinding` / `Ref`. Mirrors the -/// design.md §6 batched-queries example (line ~1326) — within the L4 -/// IR, fan-in is expressible as a `LetBinding` whose bound expression -/// is the shared `SketchAgg`. -#[test] -fn let_binding_ref_through_sketch_dag() { - let shared_agg = SketchExpr::SketchAgg { - sketch_type: SketchKind::Kll, - params: SketchParams::Kll(KllParams { k: 200 }), - child: Box::new(SketchExpr::Logical(windowed_scan())), - }; - let expr = SketchExpr::LetBinding { - name: BindingName::new("kll_state"), - expr: Box::new(shared_agg), - child: Box::new(SketchExpr::SketchMerge { - algebra: MergeAlgebra::Union, - // Two `SketchEstimate` parents reading the shared sketch via - // `Ref` — the design.md §6 line ~1339 two-tier fan-in shape. - children: vec![ - SketchExpr::SketchEstimate { - op: EstimateOp::Quantile { q: 0.99 }, - child: Box::new(SketchExpr::Ref { - name: BindingName::new("kll_state"), - }), - }, - SketchExpr::SketchEstimate { - op: EstimateOp::Quantile { q: 0.95 }, - child: Box::new(SketchExpr::Ref { - name: BindingName::new("kll_state"), - }), - }, - ], - }), - }; - // Round-trip the DAG through serde to verify the multi-parent fan-in - // shape survives wire encoding (the L4 type checker, when it lands, - // will assert the matching sketch-state schema on each `Ref` reader). - let json = serde_json::to_string(&expr).unwrap(); - let back: SketchExpr = serde_json::from_str(&json).unwrap(); - assert_eq!(expr, back); -} - -// ── Phase β: pattern-migration coverage ─────────────────────────────────────── -// -// The five PromQL pattern shapes defined in `asap-planner-rs/src/planner/ -// patterns.rs` each have a controller L3/L4 equivalent. These tests are the -// per-shape cross-reference asserting the L1→L3→L4 path produces a -// matching binding without going back through asap-planner-rs. - -/// `ONLY_TEMPORAL` — `quantile_over_time(0.99, m[5m])`. -/// asap-planner-rs path: ONLY_TEMPORAL pattern 1 → KLL/DDSketch sketch. -/// Controller path: `Aggregate{Quantile{0.99}}` over `Window` → -/// `BindKllOnQuantile` (or DDSketch) → `SketchAgg{KLL/DDSketch}`. -#[test] -fn phase_b_pattern_only_temporal_quantile_binds_to_sketch() { - let expr = QueryExpr::Aggregate { - by: vec![], - aggs: vec![AggIntent::Quantile { - q: 0.99, - accuracy: AccuracyTarget::Epsilon(0.01), - }], - having: None, - child: Box::new(windowed_scan()), - }; - let bound = bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)).unwrap(); - match bound { - SketchExpr::SketchEstimate { op, child } => { - assert!(matches!(op, EstimateOp::Quantile { .. })); - match *child { - SketchExpr::SketchAgg { sketch_type, .. } => { - assert!(matches!( - sketch_type, - SketchKind::Kll | SketchKind::DDSketch - )); - } - other => panic!("expected SketchAgg under SketchEstimate, got {other:?}"), - } - } - other => panic!("expected SketchEstimate, got {other:?}"), - } -} - -/// `ONLY_TEMPORAL` — `sum_over_time(m[5m])` (and the count/avg/min/max -/// variants that legacy `single_query.rs` accepts). -/// Controller path: `Aggregate{Sum}` over `Window` → no warm-tier rule -/// fires (no streaming sum sketch); falls through to `Logical`. The -/// existing `algebra::directory` / `algebra::physical` engine handles the -/// exact aggregate. -#[test] -fn phase_b_pattern_only_temporal_sum_falls_through_to_logical() { - let expr = QueryExpr::Aggregate { - by: vec![], - aggs: vec![AggIntent::Sum], - having: None, - child: Box::new(windowed_scan()), - }; - let bound = bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)).unwrap(); - // Sum is exact → no SketchAgg, just a Logical pass-through. - assert!(matches!(bound, SketchExpr::Logical(_))); -} - -/// `ONLY_SPATIAL` — `sum by (host) (m)`. -/// Controller path: `Aggregate{Sum, by=[host]}` over a bare `Scan` (no -/// `Window`). Sum is exact → Logical pass-through. The point of the test -/// is the by-clause survives binding intact. -#[test] -fn phase_b_pattern_only_spatial_aggregate_preserves_by_clause() { - let expr = QueryExpr::Aggregate { - by: vec![1], // service column - aggs: vec![AggIntent::Sum], - having: None, - child: Box::new(ts_scan()), - }; - let bound = bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)).unwrap(); - match bound { - SketchExpr::Logical(QueryExpr::Aggregate { by, .. }) => { - assert_eq!(by, vec![1]); - } - other => panic!("expected Logical(Aggregate), got {other:?}"), - } -} - -/// `ONE_TEMPORAL_ONE_SPATIAL` — `sum by (host) (rate(m[5m]))`. -/// Controller path: combined `Aggregate{Sum, by=[host]}` over `Window` — -/// the L3 algebra captures both axes natively without needing the legacy -/// pattern's `One*One*` enum. -#[test] -fn phase_b_pattern_temporal_and_spatial_combined() { - let expr = QueryExpr::Aggregate { - by: vec![1], - aggs: vec![AggIntent::Rate { - window: Duration::from_secs(300), - }], - having: None, - child: Box::new(windowed_scan()), - }; - let bound = bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)).unwrap(); - // Rate has no warm-tier sketch family today — expect Logical. - assert!(matches!(bound, SketchExpr::Logical(_))); -} - -/// `histogram_quantile(φ, …)` — Phase β archive-only addition (not in -/// `patterns.rs`'s 5 entries; the legacy planner refused these via -/// `is_supported() == false`). Controller path: `BindArchiveOnly` matches -/// → `Logical` pass-through, and the L5 emitter / Phase α routing reads -/// `AggIntent::archive_only() == true` to flag the StreamingConfig entry -/// for the archive tier. -#[test] -fn phase_b_pattern_histogram_quantile_routes_to_archive() { - let intent = AggIntent::HistogramQuantile { q: 0.99 }; - assert!(intent.archive_only(), "Phase β intent must flag archive"); - let expr = QueryExpr::Aggregate { - by: vec![], - aggs: vec![intent.clone()], - having: None, - child: Box::new(windowed_scan()), - }; - let bound = bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)).unwrap(); - // The archive-only rule's output is a Logical pass-through carrying - // the original Aggregate. Downstream emitters check archive_only(). - match bound { - SketchExpr::Logical(QueryExpr::Aggregate { aggs, .. }) => { - assert_eq!(aggs, vec![intent]); - } - other => panic!("expected Logical(Aggregate(HistogramQuantile)), got {other:?}"), - } -} - -// ── Phase β: end-to-end pattern equivalence with asap-planner-rs ───────────── -// -// The asap-planner-rs test suite drives a set of canonical PromQL workloads -// (`tests/comparison/test_data/configs/*.yaml`). For each, the legacy -// planner produces a StreamingConfig with one or more `aggregation_id` -// entries keyed on (sketch_kind, sketch_params). -// -// Phase β asserts the CONTROLLER's L1→L3→L4 path produces a functionally -// equivalent set of bound aggregations for the same input strings. We -// don't load the YAML files — that would couple the controller to the -// asap-planner-rs test fixture layout. Instead each test embeds the -// representative query string from the corresponding fixture YAML and -// pins the expected (sketch_kind | archive-only) outcome. - -/// Helper: parse a PromQL string, lower to L3, bind to L4. Returns the -/// produced `SketchExpr` for assertion. The controller's `parse_query` -/// returns a `ParsedQuery`; `lower_parsed_query` builds the L3 IR from -/// it under the supplied accuracy target; `bind_query_expr` is the L3→L4 -/// bottom-up walk. -fn pipeline_l1_to_l4(query: &str, accuracy: AccuracyTarget) -> SketchExpr { - let parsed = crate::query_parser::parse_query(query) - .unwrap_or_else(|e| panic!("parse {query}: {e}")); - let qe = crate::intent_algebra::lower_parsed_query(&parsed, accuracy.clone()) - .unwrap_or_else(|e| panic!("lower {query}: {e}")); - bind_query_expr(&qe, accuracy).unwrap_or_else(|e| panic!("bind {query}: {e}")) -} - -/// Walk a `SketchExpr` and collect every `SketchAgg`'s sketch_kind. The -/// number of entries + the kind set is the wire-equivalent of -/// asap-planner-rs's "aggregation_id rows in StreamingConfig output". -fn collect_sketch_kinds(expr: &SketchExpr) -> Vec { - let mut out = Vec::new(); - fn walk(e: &SketchExpr, out: &mut Vec) { - match e { - SketchExpr::SketchAgg { sketch_type, child, .. } => { - out.push(sketch_type.clone()); - walk(child, out); - } - SketchExpr::SketchEstimate { child, .. } => walk(child, out), - SketchExpr::SketchMerge { children, .. } => { - for c in children { - walk(c, out); - } - } - SketchExpr::LetBinding { expr, child, .. } => { - walk(expr, out); - walk(child, out); - } - SketchExpr::Logical(_) | SketchExpr::Ref { .. } => {} - // Phase ε.1 — the new placement variants don't carry a - // SketchAgg child the legacy walk recognises. Mode 2 records - // its own family directly; Mode 3 has no sketch at all. - SketchExpr::RawAtEdgeSketchAtBackend { family, child, .. } => { - out.push(family.clone()); - walk(child, out); - } - SketchExpr::RawAtEdgePrometheusArchive { .. } => {} - } - } - walk(expr, &mut out); - out -} - -/// Walk a `SketchExpr` and detect whether the binding ended in a -/// `Logical`-wrapped `Aggregate` carrying an archive-only intent. This is -/// the L4 signal that the L5 emitter routes the StreamingConfig entry -/// to the cold tier rather than the warm one. -fn binding_is_archive(expr: &SketchExpr) -> bool { - match expr { - SketchExpr::Logical(QueryExpr::Aggregate { aggs, .. }) => { - aggs.iter().any(|a| a.archive_only()) - } - SketchExpr::Logical(_) => false, - SketchExpr::SketchEstimate { child, .. } => binding_is_archive(child), - SketchExpr::SketchAgg { child, .. } => binding_is_archive(child), - SketchExpr::SketchMerge { children, .. } => children.iter().any(binding_is_archive), - SketchExpr::LetBinding { expr, child, .. } => { - binding_is_archive(expr) || binding_is_archive(child) - } - SketchExpr::Ref { .. } => false, - // Phase ε.1 — Mode 3 routes to the prometheus_remote engine - // (its own engine ID), which the L5 emitter handles via - // emit_backend_storage_routing rather than the warm-vs-archive - // gate this helper guards. Treat as not-archive: this helper is - // about cold-tier scan-vs-warm-tier-sketch decisions, not Mode 3. - SketchExpr::RawAtEdgeSketchAtBackend { child, .. } => binding_is_archive(child), - SketchExpr::RawAtEdgePrometheusArchive { .. } => false, - } -} - -/// `quantile_over_time.yaml` — the asap-planner-rs `quantile_over_time` -/// fixture maps to a KLL or DDSketch StreamingConfig row. The controller -/// path: L1 PromQL parse → L3 `Aggregate{Quantile{0.99}}` over `Window` → -/// L4 `BindKllOnQuantile` (default) or `BindDDSketchOnQuantile`. Either -/// is functionally equivalent — both are quantile sketches. -#[test] -fn phase_b_e2e_quantile_over_time_binds_to_quantile_sketch() { - let bound = pipeline_l1_to_l4( - "quantile_over_time(0.99, http_request_duration_seconds[5m])", - AccuracyTarget::Epsilon(0.01), - ); - let kinds = collect_sketch_kinds(&bound); - assert_eq!(kinds.len(), 1, "expected 1 sketch agg, got {kinds:?}"); - assert!( - matches!(kinds[0], SketchKind::Kll | SketchKind::DDSketch), - "expected quantile sketch family, got {:?}", - kinds[0] - ); - assert!( - !binding_is_archive(&bound), - "warm-tier quantile must not flag archive" - ); -} - -/// `sum_over_time.yaml` — the legacy planner produces an exact-sum -/// aggregation row (no sketch). Controller path: `Aggregate{Sum}` over -/// `Window` → no warm-tier rule fires → `Logical` pass-through. -/// Functional equivalence: both produce a single non-sketch row. -#[test] -fn phase_b_e2e_sum_over_time_falls_through_to_logical() { - let bound = pipeline_l1_to_l4( - "sum_over_time(http_requests_total[5m])", - AccuracyTarget::Epsilon(0.01), - ); - let kinds = collect_sketch_kinds(&bound); - assert!( - kinds.is_empty(), - "sum_over_time should not produce a sketch agg, got {kinds:?}" - ); - assert!( - !binding_is_archive(&bound), - "Sum is exact-warm, not archive — bind output should stay Logical without archive flag" - ); -} - -/// `sum_by.yaml` — `sum by (label) (sum_over_time(...))`. Spatial-and- -/// temporal aggregation; the legacy planner emits an exact-sum row keyed -/// on the by-label. Controller path: `Aggregate{Sum, by=[…]}` over -/// `Window` → no warm-tier rule fires → `Logical` pass-through. The -/// by-label is preserved on the L3 group-by-id list, which Phase α's -/// routing emit reads to build the per-label rollup partition. -#[test] -fn phase_b_e2e_sum_by_preserves_grouping_label() { - let bound = pipeline_l1_to_l4( - "sum by (instance) (sum_over_time(http_requests_total[5m]))", - AccuracyTarget::Epsilon(0.01), - ); - // No sketch family for plain Sum. - assert!(collect_sketch_kinds(&bound).is_empty()); - // The end shape may be Logical(Aggregate{by, ...}) when the Aggregate - // node survives the lowering, or Logical(Window{...}) when the - // ParsedQuery → QueryExpr lowering drops the Aggregate (legacy - // ParsedQuery only carries `aggregations: Vec` not the - // by-axis directly). In either case the metric name + label survive - // somewhere in the L3 sub-tree — assert that. - let json = serde_json::to_string(&bound).unwrap(); - assert!( - json.contains("http_requests_total"), - "metric name lost through pipeline: {json}" - ); - assert!( - json.contains("instance"), - "by-label `instance` lost through pipeline: {json}" - ); -} - -/// `rate_increase.yaml` — the legacy planner emits a MultipleIncrease -/// (counter-reset adjusted) row. Controller path: `Aggregate{Rate}` over -/// `Window` → no streaming-rate sketch family today → `Logical`. Both -/// paths produce a single non-sketch streaming row; the L5 emitter is -/// the one that picks the actual MultipleIncrease processor. -#[test] -fn phase_b_e2e_rate_falls_through_to_logical() { - let bound = pipeline_l1_to_l4( - "rate(http_requests_total[5m])", - AccuracyTarget::Epsilon(0.01), - ); - assert!(collect_sketch_kinds(&bound).is_empty()); - assert!(!binding_is_archive(&bound), "Rate is warm-tier, not archive"); -} - -/// `topk.yaml` — `topk(10, sum by (label) (rate(...))`. The legacy -/// planner emits a CountSketch+heap row. Controller path: the parser -/// recognises `topk` as a special node that lowers to `AggIntent::TopK`. -/// At the time of writing, the controller's `parse_query` may flatten -/// `topk` differently (no `inside_topk` propagation through TopK + -/// nested aggregate). The test asserts the END-STATE: either a -/// CountSketch sketch fired, OR a Logical pass-through (which Phase γ -/// can decide whether to refine). The contract Phase β cares about is -/// that the bound expression is well-formed. -#[test] -fn phase_b_e2e_topk_well_formed() { - let bound = pipeline_l1_to_l4( - "topk(10, sum by (instance) (rate(http_requests_total[5m])))", - AccuracyTarget::Epsilon(0.05), - ); - // Either a CountSketch / KLL / DDSketch fires (warm path) or it's a - // Logical pass-through (engine handles it). Both are accepted L4 - // shapes — Phase β's contract is just "doesn't panic, produces a - // legitimate SketchExpr". - let _ = collect_sketch_kinds(&bound); -} - -/// `histogram_quantile` — Phase β archive routing through the full L1→ -/// L3→L4 pipeline. Asserts the expected functional equivalent of -/// asap-planner-rs's previous `is_supported() == false` behavior -/// (refused outright); the controller now lifts these to L3 with -/// `archive_only() == true` and the binder emits a Logical pass-through. -#[test] -fn phase_b_e2e_histogram_quantile_e2e_through_parser() { - // The PromQL parser produces a `HistogramQuantile` QueryExpr node - // (not a flat `Aggregate{Quantile}`), so the L3 lowering of - // ParsedQuery cannot fully express it via the legacy AggType axis. - // We assert the SHAPE of the bound expression directly here: an - // archive-only intent under any `Aggregate` survives through bind. - let intent = AggIntent::HistogramQuantile { q: 0.99 }; - let expr = QueryExpr::Aggregate { - by: vec![], - aggs: vec![intent.clone()], - having: None, - child: Box::new(windowed_scan()), - }; - let bound = bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)).unwrap(); - assert!( - binding_is_archive(&bound), - "histogram_quantile must surface archive flag through L4 binding" - ); - // No warm-tier sketch fires for HistogramQuantile. - assert!(collect_sketch_kinds(&bound).is_empty()); -} - -/// Cross-cutting: every Phase β archive-only intent reaches -/// `bind_query_expr` and lands as a `Logical` pass-through whose contents -/// the L5 emitter can route via `archive_only()`. Mirrors Phase γ's -/// "delete asap-planner-rs without losing coverage" goal — none of these -/// raise an error or panic; all produce a valid L4 expression. -#[test] -fn phase_b_archive_only_intents_round_trip_through_binder() { - let intents = vec![ - AggIntent::HistogramQuantile { q: 0.5 }, - AggIntent::Absent, - AggIntent::Present, - AggIntent::Delta { - window: Duration::from_secs(60), - }, - AggIntent::Deriv { - window: Duration::from_secs(60), - }, - AggIntent::PredictLinear { - window: Duration::from_secs(300), - ahead: Duration::from_secs(60), - }, - AggIntent::HoltWinters { - window: Duration::from_secs(300), - smoothing_factor: 0.3, - trend_factor: 0.3, - }, - AggIntent::Idelta { - window: Duration::from_secs(60), - }, - AggIntent::Irate { - window: Duration::from_secs(60), - }, - AggIntent::Resets { - window: Duration::from_secs(300), - }, - AggIntent::Changes { - window: Duration::from_secs(300), - }, - ]; - for intent in intents { - let expr = QueryExpr::Aggregate { - by: vec![], - aggs: vec![intent.clone()], - having: None, - child: Box::new(windowed_scan()), - }; - let bound = - bind_query_expr(&expr, AccuracyTarget::Epsilon(0.01)).expect("bind should succeed"); - match bound { - SketchExpr::Logical(QueryExpr::Aggregate { aggs, .. }) => { - assert_eq!(aggs.len(), 1); - assert!( - aggs[0].archive_only(), - "{intent:?} should preserve archive_only() flag through bind" - ); - } - other => panic!("expected Logical(Aggregate({intent:?})), got {other:?}"), - } - } -} diff --git a/controller/src/stage_split/allocator.rs b/controller/src/stage_split/allocator.rs deleted file mode 100644 index 9113228c..00000000 --- a/controller/src/stage_split/allocator.rs +++ /dev/null @@ -1,310 +0,0 @@ -//! L5 stage allocator — colours a `SketchExpr` DAG by `StageId`. -//! -//! Per `controller/docs/design.md` §6 (line ~810): -//! -//! ```ignore -//! // generic stage allocator — given a SketchExpr tree + a topology, decide which -//! // ops land on which stage subject to constraints. Stage-level only; per-executor -//! // fan-out happens in the deployment model's PhysicalPlanner using the executor -//! // list from `DeploymentConstraints::executors()`. -//! pub struct StageAllocator; -//! impl StageAllocator { -//! pub fn allocate( -//! &self, exprs: &[QueryExpr], topology: &T, c: &DeploymentConstraints, -//! ) -> Result, PlanError>; -//! } -//! ``` -//! -//! Phase E surfaces the `Topology::ThreeStage` colouring; the rules -//! mirror design.md §6 batched-queries example (line ~1380): -//! -//! | Node | StageId | Why | -//! |---|---|---| -//! | `Logical(Scan)` | Edge | scrape happens at the agent host | -//! | `Logical(Window)` | Edge | windowing at edge keeps bandwidth low | -//! | `SketchAgg` | Edge | sketch building at the edge — the bandwidth claim | -//! | `Logical(Aggregate{exact})` over `Window` | Edge | per-row state; same logic as `SketchAgg` | -//! | `SketchMerge` | Gateway | merge edge sketches across hosts | -//! | `Logical(Aggregate{exact})` over `Merge`-shape | Backend | final readout (root of q3) | -//! | `SketchEstimate` | Backend | the query-readout side | -//! | `LetBinding` / `Ref` | (color of bound expr) | scope-resolved | -//! -//! The allocator does not own constraint logic (no memory budget / cost -//! threshold inputs in Phase E) — those come back as Phase G's -//! `DeploymentConstraints` plumbing. The Phase E colouring is purely -//! structural per the design.md table. - -#![allow(dead_code)] - -use std::collections::HashMap; - -use crate::sketch_algebra::SketchExpr; -use crate::stage_split::colored_dag::{ColoredDag, ColoredNode, NodeId}; -use crate::stage_split::stage_id::{StageId, Topology}; -use crate::types_v2::BindingName; - -/// Errors surfaced by [`StageAllocator::allocate`]. -#[derive(Debug, thiserror::Error, PartialEq)] -pub enum AllocateError { - /// The supplied topology is not implemented in Phase E. - #[error("unsupported topology in Phase E (only ThreeStage is implemented): {0:?}")] - UnsupportedTopology(Topology), - /// `Ref(name)` did not resolve against any in-scope `LetBinding`. - #[error("unresolved Ref: {0}")] - UnresolvedRef(String), -} - -/// L5 stage allocator. Stateless — Phase E exposes a unit struct so the -/// API matches design.md (`pub struct StageAllocator;`). -#[derive(Debug, Default, Clone, Copy)] -pub struct StageAllocator; - -impl StageAllocator { - /// Colour `expr` against `topology`. Returns the colored DAG ready - /// for emitter consumption. - /// - /// Phase E only implements `Topology::ThreeStage`; other variants - /// return [`AllocateError::UnsupportedTopology`]. - pub fn allocate( - &self, - expr: &SketchExpr, - topology: Topology, - ) -> Result { - match topology { - Topology::ThreeStage => { - let mut walker = ThreeStageWalker::default(); - walker.dag.topology = topology; - walker.visit(expr)?; - Ok(walker.dag) - } - other => Err(AllocateError::UnsupportedTopology(other)), - } - } -} - -// ── Three-stage colouring walker ────────────────────────────────────────────── - -#[derive(Default)] -struct ThreeStageWalker { - dag: ColoredDag, - /// Lexical scope: binding name → colored stage of the bound expression's - /// root. - scope: HashMap, -} - -impl ThreeStageWalker { - /// Recursively visit `expr`, append its colored node to the DAG, - /// and return its `(NodeId, StageId)`. - fn visit(&mut self, expr: &SketchExpr) -> Result<(NodeId, StageId), AllocateError> { - // Reserve a slot for this node up-front so child IDs are - // strictly larger than the parent's; downstream `cut_edges` - // analysis assumes parents come before children in `nodes`. - let id = NodeId(self.dag.nodes.len()); - self.dag.nodes.push(ColoredNode { - id, - expr: expr.clone(), - // Placeholder — overwritten below once children are coloured. - stage: StageId::Edge, - }); - - let stage = match expr { - // ── Logical pass-through — colour by inspecting the wrapped - // L3 QueryExpr. `Scan` / `Window` always land on edge; - // `Aggregate{exact}` lands on edge if its child is an edge - // (scrape locality); `Ref` resolves through the lexical - // scope map. - SketchExpr::Logical(qe) => self.colour_logical(qe)?, - - // ── SketchAgg: always edge per design.md §6 batched-queries - // table. The "SketchAgg whose child is a Scan MUST be on - // Edge" invariant is automatically satisfied. - SketchExpr::SketchAgg { child, .. } => { - let (cid, _) = self.visit(child)?; - self.dag.edges.push((id, cid)); - StageId::Edge - } - - // ── SketchEstimate: always backend per design.md §6. - // The "SketchEstimate MUST be on the same stage as its - // consumers (typically Backend)" invariant is satisfied - // because consumers above SketchEstimate are also backend. - SketchExpr::SketchEstimate { child, .. } => { - let (cid, child_stage) = self.visit(child)?; - self.dag.edges.push((id, cid)); - // If child is on edge or gateway, this is a cross-stage - // edge — that's expected (the wire-format hop). - let _ = child_stage; - StageId::Backend - } - - // ── SketchMerge: gateway under three-stage. Children are - // edge SketchAgg outputs. - SketchExpr::SketchMerge { children, .. } => { - for child in children { - let (cid, _) = self.visit(child)?; - self.dag.edges.push((id, cid)); - } - StageId::Gateway - } - - // ── LetBinding: colour by the bound expression's stage, - // and bring the binding into scope before walking the body. - SketchExpr::LetBinding { name, expr, child } => { - let (eid, expr_stage) = self.visit(expr)?; - self.dag.edges.push((id, eid)); - self.scope.insert(name.as_str().to_string(), expr_stage); - let (bid, _) = self.visit(child)?; - self.dag.edges.push((id, bid)); - expr_stage - } - - // ── Ref: colour matches the binding's stage. Unresolved - // refs bubble up as `AllocateError::UnresolvedRef`. - SketchExpr::Ref { name } => self - .scope - .get(name.as_str()) - .copied() - .ok_or_else(|| AllocateError::UnresolvedRef(name.as_str().to_string()))?, - - // ── Phase ε.1 Mode 2: raw at edge, sketch built at backend. - // Edge ships raw OTLP — we stage as Edge so the L5 emitter's - // edge-side YAML pipeline picks it up; the sketch construction - // itself happens at the backend (no edge sketch processor). - SketchExpr::RawAtEdgeSketchAtBackend { child, .. } => { - let (cid, _) = self.visit(child)?; - self.dag.edges.push((id, cid)); - StageId::Edge - } - - // ── Phase ε.1 Mode 3: raw at edge, ships directly to - // Prometheus's native OTLP receiver. The agent pipeline picks - // this up via `asap.mode=prometheus_archive` routing. - SketchExpr::RawAtEdgePrometheusArchive { .. } => StageId::Edge, - }; - - // Patch in the resolved stage now that children have been visited. - self.dag.nodes[id.0].stage = stage; - - Ok((id, stage)) - } - - /// Colour a `Logical(QueryExpr)` node per the three-stage rules. - /// Per design.md §6: Scan / Window → Edge; Aggregate over Window → - /// Edge (per-row exact aggregation, e.g. `Max`); Aggregate over a - /// gateway-coloured input → Backend (final readout root). - /// `LetBinding`/`Ref` at the L3 level reuse the same scope map. - fn colour_logical( - &mut self, - qe: &crate::intent_algebra::QueryExpr, - ) -> Result { - use crate::intent_algebra::QueryExpr as QE; - match qe { - QE::Scan { .. } => Ok(StageId::Edge), - QE::Window { .. } => Ok(StageId::Edge), - // Aggregate at L3-in-L4: the design.md L5 table says - // `Aggregate{exact}` (e.g. `Max`) → Edge, and the *root of - // q3* (the same Aggregate after a SketchMerge / Merge) → - // Backend. Phase E's Logical wrapper does not surface a - // SketchExpr-level Merge over exact streams, so the L3 - // Aggregate node reachable here is always the per-window - // edge form. Final-readout placement happens at the - // SketchExpr-level (root of q3 wrapped in a SketchMerge - // sibling structure) — Phase G+ adds an explicit - // `Logical(Merge)` SketchExpr variant for the gateway hop. - QE::Aggregate { .. } => Ok(StageId::Edge), - // Lexical scope for L3 LetBinding / Ref — mirrors the - // SketchExpr-level handling. - QE::LetBinding { name, expr, child } => { - let expr_stage = self.colour_logical(expr)?; - self.scope.insert(name.as_str().to_string(), expr_stage); - self.colour_logical(child) - } - QE::Ref { name } => self - .scope - .get(name.as_str()) - .copied() - .ok_or_else(|| AllocateError::UnresolvedRef(name.as_str().to_string())), - } - } -} - -// Convenience helper used by tests / external callers that only need a -// stage lookup keyed by binding name. -pub(crate) fn binding_stage(dag: &ColoredDag, name: &BindingName) -> Option { - dag.nodes.iter().find_map(|n| match &n.expr { - SketchExpr::LetBinding { name: n2, .. } if n2 == name => Some(n.stage), - _ => None, - }) -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use crate::intent_algebra::schema::{Column, DataType}; - use crate::intent_algebra::{QueryExpr, Schema, Source, WindowKind}; - use crate::sketch_algebra::params::{KllParams, SketchKind, SketchParams}; - use crate::sketch_algebra::sketch_expr::EstimateOp; - use std::time::Duration; - - fn ts_scan() -> QueryExpr { - QueryExpr::Scan { - source: Source::TimeSeries { - metric: "http_request_duration_seconds".into(), - }, - label_filters: vec![], - schema: Schema::with_time_index( - vec![ - Column { - name: "ts".into(), - dtype: DataType::Timestamp, - nullable: false, - }, - Column { - name: "value".into(), - dtype: DataType::Float64, - nullable: false, - }, - ], - 0, - vec![vec![0]], - ), - } - } - - fn windowed_scan() -> QueryExpr { - QueryExpr::Window { - kind: WindowKind::Sliding, - size: Duration::from_secs(300), - slide: None, - child: Box::new(ts_scan()), - } - } - - #[test] - fn allocate_unsupported_topology_errors() { - let leaf = SketchExpr::Logical(ts_scan()); - let err = StageAllocator - .allocate(&leaf, Topology::SingleStage) - .unwrap_err(); - assert_eq!(err, AllocateError::UnsupportedTopology(Topology::SingleStage)); - } - - #[test] - fn three_stage_quantile_dag_basic() { - let expr = SketchExpr::estimate_over_agg( - EstimateOp::Quantile { q: 0.99 }, - SketchKind::Kll, - SketchParams::Kll(KllParams { k: 200 }), - windowed_scan(), - ); - let dag = StageAllocator.allocate(&expr, Topology::ThreeStage).unwrap(); - // root = SketchEstimate → Backend - assert_eq!(dag.root().unwrap().stage, StageId::Backend); - // node 1 = SketchAgg → Edge - assert_eq!(dag.nodes[1].stage, StageId::Edge); - // node 2 = Logical(Window) → Edge - assert_eq!(dag.nodes[2].stage, StageId::Edge); - } -} diff --git a/controller/src/stage_split/colored_dag.rs b/controller/src/stage_split/colored_dag.rs deleted file mode 100644 index eff1167f..00000000 --- a/controller/src/stage_split/colored_dag.rs +++ /dev/null @@ -1,223 +0,0 @@ -//! L5 colored DAG — `SketchExpr` nodes painted with `StageId`. -//! -//! Per `controller/docs/design.md` §3 (line ~123): "A 'stage assignment' -//! is a colouring of the L4-bound `SketchExpr` DAG by `StageId`, with -//! sketch-merge / data-shipping nodes inserted on the cut edges." -//! -//! [`ColoredDag`] is the IR the [`crate::stage_split::Emitter`] -//! consumes. It carries: -//! -//! - the `nodes` vector (every visited `SketchExpr` node keyed by -//! [`NodeId`], with its assigned `StageId`), -//! - the `edges` vector (parent → child references from the DAG walk), -//! carried so future Phase G+ work can detect cross-stage cut edges -//! for explicit data-shipping op insertion. -//! -//! Today the emitter consumes the per-stage *bucket* of nodes (`nodes -//! filtered by stage`) plus the root sketch-state metadata; cut-edge -//! reasoning is left as a Phase G hook (the `edges` vector is the data -//! it needs). - -#![allow(dead_code)] - -use serde::{Deserialize, Serialize}; - -use crate::sketch_algebra::SketchExpr; -use crate::stage_split::stage_id::{StageId, Topology}; - -/// Stable position-based identifier for a node within a `ColoredDag`. -/// `NodeId(0)` is the root; depth-first walk order otherwise. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(transparent)] -pub struct NodeId(pub usize); - -impl std::fmt::Display for NodeId { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "n{}", self.0) - } -} - -/// One entry in the colored DAG: a `SketchExpr` node + its assigned -/// `StageId`. -/// -/// `expr` is a clone of the node's surface variant (children are NOT -/// recursively cloned — the `child` payload is replaced with a sentinel -/// to keep the colored-DAG flat; structural information lives in -/// [`ColoredDag::edges`]). Test-friendly variant: when callers want the -/// full sub-tree they can rebuild from the original `SketchExpr` using -/// `NodeId` as the index. -#[derive(Debug, Clone, PartialEq)] -pub struct ColoredNode { - /// Position-based identifier — index into `ColoredDag::nodes`. - pub id: NodeId, - /// The `SketchExpr` node (full sub-tree as originally walked — Phase - /// E does not strip children, so emitters can read what they need). - pub expr: SketchExpr, - /// Stage this node was painted with. - pub stage: StageId, -} - -/// L5 colored DAG. The output of [`crate::stage_split::StageAllocator::allocate`]. -/// -/// Per design.md §6: "L5 colors the DAG by `StageId` and emits -/// per-executor configs. Same `SketchExpr` input; topology and emitter -/// differ per deployment model." -/// -/// `Default::default()` returns the empty `Topology::ThreeStage` shape — -/// that's the only topology Phase E ships, so the default is safe and -/// useful for the allocator's incremental-build path. -#[derive(Debug, Clone)] -pub struct ColoredDag { - /// Topology this colouring was produced under. - pub topology: Topology, - /// Node table — depth-first walk order, root at index 0. - pub nodes: Vec, - /// Parent → child edges (DAG structure). Carried for future - /// cut-edge analysis; today the emitter consumes per-stage buckets. - pub edges: Vec<(NodeId, NodeId)>, -} - -impl ColoredDag { - /// Empty colored DAG for the supplied topology — the allocator - /// builds the contents. - pub fn new(topology: Topology) -> Self { - Self { - topology, - nodes: Vec::new(), - edges: Vec::new(), - } - } -} - -impl Default for ColoredDag { - fn default() -> Self { - ColoredDag::new(Topology::ThreeStage) - } -} - -impl ColoredDag { - - /// Root node (the original `SketchExpr` root). `None` only for the - /// degenerate empty DAG. - pub fn root(&self) -> Option<&ColoredNode> { - self.nodes.first() - } - - /// All nodes painted with `stage`. - pub fn nodes_in_stage(&self, stage: StageId) -> impl Iterator { - self.nodes.iter().filter(move |n| n.stage == stage) - } - - /// Set of `StageId`s actually present in this colouring (subset of - /// `topology.stages()`). - pub fn occupied_stages(&self) -> Vec { - let mut seen: Vec = Vec::new(); - for n in &self.nodes { - if !seen.contains(&n.stage) { - seen.push(n.stage); - } - } - seen - } - - /// Cut edges — edges where parent and child end up on different - /// stages. These are the wire-format hops (OpAMP push / OTLP / - /// `StreamingConfig`) the emitter must materialize. Phase E reports - /// them; Phase G+ inserts explicit data-shipping ops on them. - pub fn cut_edges(&self) -> Vec<(NodeId, NodeId)> { - self.edges - .iter() - .copied() - .filter(|(p, c)| { - let ps = self.nodes.get(p.0).map(|n| n.stage); - let cs = self.nodes.get(c.0).map(|n| n.stage); - match (ps, cs) { - (Some(a), Some(b)) => a != b, - _ => false, - } - }) - .collect() - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use crate::sketch_algebra::SketchExpr; - use crate::sketch_algebra::params::{KllParams, SketchKind, SketchParams}; - use crate::sketch_algebra::sketch_expr::EstimateOp; - use crate::intent_algebra::QueryExpr; - - fn dummy_logical() -> SketchExpr { - SketchExpr::Logical(QueryExpr::Ref { - name: crate::types_v2::BindingName::new("dummy"), - }) - } - - fn dummy_agg() -> SketchExpr { - SketchExpr::SketchAgg { - sketch_type: SketchKind::Kll, - params: SketchParams::Kll(KllParams { k: 200 }), - child: Box::new(dummy_logical()), - } - } - - fn dummy_estimate() -> SketchExpr { - SketchExpr::SketchEstimate { - op: EstimateOp::Quantile { q: 0.99 }, - child: Box::new(dummy_agg()), - } - } - - #[test] - fn empty_dag_has_no_root() { - let d = ColoredDag::new(Topology::ThreeStage); - assert!(d.root().is_none()); - assert!(d.occupied_stages().is_empty()); - assert!(d.cut_edges().is_empty()); - } - - #[test] - fn occupied_stages_dedupe() { - let mut d = ColoredDag::new(Topology::ThreeStage); - d.nodes.push(ColoredNode { - id: NodeId(0), - expr: dummy_estimate(), - stage: StageId::Backend, - }); - d.nodes.push(ColoredNode { - id: NodeId(1), - expr: dummy_agg(), - stage: StageId::Edge, - }); - d.nodes.push(ColoredNode { - id: NodeId(2), - expr: dummy_agg(), - stage: StageId::Edge, - }); - let stages = d.occupied_stages(); - assert!(stages.contains(&StageId::Edge)); - assert!(stages.contains(&StageId::Backend)); - assert_eq!(stages.len(), 2); - } - - #[test] - fn cut_edges_detected_across_stages() { - let mut d = ColoredDag::new(Topology::ThreeStage); - d.nodes.push(ColoredNode { - id: NodeId(0), - expr: dummy_estimate(), - stage: StageId::Backend, - }); - d.nodes.push(ColoredNode { - id: NodeId(1), - expr: dummy_agg(), - stage: StageId::Edge, - }); - d.edges.push((NodeId(0), NodeId(1))); - let cut = d.cut_edges(); - assert_eq!(cut, vec![(NodeId(0), NodeId(1))]); - } -} diff --git a/controller/src/stage_split/emitter.rs b/controller/src/stage_split/emitter.rs deleted file mode 100644 index e6d357c6..00000000 --- a/controller/src/stage_split/emitter.rs +++ /dev/null @@ -1,680 +0,0 @@ -//! L5 emitter — turns a [`ColoredDag`] into per-stage configs. -//! -//! Per `controller/docs/design.md` §1219: "L4 chose the sketch family + -//! params. L5 colors the DAG by `StageId` and emits per-executor -//! configs. Same `SketchExpr` input; topology and emitter differ per -//! deployment model." -//! -//! Phase E ships [`ThreeStageEmitter`] for the DC topology (edge → -//! gateway → backend). Each per-stage [`StageConfig`] is a structured -//! description that the OpAMP push (Phase G+) and the backend client -//! (Phase G+) materialise into wire bytes: -//! -//! - [`StageConfig::Edge`] — the agent OpAMP YAML's logical content: -//! scrape source, optional window, the chosen sketch processor, and -//! the OTLP exporter pointer to gateway. -//! - [`StageConfig::Gateway`] — the gateway OpAMP YAML's logical -//! content: an OTLP receiver, the sketch-merge processor list, and -//! the OTLP exporter pointer to backend. -//! - [`StageConfig::Backend`] — the backend `StreamingConfig`'s logical -//! content: a list of `(aggregation_id → (sketch_kind, params))` -//! tuples plus the readout query catalog. -//! -//! Emitters do NOT push to executors here — they produce the structured -//! output. The actual push (`crate::opamp::OpampServer::push_to_role`, -//! `crate::backend_client::StreamingConfigClient::post`) is wired in -//! Phase G+. - -#![allow(dead_code)] - -use std::collections::HashMap; - -use serde::{Deserialize, Serialize}; - -use crate::sketch_algebra::params::{SketchKind, SketchParams}; -use crate::sketch_algebra::sketch_expr::{EstimateOp, SketchExpr}; -use crate::stage_split::colored_dag::ColoredDag; -use crate::stage_split::stage_id::{StageId, Topology}; - -/// Errors surfaced by [`Emitter::emit_per_stage`]. -#[derive(Debug, thiserror::Error, PartialEq)] -pub enum EmitError { - /// Topology shape isn't supported by this emitter — see - /// `controller/docs/design.md` §6 for the per-deployment-model - /// emitter list. - #[error("unsupported topology for this emitter: {0:?} (expected {1:?})")] - UnsupportedTopology(Topology, Topology), - /// A sketch processor name could not be derived for the supplied - /// `SketchKind`. Should not occur with the catalog ranges shipped - /// in Phase C — kept as a defensive error for future kinds. - #[error("no edge processor known for sketch kind {0:?}")] - NoEdgeProcessor(SketchKind), - /// Backend would emit an empty StreamingConfig because no sketch - /// state ever reaches it (e.g. a colouring with only `Logical` - /// nodes). Surfaced as a clean error so callers can fall back to - /// the legacy planner output rather than POST an empty payload. - #[error("backend has no sketch consumers; nothing to wire")] - BackendEmpty, -} - -/// Generic emitter trait — Phase E ships only [`ThreeStageEmitter`]; future -/// phases add `SingleStageEmitter` (asap-query), `ZeroStageEmitter` -/// (asap-fusion), and friends. The trait keeps the dispatch surface -/// uniform so `planner::stage_split` can pick at runtime. -pub trait Emitter { - /// Lower a colored DAG into one [`StageConfig`] per occupied stage. - /// Returns a map keyed by `StageId` for stable consumer access; any - /// stage not occupied in the DAG is omitted. - fn emit_per_stage(&self, dag: &ColoredDag) -> Result, EmitError>; -} - -/// Per-stage emitter output for the DC three-stage topology. -/// -/// The variants are deliberately struct-shaped (named fields) so future -/// downstream consumers can pattern-match without relying on tuple-index -/// stability. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(tag = "stage", rename_all = "snake_case")] -pub enum StageConfig { - /// Edge agent's logical config — what the OpAMP push for this - /// agent will need to materialise into OTel collector YAML. - Edge(EdgeStageConfig), - /// Gateway aggregator's logical config — receivers, sketch merge, - /// onward exporter. - Gateway(GatewayStageConfig), - /// Backend `StreamingConfig` logical content — the - /// (aggregation_id, sketch_type, params) bindings the backend's - /// `OtlpReceiver` + readout catalog need. - Backend(BackendStageConfig), -} - -impl StageConfig { - /// Stage this config corresponds to (mirror of the variant tag). - pub fn stage(&self) -> StageId { - match self { - StageConfig::Edge(_) => StageId::Edge, - StageConfig::Gateway(_) => StageId::Gateway, - StageConfig::Backend(_) => StageId::Backend, - } - } -} - -/// Logical content of an edge agent's per-stage config. -/// -/// Mirrors the surface of `crate::types::AgentCollectorConfig` minus the -/// wire-format details (delta encoding, series-id TTL, sink addressing) -/// — those are emitter-side decisions Phase G+ owns. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct EdgeStageConfig { - /// Source metric name (from the L3 `Scan{Source::TimeSeries}` - /// node). `None` only for synthetic colourings used in tests. - pub source_metric: Option, - /// Equality label filters from `Scan` (`{service="api"}` etc.). - /// Carried as `(label, equals)` tuples — the emitter's wire layer - /// converts them to OTel YAML `attributes/include` matchers. - pub label_filters: Vec<(String, String)>, - /// Window size in seconds, when a `Window` node landed on edge. - pub window_secs: Option, - /// Sketch processor list — one per `SketchAgg` rooted at edge. - /// For the canonical KLL quantile DAG this is exactly one entry. - pub sketch_processors: Vec, - /// OTLP exporter target — the gateway endpoint. Phase E does not - /// resolve a concrete address (no `DeploymentConstraints` plumbed - /// in); emitters produce the abstract `Self` and downstream code - /// fills in `gateway:4317` / similar. - pub exporter_target: ExportTarget, - /// Phase ε.1 — Mode 3 routing destinations, when one or more - /// `RawAtEdgePrometheusArchive` nodes coloured to this edge stage. - /// Each entry produces a separate `otlphttp/prometheus` exporter + - /// pipeline tagged `asap.mode=prometheus_archive` so the agent's - /// routing processor dispatches per-metric. - /// - /// Empty list = no Mode 3 metrics → no `otlphttp/prometheus` - /// exporter is emitted (the YAML is identical to Phase β). - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub prometheus_archive_metrics: Vec, - /// Phase 3.2.5 — archive-tier metrics that should flow through the - /// `gorillas3` processor at the edge agent (write a Gorilla-S3 - /// chunk + Prometheus TSDB block to MinIO so the warm-tier query - /// engine and the Thanos store-gateway can both serve them). - /// - /// Empty list = no archive-tier metrics → no `gorillas3` processor - /// block in the emitted YAML (matches pre-Phase 3.2.5 behaviour - /// for plans that route nothing to the archive). - /// - /// Populated by [`ThreeStageEmitter`] from any `Logical(Scan)` / - /// `RawAtEdgePrometheusArchive` node whose metric is on the - /// archive list (e.g. the freshness probes), and from explicit - /// out-of-DAG opt-ins by callers that don't go through stage-split - /// (the freshness probe path is the canonical example: it doesn't - /// drop a `SketchExpr` node, but the agent still has to land its - /// counter samples in MinIO so the Gorilla-S3 / Thanos archive - /// can answer `last_over_time(...)`). - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub archive_tier_metrics: Vec, - /// Phase 3.2.5 — metrics that must be carried through the - /// warm-tier pipeline WITHOUT the family-specific sketch processor - /// renaming them. The freshness probes are timestamp counters by - /// design (the wire value `unix_ts_ms_of_emission` IS the freshness - /// signal); the DDSketch processor's `_quantile` suffix would - /// rename `http_freshness_probe_warm` to - /// `http_freshness_probe_warm_quantile` and break the replay - /// client's `last_over_time(http_freshness_probe_warm[10s])` query. - /// - /// When non-empty the L5 emitter adds a `routing` processor that - /// dispatches by `metric.name`: matching metrics route to a - /// `metrics/warm_passthrough` pipeline (gorillas3 if archive is - /// declared, then exporter — NO sketch processor); everything - /// else takes the existing `metrics/warm_tier` pipeline. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub warm_passthrough_metrics: Vec, - /// MVP §46 — per-metric → sketch-family mapping populated by the - /// planner from the workload spec. When non-empty, the L5 edge - /// emitter switches to the **5-sketch routing-connector** wire - /// shape: it loads all referenced sketch processors and uses the - /// OTel `routing` *connector* (NOT the deprecated routing - /// processor) to dispatch each metric to the right per-family - /// pipeline. Metrics absent from this map fall through to the - /// `metrics/raw_passthrough` default pipeline. - /// - /// The field is named `metric_to_family` and the value type is - /// [`SketchKind`] — agreed convention with the planner agent - /// shipping in parallel (`SketchFamily` is a controller-side - /// alias for `SketchKind` per `sketch_algebra::params`). Empty - /// map ⇒ legacy single-pipeline / Mode-3 / warm-passthrough wire - /// shapes are emitted unchanged (backward-compat). - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub metric_to_family: HashMap, -} - -/// Phase 3.2.5 — one archive-tier metric the agent should land in -/// MinIO via the `gorillas3` processor (Gorilla-S3 chunks + Prometheus -/// TSDB blocks for the Thanos store-gateway). The `metric` field is -/// used both for the controller-side bookkeeping and (downstream) for -/// the gorillas3 processor's per-metric prefix template — but the -/// processor today flushes every series it sees, so the field is -/// informational at the YAML layer. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct ArchiveTierMetric { - /// Metric name as it appears at the edge. - pub metric: String, - /// Optional flush window in seconds. Mirrors the planner's - /// `gorilla_window_secs` (see `mvp-freshness-probes.yaml`); the - /// L5 emitter uses the smallest non-None entry to size the - /// `gorillas3.window_interval` knob. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub window_secs: Option, -} - -/// Phase ε.1 — one Mode-3 metric the agent forwards to Prometheus's -/// native OTLP receiver. The agent's `routing` processor matches on -/// `attributes["asap.mode"] == "prometheus_archive"` and dispatches to -/// the `otlphttp/prometheus` exporter. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct PrometheusArchiveMetric { - /// Metric name as it appears at the edge. - pub metric: String, - /// Optional window — informational; Prometheus stores raw samples - /// regardless. The L5 emitter uses this to pick a scrape interval - /// consistent with the planner's intent. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub window_secs: Option, - /// Resource-attribute label projection — labels Prometheus's - /// `otlp.promote_resource_attributes` will promote. Defaults to - /// `["service.name", "service.namespace", "service.instance.id"]`. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub label_proj: Vec, -} - -/// One sketch processor configured at an edge agent. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct EdgeSketchProcessor { - /// OTel processor component id — `KLL`, `ddsketch`, `HLL`, - /// `countmin`, etc. Maps 1:1 from `SketchKind`. - pub processor_name: String, - /// Sketch family (mirror of the `SketchAgg::sketch_type` field). - pub sketch_kind: SketchKind, - /// Sketch parameters (mirror of the `SketchAgg::params` field). - pub sketch_params: SketchParams, - /// Stable `aggregation_id` the backend uses to look up the - /// `(sketch_kind, params)` pair when receiving the corresponding - /// OTLP stream. Phase E derives a deterministic id from the - /// processor name + a position counter; downstream callers may - /// override. - pub aggregation_id: String, -} - -/// Logical content of a gateway aggregator's per-stage config. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct GatewayStageConfig { - /// OTLP receiver port — Phase E surfaces the abstract `Default` - /// (`4317`); deployment-specific overrides happen at Phase G. - pub otlp_receiver_port: u16, - /// One merge processor per `SketchMerge` rooted at gateway. - pub merge_processors: Vec, - /// OTLP exporter target — typically the backend's OTLP endpoint. - pub exporter_target: ExportTarget, -} - -/// One sketch-merge processor configured at the gateway. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct GatewayMergeProcessor { - /// OTel processor name — `sketchmergeprocessor`. - pub processor_name: String, - /// Sketch family being merged. All inputs to the merge agree on - /// this (L4 type checker enforces it; design.md §6.4). - pub sketch_kind: SketchKind, - /// Aggregation id — matches the upstream edge's - /// `EdgeSketchProcessor::aggregation_id` so the gateway routes - /// streams correctly. - pub aggregation_id: String, -} - -/// Logical content of the backend `StreamingConfig`. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct BackendStageConfig { - /// One entry per readout query the backend must serve. The - /// `aggregation_id` in each routing entry is the backend's - /// `OtlpReceiver` lookup key. - pub aggregations: Vec, - /// One readout per `SketchEstimate` node — what the backend - /// returns to the inference YAML's PromQL evaluator. - pub readouts: Vec, -} - -/// One sketch source the backend must accept. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct BackendAggregation { - /// Stable id matching the upstream gateway's `aggregation_id`. - pub aggregation_id: String, - /// Sketch family. - pub sketch_kind: SketchKind, - /// Sketch parameters — the backend uses these to build its - /// per-aggregation `Sketch` instance (KLL with the right `k`, - /// DDSketch with the right `alpha`, etc.). - pub sketch_params: SketchParams, - /// Phase ε.1 — what shape the backend ingests for this - /// aggregation. Mode 1 (sketch at edge) / sketch_envelope is the - /// default (the wire payload is a sketch state already). Mode 2 - /// (raw at edge → sketch at backend) sets this to `raw` so the - /// backend builds the sketch from raw OTLP samples at ingest. The - /// backend's `StreamingConfig` consumer interprets the field — - /// Phase ε.2 implements the raw-input ingest path. - #[serde(default)] - pub aggregation_input: AggregationInput, -} - -/// Phase ε.1 — what wire shape the backend ingests for an aggregation. -/// Determines whether the backend builds the sketch from raw samples -/// (Mode 2) or accepts pre-built sketch state from upstream (Mode 1). -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum AggregationInput { - /// Mode 1 — backend receives sketch state envelopes (gateway-merged - /// or direct from edge). The default for legacy plans. - #[default] - SketchEnvelope, - /// Mode 2 — backend receives raw OTLP samples and builds the sketch - /// at ingest. New in Phase ε.1; ingest path lands in Phase ε.2. - Raw, -} - -/// One readout entry — what the backend's inference YAML asks for. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct BackendReadout { - /// Aggregation this readout reads from. - pub aggregation_id: String, - /// Readout op (mirror of `SketchExpr::SketchEstimate::op`). - pub op: EstimateOp, -} - -/// Abstract OTLP / HTTP endpoint description. Phase E does not resolve -/// to a concrete URL — the emitter ships symbolic names that the -/// downstream OpAMP / backend-client wiring (Phase G+) materialises -/// using `DeploymentConstraints::executors()`. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum ExportTarget { - /// Symbolic stage role — "ship to whichever gateway is registered". - Stage(StageId), - /// Concrete endpoint, e.g. `gateway:4317`. - Endpoint(String), -} - -impl Default for ExportTarget { - fn default() -> Self { - ExportTarget::Stage(StageId::Backend) - } -} - -// ── ThreeStageEmitter ───────────────────────────────────────────────────────── - -/// DC lifecycle emitter — colours match `Topology::ThreeStage`. -#[derive(Debug, Default, Clone, Copy)] -pub struct ThreeStageEmitter; - -impl ThreeStageEmitter { - /// Convenience alias for [`Emitter::emit_per_stage`] when callers - /// already hold a `ThreeStageEmitter` value. - pub fn emit(&self, dag: &ColoredDag) -> Result, EmitError> { - self.emit_per_stage(dag) - } -} - -impl Emitter for ThreeStageEmitter { - fn emit_per_stage(&self, dag: &ColoredDag) -> Result, EmitError> { - if dag.topology != Topology::ThreeStage { - return Err(EmitError::UnsupportedTopology( - dag.topology, - Topology::ThreeStage, - )); - } - - // ── Edge config ──────────────────────────────────────────────── - let mut edge = EdgeStageConfig { - source_metric: None, - label_filters: Vec::new(), - window_secs: None, - sketch_processors: Vec::new(), - exporter_target: ExportTarget::Stage(StageId::Gateway), - prometheus_archive_metrics: Vec::new(), - archive_tier_metrics: Vec::new(), - warm_passthrough_metrics: Vec::new(), - metric_to_family: HashMap::new(), - }; - let mut backend_aggregations: Vec = Vec::new(); - let mut gateway_processors: Vec = Vec::new(); - let mut readouts: Vec = Vec::new(); - - // Walk the DAG once, gathering per-stage facts. We rely on the - // node table being depth-first walk order so the SketchAgg / - // SketchEstimate / SketchMerge chain is linkable by position. - // Aggregation ids are deterministic: `agg{N}` per SketchAgg - // index in the DAG. - let mut next_agg_index: usize = 0; - // SketchAgg node id → aggregation_id. Reused by SketchMerge - // (gateway) and SketchEstimate (backend) to thread the same id - // through. - let mut sketch_agg_ids: HashMap = HashMap::new(); - - // Pass 1 — assign deterministic aggregation_ids to every - // SketchAgg up-front so SketchMerge / SketchEstimate emission - // (pass 2) can resolve them regardless of node-table order. - for node in &dag.nodes { - if let (SketchExpr::SketchAgg { .. }, StageId::Edge) = (&node.expr, node.stage) { - let aggregation_id = format!("agg{next_agg_index}"); - next_agg_index += 1; - sketch_agg_ids.insert(node.id.0, aggregation_id); - } - } - - // Pass 2 — emit per-stage facts. - for node in &dag.nodes { - match (&node.expr, node.stage) { - // Edge: source metric + label filters from Logical - // — the wrapped L3 sub-tree may be Scan, Window{Scan}, - // Aggregate{Window{Scan}} etc., so descend recursively. - (SketchExpr::Logical(qe), StageId::Edge) => { - extract_edge_facts(qe, &mut edge); - } - // Edge: SketchAgg becomes one EdgeSketchProcessor. - ( - SketchExpr::SketchAgg { - sketch_type, - params, - .. - }, - StageId::Edge, - ) => { - let processor_name = edge_processor_name(sketch_type)?; - let aggregation_id = sketch_agg_ids - .get(&node.id.0) - .cloned() - .unwrap_or_else(|| format!("agg{}", node.id.0)); - edge.sketch_processors.push(EdgeSketchProcessor { - processor_name, - sketch_kind: sketch_type.clone(), - sketch_params: params.clone(), - aggregation_id: aggregation_id.clone(), - }); - backend_aggregations.push(BackendAggregation { - aggregation_id, - sketch_kind: sketch_type.clone(), - sketch_params: params.clone(), - // Mode 1 — sketch built at edge, ships envelope. - aggregation_input: AggregationInput::SketchEnvelope, - }); - } - // Gateway: SketchMerge over edge sketches → one merge - // processor per merged-sketch family. Aggregation id - // inherited from the merge's first SketchAgg child - // (looked up via the DAG's edges table so identical - // child sub-trees don't collide on a position-by-expr - // search). - (SketchExpr::SketchMerge { .. }, StageId::Gateway) => { - if let Some((kind, aid)) = - first_sketch_child_via_edges(dag, node.id, &sketch_agg_ids) - { - gateway_processors.push(GatewayMergeProcessor { - processor_name: "sketchmergeprocessor".into(), - sketch_kind: kind, - aggregation_id: aid, - }); - } - } - // Backend: SketchEstimate → one readout entry. The - // matching aggregation_id comes from the descendant - // SketchAgg (resolved by walking the DAG edges table). - (SketchExpr::SketchEstimate { op, .. }, StageId::Backend) => { - let aid = resolve_descendant_agg_id_via_edges(dag, node.id, &sketch_agg_ids) - .unwrap_or_else(|| format!("agg{}", readouts.len())); - readouts.push(BackendReadout { - aggregation_id: aid, - op: op.clone(), - }); - } - // ── Phase ε.1 Mode 3: edge raw → Prometheus OTLP receiver. - // Records a `PrometheusArchiveMetric` so the L5 emitter - // adds the `otlphttp/prometheus` exporter + routing - // pipeline. Backend gets a `prometheus_remote` storage - // routing target (no aggregation entry). - ( - SketchExpr::RawAtEdgePrometheusArchive { - metric, - window, - label_proj, - }, - StageId::Edge, - ) => { - edge.prometheus_archive_metrics - .push(PrometheusArchiveMetric { - metric: metric.clone(), - window_secs: window.map(|d| d.as_secs()), - label_proj: label_proj.clone(), - }); - // Phase 3.2.5 (Bug a): Mode-3 metrics also land in - // the Gorilla-S3 archive so the warm-tier - // sketch-engine and the Thanos store-gateway can - // both serve them. The `gorillas3` processor block - // is emitted by the L5 emitter when this list is - // non-empty. - edge.archive_tier_metrics.push(ArchiveTierMetric { - metric: metric.clone(), - window_secs: window.map(|d| d.as_secs()), - }); - } - // ── Phase ε.1 Mode 2: edge raw → backend builds sketch. - // Edge-side: no sketch processor. Backend-side: a - // BackendAggregation with the family the backend will - // build at ingest. The aggregation_input=raw flag is - // emitted by `emit_backend_config_json`. - (SketchExpr::RawAtEdgeSketchAtBackend { family, params, .. }, StageId::Edge) => { - let aid = format!("agg{next_agg_index}"); - next_agg_index += 1; - backend_aggregations.push(BackendAggregation { - aggregation_id: aid, - sketch_kind: family.clone(), - sketch_params: params.clone(), - // Mode 2 — backend builds sketch from raw OTLP. - aggregation_input: AggregationInput::Raw, - }); - } - _ => {} - } - } - - let mut out: HashMap = HashMap::new(); - if dag.occupied_stages().contains(&StageId::Edge) { - out.insert(StageId::Edge, StageConfig::Edge(edge)); - } - if dag.occupied_stages().contains(&StageId::Gateway) { - out.insert( - StageId::Gateway, - StageConfig::Gateway(GatewayStageConfig { - otlp_receiver_port: 4317, - merge_processors: gateway_processors, - exporter_target: ExportTarget::Stage(StageId::Backend), - }), - ); - } - if dag.occupied_stages().contains(&StageId::Backend) { - if backend_aggregations.is_empty() && readouts.is_empty() { - return Err(EmitError::BackendEmpty); - } - out.insert( - StageId::Backend, - StageConfig::Backend(BackendStageConfig { - aggregations: backend_aggregations, - readouts, - }), - ); - } - Ok(out) - } -} - -// ── Helpers ─────────────────────────────────────────────────────────────────── - -/// Map a `SketchKind` to the OTel collector processor name. Mirrors the -/// names the existing OpAMP YAML emitter (and the per-sketch processor -/// crates in `opentelemetry-collector-contrib`) already use. -pub(crate) fn edge_processor_name(kind: &SketchKind) -> Result { - Ok(match kind { - SketchKind::Kll => "KLL".into(), - SketchKind::DDSketch => "ddsketch".into(), - SketchKind::Hll => "HLL".into(), - SketchKind::Cms => "countmin".into(), - SketchKind::CountSketch => "countsketch".into(), - }) -} - -/// Recursively descend an L3 [`crate::intent_algebra::QueryExpr`] -/// gathering edge-stage facts (source metric name, label filters, -/// window size). The L3 sub-tree wrapped in a `SketchExpr::Logical` -/// can be `Scan`, `Window{Scan}`, `Aggregate{Window{Scan}}`, etc., -/// so a recursive descent is necessary to surface the leaf metric. -fn extract_edge_facts(qe: &crate::intent_algebra::QueryExpr, edge: &mut EdgeStageConfig) { - use crate::intent_algebra::{QueryExpr as QE, Source}; - match qe { - QE::Scan { - source, - label_filters, - .. - } => { - if let Source::TimeSeries { metric } = source { - if edge.source_metric.is_none() { - edge.source_metric = Some(metric.clone()); - } - } - for f in label_filters { - let pair = (f.label.clone(), f.equals.clone()); - if !edge.label_filters.contains(&pair) { - edge.label_filters.push(pair); - } - } - } - QE::Window { size, child, .. } => { - if edge.window_secs.is_none() { - edge.window_secs = Some(size.as_secs()); - } - extract_edge_facts(child, edge); - } - QE::Aggregate { child, .. } => { - extract_edge_facts(child, edge); - } - QE::LetBinding { expr, child, .. } => { - extract_edge_facts(expr, edge); - extract_edge_facts(child, edge); - } - QE::Ref { .. } => {} - } -} - -/// Children of `parent` per the DAG's edges table. The colouring walker -/// emits parent → child edges in visit order so this iterator is -/// deterministic. -fn children_of<'a>( - dag: &'a ColoredDag, - parent: crate::stage_split::colored_dag::NodeId, -) -> impl Iterator + 'a { - dag.edges - .iter() - .filter(move |(p, _)| *p == parent) - .map(|(_, c)| *c) -} - -/// First descendant `SketchAgg` reachable from `parent` via the DAG's -/// edges table — used by gateway-merge / backend-readout wiring to -/// pull the right aggregation_id from `sketch_agg_ids`. -fn first_sketch_child_via_edges( - dag: &ColoredDag, - parent: crate::stage_split::colored_dag::NodeId, - sketch_agg_ids: &HashMap, -) -> Option<(SketchKind, String)> { - for cid in children_of(dag, parent) { - let cnode = dag.nodes.get(cid.0)?; - match &cnode.expr { - SketchExpr::SketchAgg { sketch_type, .. } => { - if let Some(aid) = sketch_agg_ids.get(&cid.0) { - return Some((sketch_type.clone(), aid.clone())); - } - } - SketchExpr::LetBinding { .. } | SketchExpr::SketchMerge { .. } => { - if let Some(found) = first_sketch_child_via_edges(dag, cid, sketch_agg_ids) { - return Some(found); - } - } - SketchExpr::Ref { name } => { - // Resolve the ref to its binding's expr id, then recurse. - if let Some(bid) = dag - .nodes - .iter() - .enumerate() - .find_map(|(i, n)| match &n.expr { - SketchExpr::LetBinding { name: n2, .. } if n2 == name => Some(i), - _ => None, - }) - { - let bnode_id = crate::stage_split::colored_dag::NodeId(bid); - if let Some(found) = first_sketch_child_via_edges(dag, bnode_id, sketch_agg_ids) - { - return Some(found); - } - } - } - _ => {} - } - } - None -} - -/// Aggregation_id of the first SketchAgg reachable from `parent` — -/// shorthand around [`first_sketch_child_via_edges`] for the readout -/// path (we only need the id, not the kind). -fn resolve_descendant_agg_id_via_edges( - dag: &ColoredDag, - parent: crate::stage_split::colored_dag::NodeId, - sketch_agg_ids: &HashMap, -) -> Option { - first_sketch_child_via_edges(dag, parent, sketch_agg_ids).map(|(_, aid)| aid) -} diff --git a/controller/src/stage_split/mod.rs b/controller/src/stage_split/mod.rs deleted file mode 100644 index 368d3338..00000000 --- a/controller/src/stage_split/mod.rs +++ /dev/null @@ -1,54 +0,0 @@ -//! Layer 5 — `stage_split` (StageAllocator + Emitter framework). -//! -//! Per `controller/docs/design.md` §6 (line ~765 `core::physical`) and -//! the §6 batched-queries example (line ~1376) — Phase E lands the -//! typed L5 colouring + emitter for the DC three-stage topology -//! (edge → gateway → backend). -//! -//! Pipeline position. L4 [`crate::sketch_algebra::SketchExpr`] is the -//! input — sketch-bound, language-orthogonal, deployment-independent. -//! L5 paints each `SketchExpr` node with a [`StageId`] and emits one -//! [`emitter::StageConfig`] per occupied stage. The configs become the -//! OpAMP `RemoteConfig` payload (edge / gateway) and the backend -//! `StreamingConfig` (backend) — Phase G+ wires the actual push. -//! -//! Module layout: -//! -//! - [`stage_id`] — [`StageId`] enum + [`Topology`] enum. -//! - [`colored_dag`] — [`ColoredDag`] IR (the allocator's output). -//! - [`allocator`] — [`StageAllocator`] + [`AllocateError`]. -//! - [`emitter`] — [`Emitter`] trait + [`ThreeStageEmitter`]. -//! -//! Scope reduction (per orchestrator spec): Phase E ships only the DC -//! lifecycle topology. `Topology::SingleStage` / -//! `Topology::ZeroStage` are surfaced as variants but -//! [`StageAllocator::allocate`] returns -//! [`AllocateError::UnsupportedTopology`] for them — the -//! single-stage / zero-stage allocators land with their respective -//! deployment-model crates (asap-query / asap-fusion) in later phases. -//! -//! Wire-up state. The typed L5 path is opt-in via the -//! `USE_TYPED_STAGE_SPLIT` env var consulted by the existing -//! `planner::stage_split` module — see [`crate::planner::stage_split`] -//! for the additive call path. Existing untyped callers keep working -//! unchanged. - -#![allow(dead_code, unused_imports)] - -pub mod allocator; -pub mod colored_dag; -pub mod emitter; -pub mod stage_id; - -#[cfg(test)] -mod tests; - -// Re-exports — `crate::stage_split::*` for downstream callers. -pub use allocator::{AllocateError, StageAllocator}; -pub use colored_dag::{ColoredDag, ColoredNode, NodeId}; -pub use emitter::{ - ArchiveTierMetric, BackendAggregation, BackendReadout, BackendStageConfig, EdgeSketchProcessor, - EdgeStageConfig, EmitError, Emitter, ExportTarget, GatewayMergeProcessor, GatewayStageConfig, - PrometheusArchiveMetric, StageConfig, ThreeStageEmitter, -}; -pub use stage_id::{StageId, Topology}; diff --git a/controller/src/stage_split/stage_id.rs b/controller/src/stage_split/stage_id.rs deleted file mode 100644 index 92493119..00000000 --- a/controller/src/stage_split/stage_id.rs +++ /dev/null @@ -1,145 +0,0 @@ -//! L5 stage identifiers + topology descriptors. -//! -//! Per `controller/docs/design.md` §6 `core::physical` (around line ~765): -//! -//! - [`StageId`] — the categorical *tier* in the data lifecycle. Topology -//! declares which stages exist; allocator paints `SketchExpr` nodes -//! with one of these. -//! - [`Topology`] — the deployment-model topology shape. Phase E surfaces -//! only [`Topology::ThreeStage`] (DC lifecycle: edge → gateway → -//! backend) per the orchestrator's scope reduction. Single-stage and -//! zero-stage are reserved for asap-query / asap-fusion deployments -//! not in this phase. -//! -//! `StageId` is intentionally an enum (not the `pub struct StageId(pub -//! String)` shape from design.md line 787): Phase E ships only the -//! 3-stage DC topology, so the variants are closed and exhaustive -//! `match`es catch typos at compile time. The string-shaped form from -//! design.md is the right surface once a deployment model registers a -//! custom stage role; until that happens, the enum is the safer choice. - -#![allow(dead_code)] - -use serde::{Deserialize, Serialize}; - -/// L5 stage role — a categorical tier in the data lifecycle. -/// -/// Per design.md §3 (line ~120): "Stage (`StageId`) — a categorical tier -/// in the data lifecycle (edge / gateway / backend / in-process). The -/// topology declares which stages exist (3-stage / 1-stage / 0-stage). -/// Stages are roles, not instances." -/// -/// Multiple `Executor`s may share a `StageId` (e.g. a 50-host edge fleet -/// has 50 executors all carrying `StageId::Edge`). Phase E operates at -/// stage granularity; per-executor fan-out is downstream (Phase G+). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum StageId { - /// Edge / agent collector — per-host scrape + per-host sketch - /// building. Bandwidth claim: KLL / DDSketch / HLL build at the - /// edge ships state across the cut, not raw samples. - Edge, - /// Gateway / aggregation collector — receives N edge streams and - /// merges them. `SketchMerge` lives here under `Topology::ThreeStage`. - Gateway, - /// Backend / readout — query-engine wiring; `SketchEstimate` final - /// readouts; final aggregation roots. - Backend, -} - -impl StageId { - /// Stable lowercase identifier for diagnostics + emitter routing - /// keys. Matches `crate::opamp::AgentRole` strings where possible - /// (`"agent"` ↔ `Edge`; `"backend"` ↔ `Backend`). - pub fn as_str(&self) -> &'static str { - match self { - StageId::Edge => "edge", - StageId::Gateway => "gateway", - StageId::Backend => "backend", - } - } -} - -impl std::fmt::Display for StageId { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(self.as_str()) - } -} - -/// Deployment-model topology descriptor. -/// -/// Per design.md §6 `core::physical::topology` (line ~779): -/// ```ignore -/// pub mod topology { -/// pub struct ThreeStage { /* edge → gateway → backend */ } -/// pub struct SingleStage { /* backend-only */ } -/// pub struct ZeroStage; /* in-process */ -/// } -/// ``` -/// -/// Phase E surfaces only `ThreeStage` (DC lifecycle scope-reduction); -/// the other variants are reserved as future-proofing — adding them is -/// purely additive. -/// -/// (`clippy::enum_variant_names` is silenced — the `*Stage` suffix is -/// part of the design.md naming, not a typo carrying redundant -/// prefix.) -#[allow(clippy::enum_variant_names)] -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum Topology { - /// DC lifecycle topology: edge → gateway → backend. - ThreeStage, - /// Reserved for asap-query (backend-only). - SingleStage, - /// Reserved for asap-fusion (in-process). - ZeroStage, -} - -impl Topology { - /// The set of `StageId`s declared by this topology, in pipeline - /// flow order (upstream first). The allocator uses this to validate - /// every node lands on a declared stage. - pub fn stages(&self) -> &'static [StageId] { - match self { - Topology::ThreeStage => &[StageId::Edge, StageId::Gateway, StageId::Backend], - Topology::SingleStage => &[StageId::Backend], - Topology::ZeroStage => &[], - } - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn stage_id_strings_are_stable() { - assert_eq!(StageId::Edge.as_str(), "edge"); - assert_eq!(StageId::Gateway.as_str(), "gateway"); - assert_eq!(StageId::Backend.as_str(), "backend"); - } - - #[test] - fn three_stage_topology_lists_three_stages_in_order() { - let s = Topology::ThreeStage.stages(); - assert_eq!(s, &[StageId::Edge, StageId::Gateway, StageId::Backend]); - } - - #[test] - fn single_and_zero_stage_topology_shapes() { - assert_eq!(Topology::SingleStage.stages(), &[StageId::Backend]); - assert!(Topology::ZeroStage.stages().is_empty()); - } - - #[test] - fn stage_id_serde_roundtrip() { - for s in [StageId::Edge, StageId::Gateway, StageId::Backend] { - let json = serde_json::to_string(&s).unwrap(); - let back: StageId = serde_json::from_str(&json).unwrap(); - assert_eq!(s, back); - } - } -} diff --git a/controller/src/stage_split/tests.rs b/controller/src/stage_split/tests.rs deleted file mode 100644 index 4331b377..00000000 --- a/controller/src/stage_split/tests.rs +++ /dev/null @@ -1,426 +0,0 @@ -//! Module-level integration tests for the L5 stage_split framework. -//! -//! Per `controller/docs/design.md` §6 batched-queries example -//! (line ~1376) and the §6 single-query trace (line ~1217) — these tests -//! exercise the StageAllocator coloring rules + ThreeStageEmitter output -//! against the canonical inputs the orchestrator's spec calls out. - -#![cfg(test)] - -use std::time::Duration; - -use crate::intent_algebra::schema::{Column, DataType}; -use crate::intent_algebra::{LabelFilter, QueryExpr, Schema, Source, WindowKind}; -use crate::sketch_algebra::params::{ - DDSketchParams, HllParams, KllParams, SketchKind, SketchParams, -}; -use crate::sketch_algebra::sketch_expr::{EstimateOp, MergeAlgebra, SketchExpr}; -use crate::stage_split::allocator::StageAllocator; -use crate::stage_split::emitter::{EmitError, Emitter, StageConfig, ThreeStageEmitter}; -use crate::stage_split::stage_id::{StageId, Topology}; -use crate::types_v2::{AccuracyTarget, BindingName}; - -// ── Test fixtures ───────────────────────────────────────────────────────────── - -fn ts_scan(metric: &str, label: Option<(&str, &str)>) -> QueryExpr { - QueryExpr::Scan { - source: Source::TimeSeries { - metric: metric.into(), - }, - label_filters: label - .map(|(k, v)| { - vec![LabelFilter { - label: k.into(), - equals: v.into(), - }] - }) - .unwrap_or_default(), - schema: Schema::with_time_index( - vec![ - Column { - name: "ts".into(), - dtype: DataType::Timestamp, - nullable: false, - }, - Column { - name: "service".into(), - dtype: DataType::Utf8, - nullable: false, - }, - Column { - name: "value".into(), - dtype: DataType::Float64, - nullable: false, - }, - ], - 0, - vec![vec![0, 1]], - ), - } -} - -fn windowed_scan() -> QueryExpr { - QueryExpr::Window { - kind: WindowKind::Sliding, - size: Duration::from_secs(300), - slide: None, - child: Box::new(ts_scan( - "http_request_duration_seconds", - Some(("service", "api")), - )), - } -} - -/// `SketchEstimate{Quantile{0.99}}{SketchAgg{KLL}{Logical(Window{Scan})}}` -/// — the §6 single-query trace input. -fn quantile_kll_dag() -> SketchExpr { - SketchExpr::estimate_over_agg( - EstimateOp::Quantile { q: 0.99 }, - SketchKind::Kll, - SketchParams::Kll(KllParams { k: 200 }), - windowed_scan(), - ) -} - -// ── Allocator: per-rule + edge-case tests ───────────────────────────────────── - -#[test] -fn allocator_three_stage_basic() { - let dag = StageAllocator - .allocate(&quantile_kll_dag(), Topology::ThreeStage) - .expect("allocate ok"); - // root = SketchEstimate → Backend - assert_eq!(dag.root().unwrap().stage, StageId::Backend); - // 4 nodes total: SketchEstimate, SketchAgg, Logical(Window), - // Logical(Scan) — Phase E walks the Logical(Window) child via the - // SketchAgg path; the inner Scan only surfaces if Logical is - // recursively unfolded. Today Logical wraps the entire L3 sub-tree - // as a single SketchExpr node, so the count is 3. - assert!( - dag.nodes.len() >= 3, - "expected at least 3 nodes, got {}", - dag.nodes.len() - ); - // SketchAgg is colored Edge. - let agg = dag - .nodes - .iter() - .find(|n| matches!(n.expr, SketchExpr::SketchAgg { .. })) - .expect("SketchAgg present"); - assert_eq!(agg.stage, StageId::Edge); - // Logical wrapper of Window is colored Edge. - let win_or_scan = dag - .nodes - .iter() - .find(|n| matches!(n.expr, SketchExpr::Logical(_))) - .expect("Logical present"); - assert_eq!(win_or_scan.stage, StageId::Edge); -} - -#[test] -fn allocator_sketch_agg_under_scan_pinned_edge() { - // Exact design.md §6 invariant: a SketchAgg whose child is a Scan - // (wrapped in Logical) MUST land on Edge. - let expr = SketchExpr::SketchAgg { - sketch_type: SketchKind::Hll, - params: SketchParams::Hll(HllParams { precision: 14 }), - child: Box::new(SketchExpr::Logical(ts_scan("events", None))), - }; - let dag = StageAllocator - .allocate(&expr, Topology::ThreeStage) - .unwrap(); - assert_eq!(dag.root().unwrap().stage, StageId::Edge); - assert_eq!(dag.nodes[1].stage, StageId::Edge); -} - -#[test] -fn allocator_sketch_estimate_pinned_backend() { - // SketchEstimate MUST be on Backend (the readout side). - let dag = StageAllocator - .allocate(&quantile_kll_dag(), Topology::ThreeStage) - .unwrap(); - let est = dag - .nodes - .iter() - .find(|n| matches!(n.expr, SketchExpr::SketchEstimate { .. })) - .expect("SketchEstimate present"); - assert_eq!(est.stage, StageId::Backend); -} - -#[test] -fn allocator_let_binding_color_propagates() { - // LetBinding takes the bound expression's stage. Bind a - // SketchAgg{KLL} (edge) and verify the LetBinding node colors edge. - let inner_agg = SketchExpr::SketchAgg { - sketch_type: SketchKind::Kll, - params: SketchParams::Kll(KllParams { k: 200 }), - child: Box::new(SketchExpr::Logical(windowed_scan())), - }; - let bind = SketchExpr::LetBinding { - name: BindingName::new("kll_state"), - expr: Box::new(inner_agg), - child: Box::new(SketchExpr::SketchEstimate { - op: EstimateOp::Quantile { q: 0.95 }, - child: Box::new(SketchExpr::Ref { - name: BindingName::new("kll_state"), - }), - }), - }; - let dag = StageAllocator - .allocate(&bind, Topology::ThreeStage) - .unwrap(); - let let_node = dag - .nodes - .iter() - .find(|n| matches!(n.expr, SketchExpr::LetBinding { .. })) - .expect("LetBinding present"); - // LetBinding takes its expr's stage → Edge. - assert_eq!(let_node.stage, StageId::Edge); -} - -#[test] -fn allocator_ref_resolves_to_binding_stage() { - // Ref takes the stage of its binding. Same fixture as above; Ref - // child of SketchEstimate must color Edge (the binding's stage). - let inner_agg = SketchExpr::SketchAgg { - sketch_type: SketchKind::Kll, - params: SketchParams::Kll(KllParams { k: 200 }), - child: Box::new(SketchExpr::Logical(windowed_scan())), - }; - let bind = SketchExpr::LetBinding { - name: BindingName::new("shared"), - expr: Box::new(inner_agg), - child: Box::new(SketchExpr::SketchEstimate { - op: EstimateOp::Quantile { q: 0.5 }, - child: Box::new(SketchExpr::Ref { - name: BindingName::new("shared"), - }), - }), - }; - let dag = StageAllocator - .allocate(&bind, Topology::ThreeStage) - .unwrap(); - let ref_node = dag - .nodes - .iter() - .find(|n| matches!(n.expr, SketchExpr::Ref { .. })) - .expect("Ref present"); - assert_eq!(ref_node.stage, StageId::Edge); -} - -#[test] -fn allocator_sketch_merge_lands_gateway() { - // SketchMerge over edge-built KLL sketches → Gateway. - let one_agg = || SketchExpr::SketchAgg { - sketch_type: SketchKind::Kll, - params: SketchParams::Kll(KllParams { k: 200 }), - child: Box::new(SketchExpr::Logical(windowed_scan())), - }; - let merge = SketchExpr::SketchMerge { - algebra: MergeAlgebra::Union, - children: vec![one_agg(), one_agg()], - }; - let with_estimate = SketchExpr::SketchEstimate { - op: EstimateOp::Quantile { q: 0.99 }, - child: Box::new(merge), - }; - let dag = StageAllocator - .allocate(&with_estimate, Topology::ThreeStage) - .unwrap(); - let merge_node = dag - .nodes - .iter() - .find(|n| matches!(n.expr, SketchExpr::SketchMerge { .. })) - .expect("SketchMerge present"); - assert_eq!(merge_node.stage, StageId::Gateway); - assert_eq!(dag.root().unwrap().stage, StageId::Backend); -} - -// ── Emitter tests ───────────────────────────────────────────────────────────── - -#[test] -fn emitter_three_stage_emits_three_configs() { - // Build a DAG with all three stages occupied: SketchEstimate over - // SketchMerge over two SketchAggs. - let one_agg = || SketchExpr::SketchAgg { - sketch_type: SketchKind::Kll, - params: SketchParams::Kll(KllParams { k: 200 }), - child: Box::new(SketchExpr::Logical(windowed_scan())), - }; - let merge = SketchExpr::SketchMerge { - algebra: MergeAlgebra::Union, - children: vec![one_agg(), one_agg()], - }; - let root = SketchExpr::SketchEstimate { - op: EstimateOp::Quantile { q: 0.99 }, - child: Box::new(merge), - }; - let dag = StageAllocator - .allocate(&root, Topology::ThreeStage) - .unwrap(); - let configs = ThreeStageEmitter.emit_per_stage(&dag).unwrap(); - assert!(configs.contains_key(&StageId::Edge)); - assert!(configs.contains_key(&StageId::Gateway)); - assert!(configs.contains_key(&StageId::Backend)); - assert_eq!(configs.len(), 3); -} - -#[test] -fn emitter_edge_config_has_correct_processor_kll() { - let dag = StageAllocator - .allocate(&quantile_kll_dag(), Topology::ThreeStage) - .unwrap(); - let configs = ThreeStageEmitter.emit_per_stage(&dag).unwrap(); - match configs.get(&StageId::Edge).expect("edge config") { - StageConfig::Edge(e) => { - assert_eq!(e.sketch_processors.len(), 1); - assert_eq!(e.sketch_processors[0].processor_name, "KLL"); - assert_eq!(e.sketch_processors[0].sketch_kind, SketchKind::Kll); - assert_eq!( - e.source_metric.as_deref(), - Some("http_request_duration_seconds") - ); - assert_eq!(e.window_secs, Some(300)); - } - other => panic!("expected Edge config, got {other:?}"), - } -} - -#[test] -fn emitter_edge_config_has_correct_processor_ddsketch() { - let expr = SketchExpr::estimate_over_agg( - EstimateOp::Quantile { q: 0.99 }, - SketchKind::DDSketch, - SketchParams::DDSketch(DDSketchParams { alpha: 0.01 }), - windowed_scan(), - ); - let dag = StageAllocator - .allocate(&expr, Topology::ThreeStage) - .unwrap(); - let configs = ThreeStageEmitter.emit_per_stage(&dag).unwrap(); - match configs.get(&StageId::Edge).expect("edge config") { - StageConfig::Edge(e) => { - assert_eq!(e.sketch_processors[0].processor_name, "ddsketch"); - } - other => panic!("expected Edge config, got {other:?}"), - } -} - -#[test] -fn emitter_backend_config_routes_aggregation_id() { - let dag = StageAllocator - .allocate(&quantile_kll_dag(), Topology::ThreeStage) - .unwrap(); - let configs = ThreeStageEmitter.emit_per_stage(&dag).unwrap(); - let edge_aid = match configs.get(&StageId::Edge).unwrap() { - StageConfig::Edge(e) => e.sketch_processors[0].aggregation_id.clone(), - _ => unreachable!(), - }; - match configs.get(&StageId::Backend).expect("backend config") { - StageConfig::Backend(b) => { - assert_eq!(b.aggregations.len(), 1); - assert_eq!(b.aggregations[0].aggregation_id, edge_aid); - assert_eq!(b.aggregations[0].sketch_kind, SketchKind::Kll); - assert_eq!(b.readouts.len(), 1); - assert_eq!(b.readouts[0].aggregation_id, edge_aid); - assert_eq!(b.readouts[0].op, EstimateOp::Quantile { q: 0.99 }); - } - other => panic!("expected Backend config, got {other:?}"), - } -} - -#[test] -fn emitter_unsupported_topology_errors_cleanly() { - // Build a colored DAG that claims SingleStage topology and pass it - // to ThreeStageEmitter — the emitter rejects it cleanly. - let mut dag = StageAllocator - .allocate(&quantile_kll_dag(), Topology::ThreeStage) - .unwrap(); - dag.topology = Topology::SingleStage; - let err = ThreeStageEmitter.emit_per_stage(&dag).unwrap_err(); - assert_eq!( - err, - EmitError::UnsupportedTopology(Topology::SingleStage, Topology::ThreeStage) - ); -} - -// ── End-to-end: design.md §6 batched-queries example ────────────────────────── - -#[test] -fn end_to_end_quantile_workload() { - // Two quantile queries (q=0.99, q=0.95) and one max — the §6 - // batched example. After CSE they share Window+Scan; after sketch - // reuse they share one SketchAgg{KLL}; q3 (Max) takes a separate - // exact path. Phase C/B don't yet wire CSE through the typed path, - // so this test models the post-rule structure by hand. - // - // Structure: - // Backend: SketchEstimate{q=0.99} SketchEstimate{q=0.95} - // \ / - // \ / - // Gateway: SketchMerge{KLL} (and another SketchMerge for q3) - // Edge: SketchAgg{KLL} SketchAgg{KLL} SketchAgg{KLL} - // (Window + Scan shared in real DAG; for the - // test we materialise three Logical wrappers.) - // - // The test asserts the per-stage bucketing matches the design.md - // table (Edge: SketchAgg + Logical(Scan/Window/Aggregate{Max}); - // Gateway: SketchMerge + Merge; Backend: SketchEstimate + final - // root). - let agg = || SketchExpr::SketchAgg { - sketch_type: SketchKind::Kll, - params: SketchParams::Kll(KllParams { k: 200 }), - child: Box::new(SketchExpr::Logical(windowed_scan())), - }; - let merge_kll = SketchExpr::SketchMerge { - algebra: MergeAlgebra::Union, - children: vec![agg(), agg(), agg()], - }; - // Two SketchEstimate readouts hanging off the merge — the typed - // SketchExpr is single-rooted, so we model the workload as the - // higher of the two readouts (q=0.99) and assert the underlying - // colouring is correct. The second readout (q=0.95) is exercised - // by `allocator_let_binding_color_propagates` and the per-rule - // tests above. - let q99 = SketchExpr::SketchEstimate { - op: EstimateOp::Quantile { q: 0.99 }, - child: Box::new(merge_kll), - }; - let dag = StageAllocator.allocate(&q99, Topology::ThreeStage).unwrap(); - let configs = ThreeStageEmitter.emit_per_stage(&dag).unwrap(); - // Edge: 3 SketchAgg processors. - match configs.get(&StageId::Edge).unwrap() { - StageConfig::Edge(e) => { - assert_eq!(e.sketch_processors.len(), 3); - for p in &e.sketch_processors { - assert_eq!(p.processor_name, "KLL"); - } - } - _ => unreachable!(), - } - // Gateway: at least one merge processor. - match configs.get(&StageId::Gateway).unwrap() { - StageConfig::Gateway(g) => { - assert!(!g.merge_processors.is_empty()); - assert_eq!(g.merge_processors[0].processor_name, "sketchmergeprocessor"); - assert_eq!(g.merge_processors[0].sketch_kind, SketchKind::Kll); - } - _ => unreachable!(), - } - // Backend: one readout for q=0.99 + 3 aggregations (one per edge SketchAgg). - match configs.get(&StageId::Backend).unwrap() { - StageConfig::Backend(b) => { - assert_eq!(b.aggregations.len(), 3); - assert_eq!(b.readouts.len(), 1); - assert_eq!(b.readouts[0].op, EstimateOp::Quantile { q: 0.99 }); - } - _ => unreachable!(), - } -} - -// Quiet the unused-import lint when AccuracyTarget is gated only by -// future workflow tests. -#[allow(dead_code)] -fn _force_accuracy_target_use() -> AccuracyTarget { - AccuracyTarget::Epsilon(0.01) -} diff --git a/controller/src/store/mod.rs b/controller/src/store/mod.rs deleted file mode 100644 index 3889c427..00000000 --- a/controller/src/store/mod.rs +++ /dev/null @@ -1,268 +0,0 @@ -pub mod workload; -pub use workload::WorkloadStore; - -use chrono::{DateTime, Utc}; -use std::collections::HashMap; -use std::sync::RwLock; - -use crate::types::CollectionPlan; - -#[derive(Debug)] -pub struct PlanStore { - inner: RwLock, -} - -#[derive(Debug, Default)] -struct StoreInner { - entries: HashMap, -} - -#[derive(Debug, Clone)] -struct Entry { - current: CollectionPlan, - previous: Option, - updated_at: DateTime, -} - -#[derive(Debug, thiserror::Error)] -pub enum StoreError { - #[error("plan not found for metric {0:?}")] - NotFound(String), - #[error("no previous plan for metric {0:?}")] - NoPrevious(String), -} - -impl PlanStore { - pub fn new() -> Self { - Self { - inner: RwLock::new(StoreInner::default()), - } - } - - pub fn set(&self, metric: impl Into, plan: CollectionPlan) { - let metric = metric.into(); - let mut inner = self.inner.write().unwrap(); - match inner.entries.get_mut(&metric) { - None => { - inner.entries.insert( - metric, - Entry { - current: plan, - previous: None, - updated_at: Utc::now(), - }, - ); - } - Some(e) => { - let prev = e.current.clone(); - e.previous = Some(prev); - e.current = plan; - e.updated_at = Utc::now(); - } - } - } - - pub fn get(&self, metric: &str) -> Result { - self.inner - .read() - .unwrap() - .entries - .get(metric) - .map(|e| e.current.clone()) - .ok_or_else(|| StoreError::NotFound(metric.to_string())) - } - - pub fn rollback(&self, metric: &str) -> Result { - let mut inner = self.inner.write().unwrap(); - let e = inner - .entries - .get_mut(metric) - .ok_or_else(|| StoreError::NotFound(metric.to_string()))?; - - let prev = e - .previous - .take() - .ok_or_else(|| StoreError::NoPrevious(metric.to_string()))?; - e.current = prev.clone(); - e.updated_at = Utc::now(); - Ok(prev) - } - - pub fn metrics(&self) -> Vec { - self.inner.read().unwrap().entries.keys().cloned().collect() - } - - pub fn expired(&self, now: DateTime) -> Vec { - self.inner - .read() - .unwrap() - .entries - .iter() - .filter(|(_, e)| e.current.valid_until < now) - .map(|(k, _)| k.clone()) - .collect() - } - - /// Returns a diff between the current and previous plan for `metric`. - /// Returns `None` if no previous plan exists. - pub fn diff(&self, metric: &str) -> Result, StoreError> { - let inner = self.inner.read().unwrap(); - let e = inner.entries.get(metric) - .ok_or_else(|| StoreError::NotFound(metric.to_string()))?; - let Some(prev) = &e.previous else { return Ok(None) }; - let curr = &e.current; - let diff = PlanDiff { - sketch_type_changed: prev.agent_config.sketch_type != curr.agent_config.sketch_type, - prev_sketch_type: prev.agent_config.sketch_type.to_string(), - curr_sketch_type: curr.agent_config.sketch_type.to_string(), - delta_transmission_changed: - prev.agent_config.delta_transmission != curr.agent_config.delta_transmission, - prev_delta_transmission: prev.agent_config.delta_transmission, - curr_delta_transmission: curr.agent_config.delta_transmission, - mode_changed: prev.agent_config.mode != curr.agent_config.mode, - prev_mode: prev.agent_config.mode.to_string(), - curr_mode: curr.agent_config.mode.to_string(), - updated_at: e.updated_at, - }; - Ok(Some(diff)) - } -} - -/// A human-readable summary of what changed between the current and previous plan. -#[derive(Debug, Clone, serde::Serialize)] -pub struct PlanDiff { - pub sketch_type_changed: bool, - pub prev_sketch_type: String, - pub curr_sketch_type: String, - pub delta_transmission_changed: bool, - pub prev_delta_transmission: bool, - pub curr_delta_transmission: bool, - pub mode_changed: bool, - pub prev_mode: String, - pub curr_mode: String, - pub updated_at: DateTime, -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use crate::types::*; - use std::time::Duration; - - fn make_plan(valid_secs: i64) -> CollectionPlan { - let valid_until = Utc::now() + chrono::Duration::seconds(valid_secs); - CollectionPlan { - agent_config: AgentCollectorConfig { - output_mode: OutputMode::Sketch, - sketch_type: SketchType::DDSketch, - sketch_params: Default::default(), - aggregate_by: vec![], - label_matchers: vec![], - window_duration: None, - mode: ProcessorMode::Batch, - enable_self_monitoring: true, - transmit_sketch: true, - drop_original: true, - delta_transmission: false, - delta_threshold: 0.0, - enable_series_id: false, - series_id_ttl_secs: 300, - - data_sink: AgentDataSink::default(), - }, - gateway_config: GatewayCollectorConfig { passthrough: true }, - backend_config: BackendCollectorConfig { - merge_sketch_type: SketchType::DDSketch, - group_by: vec![], - }, - precompute: vec![], - valid_until, - delta_decision: DeltaDecision::default(), - transmission_cost_summary: TransmissionCostSummary::default(), - staged_plan: None, - } - } - - #[test] - fn set_and_get() { - let s = PlanStore::new(); - let plan = make_plan(600); - s.set("latency", plan.clone()); - let got = s.get("latency").unwrap(); - assert_eq!(got.valid_until, plan.valid_until); - } - - #[test] - fn get_not_found() { - let s = PlanStore::new(); - assert!(matches!(s.get("missing"), Err(StoreError::NotFound(_)))); - } - - #[test] - fn rollback() { - let s = PlanStore::new(); - let p1 = make_plan(100); - let p2 = make_plan(200); - s.set("m", p1.clone()); - s.set("m", p2.clone()); - let rolled = s.rollback("m").unwrap(); - assert_eq!(rolled.valid_until, p1.valid_until); - // After rollback, Get should return p1. - assert_eq!(s.get("m").unwrap().valid_until, p1.valid_until); - } - - #[test] - fn rollback_no_previous() { - let s = PlanStore::new(); - s.set("m", make_plan(600)); - assert!(matches!(s.rollback("m"), Err(StoreError::NoPrevious(_)))); - } - - #[test] - fn rollback_not_found() { - let s = PlanStore::new(); - assert!(matches!(s.rollback("x"), Err(StoreError::NotFound(_)))); - } - - #[test] - fn metrics_list() { - let s = PlanStore::new(); - s.set("a", make_plan(600)); - s.set("b", make_plan(600)); - let mut m = s.metrics(); - m.sort(); - assert_eq!(m, vec!["a", "b"]); - } - - #[test] - fn expired() { - let s = PlanStore::new(); - s.set("old", make_plan(-1)); // already expired - s.set("active", make_plan(600)); - let exp = s.expired(Utc::now()); - assert_eq!(exp, vec!["old"]); - } - - #[test] - fn concurrent_access() { - use std::sync::Arc; - let s = Arc::new(PlanStore::new()); - s.set("m", make_plan(600)); - - let handles: Vec<_> = (0..8) - .map(|_| { - let s = Arc::clone(&s); - std::thread::spawn(move || { - for _ in 0..100 { - let _ = s.get("m"); - } - }) - }) - .collect(); - for h in handles { - h.join().unwrap(); - } - } -} diff --git a/controller/src/store/workload.rs b/controller/src/store/workload.rs deleted file mode 100644 index 38113a9a..00000000 --- a/controller/src/store/workload.rs +++ /dev/null @@ -1,87 +0,0 @@ -//! Persists the `QueryWorkload` + `WorkloadCharacteristics` associated with -//! each planned metric so that the re-planner can re-run `plan()` without -//! needing the original `QuerySpec` HTTP payload. -use std::collections::HashMap; -use std::sync::RwLock; - -use crate::types::{QueryWorkload, WorkloadCharacteristics}; - -pub struct WorkloadStore { - inner: RwLock>, -} - -impl WorkloadStore { - pub fn new() -> Self { - Self { inner: RwLock::new(HashMap::new()) } - } - - pub fn set(&self, metric: impl Into, wl: QueryWorkload, wc: WorkloadCharacteristics) { - self.inner.write().unwrap().insert(metric.into(), (wl, wc)); - } - - /// Returns a clone of `(workload, characteristics)` if the metric is known. - pub fn get(&self, metric: &str) -> Option<(QueryWorkload, WorkloadCharacteristics)> { - self.inner.read().unwrap().get(metric).cloned() - } - - pub fn remove(&self, metric: &str) { - self.inner.write().unwrap().remove(metric); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::collections::HashMap; - use std::time::Duration; - use crate::types::AggType; - - fn wl(name: &str) -> QueryWorkload { - QueryWorkload { - metric_name: name.into(), - label_filters: HashMap::new(), - group_by_labels: vec![], - aggregations: vec![AggType::Quantile], - time_window: Duration::from_secs(300), - repeat_every: None, - accuracy_sla: 0.01, - latency_sla: None, - sketch_type_override: None, - exact_required: false, - quantiles: vec![], - } - } - - #[test] - fn set_and_get() { - let s = WorkloadStore::new(); - s.set("latency", wl("latency"), WorkloadCharacteristics::default()); - let (got, _) = s.get("latency").unwrap(); - assert_eq!(got.metric_name, "latency"); - } - - #[test] - fn unknown_metric_returns_none() { - let s = WorkloadStore::new(); - assert!(s.get("nope").is_none()); - } - - #[test] - fn overwrite_replaces() { - let s = WorkloadStore::new(); - s.set("m", wl("m"), WorkloadCharacteristics::default()); - let mut updated = wl("m"); - updated.accuracy_sla = 0.05; - s.set("m", updated, WorkloadCharacteristics::default()); - let (got, _) = s.get("m").unwrap(); - assert_eq!(got.accuracy_sla, 0.05); - } - - #[test] - fn remove_clears_entry() { - let s = WorkloadStore::new(); - s.set("m", wl("m"), WorkloadCharacteristics::default()); - s.remove("m"); - assert!(s.get("m").is_none()); - } -} diff --git a/controller/src/types.rs b/controller/src/types.rs deleted file mode 100644 index f5069982..00000000 --- a/controller/src/types.rs +++ /dev/null @@ -1,647 +0,0 @@ -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use std::time::Duration; - -// ── Workload characteristics ─────────────────────────────────────────────────── - -/// Hint about the statistical distribution of keys in the data stream. -/// Affects fill-rate estimation and therefore delta compression projections. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum DataDistribution { - /// Zipf-distributed keys (s ≈ 1.1). A small number of keys dominate, - /// so only a fraction of sketch cells are touched per window. This is - /// the typical production case. - #[default] - Zipf, - /// All keys are equally probable. Every window fills the sketch more - /// uniformly; delta compression benefit is lower. - Uniform, - /// Traffic arrives in bursts with a concentrated key set. Effective - /// fill rate is lower on average but spikes can reach Uniform levels. - Bursty, -} - -/// Observable characteristics of the incoming data stream. -/// -/// Callers supply these alongside a [`QueryWorkload`] so the planner can -/// compare raw vs. sketch-full vs. sketch-delta transmission costs and -/// estimate the CPU / memory overhead at the SDK or agent collector. -/// -/// All fields have conservative defaults so callers can omit the struct -/// entirely and still get a valid (if pessimistic) decision. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct WorkloadCharacteristics { - /// Number of distinct active time series for this metric. - pub series_count: u64, - /// Sample rate per series at the SDK / agent (Hz). - pub samples_per_sec_per_series: f64, - /// Wire size of one raw OTLP metric data point after protobuf encoding - /// (bytes). Typical range: 50–200 bytes. - pub bytes_per_raw_sample: u32, - /// Known distinct key values per flush period for frequency / cardinality - /// sketches. `None` → inferred analytically from inserts and distribution. - pub distinct_keys_per_window: Option, - /// Statistical distribution of keys in the stream. - pub data_distribution: DataDistribution, - /// Optional memory cap at the SDK / agent collector (bytes). - /// `None` → no budget constraint applied. - pub memory_budget_bytes: Option, -} - -impl Default for WorkloadCharacteristics { - fn default() -> Self { - Self { - series_count: 1_000, - samples_per_sec_per_series: 100.0, - bytes_per_raw_sample: 100, - distinct_keys_per_window: None, - data_distribution: DataDistribution::Zipf, - memory_budget_bytes: None, - } - } -} - -// ── Delta transmission decision ──────────────────────────────────────────────── - -/// Reason the planner chose not to enable delta encoding. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum DeltaSkipReason { - /// Estimated fill rate is so high that delta compression ratio < 2×, - /// making snapshot overhead unjustifiable. - FillRateTooHigh, - /// Snapshot memory for all series × sketches would exceed the configured - /// memory budget at the agent. - MemoryBudgetExceeded, - /// This sketch type has no delta implementation (e.g. KLL). - SketchTypeUnsupported, - /// Compression ratio fell below the minimum acceptable threshold. - CompressionRatioBelowThreshold, -} - -/// Reason the planner chose raw pass-through over sketch transmission. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum RawDataReason { - /// Series count × sample rate is so small that sketch CPU / memory - /// overhead is not justified by the bandwidth savings. - WorkloadTooSmall, -} - -/// The controller's resolved decision on transmission mode. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "mode", rename_all = "snake_case")] -pub enum DeltaDecision { - /// Enable delta-compressed sketch payloads. - UseDelta { - /// Minimum absolute cell change included in a delta payload (T). - threshold: f64, - /// Estimated compression ratio (full_bytes / delta_bytes). - estimated_compression_ratio: f64, - /// Estimated outbound bandwidth with delta enabled (bytes/sec). - estimated_delta_bytes_per_sec: f64, - /// Additional CPU at the agent per sample due to snapshot diff and - /// sparse encoding (µs/sample), amortised over the flush period. - delta_cpu_overhead_micros_per_sample: f64, - /// Additional memory at the agent for storing snapshots (bytes). - delta_memory_overhead_bytes: f64, - }, - /// Transmit full (non-delta) sketch payloads each flush. - UseFullSketch { - reason: DeltaSkipReason, - /// Estimated outbound bandwidth with full sketches (bytes/sec). - estimated_full_bytes_per_sec: f64, - }, - /// Skip sketch aggregation; pass raw OTLP samples through. - UseRaw { - reason: RawDataReason, - /// Estimated outbound bandwidth with raw samples (bytes/sec). - estimated_raw_bytes_per_sec: f64, - }, -} - -impl Default for DeltaDecision { - fn default() -> Self { - DeltaDecision::UseFullSketch { - reason: DeltaSkipReason::CompressionRatioBelowThreshold, - estimated_full_bytes_per_sec: 0.0, - } - } -} - -/// Bandwidth and overhead estimates for all three transmission strategies. -/// Carried on every [`CollectionPlan`] for observability and debugging. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct TransmissionCostSummary { - /// Raw OTLP pass-through bandwidth (bytes/sec). - pub raw_bytes_per_sec: f64, - /// Full-sketch transmission bandwidth (bytes/sec). - pub sketch_full_bytes_per_sec: f64, - /// Delta-sketch transmission bandwidth (bytes/sec); 0 if delta not viable. - pub sketch_delta_bytes_per_sec: f64, - /// Extra CPU at the agent per sample in delta mode (µs/sample). - pub delta_cpu_overhead_micros_per_sample: f64, - /// Extra memory at the agent for delta snapshots (bytes). - pub delta_memory_overhead_bytes: f64, - /// Estimated fill rate (fraction of sketch cells changed per flush). - pub estimated_fill_rate: f64, - /// Flush rate derived from window_duration or repeat_every (Hz). - pub flush_rate_hz: f64, -} - -// ── Enumerations ────────────────────────────────────────────────────────────── - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum AggType { - Quantile, - Cardinality, - Frequency, -} - -impl std::fmt::Display for AggType { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - AggType::Quantile => write!(f, "quantile"), - AggType::Cardinality => write!(f, "cardinality"), - AggType::Frequency => write!(f, "frequency"), - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum SketchType { - DDSketch, - KLL, - HLL, - CountSketch, - CountMinSketch, -} - -impl std::fmt::Display for SketchType { - /// Returns the OTel Collector component type string (must match the Go - /// factory's `component.MustNewType(…)` in each processor). - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - SketchType::DDSketch => write!(f, "ddsketch"), - SketchType::KLL => write!(f, "KLL"), - SketchType::HLL => write!(f, "HLL"), - SketchType::CountSketch => write!(f, "countsketch"), - SketchType::CountMinSketch => write!(f, "countmin"), - } - } -} - -#[derive(Debug, Clone, PartialEq)] -pub enum OutputMode { - Raw, - Sketch, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum ProcessorMode { - Batch, - Window, -} - -impl std::fmt::Display for ProcessorMode { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - ProcessorMode::Batch => write!(f, "batch"), - ProcessorMode::Window => write!(f, "window"), - } - } -} - -// ── Core types ──────────────────────────────────────────────────────────────── - -#[derive(Debug, Clone)] -pub struct QueryWorkload { - pub metric_name: String, - pub label_filters: HashMap, - pub group_by_labels: Vec, - pub aggregations: Vec, - pub time_window: Duration, - pub repeat_every: Option, - pub accuracy_sla: f64, - pub latency_sla: Option, - /// When set, the planner must use this sketch type instead of running - /// the cost model. Allows pinning for collectors that support a subset. - pub sketch_type_override: Option, - /// When true, sketches offer no benefit and the plan must use raw - /// pass-through (SP-2–SP-4 collapse to raw-preservation). - /// Set for stateful per-sample queries (RSI, MACD, stochastic, SUM). - pub exact_required: bool, - /// Quantile φ targets implied by the query (e.g. [0.5] for TWAP, - /// [0.0, 1.0] for price range). Empty for non-quantile workloads. - pub quantiles: Vec, -} - -// ── Sketch defaults (YAML-configurable) ────────────────────────────────────── - -/// Per-sketch-type default parameters. Loaded from a YAML config file at -/// startup; falls back to compile-time defaults when the file is absent. -/// -/// Example `sketch_params_default.yml`: -/// ```yaml -/// quantile_grid: [0.0, 0.25, 0.5, 0.75, 0.9, 0.99, 1.0] -/// ddsketch: -/// relative_accuracy: 0.01 -/// kll: -/// min_k: 32 -/// hll: -/// precision_coarse: 10 -/// precision_fine: 14 -/// precision_threshold: 0.02 -/// count_sketch: -/// epsilon: 0.022 -/// delta: 0.007 -/// count_min_sketch: -/// rows: 5 -/// cols: 2048 -/// metric_name: "countsketch_partition" -/// ``` -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(default)] -pub struct SketchDefaults { - /// Quantile grid used when no query-specific φ values are available. - pub quantile_grid: Vec, - pub ddsketch: DDSketchDefaults, - pub kll: KLLDefaults, - pub hll: HLLDefaults, - pub count_sketch: CountSketchDefaults, - pub count_min_sketch: CountMinSketchDefaults, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(default)] -pub struct DDSketchDefaults { - pub relative_accuracy: f64, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(default)] -pub struct KLLDefaults { - /// Minimum k value (clamped from 1/accuracy_sla). - pub min_k: u32, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(default)] -pub struct HLLDefaults { - /// Precision for coarse SLA (accuracy > threshold). - pub precision_coarse: u32, - /// Precision for fine SLA (accuracy ≤ threshold). - pub precision_fine: u32, - /// SLA boundary between coarse and fine precision. - pub precision_threshold: f64, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(default)] -pub struct CountSketchDefaults { - /// Relative error bound (ε ≈ 1/√cols). - pub epsilon: f64, - /// Error probability (δ ≈ e^(−rows)). - pub delta: f64, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(default)] -pub struct CountMinSketchDefaults { - pub rows: u32, - pub cols: u32, - pub metric_name: String, -} - -impl Default for SketchDefaults { - fn default() -> Self { - Self { - quantile_grid: vec![0.0, 0.25, 0.5, 0.75, 0.9, 0.99, 1.0], - ddsketch: DDSketchDefaults::default(), - kll: KLLDefaults::default(), - hll: HLLDefaults::default(), - count_sketch: CountSketchDefaults::default(), - count_min_sketch: CountMinSketchDefaults::default(), - } - } -} - -impl Default for DDSketchDefaults { - fn default() -> Self { Self { relative_accuracy: 0.01 } } -} - -impl Default for KLLDefaults { - fn default() -> Self { Self { min_k: 32 } } -} - -impl Default for HLLDefaults { - fn default() -> Self { - Self { precision_coarse: 10, precision_fine: 14, precision_threshold: 0.02 } - } -} - -impl Default for CountSketchDefaults { - fn default() -> Self { Self { epsilon: 0.022, delta: 0.007 } } -} - -impl Default for CountMinSketchDefaults { - fn default() -> Self { - Self { rows: 5, cols: 2048, metric_name: "countsketch_partition".into() } - } -} - -impl SketchDefaults { - /// Load from a YAML file, falling back to compiled defaults on any error. - pub fn load(path: &str) -> Self { - match std::fs::read_to_string(path) { - Ok(contents) => serde_yaml::from_str(&contents).unwrap_or_else(|e| { - tracing::warn!(path, error = %e, "invalid sketch_defaults YAML; using built-in defaults"); - Self::default() - }), - Err(_) => Self::default(), - } - } -} - -/// Per-sketch-type parameters. Each variant carries only the fields relevant -/// to that sketch family, avoiding the "bag of unrelated fields" problem. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum SketchParams { - DDSketch { - relative_accuracy: f64, - quantiles: Vec, - }, - KLL { - k: u32, - quantiles: Vec, - }, - HLL { - precision: u32, - }, - CountSketch { - /// Relative error bound (ε). - epsilon: f64, - /// Error probability (δ). - delta: f64, - }, - CountMinSketch { - rows: u32, - cols: u32, - /// Metric name required by the CMS processor. - metric_name: String, - }, -} - -impl Default for SketchParams { - fn default() -> Self { - let d = SketchDefaults::default(); - SketchParams::DDSketch { - relative_accuracy: d.ddsketch.relative_accuracy, - quantiles: d.quantile_grid, - } - } -} - -impl SketchParams { - /// Extract quantiles if this sketch type supports them. - pub fn quantiles(&self) -> &[f64] { - match self { - SketchParams::DDSketch { quantiles, .. } - | SketchParams::KLL { quantiles, .. } => quantiles, - _ => &[], - } - } -} - -#[derive(Debug, Clone)] -pub struct AgentCollectorConfig { - pub output_mode: OutputMode, - pub sketch_type: SketchType, - pub sketch_params: SketchParams, - pub aggregate_by: Vec, - pub label_matchers: Vec, - pub window_duration: Option, - pub mode: ProcessorMode, - pub enable_self_monitoring: bool, - pub transmit_sketch: bool, - pub drop_original: bool, - /// Whether to enable the series ID (UID) registry on the OTLP receiver. - /// When true, the receiver caches metric name + attributes per series and - /// assigns a compact `series_id`. Subsequent exports can omit attributes - /// and send only the ID, saving ~120 bytes per sample. - pub enable_series_id: bool, - /// TTL for series ID cache entries (seconds). 0 = use receiver default. - pub series_id_ttl_secs: u64, - /// Whether the agent processor should enable delta encoding. - /// Set by the delta cost model after sketch type selection. - pub delta_transmission: bool, - /// Minimum absolute cell change included in a delta payload (T). - /// Ignored when `delta_transmission` is false. - pub delta_threshold: f64, - /// Data sink the planner wants the agent to emit to. Decoupled - /// from the planner output (which sketch / window / projection) - /// because where the data goes is a deployment-scope concern, - /// not a planning concern. The previous hardcoded - /// "prometheus exporter on :8889" approach broke the moment we - /// tried to ship sketch types — stock Prometheus exporter - /// silently drops `DDSketchDataPoint` / `HLLSketchDataPoint` - /// etc. — so emit OTLP-to-backend for sketch deployments and - /// keep the prometheus path only for legacy raw-scalar pipelines. - pub data_sink: AgentDataSink, -} - -/// What the agent's collector exports to. -/// -/// `Otlp` — the agent's pipeline ends with an OTLP exporter -/// pointed at the configured endpoint. Required for sketch -/// transport: the modified-OTLP `Data::Ddsketch` / `KLLSketch` / -/// etc. variants are carried natively over OTLP and decoded by -/// the backend's `OtlpReceiver` + the per-sketch -/// `from_sketchlib_proto_bytes` / `from_msgpack_bytes` decoders. -/// -/// `PrometheusScrape` — agent exposes `/metrics` on the listed -/// host:port for an external scraper. Loses sketch types at the -/// translation step; only useful for raw-scalar pipelines. -#[derive(Debug, Clone)] -pub enum AgentDataSink { - /// `endpoint` is an OTLP gRPC endpoint, e.g. `backend:4317`. - /// `compression` is the transport-level codec; the canonical - /// path uses `none` because the backend's tonic gRPC server - /// rejects gzip-compressed bodies (returns Unimplemented). - Otlp { endpoint: String, compression: String }, - /// Pre-existing path: prometheus exporter at `endpoint`. Kept - /// for back-compat with the legacy raw-scalar deployment. - PrometheusScrape { endpoint: String }, -} - -impl Default for AgentDataSink { - /// Default is OTLP-to-backend at the canonical compose - /// hostname. Override per-deployment via the planner's - /// `--agent-data-sink` flag (or future config push). - fn default() -> Self { - AgentDataSink::Otlp { - endpoint: "backend:4317".to_string(), - compression: "none".to_string(), - } - } -} - -#[derive(Debug, Clone)] -pub struct GatewayCollectorConfig { - pub passthrough: bool, -} - -#[derive(Debug, Clone)] -pub struct BackendCollectorConfig { - pub merge_sketch_type: SketchType, - pub group_by: Vec, -} - -#[derive(Debug, Clone)] -pub struct PrecomputeJob { - pub query_expr: String, - pub granularity: Duration, - pub sketch_source: String, - pub store_path: String, -} - -// ── SP-9: per-stage resource budgets ───────────────────────────────────────── - -/// Per-stage resource caps used by `split_expr_by_stage()` (SP-9). -/// -/// When a node's estimated memory cost exceeds the cap at its natural stage, -/// it is deferred to the next stage in the pipeline: -/// -/// `Agent OTel Collector → Backend OTel Collector → Precompute Engine` -/// -/// `None` means unbounded (no cap enforced). Typically sourced from -/// [`WorkloadCharacteristics::memory_budget_bytes`] for the agent stage and -/// from a runtime config file for backend / precompute stages. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct StageResourceBudgets { - /// Max sketch memory at the agent OTel Collector (bytes). - pub agent_memory_bytes: Option, - /// Max sketch-insertion CPU budget at the agent (µs/sample). - pub agent_cpu_micros_per_sample: Option, - /// Max sketch memory at the backend OTel Collector (bytes). - pub backend_memory_bytes: Option, - /// Max memory at the ASAPQuery Precompute Engine (bytes). - pub precompute_memory_bytes: Option, -} - -impl StageResourceBudgets { - /// Derive budgets from [`WorkloadCharacteristics`]: propagates the agent - /// memory cap; other stages default to unbounded. - pub fn from_workload_chars(wc: &WorkloadCharacteristics) -> Self { - Self { - agent_memory_bytes: wc.memory_budget_bytes, - ..Default::default() - } - } -} - -// ── SP-9: per-stage sub-plans ───────────────────────────────────────────────── - -/// Sub-plan for the **Agent OTel Collector** stage. -/// -/// Covers `QueryExpr` nodes: `Source`, `Filter`, `Window`, `Agg` (sketch ops). -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct AgentSubPlan { - /// Concrete sketch type resolved from the `Agg` node (absent when no Agg - /// node was assigned to this stage, e.g. all deferred to Backend). - pub sketch_type: Option, - pub sketch_params: SketchParams, - /// Time window in seconds (from the `Window` node). - pub window_secs: Option, - /// Partition / group-by dimensions if a `Partition` node was pushed down - /// to the agent stage. - pub aggregate_by: Vec, - /// Label-filter predicates as `"key=value"` strings (from `Filter` nodes). - pub label_filters: Vec, - /// True when a `Dedup` node was assigned to this stage. - pub has_dedup: bool, -} - -/// Sub-plan for the **Backend OTel Collector** stage. -/// -/// Covers `QueryExpr` nodes: `Partition`, `Merge`, `Dedup`, and -/// `Agg { Exact(Sum|Count|Min|Max) }` (mergeable exact ops). -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct BackendSubPlan { - /// GROUP BY dimensions from the `Partition` node. - pub group_by: Vec, - /// True when a `Dedup` node was assigned here. - pub has_dedup: bool, - /// True when a `Merge` node is at this stage (expected for all - /// multi-agent deployments). - pub has_merge: bool, -} - -/// Sub-plan for the **ASAPQuery Precompute Engine** stage. -/// -/// Covers `QueryExpr` nodes: `TopK`, and sketch `Agg` ops deferred from -/// the Agent stage due to memory budget overflow. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct PrecomputeSubPlan { - /// Top-K value when a `TopK` node was assigned to this stage. - pub topk: Option, - /// PromQL/SQL expression representing the upper sub-tree assigned here. - /// Empty string when no precompute operations are present. - pub query_expr: String, - /// True when at least one operation was assigned to this stage. - pub active: bool, -} - -/// Sub-plan for **DB-side exact computation** (ClickHouse / TSDB). -/// -/// Covers `Agg { Exact(Avg) }` — non-mergeable; cannot be precomputed across -/// distributed agents without collecting all raw data first. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct DbSubPlan { - /// PromQL/SQL expression for the exact DB-side query. - pub query_expr: String, - /// True when at least one operation was assigned to this stage. - pub active: bool, -} - -/// Result of SP-9 AST-aware stage split. -/// -/// Produced by `planner::stage_split::split_expr_by_stage()`. Attached to -/// [`CollectionPlan::staged_plan`] when the workload was supplied via -/// `query_string` (giving access to the full `QueryExpr` tree). -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct StagedPlan { - pub agent: AgentSubPlan, - pub backend: BackendSubPlan, - pub precompute: PrecomputeSubPlan, - pub db: DbSubPlan, - /// Human-readable log of deferral decisions made during the split - /// (e.g. a sketch op moved from Agent to Backend due to a memory cap). - /// Populated for observability / debugging. - pub deferral_log: Vec, -} - -// ── Collection plan ─────────────────────────────────────────────────────────── - -#[derive(Debug, Clone)] -pub struct CollectionPlan { - pub agent_config: AgentCollectorConfig, - pub gateway_config: GatewayCollectorConfig, - pub backend_config: BackendCollectorConfig, - pub precompute: Vec, - pub valid_until: DateTime, - /// Resolved delta transmission decision and rationale. - pub delta_decision: DeltaDecision, - /// Bandwidth and overhead estimates for all three transmission strategies. - pub transmission_cost_summary: TransmissionCostSummary, - /// SP-9: AST-aware per-stage sub-plans. - /// - /// `Some` when the workload was supplied via `query_string` (full - /// `QueryExpr` tree available). `None` when built from explicit - /// aggregation fields — the SP-3 flat assignment is used as fallback. - pub staged_plan: Option, -} diff --git a/controller/src/types_v2.rs b/controller/src/types_v2.rs deleted file mode 100644 index 64f768f4..00000000 --- a/controller/src/types_v2.rs +++ /dev/null @@ -1,382 +0,0 @@ -//! Typed schema fragments converging toward `controller/docs/design.md`. -//! -//! The design defines a typed `QuerySpec` with `id`, `language`, -//! `accuracy: AccuracyTarget`, `shape: QueryShape`, `data: DataShape`, etc. -//! The current `analyzer::QuerySpec` is a JSON-friendly impl that pre-dates -//! that design. -//! -//! Strategy: **additive extension**. The new types live here and are folded -//! into `analyzer::QuerySpec` as `#[serde(default)] Option<…>` fields so -//! existing JSON callers (the `/plan` HTTP endpoint, planner pre-population -//! from `workloads.yaml`, every test that constructs a `QuerySpec`) keep -//! working byte-for-byte. Code that wants to *consume* the new fields can -//! pattern-match on them; code that doesn't care can ignore them. -//! -//! Nothing in this module is yet load-bearing for cost / binding decisions -//! in `planner/`. That's a separate downstream change once the planner has -//! an L4 rule engine to pivot on `AccuracyTarget` and a stage allocator -//! that respects `QueryShape::Streaming`. - -// Several types in this module (`BindingName`, `WorkloadPlan`, -// `QueryExprPlaceholder`, the `new` / `as_str` helpers on `QueryId` -// and `BindingName`) are intentionally part of the public surface but -// have no in-tree consumers yet — they're targets for the downstream -// PR that wires the planner to consume the typed schema. Suppress the -// dead_code warnings until then so this module doesn't visually -// regress the lint baseline. -#![allow(dead_code)] - -use std::time::Duration; - -use serde::{Deserialize, Serialize}; - -// ── QueryLanguage ───────────────────────────────────────────────────────────── - -/// Source language the raw query string is written in. Drives which L1 -/// parser the controller dispatches to. -/// -/// Today the controller only consumes `PromQL` and `Sql` (see -/// `query_parser/{promql,sql}.rs`); `DataFusion` and `ElasticDsl` are -/// reserved for the future asap-fusion + ElasticDSL deployment models -/// described in `design.md` §3. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] -#[serde(rename_all = "snake_case")] -pub enum QueryLanguage { - /// Prometheus query language. Parsed via `promql-parser`. - #[serde(rename = "prom_ql", alias = "prom_q_l")] - PromQL, - /// SQL. Parsed via `sqlparser`. - Sql, - /// A pre-built DataFusion `LogicalPlan`. Reserved — asap-fusion's - /// L1 happens upstream in the caller's `SessionContext`. - DataFusion, - /// Elasticsearch DSL. Reserved — no L1 parser yet. - ElasticDsl, -} - -// ── AccuracyTarget ──────────────────────────────────────────────────────────── - -/// Per-target accuracy SLA, in the typed form `design.md` §6 calls for. -/// -/// Drives L4 sketch binding (`Exact` disables every `Bind*` rule, so the -/// optimiser falls back to an exact `HashAgg` / `SortAgg`; `Epsilon` and -/// `EpsilonDelta` set the ε / δ budget the cost model has to satisfy when -/// it picks a sketch family + parameters). -/// -/// The legacy `analyzer::QuerySpec.accuracy_sla: f64` field is preserved -/// for back-compat — when a caller supplies a typed `accuracy: Some(…)` -/// it takes precedence; otherwise the analyzer translates the legacy -/// fraction to `Epsilon(1.0 - accuracy_sla)`. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(tag = "kind", content = "value", rename_all = "snake_case")] -pub enum AccuracyTarget { - /// No approximation allowed. L4 must pick an exact path; sketch - /// binding rules are skipped. - Exact, - /// Bound on relative error. The L4 cost model must pick sketch - /// parameters that satisfy `error ≤ eps` with whatever default - /// confidence the sketch family provides. - Epsilon(f64), - /// Bound on relative error and the probability of exceeding it - /// (Pr[error > eps] ≤ delta). Required for sketches whose - /// guarantees are inherently probabilistic (CMS, HLL). - EpsilonDelta { - /// Relative-error bound. - eps: f64, - /// Probability of exceeding the bound. - delta: f64, - }, -} - -impl AccuracyTarget { - /// Translate the legacy `accuracy_sla: f64` field — a fractional - /// "1.0 = exact, 0.0 = anything goes" SLA — into the typed form. - /// `accuracy_sla == 1.0` round-trips to `Exact`; everything else - /// becomes `Epsilon(1.0 - accuracy_sla)` (the implied error bound). - pub fn from_legacy_accuracy_sla(accuracy_sla: f64) -> Self { - if accuracy_sla >= 1.0 { - AccuracyTarget::Exact - } else { - AccuracyTarget::Epsilon((1.0 - accuracy_sla).max(0.0)) - } - } -} - -// ── QueryShape ──────────────────────────────────────────────────────────────── - -/// How the query is *evaluated*: one-shot, continuous, or scheduled. -/// -/// Drives L4 binding (mergeable vs one-shot sketch family) and the L5 -/// wire format (`OneShot` emits config + result; `Streaming` and -/// `Periodic` emit a config that keeps running). Distinct from -/// [`DataShape`] — see `design.md` §6. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum QueryShape { - /// Evaluate once. Plan, execute, return result, discard state. - /// SQL ad-hoc queries; one-off PromQL via `POST /plan`. - #[default] - OneShot, - /// Continuous query — output stream that the executor keeps emitting - /// as new data arrives. No fixed cadence; the runtime emits whenever - /// the underlying state changes. Streaming dashboards, alerting - /// expressions evaluated by the agent rather than by a poller. - Streaming, - /// Re-evaluated at a fixed cadence — Prometheus recording rules, - /// scheduled dashboard panels, alerting evaluation cycles. The - /// planner amortises sketch / aggregate build cost across - /// evaluations within the cadence and reuses state between - /// adjacent windows. - Periodic { - /// Re-evaluation cadence. - every: Duration, - }, -} - -// ── DataShape ───────────────────────────────────────────────────────────────── - -/// Shape of the *data* feeding the query. Workload-level summary; per-leaf -/// detail rides on `Source::data_shape` per `design.md` §6 L3. -/// -/// Drives L4 binding choices: an `AppendOnlyStream` unlocks incremental, -/// mergeable sketches and retraction-free aggregation; `Batch` lets the -/// planner pick a non-mergeable estimator (e.g. exact percentile over a -/// sort) that wouldn't survive a distributed streaming setting; `Mutable` -/// requires retraction-aware operators (out of scope today — the planner -/// refuses sketch binding and falls back to re-scan). -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] -#[serde(rename_all = "snake_case")] -pub enum DataShape { - /// Bounded relation, fully materialised at plan time. SQL tables, - /// Parquet / CSV files, DataFusion in-process tables. - Batch, - /// Append-only stream — events arrive over time, never updated or - /// deleted. Metrics, logs, event streams. The common case for the - /// asap-collector / asap-query deployments and so the controller - /// default when the field is omitted. - #[default] - AppendOnlyStream, - /// Mutable relation — inserts + updates + deletes. Operational - /// databases, CRUD-style tables. Sketch binding is currently - /// refused for this shape (no retraction-aware sketches in the - /// catalog yet). - Mutable, - /// Join across sources of differing shape. The planner consults - /// `Source::data_shape` per leaf during L4; this variant exists so - /// callers don't have to flatten a workload-level summary. - Mixed, -} - -// ── QueryId / BindingName ───────────────────────────────────────────────────── - -/// Stable identifier preserved across `replan` cycles so the runtime can -/// correlate plan outputs with the originating spec, and L4 reuse rules -/// can name shared producers across consumers in the same workload. -/// -/// Idiomatically a string here — the controller already round-trips -/// metric names + agent IDs as strings (see `monitor::Endpoint`, -/// `opamp::AgentRole`), so adding a `Uuid` dependency for one field -/// with no DB-side semantics would be churn for no benefit. The HTTP -/// API is JSON; callers can supply any string they want, including a -/// stringified UUID. When omitted, `Analyzer::analyze` derives a -/// deterministic id from the parsed metric name + accuracy target. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] -#[serde(transparent)] -pub struct QueryId(pub String); - -impl QueryId { - pub fn new(id: impl Into) -> Self { - QueryId(id.into()) - } - - pub fn as_str(&self) -> &str { - &self.0 - } -} - -impl std::fmt::Display for QueryId { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(&self.0) - } -} - -/// Name of a hoisted shared sub-expression in a [`WorkloadPlan`]. Each -/// binding is referenced by ≥2 roots via `QueryExpr::Ref` (`design.md` -/// §6 batched-queries example). -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] -#[serde(transparent)] -pub struct BindingName(pub String); - -impl BindingName { - pub fn new(name: impl Into) -> Self { - BindingName(name.into()) - } - - pub fn as_str(&self) -> &str { - &self.0 - } -} - -impl std::fmt::Display for BindingName { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(&self.0) - } -} - -// ── WorkloadPlan ────────────────────────────────────────────────────────────── - -/// Multi-root DAG container, one level above `QueryExpr` (`design.md` §6 -/// "Multi-root DAGs live one level above `QueryExpr`"). `QueryExpr` -/// stays single-root; this struct holds N roots plus the hoisted -/// bindings the CSE pass shares between them. -/// -/// **Container only — no CSE pass yet.** This type is defined so the -/// `analyzer::QuerySpec.id` field has a target to dock against and so -/// the downstream planner can grow into a `WorkloadPlan` consumer -/// without another schema rev. The `bindings` and `roots` payloads use -/// `String` placeholders for the `QueryExpr` slot; the real `QueryExpr` -/// from `algebra/expr.rs` lacks `Serialize` today, and the design's L3 -/// rewrites — `LetBinding` / `Ref` / sketch-binding split — haven't -/// landed in `algebra/`. When they do, the placeholder becomes a -/// `QueryExpr` and the surrounding plumbing stays put. -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -pub struct WorkloadPlan { - /// Named shared producers, hoisted out of individual queries by - /// the CSE pass. Each binding is referenced by ≥2 roots via - /// `QueryExpr::Ref` once that lowering exists. - pub bindings: Vec<(BindingName, QueryExprPlaceholder)>, - /// One root per `QuerySpec` in the workload, in input order. - pub roots: Vec<(QueryId, QueryExprPlaceholder)>, -} - -/// Placeholder for `QueryExpr` until `algebra::expr::QueryExpr` gets a -/// `Serialize` impl + the L3 rewrites that `WorkloadPlan` consumers -/// expect (CTE-style `LetBinding` / `Ref`, sketch-binding split). Today -/// it's a string carrying the source-level query text or a debug -/// `format!("{qe:?}")` of the algebra tree — enough for the controller -/// to round-trip a `WorkloadPlan` through JSON without losing identity, -/// but not yet enough for L4 to consume. -#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)] -#[serde(transparent)] -pub struct QueryExprPlaceholder(pub String); - -impl QueryExprPlaceholder { - pub fn new(text: impl Into) -> Self { - QueryExprPlaceholder(text.into()) - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn query_language_serde_roundtrip() { - for variant in [ - QueryLanguage::PromQL, - QueryLanguage::Sql, - QueryLanguage::DataFusion, - QueryLanguage::ElasticDsl, - ] { - let json = serde_json::to_string(&variant).unwrap(); - let back: QueryLanguage = serde_json::from_str(&json).unwrap(); - assert_eq!(variant, back, "round-trip failed for {variant:?}"); - } - } - - #[test] - fn accuracy_target_serde_roundtrip() { - let cases = [ - AccuracyTarget::Exact, - AccuracyTarget::Epsilon(0.05), - AccuracyTarget::EpsilonDelta { - eps: 0.01, - delta: 0.001, - }, - ]; - for variant in cases { - let json = serde_json::to_string(&variant).unwrap(); - let back: AccuracyTarget = serde_json::from_str(&json).unwrap(); - assert_eq!(variant, back, "round-trip failed for {variant:?}"); - } - } - - #[test] - fn accuracy_target_from_legacy() { - // 1.0 means exact in the legacy schema. - assert_eq!( - AccuracyTarget::from_legacy_accuracy_sla(1.0), - AccuracyTarget::Exact - ); - // 0.99 SLA → ε = 0.01. - match AccuracyTarget::from_legacy_accuracy_sla(0.99) { - AccuracyTarget::Epsilon(eps) => { - assert!((eps - 0.01).abs() < 1e-9, "got eps={eps}"); - } - other => panic!("expected Epsilon, got {other:?}"), - } - // Out-of-range guard — analyzer rejects these upstream, but the - // helper itself must not panic on a 0.0 SLA. - match AccuracyTarget::from_legacy_accuracy_sla(0.0) { - AccuracyTarget::Epsilon(eps) => assert!((eps - 1.0).abs() < 1e-9), - other => panic!("expected Epsilon, got {other:?}"), - } - } - - #[test] - fn query_shape_serde_roundtrip_and_default() { - assert_eq!(QueryShape::default(), QueryShape::OneShot); - let cases = [ - QueryShape::OneShot, - QueryShape::Streaming, - QueryShape::Periodic { - every: Duration::from_secs(60), - }, - ]; - for variant in cases { - let json = serde_json::to_string(&variant).unwrap(); - let back: QueryShape = serde_json::from_str(&json).unwrap(); - assert_eq!(variant, back, "round-trip failed for {variant:?}"); - } - } - - #[test] - fn data_shape_serde_roundtrip_and_default() { - assert_eq!(DataShape::default(), DataShape::AppendOnlyStream); - for variant in [ - DataShape::Batch, - DataShape::AppendOnlyStream, - DataShape::Mutable, - DataShape::Mixed, - ] { - let json = serde_json::to_string(&variant).unwrap(); - let back: DataShape = serde_json::from_str(&json).unwrap(); - assert_eq!(variant, back, "round-trip failed for {variant:?}"); - } - } - - #[test] - fn query_id_transparent_string_serde() { - let id = QueryId::new("metric-x@0.99"); - let json = serde_json::to_string(&id).unwrap(); - // `#[serde(transparent)]` should serialize as a bare string. - assert_eq!(json, "\"metric-x@0.99\""); - let back: QueryId = serde_json::from_str(&json).unwrap(); - assert_eq!(id, back); - assert_eq!(id.as_str(), "metric-x@0.99"); - } - - #[test] - fn workload_plan_default_is_empty() { - let wp = WorkloadPlan::default(); - assert!(wp.bindings.is_empty()); - assert!(wp.roots.is_empty()); - // Round-trip an empty WorkloadPlan as JSON. - let json = serde_json::to_string(&wp).unwrap(); - let back: WorkloadPlan = serde_json::from_str(&json).unwrap(); - assert!(back.bindings.is_empty()); - assert!(back.roots.is_empty()); - } -} diff --git a/deploy/configs/asap-otel-agent-allsketches.yaml b/deploy/configs/asap-otel-agent-allsketches.yaml index 09469bc1..a9c9f5c7 100644 --- a/deploy/configs/asap-otel-agent-allsketches.yaml +++ b/deploy/configs/asap-otel-agent-allsketches.yaml @@ -79,7 +79,7 @@ extensions: opamp: server: ws: - endpoint: ws://controller:4320/v1/opamp + endpoint: ws://backend:4320/v1/opamp # Header-based agent identification — see # `asap-otel-agent-b3-delta-direct.yaml` for the rationale # (controller's OpAMP server uses X-Agent-ID, not the diff --git a/deploy/configs/asap-otel-agent-b0-prometheus.yaml b/deploy/configs/asap-otel-agent-b0-prometheus.yaml index b81373fc..b15c427a 100644 --- a/deploy/configs/asap-otel-agent-b0-prometheus.yaml +++ b/deploy/configs/asap-otel-agent-b0-prometheus.yaml @@ -24,7 +24,7 @@ extensions: opamp: server: ws: - endpoint: ws://controller:4320/v1/opamp + endpoint: ws://backend:4320/v1/opamp headers: X-Agent-ID: "agent-1" @@ -32,6 +32,7 @@ receivers: otlp: protocols: grpc: + max_recv_msg_size_mib: 4096 endpoint: 0.0.0.0:4317 http: endpoint: 0.0.0.0:4318 @@ -55,7 +56,7 @@ exporters: # service that runs with `--web.enable-remote-write-receiver`). # The base.yml `prometheus` doesn't have remote-write enabled, so # writing there returns 404 + drops every batch. - endpoint: "http://prometheus-b0:9090/api/v1/write" + endpoint: "http://victoriametrics:8428/api/v1/write" tls: insecure: true target_info: diff --git a/deploy/configs/asap-otel-agent-b1-serf-prometheus.yaml b/deploy/configs/asap-otel-agent-b1-serf-prometheus.yaml index c263b9e1..b04a8daa 100644 --- a/deploy/configs/asap-otel-agent-b1-serf-prometheus.yaml +++ b/deploy/configs/asap-otel-agent-b1-serf-prometheus.yaml @@ -34,7 +34,7 @@ extensions: opamp: server: ws: - endpoint: ws://controller:4320/v1/opamp + endpoint: ws://backend:4320/v1/opamp headers: X-Agent-ID: "agent-1" @@ -42,6 +42,7 @@ receivers: otlp: protocols: grpc: + max_recv_msg_size_mib: 4096 endpoint: 0.0.0.0:4317 http: endpoint: 0.0.0.0:4318 @@ -62,7 +63,7 @@ processors: exporters: prometheusremotewrite: - endpoint: "http://prometheus:9090/api/v1/write" + endpoint: "http://victoriametrics:8428/api/v1/write" tls: insecure: true target_info: diff --git a/deploy/configs/asap-otel-agent-b3-delta-direct.yaml b/deploy/configs/asap-otel-agent-b3-delta-direct.yaml index a3548e97..90957267 100644 --- a/deploy/configs/asap-otel-agent-b3-delta-direct.yaml +++ b/deploy/configs/asap-otel-agent-b3-delta-direct.yaml @@ -14,7 +14,7 @@ extensions: opamp: server: ws: - endpoint: ws://controller:4320/v1/opamp + endpoint: ws://backend:4320/v1/opamp # The controller's OpAMP server requires an X-Agent-ID # header to dispatch pushed configs to the right # connection. Stock OTel opampextension carries InstanceUid diff --git a/deploy/configs/asap-otel-agent-b5-gorilla-prometheus.yaml b/deploy/configs/asap-otel-agent-b5-gorilla-prometheus.yaml index 3a77c011..0fb4803c 100644 --- a/deploy/configs/asap-otel-agent-b5-gorilla-prometheus.yaml +++ b/deploy/configs/asap-otel-agent-b5-gorilla-prometheus.yaml @@ -35,7 +35,7 @@ extensions: opamp: server: ws: - endpoint: ws://controller:4320/v1/opamp + endpoint: ws://backend:4320/v1/opamp headers: X-Agent-ID: "agent-1" diff --git a/deploy/configs/asap-otel-agent-b6-asap-single-sketch.yaml b/deploy/configs/asap-otel-agent-b6-asap-single-sketch.yaml index d67723e6..482aaf89 100644 --- a/deploy/configs/asap-otel-agent-b6-asap-single-sketch.yaml +++ b/deploy/configs/asap-otel-agent-b6-asap-single-sketch.yaml @@ -67,7 +67,7 @@ extensions: opamp: server: ws: - endpoint: ws://controller:4320/v1/opamp + endpoint: ws://backend:4320/v1/opamp # Header-based agent identification — see # `asap-otel-agent-b3-delta-direct.yaml` for the rationale # (controller's OpAMP server uses X-Agent-ID, not the @@ -84,7 +84,7 @@ receivers: # 500 user_ids) don't trip the 4 MiB grpc default and get # rejected by the agent. Mirrors the gateway receiver knob and # the controller emit. - max_recv_msg_size_mib: 64 + max_recv_msg_size_mib: 4096 http: endpoint: 0.0.0.0:4318 diff --git a/deploy/configs/asap-otel-agent-b6-gorilla-s3.yaml b/deploy/configs/asap-otel-agent-b6-gorilla-s3.yaml index 3d69b0c7..1d46fc99 100644 --- a/deploy/configs/asap-otel-agent-b6-gorilla-s3.yaml +++ b/deploy/configs/asap-otel-agent-b6-gorilla-s3.yaml @@ -28,7 +28,7 @@ extensions: opamp: server: ws: - endpoint: ws://controller:4320/v1/opamp + endpoint: ws://backend:4320/v1/opamp headers: X-Agent-ID: "agent-1" diff --git a/deploy/configs/asap-otel-agent-cms-direct.yaml b/deploy/configs/asap-otel-agent-cms-direct.yaml index 1bb1c127..db1fa36a 100644 --- a/deploy/configs/asap-otel-agent-cms-direct.yaml +++ b/deploy/configs/asap-otel-agent-cms-direct.yaml @@ -3,7 +3,7 @@ extensions: opamp: server: ws: - endpoint: ws://controller:4320/v1/opamp + endpoint: ws://backend:4320/v1/opamp headers: X-Agent-ID: "agent-1" diff --git a/deploy/configs/asap-otel-agent-cs-direct.yaml b/deploy/configs/asap-otel-agent-cs-direct.yaml index b1a68eff..95c550db 100644 --- a/deploy/configs/asap-otel-agent-cs-direct.yaml +++ b/deploy/configs/asap-otel-agent-cs-direct.yaml @@ -3,7 +3,7 @@ extensions: opamp: server: ws: - endpoint: ws://controller:4320/v1/opamp + endpoint: ws://backend:4320/v1/opamp headers: X-Agent-ID: "agent-1" diff --git a/deploy/configs/asap-otel-agent-hll-direct.yaml b/deploy/configs/asap-otel-agent-hll-direct.yaml index d10660f0..26cd6fef 100644 --- a/deploy/configs/asap-otel-agent-hll-direct.yaml +++ b/deploy/configs/asap-otel-agent-hll-direct.yaml @@ -3,7 +3,7 @@ extensions: opamp: server: ws: - endpoint: ws://controller:4320/v1/opamp + endpoint: ws://backend:4320/v1/opamp headers: X-Agent-ID: "agent-1" diff --git a/deploy/configs/asap-otel-agent-kll-direct.yaml b/deploy/configs/asap-otel-agent-kll-direct.yaml index 241bcd8f..eb0d8feb 100644 --- a/deploy/configs/asap-otel-agent-kll-direct.yaml +++ b/deploy/configs/asap-otel-agent-kll-direct.yaml @@ -6,7 +6,7 @@ extensions: opamp: server: ws: - endpoint: ws://controller:4320/v1/opamp + endpoint: ws://backend:4320/v1/opamp headers: X-Agent-ID: "agent-1" diff --git a/deploy/configs/asap-otel-gateway-mvp-placeholder.yaml b/deploy/configs/asap-otel-gateway-mvp-placeholder.yaml index 57ccaac4..4adf2e6d 100644 --- a/deploy/configs/asap-otel-gateway-mvp-placeholder.yaml +++ b/deploy/configs/asap-otel-gateway-mvp-placeholder.yaml @@ -58,6 +58,10 @@ exporters: endpoint: backend:4317 tls: insecure: true + # Backend's OTLP receiver doesn't decompress gzip — disable client-side + # compression to avoid "rpc error: code = Unimplemented desc = Content is + # compressed with `gzip`" drops. + compression: none sending_queue: enabled: true num_consumers: 4 diff --git a/deploy/configs/backend-storage-routing.yaml b/deploy/configs/backend-storage-routing.yaml index 83d49fdc..bb924066 100644 --- a/deploy/configs/backend-storage-routing.yaml +++ b/deploy/configs/backend-storage-routing.yaml @@ -68,13 +68,34 @@ default: sketch_warm_tier # OR count-on-archive, never both); the multi-target form lets a # single metric serve both classes from the engine each is best at. routes: + # Refactor-2026-05 (Phase 6): universal warm-first / archive-fallthrough + # for raw-named metrics. The warm tier serves any query where a + # registered SketchInstanceMetadata + sketch state hits (Phases 4-5); + # any miss — whether the controller hasn't planned for the query + # shape, or the sid is a ghost (registered identity whose state was + # merged away upstream), or backend's cache is empty after restart — + # falls through to gorilla_s3_archive (Thanos forward, raw archive). + # + # The previous per-shape allow-list (`[count, topk, rate_post_hoc]`) + # is removed; routing decisions are now entirely shape-agnostic at + # this layer. Per-shape selectivity at the runtime is determined by + # the warm tier's own SidLookup classification (Hit / Ghost / + # Unknown), not by a static YAML allow-list. - metric: http_requests_total targets: - backend: sketch_warm_tier - # default slot — every shape not explicitly claimed below - # routes here (Quantile, Sum, LastOverTime, Other). + # default slot — try warm tier for every shape + - backend: gorilla_s3_archive + # universal fallthrough — runtime engages this slot whenever + # the warm tier returns SidLookup::Ghost or SidLookup::Unknown + # (and on capability mismatch, when the controller's plan + # didn't cover this query shape for this metric). + applies_to_query_shape: [count, topk, rate_post_hoc, quantile, sum, last_over_time, other] + - metric: http_requests_total_latency_ms + targets: + - backend: sketch_warm_tier - backend: gorilla_s3_archive - applies_to_query_shape: [count, topk, rate_post_hoc] + applies_to_query_shape: [count, topk, rate_post_hoc, quantile, sum, last_over_time, other] # The latency-quantile metric stays warm-tier-only. The cold # engine returns NaN for empty / partial 60s windows diff --git a/deploy/configs/backend-streaming.yaml b/deploy/configs/backend-streaming.yaml index 7025fd39..65b89567 100644 --- a/deploy/configs/backend-streaming.yaml +++ b/deploy/configs/backend-streaming.yaml @@ -2,6 +2,14 @@ # the agent emits sketchlib-go's `SketchEnvelope{DDSketchState}` # wire format and the backend's `DDSketchAccumulator` decodes it # directly. +# +# Refactor-2026-05: agent's ddsketchprocessor no longer appends a +# "_quantile" suffix at emit. The metric name on the wire is the raw +# input metric (`http_requests_total_latency_ms`, +# `http_requests_total`); the DDSketch encoding is identified by the +# OTLP pdata variant tag, not by a name suffix. Backend ingests the +# sketch under the original name. PromQL fired against the raw name +# resolves directly against the stored sketch state. aggregations: - aggregationId: 1 aggregationType: DDSketch @@ -10,7 +18,7 @@ aggregations: grouping: [zone] rollup: [rack, node, pod] aggregated: [] - metric: http_requests_total_latency_ms_quantile + metric: http_requests_total_latency_ms parameters: relativeAccuracy: 0.01 windowSize: 30 @@ -23,19 +31,19 @@ aggregations: grouping: [zone] rollup: [rack, node, pod] aggregated: [] - metric: http_requests_total_quantile + metric: http_requests_total parameters: relativeAccuracy: 0.01 windowSize: 30 windowType: tumbling spatialFilter: '' metrics: - http_requests_total_latency_ms_quantile: + http_requests_total_latency_ms: - zone - rack - node - pod - http_requests_total_quantile: + http_requests_total: - zone - rack - node diff --git a/deploy/docker-compose/trace-replay.yml b/deploy/docker-compose/trace-replay.yml index 577eb1ed..8270a1ad 100644 --- a/deploy/docker-compose/trace-replay.yml +++ b/deploy/docker-compose/trace-replay.yml @@ -10,7 +10,7 @@ # -f baseline-b3-delta.yml -f trace-replay.yml up -d # # The CSV is bind-mounted from the host repo at -# `deploy/fake-exporter/traces/`. Drop your own preprocessed +# `deploy/fake-exporter/replay-data/`. Drop your own preprocessed # Google-cluster-trace CSV in there (same schema as # `demo-trace.csv`) and point `EXPORTER_TRACE_FILE` at it to run # real workloads against the baselines. @@ -22,4 +22,4 @@ services: EXPORTER_TRACE_SCALE: "${EXPORTER_TRACE_SCALE:-1.0}" EXPORTER_TRACE_LOOP: "${EXPORTER_TRACE_LOOP:-true}" volumes: - - ../fake-exporter/traces:/trace:ro + - ../fake-exporter/replay-data:/trace:ro diff --git a/deploy/docker/Dockerfile.controller b/deploy/docker/Dockerfile.controller deleted file mode 100644 index 7f0d6979..00000000 --- a/deploy/docker/Dockerfile.controller +++ /dev/null @@ -1,44 +0,0 @@ -# Multi-stage build for the controller binary. Builds cleanly -# from the top-level repo; run: -# -# docker build -f deploy/docker/Dockerfile.controller -t asap/controller:dev . -# -# The `:dev` tag is what `deploy/docker-compose/base.yml` -# references via its `build:` block — you don't normally invoke -# this file directly. -FROM rust:1.90-bookworm AS build -WORKDIR /src - -# The `opamp.proto` build step uses plain `prost_build::compile_protos` -# which requires `protoc` on PATH. (`tonic_build` pulls -# `protoc-bin-vendored` for our `feedback.proto` so doesn't need -# this, but prost_build alone does.) -RUN apt-get update && \ - apt-get install -y --no-install-recommends protobuf-compiler && \ - rm -rf /var/lib/apt/lists/* - -# `tonic-build` invokes `protoc` via the `protoc-bin-vendored` -# crate, so no apt install is needed — but we do need the full -# controller tree (proto + build.rs + src) available in one -# layer because `cargo fetch` alone doesn't exercise the build -# script. Skip the two-stage manifest-cache trick and copy -# everything up front; release builds aren't hot-path. -COPY controller controller -RUN cd controller && cargo build --release --bin controller - -# Runtime image kept small — just the binary + ca-certs for -# outbound HTTPS (controller talks to the backend + any S3 -# endpoint the planner's future raw-data analyzer hits). -FROM debian:bookworm-slim -RUN apt-get update && \ - apt-get install -y --no-install-recommends ca-certificates && \ - rm -rf /var/lib/apt/lists/* -COPY --from=build /src/controller/target/release/controller /usr/local/bin/controller - -ENV CONTROLLER_ADDR=0.0.0.0:8080 \ - CONTROLLER_OPAMP_ADDR=0.0.0.0:4320 \ - CONTROLLER_GRPC_ADDR=0.0.0.0:4321 \ - RUST_LOG=info - -EXPOSE 8080 4320 4321 -ENTRYPOINT ["/usr/local/bin/controller"] diff --git a/deploy/fake-exporter/main.go b/deploy/fake-exporter/main.go index 3a1d0692..3e3a5a03 100644 --- a/deploy/fake-exporter/main.go +++ b/deploy/fake-exporter/main.go @@ -451,6 +451,17 @@ func runSynthetic(ctx context.Context, meter metric.Meter, metricName string) { } labelSets := buildLabelSets(cardinality, zoneVals, rackVals, nodeVals, podVals) + // Multi-producer demos: prepend EXPORTER_PRODUCER_ID as a label so + // each producer's series stay distinct at the backend instead of + // collapsing onto the same {zone,rack,node,pod} buckets. + if pid := os.Getenv("EXPORTER_PRODUCER_ID"); pid != "" { + for i := range labelSets { + labelSets[i] = append( + []attribute.KeyValue{attribute.String("producer_id", pid)}, + labelSets[i]..., + ) + } + } period := time.Duration(float64(time.Second) / freqHz) // Five-sketch MVP workload (issue #46) — emits the four new diff --git a/deploy/fake-exporter/traces/README.md b/deploy/fake-exporter/replay-data/README.md similarity index 97% rename from deploy/fake-exporter/traces/README.md rename to deploy/fake-exporter/replay-data/README.md index 59ae0bcc..7de61fef 100644 --- a/deploy/fake-exporter/traces/README.md +++ b/deploy/fake-exporter/replay-data/README.md @@ -27,7 +27,7 @@ EXPORTER_TRACE_FILE=/trace/demo-trace.csv \ -f base.yml -f agents-N1.yml -f baseline-b3-delta.yml up -d ``` -The compose stack already mounts `deploy/fake-exporter/traces/` +The compose stack already mounts `deploy/fake-exporter/replay-data/` at `/trace/` in each fake-exporter container (see the `baseline-*-trace.yml` overlays). diff --git a/deploy/fake-exporter/traces/demo-trace.csv b/deploy/fake-exporter/replay-data/demo-trace.csv similarity index 100% rename from deploy/fake-exporter/traces/demo-trace.csv rename to deploy/fake-exporter/replay-data/demo-trace.csv diff --git a/deploy/fake-exporter/traces/gen-demo-trace.py b/deploy/fake-exporter/replay-data/gen-demo-trace.py similarity index 100% rename from deploy/fake-exporter/traces/gen-demo-trace.py rename to deploy/fake-exporter/replay-data/gen-demo-trace.py diff --git a/opentelemetry-collector-contrib-patch/processor/countminsketchprocessor/factory.go b/opentelemetry-collector-contrib-patch/processor/countminsketchprocessor/factory.go index 57ec74c5..4501b611 100644 --- a/opentelemetry-collector-contrib-patch/processor/countminsketchprocessor/factory.go +++ b/opentelemetry-collector-contrib-patch/processor/countminsketchprocessor/factory.go @@ -25,7 +25,14 @@ func NewFactory() processor.Factory { func createDefaultConfig() component.Config { return &Config{ - MetricName: "countmin_sketch", + // Refactor-2026-05: empty default — output metric name is + // taken from each envelope's input metric name. The + // `if name == "" { name = p.cfg.MetricName }` fallback in + // shim_helpers.go remains as a safety net for envelopes + // that arrive without a name (shouldn't happen in practice + // since the precompute runtime tags every envelope with the + // input metric name). + MetricName: "", Rows: 5, Columns: 1024, // Power of two required by new lib EnableSelfMonitoring: true, diff --git a/opentelemetry-collector-contrib-patch/processor/countminsketchprocessor/shim_helpers.go b/opentelemetry-collector-contrib-patch/processor/countminsketchprocessor/shim_helpers.go index 0a061530..d4c0dc43 100644 --- a/opentelemetry-collector-contrib-patch/processor/countminsketchprocessor/shim_helpers.go +++ b/opentelemetry-collector-contrib-patch/processor/countminsketchprocessor/shim_helpers.go @@ -148,16 +148,21 @@ func (p *cmsProcessor) encodeTypedSketch(metrics pmetric.MetricSlice, envs []*pr m = metrics.AppendEmpty() m.SetName(name) m.SetUnit("1") - m.SetEmptyCountMinSketch().SetAggregationTemporality( - pmetric.AggregationTemporalityDelta) + parent := m.SetEmptyCountMinSketch() + parent.SetAggregationTemporality(pmetric.AggregationTemporalityDelta) + // Refactor-2026-05: rows/cols are sketch-instance config — + // constant across all DPs in this Metric — so they lift to + // the parent CountMinSketch container, sent ONCE per emit + // instead of duplicated per DataPoint. + parent.SetRows(int32(p.cfg.Rows)) + parent.SetCols(int32(p.cfg.Columns)) byName[name] = m } dp := m.CountMinSketch().DataPoints().AppendEmpty() dp.SetTimestamp(pcommon.Timestamp(env.WindowEndMs * 1_000_000)) labelsToAttrs(env.Labels, dp.Attributes()) - dp.SetSampleCount(env.Count) - dp.SetRows(int32(p.cfg.Rows)) - dp.SetCols(int32(p.cfg.Columns)) + // Refactor-2026-05: per-DP sample_count is removed; it is + // derivable from the sketch row sums at the receiver. dp.SetSketch(env.Payload) dp.SetEncoding(cmsEncodingFor(env.Encoding, p.cfg.Encoding)) } diff --git a/opentelemetry-collector-contrib-patch/processor/countsketchprocessor/shim_helpers.go b/opentelemetry-collector-contrib-patch/processor/countsketchprocessor/shim_helpers.go index 51a9a689..05518e47 100644 --- a/opentelemetry-collector-contrib-patch/processor/countsketchprocessor/shim_helpers.go +++ b/opentelemetry-collector-contrib-patch/processor/countsketchprocessor/shim_helpers.go @@ -181,12 +181,19 @@ func (p *countSketchProcessor) encodeSketchMetrics(envs []*precompute.SketchEnve // stampDPMetadata walks the encoded pmetric output in encode-order // (groupOrder by ResourceLabels, envelopes within group preserved) -// and copies legacy-typed CountSketchDataPoint fields onto each DP: -// Dimension (the partition key), Epsilon, Delta, and the -// PROTO_FULL-vs-PROTO_DELTA encoding tag (the runtime adapter always -// writes Proto). Also stamps AggregationTemporality from the -// envelope onto the parent CountSketch metric. +// and stamps the CountSketch parent's AggregationTemporality plus +// Rows/Cols (sketch matrix dimensions, sent ONCE per Metric emit +// instead of duplicated per DataPoint as of refactor-2026-05) and +// stamps each DP's Encoding tag. +// +// Refactor-2026-05: per-DP `dimension`, `epsilon`, `delta` are +// removed from CountSketchDataPoint: +// - dimension was a legacy partition-key descriptor, now replaced +// by the attribute set on the DP (group-by labels); +// - epsilon/delta are derivable from rows/cols on the parent +// container, so they no longer ride per-DP. func (p *countSketchProcessor) stampDPMetadata(md pmetric.Metrics, envs []*precompute.SketchEnvelope) { + rows, cols := configDimensions(p.config) idx := 0 rms := md.ResourceMetrics() for i := 0; i < rms.Len() && idx < len(envs); i++ { @@ -201,13 +208,12 @@ func (p *countSketchProcessor) stampDPMetadata(md pmetric.Metrics, envs []*preco } cs := m.CountSketch() cs.SetAggregationTemporality(pmetric.AggregationTemporality(envs[idx].AggregationTemporality)) + cs.SetRows(int32(rows)) + cs.SetCols(int32(cols)) dps := cs.DataPoints() for l := 0; l < dps.Len() && idx < len(envs); l++ { dp := dps.At(l) env := envs[idx] - dp.SetDimension(partitionKeyFromEnvelope(env, p.config)) - dp.SetEpsilon(p.config.Epsilon) - dp.SetDelta(p.config.Delta) dp.SetEncoding(hostNeutralToTypedEncoding(env.Encoding, p.config.Encoding)) idx++ } @@ -228,7 +234,15 @@ func (p *countSketchProcessor) encodeGaugeMetrics(envs []*precompute.SketchEnvel sm.Scope().SetName("otelcol/countsketch") for _, env := range envs { m := sm.Metrics().AppendEmpty() - m.SetName(outputMetricName) + // Refactor-2026-05: prefer input metric name from the + // envelope; fall back to outputMetricName only if the + // envelope arrived without a name (shouldn't happen in + // practice). + name := env.MetricName + if name == "" { + name = outputMetricName + } + m.SetName(name) m.SetUnit("1") gauge := m.SetEmptyGauge() dp := gauge.DataPoints().AppendEmpty() diff --git a/opentelemetry-collector-contrib-patch/processor/ddsketchprocessor/config.go b/opentelemetry-collector-contrib-patch/processor/ddsketchprocessor/config.go index 33179a68..1eedd7dc 100644 --- a/opentelemetry-collector-contrib-patch/processor/ddsketchprocessor/config.go +++ b/opentelemetry-collector-contrib-patch/processor/ddsketchprocessor/config.go @@ -91,7 +91,10 @@ func createDefaultConfig() component.Config { WindowDuration: 60 * time.Second, RelativeAccuracy: 0.01, Quantiles: []float64{0.5, 0.9, 0.99}, - MetricSuffix: "_ddsketch", + // Refactor-2026-05: default empty — name is preserved end-to-end. + // Any non-empty value set in YAML is now ignored at emit time + // (see shim_helpers.go) but is still parsed for backward compat. + MetricSuffix: "", EnableSelfMonitoring: true, TransmitSketch: true, // Delta-encoded transmission is the operational default for the diff --git a/opentelemetry-collector-contrib-patch/processor/ddsketchprocessor/config_translate.go b/opentelemetry-collector-contrib-patch/processor/ddsketchprocessor/config_translate.go index e61d3c34..4a661cea 100644 --- a/opentelemetry-collector-contrib-patch/processor/ddsketchprocessor/config_translate.go +++ b/opentelemetry-collector-contrib-patch/processor/ddsketchprocessor/config_translate.go @@ -54,7 +54,8 @@ func toPrecomputeConfig(cfg *Config, metricName string) *precompute.PrecomputeCo DeltaTransmission: cfg.DeltaTransmission, DeltaThreshold: cfg.DeltaThreshold, Encoding: precompute.EncodingProtoFull, - MetricName: metricName + cfg.MetricSuffix, + // Refactor-2026-05: precompute config carries the raw input name. + MetricName: metricName, Temporality: int32(pmetric.AggregationTemporalityDelta), } } diff --git a/opentelemetry-collector-contrib-patch/processor/ddsketchprocessor/shim_helpers.go b/opentelemetry-collector-contrib-patch/processor/ddsketchprocessor/shim_helpers.go index beecf829..f0d41f80 100644 --- a/opentelemetry-collector-contrib-patch/processor/ddsketchprocessor/shim_helpers.go +++ b/opentelemetry-collector-contrib-patch/processor/ddsketchprocessor/shim_helpers.go @@ -132,9 +132,13 @@ func patchEnvelopeMetadata(md pmetric.Metrics, obs []precompute.Observation) { case pmetric.MetricTypeDDSketch: dps := m.DDSketch().DataPoints() temp := int32(m.DDSketch().AggregationTemporality()) + // Refactor-2026-05: Count is no longer carried per-DP; + // the sketch payload itself is the canonical source. + // Only Temporality (still on the parent container) is + // stamped onto observation envelopes here. for n := 0; n < dps.Len(); n++ { + _ = n if idx < len(obs) && obs[idx].Value.Envelope != nil { - obs[idx].Value.Envelope.Count = dps.At(n).Count() obs[idx].Value.Envelope.AggregationTemporality = temp } idx++ @@ -147,10 +151,15 @@ func patchEnvelopeMetadata(md pmetric.Metrics, obs []precompute.Observation) { // stampDPMetadata walks encoded in encode-order (RM groupOrder by // ResourceLabels, envelopes in slice order within each group) and -// copies Count + Temporality from envs onto each DDSketch data point. -// Also strips the runtime scope name so mergeAppend folds sketches -// into the input's empty-scope SM. -func stampDPMetadata(encoded pmetric.Metrics, envs []*precompute.SketchEnvelope) { +// copies Temporality + RelativeAccuracy onto each DDSketch parent +// container. Also strips the runtime scope name so mergeAppend folds +// sketches into the input's empty-scope SM. +// +// Refactor-2026-05: per-DP Count was removed from DDSketchDataPoint; +// it is derivable from the sketch payload, so we no longer stamp it. +// RelativeAccuracy is now a parent-container field set per Metric +// emit from the processor's configured alpha. +func (p *ddsketchProcessor) stampDPMetadata(encoded pmetric.Metrics, envs []*precompute.SketchEnvelope) { idx := 0 rms := encoded.ResourceMetrics() for i := 0; i < rms.Len(); i++ { @@ -167,9 +176,7 @@ func stampDPMetadata(encoded pmetric.Metrics, envs []*precompute.SketchEnvelope) idx++ dst := m.DDSketch() dst.SetAggregationTemporality(pmetric.AggregationTemporality(env.AggregationTemporality)) - if dps := dst.DataPoints(); dps.Len() > 0 { - dps.At(0).SetCount(env.Count) - } + dst.SetRelativeAccuracy(p.cfg.RelativeAccuracy) } } } @@ -177,9 +184,16 @@ func stampDPMetadata(encoded pmetric.Metrics, envs []*precompute.SketchEnvelope) // appendSketchMetrics encodes envs as DDSketch-typed metrics and // merges them into out, in place where possible. +// +// Refactor-2026-05: metric name is PRESERVED from the input metric. +// MetricSuffix is intentionally NOT applied — the sketch type is +// carried by the OTLP pdata.Metric variant tag (DDSketch), so the +// downstream backend can identify the encoding without a name suffix +// and PromQL queries fired against the raw input metric name resolve +// directly against the stored sketch state. func (p *ddsketchProcessor) appendSketchMetrics(out pmetric.Metrics, envs []*precompute.SketchEnvelope, inputName string) { for _, env := range envs { - env.MetricName = inputName + p.cfg.MetricSuffix + env.MetricName = inputName } encoded, err := otelpre.Encode(envs, &otelpre.AdapterConfig{}) if err != nil { @@ -188,7 +202,7 @@ func (p *ddsketchProcessor) appendSketchMetrics(out pmetric.Metrics, envs []*pre } return } - stampDPMetadata(encoded, envs) + p.stampDPMetadata(encoded, envs) mergeAppend(out, encoded) } @@ -204,7 +218,9 @@ func (p *ddsketchProcessor) appendQuantileMetrics(out pmetric.Metrics, envs []*p otelpre.KeyValuesToAttributes(envs[0].ResourceLabels, rm.Resource().Attributes()) sm := rm.ScopeMetrics().AppendEmpty() metric := sm.Metrics().AppendEmpty() - metric.SetName(inputName + p.cfg.MetricSuffix) + // Refactor-2026-05: name preserved from input (sketch encoding lives + // in the pdata variant tag, not in a name suffix). + metric.SetName(inputName) dps := metric.SetEmptyGauge().DataPoints() for _, env := range envs { sk, err := decodeDDSketchEnvelope(env.Payload) diff --git a/opentelemetry-collector-contrib-patch/processor/hllprocessor/encode.go b/opentelemetry-collector-contrib-patch/processor/hllprocessor/encode.go index 41570294..54e5e858 100644 --- a/opentelemetry-collector-contrib-patch/processor/hllprocessor/encode.go +++ b/opentelemetry-collector-contrib-patch/processor/hllprocessor/encode.go @@ -67,37 +67,33 @@ func (p *hllProcessor) encodeTypedSketch(metrics pmetric.MetricSlice, inputName p.logger.Debug("hllprocessor: msgpack re-encode failed; falling back to proto") } } - // Cardinality reconstruction: the legacy emit set - // dp.SetCardinality(uint64(series.sketch.Estimate())) from - // the live sketch object. The runtime drops the live handle - // after Tick, so we maintain a per-(metric, series-attrs) - // snapshot of the current proto state to recompute the - // estimate on each emit. On a full-state envelope we cache - // directly; on a delta-state envelope we apply onto the - // previous cached state (HLL max semantics) and re-cache. + // Refactor-2026-05: per-DP `count`, `cardinality`, and + // `precision` are removed from HLLSketchDataPoint: + // - count is derivable from sketch register state at the + // receiver; + // - cardinality is a precomputed cache of sketch.estimate() + // and would drift from the payload-derived value, so the + // receiver evaluates it on demand; + // - precision lifts to the parent HLLSketch container, sent + // ONCE per Metric emit instead of duplicated per DP. + // The cardinality snapshot cache is still maintained because + // downstream consumers may eventually want it as an attribute, + // but it is no longer stamped onto the typed-sketch DP. cardKey := inputName + "::" + precompute.AttributesKey(env.Labels, nil) - cardinality := p.updateCardSnapshot(cardKey, env.Payload, isDelta) + _ = p.updateCardSnapshot(cardKey, env.Payload, isDelta) if !created { m = metrics.AppendEmpty() m.SetName(p.cardinalityMetricName(inputName)) - m.SetEmptyHLLSketch().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + parent := m.SetEmptyHLLSketch() + parent.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + parent.SetPrecision(uint32(hll.HLLPrecision)) created = true } dp := m.HLLSketch().DataPoints().AppendEmpty() labelsToAttrs(env.Labels, dp.Attributes()) dp.SetTimestamp(now) - // Legacy hllprocessor stamped Count from the accumulator in - // batch mode; in window mode the legacy emit set Count=0 - // (its window store didn't track admit count separately). - // The runtime tracks admitted observations on env.Count, so - // batch parity is preserved (both pipelines set the same - // observation count). Window-mode delta tests don't assert - // on Count. - dp.SetCount(env.Count) - dp.SetCardinality(cardinality) dp.SetSketch(payload) dp.SetEncoding(encTag) - dp.SetPrecision(uint32(hll.HLLPrecision)) } } @@ -171,14 +167,13 @@ func (p *hllProcessor) encodeCardinalityGauge(metrics pmetric.MetricSlice, input } } -// cardinalityMetricName returns the output metric name. Mirrors the -// legacy emit's behavior: MetricSuffix overrides the implicit -// "_hll_cardinality" suffix entirely. +// cardinalityMetricName returns the output metric name. Refactor-2026-05: +// the input metric name is preserved end-to-end. The HLL encoding lives +// in the OTLP pdata.Metric variant tag (HLLSketch) for the +// TransmitSketch=true emit path; the cardinality-gauge emit path keeps +// the same name with the sketch type implicit in pdata. func (p *hllProcessor) cardinalityMetricName(base string) string { - if p.cfg.MetricSuffix != "" { - return base + p.cfg.MetricSuffix - } - return base + "_hll_cardinality" + return base } // labelsToAttrs copies host-neutral KeyValues into a pcommon.Map. diff --git a/opentelemetry-collector-contrib-patch/processor/kllprocessor/encode.go b/opentelemetry-collector-contrib-patch/processor/kllprocessor/encode.go index d3ea4625..fba53f61 100644 --- a/opentelemetry-collector-contrib-patch/processor/kllprocessor/encode.go +++ b/opentelemetry-collector-contrib-patch/processor/kllprocessor/encode.go @@ -35,16 +35,20 @@ func (p *kllProcessor) encodeEnvelopes(metrics pmetric.MetricSlice, inputName st func (p *kllProcessor) encodeTypedSketch(metrics pmetric.MetricSlice, inputName string, envs []*precompute.SketchEnvelope, now pcommon.Timestamp) { m := metrics.AppendEmpty() m.SetName(p.sketchMetricName(inputName)) - m.SetEmptyKLLSketch().SetAggregationTemporality(pmetric.AggregationTemporalityDelta) + parent := m.SetEmptyKLLSketch() + parent.SetAggregationTemporality(pmetric.AggregationTemporalityDelta) + // Refactor-2026-05: KLL parameter k is sent once per Metric emit on + // the parent KLLSketch container instead of per DataPoint. + parent.SetK(uint32(p.cfg.K)) for _, env := range envs { if env == nil || len(env.Payload) == 0 { continue } - dp := m.KLLSketch().DataPoints().AppendEmpty() + dp := parent.DataPoints().AppendEmpty() labelsToAttrs(env.Labels, dp.Attributes()) - dp.Attributes().PutInt("kll.k", int64(p.cfg.K)) dp.SetTimestamp(now) - dp.SetCount(env.Count) + // Refactor-2026-05: per-DP Count is removed; it is derivable + // from the sketch payload at the receiver. dp.SetSketch(env.Payload) dp.SetEncoding(pmetric.KLLSketchEncodingProto) } @@ -97,13 +101,14 @@ func (p *kllProcessor) encodeQuantileGauges(metrics pmetric.MetricSlice, inputNa } // sketchMetricName returns the output metric name for the -// TransmitSketch=true path. Mirrors the legacy emit's behavior: -// MetricSuffix overrides the implicit "_kll" suffix entirely. +// TransmitSketch=true path. Refactor-2026-05: the input metric name +// is preserved end-to-end. The KLL encoding lives in the OTLP +// pdata.Metric variant tag (KLLSketch), so the downstream backend +// can identify the encoding without a name suffix and PromQL fired +// against the raw input metric name resolves directly against the +// stored sketch state. func (p *kllProcessor) sketchMetricName(base string) string { - if p.cfg.MetricSuffix != "" { - return base + p.cfg.MetricSuffix - } - return base + "_kll" + return base } // labelsToAttrs copies host-neutral KeyValues into a pcommon.Map. diff --git a/opentelemetry-collector-patch/internal/cmd/pdatagen/internal/pdata/pmetric_package.go b/opentelemetry-collector-patch/internal/cmd/pdatagen/internal/pdata/pmetric_package.go index 229dda37..00b744a9 100644 --- a/opentelemetry-collector-patch/internal/cmd/pdatagen/internal/pdata/pmetric_package.go +++ b/opentelemetry-collector-patch/internal/cmd/pdatagen/internal/pdata/pmetric_package.go @@ -53,6 +53,10 @@ var pmetric = &Package{ histogram, exponentialHistogram, ddsketch, + kllsketch, + hllsketch, + countsketch, + countminsketch, summary, numberDataPointSlice, numberDataPoint, @@ -62,6 +66,14 @@ var pmetric = &Package{ exponentialHistogramDataPoint, ddsketchDataPointSlice, ddsketchDataPoint, + kllsketchDataPointSlice, + kllsketchDataPoint, + hllsketchDataPointSlice, + hllsketchDataPoint, + countsketchDataPointSlice, + countsketchDataPoint, + countminsketchDataPointSlice, + countminsketchDataPoint, bucketsValues, summaryDataPointSlice, summaryDataPoint, @@ -73,6 +85,10 @@ var pmetric = &Package{ enums: []*proto.Enum{ aggregationTemporalityEnum, ddsketchEncodingEnum, + kllsketchEncodingEnum, + hllsketchEncodingEnum, + countsketchEncodingEnum, + countminsketchEncodingEnum, }, } @@ -243,6 +259,26 @@ var metric = &messageStruct{ protoID: 11, returnMessage: summary, }, + &OneOfMessageValue{ + fieldName: "KLLSketch", + protoID: 14, + returnMessage: kllsketch, + }, + &OneOfMessageValue{ + fieldName: "CountSketch", + protoID: 15, + returnMessage: countsketch, + }, + &OneOfMessageValue{ + fieldName: "CountMinSketch", + protoID: 16, + returnMessage: countminsketch, + }, + &OneOfMessageValue{ + fieldName: "HLLSketch", + protoID: 17, + returnMessage: hllsketch, + }, }, }, &SliceField{ @@ -352,6 +388,121 @@ var ddsketch = &messageStruct{ protoID: 2, returnType: aggregationTemporalityType, }, + &PrimitiveField{ + fieldName: "RelativeAccuracy", + protoID: 3, + protoType: proto.TypeDouble, + }, + }, +} + +var kllsketch = &messageStruct{ + structName: "KLLSketch", + description: "// KLLSketch represents the type of a metric encoded using the KLL quantile sketch algorithm.", + protoName: "KLLSketch", + upstreamProto: "gootlpmetrics.KLLSketch", + fields: []Field{ + &SliceField{ + fieldName: "DataPoints", + protoID: 1, + protoType: proto.TypeMessage, + returnSlice: kllsketchDataPointSlice, + }, + &TypedField{ + fieldName: "AggregationTemporality", + protoID: 2, + returnType: aggregationTemporalityType, + }, + &PrimitiveField{ + fieldName: "K", + protoID: 3, + protoType: proto.TypeUint32, + }, + }, +} + +var hllsketch = &messageStruct{ + structName: "HLLSketch", + description: "// HLLSketch represents the type of a metric encoded using HyperLogLog cardinality estimation.", + protoName: "HLLSketch", + upstreamProto: "gootlpmetrics.HLLSketch", + fields: []Field{ + &SliceField{ + fieldName: "DataPoints", + protoID: 1, + protoType: proto.TypeMessage, + returnSlice: hllsketchDataPointSlice, + }, + &TypedField{ + fieldName: "AggregationTemporality", + protoID: 2, + returnType: aggregationTemporalityType, + }, + &PrimitiveField{ + fieldName: "Precision", + protoID: 3, + protoType: proto.TypeUint32, + }, + }, +} + +var countsketch = &messageStruct{ + structName: "CountSketch", + description: "// CountSketch represents the type of a metric encoded using the CountSketch frequency estimation algorithm.", + protoName: "CountSketch", + upstreamProto: "gootlpmetrics.CountSketch", + fields: []Field{ + &SliceField{ + fieldName: "DataPoints", + protoID: 1, + protoType: proto.TypeMessage, + returnSlice: countsketchDataPointSlice, + }, + &TypedField{ + fieldName: "AggregationTemporality", + protoID: 2, + returnType: aggregationTemporalityType, + }, + &PrimitiveField{ + fieldName: "Rows", + protoID: 3, + protoType: proto.TypeInt32, + }, + &PrimitiveField{ + fieldName: "Cols", + protoID: 4, + protoType: proto.TypeInt32, + }, + }, +} + +var countminsketch = &messageStruct{ + structName: "CountMinSketch", + description: "// CountMinSketch represents the type of a metric encoded using the Count-Min Sketch frequency estimation algorithm.", + protoName: "CountMinSketch", + upstreamProto: "gootlpmetrics.CountMinSketch", + fields: []Field{ + &SliceField{ + fieldName: "DataPoints", + protoID: 1, + protoType: proto.TypeMessage, + returnSlice: countminsketchDataPointSlice, + }, + &TypedField{ + fieldName: "AggregationTemporality", + protoID: 2, + returnType: aggregationTemporalityType, + }, + &PrimitiveField{ + fieldName: "Rows", + protoID: 3, + protoType: proto.TypeInt32, + }, + &PrimitiveField{ + fieldName: "Cols", + protoID: 4, + protoType: proto.TypeInt32, + }, }, } @@ -687,86 +838,251 @@ var ddsketchDataPoint = &messageStruct{ returnType: timestampType, }, &PrimitiveField{ - fieldName: "Count", - protoID: 4, - protoType: proto.TypeFixed64, + fieldName: "Sketch", + protoID: 8, + protoType: proto.TypeBytes, }, - &OneOfField{ - typeName: "DDSketchDataPointSumType", - originFieldName: "Sum", - testValueIdx: 0, - values: []oneOfValue{ - &OneOfPrimitiveValue{ - fieldName: "Double", - protoID: 5, - originFieldName: "SumAsDouble", - protoType: proto.TypeDouble, - }, - &OneOfPrimitiveValue{ - fieldName: "Int", - protoID: 12, - originFieldName: "SumAsInt", - protoType: proto.TypeSFixed64, - }, + &TypedField{ + fieldName: "Encoding", + protoID: 10, + returnType: ddsketchEncodingType, + }, + &SliceField{ + fieldName: "Exemplars", + protoID: 11, + protoType: proto.TypeMessage, + returnSlice: exemplarSlice, + }, + &TypedField{ + fieldName: "Flags", + protoID: 15, + returnType: &TypedType{ + structName: "DataPointFlags", + protoType: proto.TypeUint32, + defaultVal: "0", + testVal: "1", }, }, - &OneOfField{ - typeName: "DDSketchDataPointMinType", - originFieldName: "Min", - testValueIdx: 0, - values: []oneOfValue{ - &OneOfPrimitiveValue{ - fieldName: "Double", - protoID: 6, - originFieldName: "MinAsDouble", - protoType: proto.TypeDouble, - }, - &OneOfPrimitiveValue{ - fieldName: "Int", - protoID: 13, - originFieldName: "MinAsInt", - protoType: proto.TypeSFixed64, - }, + }, +} + +var kllsketchDataPointSlice = &messageSlice{ + structName: "KLLSketchDataPointSlice", + elementNullable: true, + element: kllsketchDataPoint, +} + +var kllsketchDataPoint = &messageStruct{ + structName: "KLLSketchDataPoint", + description: "// KLLSketchDataPoint is a single data point that encodes a distribution using the KLL sketch format.", + protoName: "KLLSketchDataPoint", + upstreamProto: "gootlpmetrics.KLLSketchDataPoint", + fields: []Field{ + &SliceField{ + fieldName: "Attributes", + protoID: 1, + protoType: proto.TypeMessage, + returnSlice: mapStruct, + }, + &TypedField{ + fieldName: "StartTimestamp", + originFieldName: "StartTimeUnixNano", + protoID: 2, + returnType: timestampType, + }, + &TypedField{ + fieldName: "Timestamp", + originFieldName: "TimeUnixNano", + protoID: 3, + returnType: timestampType, + }, + &PrimitiveField{ + fieldName: "Sketch", + protoID: 8, + protoType: proto.TypeBytes, + }, + &TypedField{ + fieldName: "Encoding", + protoID: 9, + returnType: kllsketchEncodingType, + }, + &TypedField{ + fieldName: "Flags", + protoID: 10, + returnType: &TypedType{ + structName: "DataPointFlags", + protoType: proto.TypeUint32, + defaultVal: "0", + testVal: "1", }, }, - &OneOfField{ - typeName: "DDSketchDataPointMaxType", - originFieldName: "Max", - testValueIdx: 0, - values: []oneOfValue{ - &OneOfPrimitiveValue{ - fieldName: "Double", - protoID: 7, - originFieldName: "MaxAsDouble", - protoType: proto.TypeDouble, - }, - &OneOfPrimitiveValue{ - fieldName: "Int", - protoID: 14, - originFieldName: "MaxAsInt", - protoType: proto.TypeSFixed64, - }, + &PrimitiveField{ + fieldName: "SeriesID", + protoID: 11, + protoType: proto.TypeUint64, + }, + }, +} + +var hllsketchDataPointSlice = &messageSlice{ + structName: "HLLSketchDataPointSlice", + elementNullable: true, + element: hllsketchDataPoint, +} + +var hllsketchDataPoint = &messageStruct{ + structName: "HLLSketchDataPoint", + description: "// HLLSketchDataPoint is a single data point that encodes cardinality estimations using HyperLogLog.", + protoName: "HLLSketchDataPoint", + upstreamProto: "gootlpmetrics.HLLSketchDataPoint", + fields: []Field{ + &SliceField{ + fieldName: "Attributes", + protoID: 1, + protoType: proto.TypeMessage, + returnSlice: mapStruct, + }, + &TypedField{ + fieldName: "StartTimestamp", + originFieldName: "StartTimeUnixNano", + protoID: 2, + returnType: timestampType, + }, + &TypedField{ + fieldName: "Timestamp", + originFieldName: "TimeUnixNano", + protoID: 3, + returnType: timestampType, + }, + &PrimitiveField{ + fieldName: "Sketch", + protoID: 6, + protoType: proto.TypeBytes, + }, + &TypedField{ + fieldName: "Encoding", + protoID: 7, + returnType: hllsketchEncodingType, + }, + &TypedField{ + fieldName: "Flags", + protoID: 9, + returnType: &TypedType{ + structName: "DataPointFlags", + protoType: proto.TypeUint32, + defaultVal: "0", + testVal: "1", }, }, + &PrimitiveField{ + fieldName: "SeriesID", + protoID: 10, + protoType: proto.TypeUint64, + }, + }, +} + +var countsketchDataPointSlice = &messageSlice{ + structName: "CountSketchDataPointSlice", + elementNullable: true, + element: countsketchDataPoint, +} + +var countsketchDataPoint = &messageStruct{ + structName: "CountSketchDataPoint", + description: "// CountSketchDataPoint is a single data point that encodes frequency estimations using CountSketch.", + protoName: "CountSketchDataPoint", + upstreamProto: "gootlpmetrics.CountSketchDataPoint", + fields: []Field{ + &SliceField{ + fieldName: "Attributes", + protoID: 1, + protoType: proto.TypeMessage, + returnSlice: mapStruct, + }, + &TypedField{ + fieldName: "StartTimestamp", + originFieldName: "StartTimeUnixNano", + protoID: 2, + returnType: timestampType, + }, + &TypedField{ + fieldName: "Timestamp", + originFieldName: "TimeUnixNano", + protoID: 3, + returnType: timestampType, + }, &PrimitiveField{ fieldName: "Sketch", - protoID: 8, + protoID: 4, protoType: proto.TypeBytes, }, &TypedField{ fieldName: "Encoding", - protoID: 10, - returnType: ddsketchEncodingType, + protoID: 5, + returnType: countsketchEncodingType, + }, + &TypedField{ + fieldName: "Flags", + protoID: 9, + returnType: &TypedType{ + structName: "DataPointFlags", + protoType: proto.TypeUint32, + defaultVal: "0", + testVal: "1", + }, }, + &PrimitiveField{ + fieldName: "SeriesID", + protoID: 10, + protoType: proto.TypeUint64, + }, + }, +} + +var countminsketchDataPointSlice = &messageSlice{ + structName: "CountMinSketchDataPointSlice", + elementNullable: true, + element: countminsketchDataPoint, +} + +var countminsketchDataPoint = &messageStruct{ + structName: "CountMinSketchDataPoint", + description: "// CountMinSketchDataPoint is a single data point that encodes frequency estimations using Count-Min Sketch.", + protoName: "CountMinSketchDataPoint", + upstreamProto: "gootlpmetrics.CountMinSketchDataPoint", + fields: []Field{ &SliceField{ - fieldName: "Exemplars", - protoID: 11, + fieldName: "Attributes", + protoID: 1, protoType: proto.TypeMessage, - returnSlice: exemplarSlice, + returnSlice: mapStruct, + }, + &TypedField{ + fieldName: "StartTimestamp", + originFieldName: "StartTimeUnixNano", + protoID: 2, + returnType: timestampType, + }, + &TypedField{ + fieldName: "Timestamp", + originFieldName: "TimeUnixNano", + protoID: 3, + returnType: timestampType, + }, + &PrimitiveField{ + fieldName: "Sketch", + protoID: 5, + protoType: proto.TypeBytes, + }, + &TypedField{ + fieldName: "Encoding", + protoID: 6, + returnType: countminsketchEncodingType, }, &TypedField{ fieldName: "Flags", - protoID: 15, + protoID: 9, returnType: &TypedType{ structName: "DataPointFlags", protoType: proto.TypeUint32, @@ -774,6 +1090,11 @@ var ddsketchDataPoint = &messageStruct{ testVal: "1", }, }, + &PrimitiveField{ + fieldName: "SeriesID", + protoID: 10, + protoType: proto.TypeUint64, + }, }, } @@ -958,5 +1279,87 @@ var ddsketchEncodingEnum = &proto.Enum{ Fields: []*proto.EnumField{ {Name: "DDSKETCH_ENCODING_UNSPECIFIED", Value: 0}, {Name: "DDSKETCH_ENCODING_PROTO", Value: 1}, + {Name: "DDSKETCH_ENCODING_PROTO_DELTA", Value: 2}, + {Name: "DDSKETCH_ENCODING_MSGPACK", Value: 3}, + {Name: "DDSKETCH_ENCODING_MSGPACK_DELTA", Value: 4}, + }, +} + +var kllsketchEncodingType = &TypedType{ + structName: "KLLSketchEncoding", + protoType: proto.TypeEnum, + messageName: "KLLSketchEncoding", + defaultVal: "KLLSketchEncoding(0)", + testVal: "KLLSketchEncoding(1)", +} + +var kllsketchEncodingEnum = &proto.Enum{ + Name: "KLLSketchEncoding", + Description: "// KLLSketchEncoding identifies how the KLL sketch payload bytes are encoded.", + Fields: []*proto.EnumField{ + {Name: "KLL_SKETCH_ENCODING_UNSPECIFIED", Value: 0}, + {Name: "KLL_SKETCH_ENCODING_PROTO", Value: 1}, + {Name: "KLL_SKETCH_ENCODING_MSGPACK", Value: 3}, + {Name: "KLL_SKETCH_ENCODING_MSGPACK_DELTA", Value: 4}, + }, +} + +var hllsketchEncodingType = &TypedType{ + structName: "HLLSketchEncoding", + protoType: proto.TypeEnum, + messageName: "HLLSketchEncoding", + defaultVal: "HLLSketchEncoding(0)", + testVal: "HLLSketchEncoding(1)", +} + +var hllsketchEncodingEnum = &proto.Enum{ + Name: "HLLSketchEncoding", + Description: "// HLLSketchEncoding identifies how the HLL sketch payload bytes are encoded.", + Fields: []*proto.EnumField{ + {Name: "HLL_SKETCH_ENCODING_UNSPECIFIED", Value: 0}, + {Name: "HLL_SKETCH_ENCODING_PROTO", Value: 1}, + {Name: "HLL_SKETCH_ENCODING_DELTA", Value: 2}, + {Name: "HLL_SKETCH_ENCODING_MSGPACK", Value: 3}, + {Name: "HLL_SKETCH_ENCODING_MSGPACK_DELTA", Value: 4}, + }, +} + +var countsketchEncodingType = &TypedType{ + structName: "CountSketchEncoding", + protoType: proto.TypeEnum, + messageName: "CountSketchEncoding", + defaultVal: "CountSketchEncoding(0)", + testVal: "CountSketchEncoding(1)", +} + +var countsketchEncodingEnum = &proto.Enum{ + Name: "CountSketchEncoding", + Description: "// CountSketchEncoding identifies how the CountSketch payload bytes are encoded.", + Fields: []*proto.EnumField{ + {Name: "COUNT_SKETCH_ENCODING_UNSPECIFIED", Value: 0}, + {Name: "COUNT_SKETCH_ENCODING_PROTO", Value: 1}, + {Name: "COUNT_SKETCH_ENCODING_DELTA", Value: 2}, + {Name: "COUNT_SKETCH_ENCODING_MSGPACK", Value: 3}, + {Name: "COUNT_SKETCH_ENCODING_MSGPACK_DELTA", Value: 4}, + }, +} + +var countminsketchEncodingType = &TypedType{ + structName: "CountMinSketchEncoding", + protoType: proto.TypeEnum, + messageName: "CountMinSketchEncoding", + defaultVal: "CountMinSketchEncoding(0)", + testVal: "CountMinSketchEncoding(1)", +} + +var countminsketchEncodingEnum = &proto.Enum{ + Name: "CountMinSketchEncoding", + Description: "// CountMinSketchEncoding identifies how the CountMinSketch payload bytes are encoded.", + Fields: []*proto.EnumField{ + {Name: "COUNT_MIN_SKETCH_ENCODING_UNSPECIFIED", Value: 0}, + {Name: "COUNT_MIN_SKETCH_ENCODING_PROTO", Value: 1}, + {Name: "COUNT_MIN_SKETCH_ENCODING_DELTA", Value: 2}, + {Name: "COUNT_MIN_SKETCH_ENCODING_MSGPACK", Value: 3}, + {Name: "COUNT_MIN_SKETCH_ENCODING_MSGPACK_DELTA", Value: 4}, }, } diff --git a/opentelemetry-collector-patch/pdata/internal/generated_proto_countminsketch.go b/opentelemetry-collector-patch/pdata/internal/generated_proto_countminsketch.go index cb9117ac..21d723d3 100644 --- a/opentelemetry-collector-patch/pdata/internal/generated_proto_countminsketch.go +++ b/opentelemetry-collector-patch/pdata/internal/generated_proto_countminsketch.go @@ -14,10 +14,12 @@ import ( "go.opentelemetry.io/collector/pdata/internal/proto" ) -// CountMinSketch represents the type of a metric encoded using Count-Min Sketch frequency estimation. +// CountMinSketch represents the type of a metric encoded using the Count-Min Sketch frequency estimation algorithm. type CountMinSketch struct { DataPoints []*CountMinSketchDataPoint AggregationTemporality AggregationTemporality + Rows int32 + Cols int32 } var ( @@ -56,6 +58,7 @@ func DeleteCountMinSketch(orig *CountMinSketch, nullable bool) { } func CopyCountMinSketch(dest, src *CountMinSketch) *CountMinSketch { + // If copying to same object, just return. if src == dest { return dest } @@ -71,6 +74,10 @@ func CopyCountMinSketch(dest, src *CountMinSketch) *CountMinSketch { dest.AggregationTemporality = src.AggregationTemporality + dest.Rows = src.Rows + + dest.Cols = src.Cols + return dest } @@ -80,6 +87,8 @@ func CopyCountMinSketchSlice(dest, src []CountMinSketch) []CountMinSketch { newDest = make([]CountMinSketch, len(src)) } else { newDest = dest[:len(src)] + // Cleanup the rest of the elements so GC can free the memory. + // This can happen when len(src) < len(dest) < cap(dest). for i := len(src); i < len(dest); i++ { DeleteCountMinSketch(&dest[i], false) } @@ -94,16 +103,22 @@ func CopyCountMinSketchPtrSlice(dest, src []*CountMinSketch) []*CountMinSketch { var newDest []*CountMinSketch if cap(dest) < len(src) { newDest = make([]*CountMinSketch, len(src)) + // Copy old pointers to re-use. copy(newDest, dest) + // Add new pointers for missing elements from len(dest) to len(srt). for i := len(dest); i < len(src); i++ { newDest[i] = NewCountMinSketch() } } else { newDest = dest[:len(src)] + // Cleanup the rest of the elements so GC can free the memory. + // This can happen when len(src) < len(dest) < cap(dest). for i := len(src); i < len(dest); i++ { DeleteCountMinSketch(dest[i], true) dest[i] = nil } + // Add new pointers for missing elements. + // This can happen when len(dest) < len(src) < cap(dest). for i := len(dest); i < len(src); i++ { newDest[i] = NewCountMinSketch() } @@ -136,6 +151,14 @@ func (orig *CountMinSketch) MarshalJSON(dest *json.Stream) { dest.WriteObjectField("aggregationTemporality") dest.WriteInt32(int32(orig.AggregationTemporality)) } + if orig.Rows != int32(0) { + dest.WriteObjectField("rows") + dest.WriteInt32(orig.Rows) + } + if orig.Cols != int32(0) { + dest.WriteObjectField("cols") + dest.WriteInt32(orig.Cols) + } dest.WriteObjectEnd() } @@ -151,6 +174,10 @@ func (orig *CountMinSketch) UnmarshalJSON(iter *json.Iterator) { case "aggregationTemporality", "aggregation_temporality": orig.AggregationTemporality = AggregationTemporality(iter.ReadEnumValue(AggregationTemporality_value)) + case "rows": + orig.Rows = iter.ReadInt32() + case "cols": + orig.Cols = iter.ReadInt32() default: iter.Skip() } @@ -168,6 +195,12 @@ func (orig *CountMinSketch) SizeProto() int { if orig.AggregationTemporality != 0 { n += 1 + proto.Sov(uint64(orig.AggregationTemporality)) } + if orig.Rows != 0 { + n += 1 + proto.Sov(uint64(orig.Rows)) + } + if orig.Cols != 0 { + n += 1 + proto.Sov(uint64(orig.Cols)) + } return n } @@ -187,6 +220,16 @@ func (orig *CountMinSketch) MarshalProto(buf []byte) int { pos-- buf[pos] = 0x10 } + if orig.Rows != 0 { + pos = proto.EncodeVarint(buf, pos, uint64(orig.Rows)) + pos-- + buf[pos] = 0x18 + } + if orig.Cols != 0 { + pos = proto.EncodeVarint(buf, pos, uint64(orig.Cols)) + pos-- + buf[pos] = 0x20 + } return len(buf) - pos } @@ -198,6 +241,7 @@ func (orig *CountMinSketch) UnmarshalProto(buf []byte) error { l := len(buf) pos := 0 for pos < l { + // If in a group parsing, move to the next tag. fieldNum, wireType, pos, err = proto.ConsumeTag(buf, pos) if err != nil { return err @@ -231,6 +275,30 @@ func (orig *CountMinSketch) UnmarshalProto(buf []byte) error { } orig.AggregationTemporality = AggregationTemporality(num) + + case 3: + if wireType != proto.WireTypeVarint { + return fmt.Errorf("proto: wrong wireType = %d for field Rows", wireType) + } + var num uint64 + num, pos, err = proto.ConsumeVarint(buf, pos) + if err != nil { + return err + } + + orig.Rows = int32(num) + + case 4: + if wireType != proto.WireTypeVarint { + return fmt.Errorf("proto: wrong wireType = %d for field Cols", wireType) + } + var num uint64 + num, pos, err = proto.ConsumeVarint(buf, pos) + if err != nil { + return err + } + + orig.Cols = int32(num) default: pos, err = proto.ConsumeUnknown(buf, pos, wireType) if err != nil { @@ -245,6 +313,8 @@ func GenTestCountMinSketch() *CountMinSketch { orig := NewCountMinSketch() orig.DataPoints = []*CountMinSketchDataPoint{{}, GenTestCountMinSketchDataPoint()} orig.AggregationTemporality = AggregationTemporality(13) + orig.Rows = int32(13) + orig.Cols = int32(13) return orig } diff --git a/opentelemetry-collector-patch/pdata/internal/generated_proto_countminsketch_test.go b/opentelemetry-collector-patch/pdata/internal/generated_proto_countminsketch_test.go new file mode 100644 index 00000000..c7076b04 --- /dev/null +++ b/opentelemetry-collector-patch/pdata/internal/generated_proto_countminsketch_test.go @@ -0,0 +1,211 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package internal + +import ( + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + gootlpmetrics "go.opentelemetry.io/proto/slim/otlp/metrics/v1" + "google.golang.org/protobuf/proto" + + "go.opentelemetry.io/collector/featuregate" + "go.opentelemetry.io/collector/pdata/internal/json" +) + +func TestCopyCountMinSketch(t *testing.T) { + for name, src := range genTestEncodingValuesCountMinSketch() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + dest := NewCountMinSketch() + CopyCountMinSketch(dest, src) + assert.Equal(t, src, dest) + CopyCountMinSketch(dest, dest) + assert.Equal(t, src, dest) + }) + } + } +} + +func TestCopyCountMinSketchSlice(t *testing.T) { + src := []CountMinSketch{} + dest := []CountMinSketch{} + // Test CopyTo empty + dest = CopyCountMinSketchSlice(dest, src) + assert.Equal(t, []CountMinSketch{}, dest) + + // Test CopyTo larger slice + src = GenTestCountMinSketchSlice() + dest = CopyCountMinSketchSlice(dest, src) + assert.Equal(t, GenTestCountMinSketchSlice(), dest) + + // Test CopyTo same size slice + dest = CopyCountMinSketchSlice(dest, src) + assert.Equal(t, GenTestCountMinSketchSlice(), dest) + + // Test CopyTo smaller size slice + dest = CopyCountMinSketchSlice(dest, []CountMinSketch{}) + assert.Len(t, dest, 0) + + // Test CopyTo larger slice with enough capacity + dest = CopyCountMinSketchSlice(dest, src) + assert.Equal(t, GenTestCountMinSketchSlice(), dest) +} + +func TestCopyCountMinSketchPtrSlice(t *testing.T) { + src := []*CountMinSketch{} + dest := []*CountMinSketch{} + // Test CopyTo empty + dest = CopyCountMinSketchPtrSlice(dest, src) + assert.Equal(t, []*CountMinSketch{}, dest) + + // Test CopyTo larger slice + src = GenTestCountMinSketchPtrSlice() + dest = CopyCountMinSketchPtrSlice(dest, src) + assert.Equal(t, GenTestCountMinSketchPtrSlice(), dest) + + // Test CopyTo same size slice + dest = CopyCountMinSketchPtrSlice(dest, src) + assert.Equal(t, GenTestCountMinSketchPtrSlice(), dest) + + // Test CopyTo smaller size slice + dest = CopyCountMinSketchPtrSlice(dest, []*CountMinSketch{}) + assert.Len(t, dest, 0) + + // Test CopyTo larger slice with enough capacity + dest = CopyCountMinSketchPtrSlice(dest, src) + assert.Equal(t, GenTestCountMinSketchPtrSlice(), dest) +} + +func TestMarshalAndUnmarshalJSONCountMinSketchUnknown(t *testing.T) { + iter := json.BorrowIterator([]byte(`{"unknown": "string"}`)) + defer json.ReturnIterator(iter) + dest := NewCountMinSketch() + dest.UnmarshalJSON(iter) + require.NoError(t, iter.Error()) + assert.Equal(t, NewCountMinSketch(), dest) +} + +func TestMarshalAndUnmarshalJSONCountMinSketch(t *testing.T) { + for name, src := range genTestEncodingValuesCountMinSketch() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + stream := json.BorrowStream(nil) + defer json.ReturnStream(stream) + src.MarshalJSON(stream) + require.NoError(t, stream.Error()) + + iter := json.BorrowIterator(stream.Buffer()) + defer json.ReturnIterator(iter) + dest := NewCountMinSketch() + dest.UnmarshalJSON(iter) + require.NoError(t, iter.Error()) + + assert.Equal(t, src, dest) + DeleteCountMinSketch(dest, true) + }) + } + } +} + +func TestMarshalAndUnmarshalProtoCountMinSketchFailing(t *testing.T) { + for name, buf := range genTestFailingUnmarshalProtoValuesCountMinSketch() { + t.Run(name, func(t *testing.T) { + dest := NewCountMinSketch() + require.Error(t, dest.UnmarshalProto(buf)) + }) + } +} + +func TestMarshalAndUnmarshalProtoCountMinSketchUnknown(t *testing.T) { + dest := NewCountMinSketch() + // message Test { required int64 field = 1313; } encoding { "field": "1234" } + require.NoError(t, dest.UnmarshalProto([]byte{0x88, 0x52, 0xD2, 0x09})) + assert.Equal(t, NewCountMinSketch(), dest) +} + +func TestMarshalAndUnmarshalProtoCountMinSketch(t *testing.T) { + for name, src := range genTestEncodingValuesCountMinSketch() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + buf := make([]byte, src.SizeProto()) + gotSize := src.MarshalProto(buf) + assert.Equal(t, len(buf), gotSize) + + dest := NewCountMinSketch() + require.NoError(t, dest.UnmarshalProto(buf)) + + assert.Equal(t, src, dest) + DeleteCountMinSketch(dest, true) + }) + } + } +} + +func TestMarshalAndUnmarshalProtoViaProtobufCountMinSketch(t *testing.T) { + for name, src := range genTestEncodingValuesCountMinSketch() { + t.Run(name, func(t *testing.T) { + buf := make([]byte, src.SizeProto()) + gotSize := src.MarshalProto(buf) + assert.Equal(t, len(buf), gotSize) + + goDest := &gootlpmetrics.CountMinSketch{} + require.NoError(t, proto.Unmarshal(buf, goDest)) + + goBuf, err := proto.Marshal(goDest) + require.NoError(t, err) + + dest := NewCountMinSketch() + require.NoError(t, dest.UnmarshalProto(goBuf)) + assert.Equal(t, src, dest) + }) + } +} + +func genTestFailingUnmarshalProtoValuesCountMinSketch() map[string][]byte { + return map[string][]byte{ + "invalid_field": {0x02}, + "DataPoints/wrong_wire_type": {0xc}, + "DataPoints/missing_value": {0xa}, + "AggregationTemporality/wrong_wire_type": {0x14}, + "AggregationTemporality/missing_value": {0x10}, + "Rows/wrong_wire_type": {0x1c}, + "Rows/missing_value": {0x18}, + "Cols/wrong_wire_type": {0x24}, + "Cols/missing_value": {0x20}, + } +} + +func genTestEncodingValuesCountMinSketch() map[string]*CountMinSketch { + return map[string]*CountMinSketch{ + "empty": NewCountMinSketch(), + "DataPoints/test": {DataPoints: []*CountMinSketchDataPoint{{}, GenTestCountMinSketchDataPoint()}}, + "AggregationTemporality/test": {AggregationTemporality: AggregationTemporality(13)}, + "Rows/test": {Rows: int32(13)}, + "Cols/test": {Cols: int32(13)}, + } +} diff --git a/opentelemetry-collector-patch/pdata/internal/generated_proto_countminsketchdatapoint.go b/opentelemetry-collector-patch/pdata/internal/generated_proto_countminsketchdatapoint.go index 3775f9be..c910d020 100644 --- a/opentelemetry-collector-patch/pdata/internal/generated_proto_countminsketchdatapoint.go +++ b/opentelemetry-collector-patch/pdata/internal/generated_proto_countminsketchdatapoint.go @@ -18,15 +18,12 @@ import ( // CountMinSketchDataPoint is a single data point that encodes frequency estimations using Count-Min Sketch. type CountMinSketchDataPoint struct { Attributes []KeyValue - SeriesID uint64 StartTimeUnixNano uint64 TimeUnixNano uint64 - SampleCount uint64 Sketch []byte Encoding CountMinSketchEncoding - Rows int32 - Cols int32 Flags uint32 + SeriesID uint64 } var ( @@ -65,6 +62,7 @@ func DeleteCountMinSketchDataPoint(orig *CountMinSketchDataPoint, nullable bool) } func CopyCountMinSketchDataPoint(dest, src *CountMinSketchDataPoint) *CountMinSketchDataPoint { + // If copying to same object, just return. if src == dest { return dest } @@ -77,16 +75,19 @@ func CopyCountMinSketchDataPoint(dest, src *CountMinSketchDataPoint) *CountMinSk dest = NewCountMinSketchDataPoint() } dest.Attributes = CopyKeyValueSlice(dest.Attributes, src.Attributes) - dest.SeriesID = src.SeriesID + dest.StartTimeUnixNano = src.StartTimeUnixNano + dest.TimeUnixNano = src.TimeUnixNano - dest.SampleCount = src.SampleCount + dest.Sketch = src.Sketch + dest.Encoding = src.Encoding - dest.Rows = src.Rows - dest.Cols = src.Cols + dest.Flags = src.Flags + dest.SeriesID = src.SeriesID + return dest } @@ -96,6 +97,8 @@ func CopyCountMinSketchDataPointSlice(dest, src []CountMinSketchDataPoint) []Cou newDest = make([]CountMinSketchDataPoint, len(src)) } else { newDest = dest[:len(src)] + // Cleanup the rest of the elements so GC can free the memory. + // This can happen when len(src) < len(dest) < cap(dest). for i := len(src); i < len(dest); i++ { DeleteCountMinSketchDataPoint(&dest[i], false) } @@ -110,16 +113,22 @@ func CopyCountMinSketchDataPointPtrSlice(dest, src []*CountMinSketchDataPoint) [ var newDest []*CountMinSketchDataPoint if cap(dest) < len(src) { newDest = make([]*CountMinSketchDataPoint, len(src)) + // Copy old pointers to re-use. copy(newDest, dest) + // Add new pointers for missing elements from len(dest) to len(srt). for i := len(dest); i < len(src); i++ { newDest[i] = NewCountMinSketchDataPoint() } } else { newDest = dest[:len(src)] + // Cleanup the rest of the elements so GC can free the memory. + // This can happen when len(src) < len(dest) < cap(dest). for i := len(src); i < len(dest); i++ { DeleteCountMinSketchDataPoint(dest[i], true) dest[i] = nil } + // Add new pointers for missing elements. + // This can happen when len(dest) < len(src) < cap(dest). for i := len(dest); i < len(src); i++ { newDest[i] = NewCountMinSketchDataPoint() } @@ -147,10 +156,6 @@ func (orig *CountMinSketchDataPoint) MarshalJSON(dest *json.Stream) { } dest.WriteArrayEnd() } - if orig.SeriesID != uint64(0) { - dest.WriteObjectField("seriesID") - dest.WriteUint64(orig.SeriesID) - } if orig.StartTimeUnixNano != uint64(0) { dest.WriteObjectField("startTimeUnixNano") dest.WriteUint64(orig.StartTimeUnixNano) @@ -159,30 +164,24 @@ func (orig *CountMinSketchDataPoint) MarshalJSON(dest *json.Stream) { dest.WriteObjectField("timeUnixNano") dest.WriteUint64(orig.TimeUnixNano) } - if orig.SampleCount != uint64(0) { - dest.WriteObjectField("sampleCount") - dest.WriteUint64(orig.SampleCount) - } + if len(orig.Sketch) > 0 { dest.WriteObjectField("sketch") dest.WriteBytes(orig.Sketch) } + if int32(orig.Encoding) != 0 { dest.WriteObjectField("encoding") dest.WriteInt32(int32(orig.Encoding)) } - if orig.Rows != int32(0) { - dest.WriteObjectField("rows") - dest.WriteInt32(orig.Rows) - } - if orig.Cols != int32(0) { - dest.WriteObjectField("cols") - dest.WriteInt32(orig.Cols) - } if orig.Flags != uint32(0) { dest.WriteObjectField("flags") dest.WriteUint32(orig.Flags) } + if orig.SeriesID != uint64(0) { + dest.WriteObjectField("seriesID") + dest.WriteUint64(orig.SeriesID) + } dest.WriteObjectEnd() } @@ -195,24 +194,19 @@ func (orig *CountMinSketchDataPoint) UnmarshalJSON(iter *json.Iterator) { orig.Attributes = append(orig.Attributes, KeyValue{}) orig.Attributes[len(orig.Attributes)-1].UnmarshalJSON(iter) } - case "seriesID", "series_id": - orig.SeriesID = iter.ReadUint64() + case "startTimeUnixNano", "start_time_unix_nano": orig.StartTimeUnixNano = iter.ReadUint64() case "timeUnixNano", "time_unix_nano": orig.TimeUnixNano = iter.ReadUint64() - case "sampleCount", "sample_count": - orig.SampleCount = iter.ReadUint64() case "sketch": orig.Sketch = iter.ReadBytes() case "encoding": orig.Encoding = CountMinSketchEncoding(iter.ReadEnumValue(CountMinSketchEncoding_value)) - case "rows": - orig.Rows = iter.ReadInt32() - case "cols": - orig.Cols = iter.ReadInt32() case "flags": orig.Flags = iter.ReadUint32() + case "seriesID", "series_id": + orig.SeriesID = iter.ReadUint64() default: iter.Skip() } @@ -220,17 +214,6 @@ func (orig *CountMinSketchDataPoint) UnmarshalJSON(iter *json.Iterator) { } func (orig *CountMinSketchDataPoint) SizeProto() int { - // Field layout: - // 1=attributes (wire 2) - // 2=start_time_unix_nano (wire 1, I64) - // 3=time_unix_nano (wire 1, I64) - // 4=sample_count (wire 0) - // 5=sketch (wire 2) - // 6=encoding (wire 0) - // 7=rows (wire 0) - // 8=cols (wire 0) - // 9=flags (wire 0) - // 10=series_id (wire 0) var n int var l int _ = l @@ -244,9 +227,6 @@ func (orig *CountMinSketchDataPoint) SizeProto() int { if orig.TimeUnixNano != 0 { n += 9 } - if orig.SampleCount != 0 { - n += 1 + proto.Sov(uint64(orig.SampleCount)) - } l = len(orig.Sketch) if l > 0 { n += 1 + proto.Sov(uint64(l)) + l @@ -254,12 +234,6 @@ func (orig *CountMinSketchDataPoint) SizeProto() int { if orig.Encoding != 0 { n += 1 + proto.Sov(uint64(orig.Encoding)) } - if orig.Rows != 0 { - n += 1 + proto.Sov(uint64(orig.Rows)) - } - if orig.Cols != 0 { - n += 1 + proto.Sov(uint64(orig.Cols)) - } if orig.Flags != 0 { n += 1 + proto.Sov(uint64(orig.Flags)) } @@ -270,17 +244,6 @@ func (orig *CountMinSketchDataPoint) SizeProto() int { } func (orig *CountMinSketchDataPoint) MarshalProto(buf []byte) int { - // Wire tag bytes: - // 1=attributes: 0x0a - // 2=start_time_unix_nano: 0x11 - // 3=time_unix_nano: 0x19 - // 4=sample_count: 0x20 - // 5=sketch: 0x2a - // 6=encoding: 0x30 - // 7=rows: 0x38 - // 8=cols: 0x40 - // 9=flags: 0x48 - // 10=series_id: 0x50 pos := len(buf) var l int _ = l @@ -289,12 +252,7 @@ func (orig *CountMinSketchDataPoint) MarshalProto(buf []byte) int { pos -= l pos = proto.EncodeVarint(buf, pos, uint64(l)) pos-- - buf[pos] = 0x0a - } - if orig.SeriesID != 0 { - pos = proto.EncodeVarint(buf, pos, uint64(orig.SeriesID)) - pos-- - buf[pos] = 0x50 + buf[pos] = 0xa } if orig.StartTimeUnixNano != 0 { pos -= 8 @@ -308,11 +266,6 @@ func (orig *CountMinSketchDataPoint) MarshalProto(buf []byte) int { pos-- buf[pos] = 0x19 } - if orig.SampleCount != 0 { - pos = proto.EncodeVarint(buf, pos, uint64(orig.SampleCount)) - pos-- - buf[pos] = 0x20 - } l = len(orig.Sketch) if l > 0 { pos -= l @@ -326,21 +279,16 @@ func (orig *CountMinSketchDataPoint) MarshalProto(buf []byte) int { pos-- buf[pos] = 0x30 } - if orig.Rows != 0 { - pos = proto.EncodeVarint(buf, pos, uint64(orig.Rows)) - pos-- - buf[pos] = 0x38 - } - if orig.Cols != 0 { - pos = proto.EncodeVarint(buf, pos, uint64(orig.Cols)) - pos-- - buf[pos] = 0x40 - } if orig.Flags != 0 { pos = proto.EncodeVarint(buf, pos, uint64(orig.Flags)) pos-- buf[pos] = 0x48 } + if orig.SeriesID != 0 { + pos = proto.EncodeVarint(buf, pos, uint64(orig.SeriesID)) + pos-- + buf[pos] = 0x50 + } return len(buf) - pos } @@ -352,6 +300,7 @@ func (orig *CountMinSketchDataPoint) UnmarshalProto(buf []byte) error { l := len(buf) pos := 0 for pos < l { + // If in a group parsing, move to the next tag. fieldNum, wireType, pos, err = proto.ConsumeTag(buf, pos) if err != nil { return err @@ -383,6 +332,7 @@ func (orig *CountMinSketchDataPoint) UnmarshalProto(buf []byte) error { if err != nil { return err } + orig.StartTimeUnixNano = uint64(num) case 3: @@ -394,18 +344,8 @@ func (orig *CountMinSketchDataPoint) UnmarshalProto(buf []byte) error { if err != nil { return err } - orig.TimeUnixNano = uint64(num) - case 4: - if wireType != proto.WireTypeVarint { - return fmt.Errorf("proto: wrong wireType = %d for field SampleCount", wireType) - } - var num uint64 - num, pos, err = proto.ConsumeVarint(buf, pos) - if err != nil { - return err - } - orig.SampleCount = uint64(num) + orig.TimeUnixNano = uint64(num) case 5: if wireType != proto.WireTypeLen { @@ -431,29 +371,8 @@ func (orig *CountMinSketchDataPoint) UnmarshalProto(buf []byte) error { if err != nil { return err } - orig.Encoding = CountMinSketchEncoding(num) - case 7: - if wireType != proto.WireTypeVarint { - return fmt.Errorf("proto: wrong wireType = %d for field Rows", wireType) - } - var num uint64 - num, pos, err = proto.ConsumeVarint(buf, pos) - if err != nil { - return err - } - orig.Rows = int32(num) - - case 8: - if wireType != proto.WireTypeVarint { - return fmt.Errorf("proto: wrong wireType = %d for field Cols", wireType) - } - var num uint64 - num, pos, err = proto.ConsumeVarint(buf, pos) - if err != nil { - return err - } - orig.Cols = int32(num) + orig.Encoding = CountMinSketchEncoding(num) case 9: if wireType != proto.WireTypeVarint { @@ -464,6 +383,7 @@ func (orig *CountMinSketchDataPoint) UnmarshalProto(buf []byte) error { if err != nil { return err } + orig.Flags = uint32(num) case 10: @@ -475,8 +395,8 @@ func (orig *CountMinSketchDataPoint) UnmarshalProto(buf []byte) error { if err != nil { return err } - orig.SeriesID = uint64(num) + orig.SeriesID = uint64(num) default: pos, err = proto.ConsumeUnknown(buf, pos, wireType) if err != nil { @@ -490,15 +410,12 @@ func (orig *CountMinSketchDataPoint) UnmarshalProto(buf []byte) error { func GenTestCountMinSketchDataPoint() *CountMinSketchDataPoint { orig := NewCountMinSketchDataPoint() orig.Attributes = []KeyValue{{}, *GenTestKeyValue()} - orig.SeriesID = uint64(13) orig.StartTimeUnixNano = uint64(13) orig.TimeUnixNano = uint64(13) - orig.SampleCount = uint64(100) orig.Sketch = []byte{1, 2, 3} - orig.Encoding = CountMinSketchEncoding(1) - orig.Rows = int32(5) - orig.Cols = int32(2048) + orig.Encoding = CountMinSketchEncoding(13) orig.Flags = uint32(13) + orig.SeriesID = uint64(13) return orig } diff --git a/opentelemetry-collector-patch/pdata/internal/generated_proto_countminsketchdatapoint_test.go b/opentelemetry-collector-patch/pdata/internal/generated_proto_countminsketchdatapoint_test.go new file mode 100644 index 00000000..e745da8d --- /dev/null +++ b/opentelemetry-collector-patch/pdata/internal/generated_proto_countminsketchdatapoint_test.go @@ -0,0 +1,220 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package internal + +import ( + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + gootlpmetrics "go.opentelemetry.io/proto/slim/otlp/metrics/v1" + "google.golang.org/protobuf/proto" + + "go.opentelemetry.io/collector/featuregate" + "go.opentelemetry.io/collector/pdata/internal/json" +) + +func TestCopyCountMinSketchDataPoint(t *testing.T) { + for name, src := range genTestEncodingValuesCountMinSketchDataPoint() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + dest := NewCountMinSketchDataPoint() + CopyCountMinSketchDataPoint(dest, src) + assert.Equal(t, src, dest) + CopyCountMinSketchDataPoint(dest, dest) + assert.Equal(t, src, dest) + }) + } + } +} + +func TestCopyCountMinSketchDataPointSlice(t *testing.T) { + src := []CountMinSketchDataPoint{} + dest := []CountMinSketchDataPoint{} + // Test CopyTo empty + dest = CopyCountMinSketchDataPointSlice(dest, src) + assert.Equal(t, []CountMinSketchDataPoint{}, dest) + + // Test CopyTo larger slice + src = GenTestCountMinSketchDataPointSlice() + dest = CopyCountMinSketchDataPointSlice(dest, src) + assert.Equal(t, GenTestCountMinSketchDataPointSlice(), dest) + + // Test CopyTo same size slice + dest = CopyCountMinSketchDataPointSlice(dest, src) + assert.Equal(t, GenTestCountMinSketchDataPointSlice(), dest) + + // Test CopyTo smaller size slice + dest = CopyCountMinSketchDataPointSlice(dest, []CountMinSketchDataPoint{}) + assert.Len(t, dest, 0) + + // Test CopyTo larger slice with enough capacity + dest = CopyCountMinSketchDataPointSlice(dest, src) + assert.Equal(t, GenTestCountMinSketchDataPointSlice(), dest) +} + +func TestCopyCountMinSketchDataPointPtrSlice(t *testing.T) { + src := []*CountMinSketchDataPoint{} + dest := []*CountMinSketchDataPoint{} + // Test CopyTo empty + dest = CopyCountMinSketchDataPointPtrSlice(dest, src) + assert.Equal(t, []*CountMinSketchDataPoint{}, dest) + + // Test CopyTo larger slice + src = GenTestCountMinSketchDataPointPtrSlice() + dest = CopyCountMinSketchDataPointPtrSlice(dest, src) + assert.Equal(t, GenTestCountMinSketchDataPointPtrSlice(), dest) + + // Test CopyTo same size slice + dest = CopyCountMinSketchDataPointPtrSlice(dest, src) + assert.Equal(t, GenTestCountMinSketchDataPointPtrSlice(), dest) + + // Test CopyTo smaller size slice + dest = CopyCountMinSketchDataPointPtrSlice(dest, []*CountMinSketchDataPoint{}) + assert.Len(t, dest, 0) + + // Test CopyTo larger slice with enough capacity + dest = CopyCountMinSketchDataPointPtrSlice(dest, src) + assert.Equal(t, GenTestCountMinSketchDataPointPtrSlice(), dest) +} + +func TestMarshalAndUnmarshalJSONCountMinSketchDataPointUnknown(t *testing.T) { + iter := json.BorrowIterator([]byte(`{"unknown": "string"}`)) + defer json.ReturnIterator(iter) + dest := NewCountMinSketchDataPoint() + dest.UnmarshalJSON(iter) + require.NoError(t, iter.Error()) + assert.Equal(t, NewCountMinSketchDataPoint(), dest) +} + +func TestMarshalAndUnmarshalJSONCountMinSketchDataPoint(t *testing.T) { + for name, src := range genTestEncodingValuesCountMinSketchDataPoint() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + stream := json.BorrowStream(nil) + defer json.ReturnStream(stream) + src.MarshalJSON(stream) + require.NoError(t, stream.Error()) + + iter := json.BorrowIterator(stream.Buffer()) + defer json.ReturnIterator(iter) + dest := NewCountMinSketchDataPoint() + dest.UnmarshalJSON(iter) + require.NoError(t, iter.Error()) + + assert.Equal(t, src, dest) + DeleteCountMinSketchDataPoint(dest, true) + }) + } + } +} + +func TestMarshalAndUnmarshalProtoCountMinSketchDataPointFailing(t *testing.T) { + for name, buf := range genTestFailingUnmarshalProtoValuesCountMinSketchDataPoint() { + t.Run(name, func(t *testing.T) { + dest := NewCountMinSketchDataPoint() + require.Error(t, dest.UnmarshalProto(buf)) + }) + } +} + +func TestMarshalAndUnmarshalProtoCountMinSketchDataPointUnknown(t *testing.T) { + dest := NewCountMinSketchDataPoint() + // message Test { required int64 field = 1313; } encoding { "field": "1234" } + require.NoError(t, dest.UnmarshalProto([]byte{0x88, 0x52, 0xD2, 0x09})) + assert.Equal(t, NewCountMinSketchDataPoint(), dest) +} + +func TestMarshalAndUnmarshalProtoCountMinSketchDataPoint(t *testing.T) { + for name, src := range genTestEncodingValuesCountMinSketchDataPoint() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + buf := make([]byte, src.SizeProto()) + gotSize := src.MarshalProto(buf) + assert.Equal(t, len(buf), gotSize) + + dest := NewCountMinSketchDataPoint() + require.NoError(t, dest.UnmarshalProto(buf)) + + assert.Equal(t, src, dest) + DeleteCountMinSketchDataPoint(dest, true) + }) + } + } +} + +func TestMarshalAndUnmarshalProtoViaProtobufCountMinSketchDataPoint(t *testing.T) { + for name, src := range genTestEncodingValuesCountMinSketchDataPoint() { + t.Run(name, func(t *testing.T) { + buf := make([]byte, src.SizeProto()) + gotSize := src.MarshalProto(buf) + assert.Equal(t, len(buf), gotSize) + + goDest := &gootlpmetrics.CountMinSketchDataPoint{} + require.NoError(t, proto.Unmarshal(buf, goDest)) + + goBuf, err := proto.Marshal(goDest) + require.NoError(t, err) + + dest := NewCountMinSketchDataPoint() + require.NoError(t, dest.UnmarshalProto(goBuf)) + assert.Equal(t, src, dest) + }) + } +} + +func genTestFailingUnmarshalProtoValuesCountMinSketchDataPoint() map[string][]byte { + return map[string][]byte{ + "invalid_field": {0x02}, + "Attributes/wrong_wire_type": {0xc}, + "Attributes/missing_value": {0xa}, + "StartTimeUnixNano/wrong_wire_type": {0x14}, + "StartTimeUnixNano/missing_value": {0x11}, + "TimeUnixNano/wrong_wire_type": {0x1c}, + "TimeUnixNano/missing_value": {0x19}, + "Sketch/wrong_wire_type": {0x2c}, + "Sketch/missing_value": {0x2a}, + "Encoding/wrong_wire_type": {0x34}, + "Encoding/missing_value": {0x30}, + "Flags/wrong_wire_type": {0x4c}, + "Flags/missing_value": {0x48}, + "SeriesID/wrong_wire_type": {0x54}, + "SeriesID/missing_value": {0x50}, + } +} + +func genTestEncodingValuesCountMinSketchDataPoint() map[string]*CountMinSketchDataPoint { + return map[string]*CountMinSketchDataPoint{ + "empty": NewCountMinSketchDataPoint(), + "Attributes/test": {Attributes: []KeyValue{{}, *GenTestKeyValue()}}, + "StartTimeUnixNano/test": {StartTimeUnixNano: uint64(13)}, + "TimeUnixNano/test": {TimeUnixNano: uint64(13)}, + "Sketch/test": {Sketch: []byte{1, 2, 3}}, + "Encoding/test": {Encoding: CountMinSketchEncoding(13)}, + "Flags/test": {Flags: uint32(13)}, + "SeriesID/test": {SeriesID: uint64(13)}, + } +} diff --git a/opentelemetry-collector-patch/pdata/internal/generated_proto_countsketch.go b/opentelemetry-collector-patch/pdata/internal/generated_proto_countsketch.go index 0eb182b9..852fdc18 100644 --- a/opentelemetry-collector-patch/pdata/internal/generated_proto_countsketch.go +++ b/opentelemetry-collector-patch/pdata/internal/generated_proto_countsketch.go @@ -18,6 +18,8 @@ import ( type CountSketch struct { DataPoints []*CountSketchDataPoint AggregationTemporality AggregationTemporality + Rows int32 + Cols int32 } var ( @@ -56,6 +58,7 @@ func DeleteCountSketch(orig *CountSketch, nullable bool) { } func CopyCountSketch(dest, src *CountSketch) *CountSketch { + // If copying to same object, just return. if src == dest { return dest } @@ -71,6 +74,10 @@ func CopyCountSketch(dest, src *CountSketch) *CountSketch { dest.AggregationTemporality = src.AggregationTemporality + dest.Rows = src.Rows + + dest.Cols = src.Cols + return dest } @@ -80,6 +87,8 @@ func CopyCountSketchSlice(dest, src []CountSketch) []CountSketch { newDest = make([]CountSketch, len(src)) } else { newDest = dest[:len(src)] + // Cleanup the rest of the elements so GC can free the memory. + // This can happen when len(src) < len(dest) < cap(dest). for i := len(src); i < len(dest); i++ { DeleteCountSketch(&dest[i], false) } @@ -94,16 +103,22 @@ func CopyCountSketchPtrSlice(dest, src []*CountSketch) []*CountSketch { var newDest []*CountSketch if cap(dest) < len(src) { newDest = make([]*CountSketch, len(src)) + // Copy old pointers to re-use. copy(newDest, dest) + // Add new pointers for missing elements from len(dest) to len(srt). for i := len(dest); i < len(src); i++ { newDest[i] = NewCountSketch() } } else { newDest = dest[:len(src)] + // Cleanup the rest of the elements so GC can free the memory. + // This can happen when len(src) < len(dest) < cap(dest). for i := len(src); i < len(dest); i++ { DeleteCountSketch(dest[i], true) dest[i] = nil } + // Add new pointers for missing elements. + // This can happen when len(dest) < len(src) < cap(dest). for i := len(dest); i < len(src); i++ { newDest[i] = NewCountSketch() } @@ -136,6 +151,14 @@ func (orig *CountSketch) MarshalJSON(dest *json.Stream) { dest.WriteObjectField("aggregationTemporality") dest.WriteInt32(int32(orig.AggregationTemporality)) } + if orig.Rows != int32(0) { + dest.WriteObjectField("rows") + dest.WriteInt32(orig.Rows) + } + if orig.Cols != int32(0) { + dest.WriteObjectField("cols") + dest.WriteInt32(orig.Cols) + } dest.WriteObjectEnd() } @@ -151,6 +174,10 @@ func (orig *CountSketch) UnmarshalJSON(iter *json.Iterator) { case "aggregationTemporality", "aggregation_temporality": orig.AggregationTemporality = AggregationTemporality(iter.ReadEnumValue(AggregationTemporality_value)) + case "rows": + orig.Rows = iter.ReadInt32() + case "cols": + orig.Cols = iter.ReadInt32() default: iter.Skip() } @@ -168,6 +195,12 @@ func (orig *CountSketch) SizeProto() int { if orig.AggregationTemporality != 0 { n += 1 + proto.Sov(uint64(orig.AggregationTemporality)) } + if orig.Rows != 0 { + n += 1 + proto.Sov(uint64(orig.Rows)) + } + if orig.Cols != 0 { + n += 1 + proto.Sov(uint64(orig.Cols)) + } return n } @@ -187,6 +220,16 @@ func (orig *CountSketch) MarshalProto(buf []byte) int { pos-- buf[pos] = 0x10 } + if orig.Rows != 0 { + pos = proto.EncodeVarint(buf, pos, uint64(orig.Rows)) + pos-- + buf[pos] = 0x18 + } + if orig.Cols != 0 { + pos = proto.EncodeVarint(buf, pos, uint64(orig.Cols)) + pos-- + buf[pos] = 0x20 + } return len(buf) - pos } @@ -198,6 +241,7 @@ func (orig *CountSketch) UnmarshalProto(buf []byte) error { l := len(buf) pos := 0 for pos < l { + // If in a group parsing, move to the next tag. fieldNum, wireType, pos, err = proto.ConsumeTag(buf, pos) if err != nil { return err @@ -231,6 +275,30 @@ func (orig *CountSketch) UnmarshalProto(buf []byte) error { } orig.AggregationTemporality = AggregationTemporality(num) + + case 3: + if wireType != proto.WireTypeVarint { + return fmt.Errorf("proto: wrong wireType = %d for field Rows", wireType) + } + var num uint64 + num, pos, err = proto.ConsumeVarint(buf, pos) + if err != nil { + return err + } + + orig.Rows = int32(num) + + case 4: + if wireType != proto.WireTypeVarint { + return fmt.Errorf("proto: wrong wireType = %d for field Cols", wireType) + } + var num uint64 + num, pos, err = proto.ConsumeVarint(buf, pos) + if err != nil { + return err + } + + orig.Cols = int32(num) default: pos, err = proto.ConsumeUnknown(buf, pos, wireType) if err != nil { @@ -245,6 +313,8 @@ func GenTestCountSketch() *CountSketch { orig := NewCountSketch() orig.DataPoints = []*CountSketchDataPoint{{}, GenTestCountSketchDataPoint()} orig.AggregationTemporality = AggregationTemporality(13) + orig.Rows = int32(13) + orig.Cols = int32(13) return orig } diff --git a/opentelemetry-collector-patch/pdata/internal/generated_proto_countsketch_test.go b/opentelemetry-collector-patch/pdata/internal/generated_proto_countsketch_test.go new file mode 100644 index 00000000..9a85f124 --- /dev/null +++ b/opentelemetry-collector-patch/pdata/internal/generated_proto_countsketch_test.go @@ -0,0 +1,211 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package internal + +import ( + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + gootlpmetrics "go.opentelemetry.io/proto/slim/otlp/metrics/v1" + "google.golang.org/protobuf/proto" + + "go.opentelemetry.io/collector/featuregate" + "go.opentelemetry.io/collector/pdata/internal/json" +) + +func TestCopyCountSketch(t *testing.T) { + for name, src := range genTestEncodingValuesCountSketch() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + dest := NewCountSketch() + CopyCountSketch(dest, src) + assert.Equal(t, src, dest) + CopyCountSketch(dest, dest) + assert.Equal(t, src, dest) + }) + } + } +} + +func TestCopyCountSketchSlice(t *testing.T) { + src := []CountSketch{} + dest := []CountSketch{} + // Test CopyTo empty + dest = CopyCountSketchSlice(dest, src) + assert.Equal(t, []CountSketch{}, dest) + + // Test CopyTo larger slice + src = GenTestCountSketchSlice() + dest = CopyCountSketchSlice(dest, src) + assert.Equal(t, GenTestCountSketchSlice(), dest) + + // Test CopyTo same size slice + dest = CopyCountSketchSlice(dest, src) + assert.Equal(t, GenTestCountSketchSlice(), dest) + + // Test CopyTo smaller size slice + dest = CopyCountSketchSlice(dest, []CountSketch{}) + assert.Len(t, dest, 0) + + // Test CopyTo larger slice with enough capacity + dest = CopyCountSketchSlice(dest, src) + assert.Equal(t, GenTestCountSketchSlice(), dest) +} + +func TestCopyCountSketchPtrSlice(t *testing.T) { + src := []*CountSketch{} + dest := []*CountSketch{} + // Test CopyTo empty + dest = CopyCountSketchPtrSlice(dest, src) + assert.Equal(t, []*CountSketch{}, dest) + + // Test CopyTo larger slice + src = GenTestCountSketchPtrSlice() + dest = CopyCountSketchPtrSlice(dest, src) + assert.Equal(t, GenTestCountSketchPtrSlice(), dest) + + // Test CopyTo same size slice + dest = CopyCountSketchPtrSlice(dest, src) + assert.Equal(t, GenTestCountSketchPtrSlice(), dest) + + // Test CopyTo smaller size slice + dest = CopyCountSketchPtrSlice(dest, []*CountSketch{}) + assert.Len(t, dest, 0) + + // Test CopyTo larger slice with enough capacity + dest = CopyCountSketchPtrSlice(dest, src) + assert.Equal(t, GenTestCountSketchPtrSlice(), dest) +} + +func TestMarshalAndUnmarshalJSONCountSketchUnknown(t *testing.T) { + iter := json.BorrowIterator([]byte(`{"unknown": "string"}`)) + defer json.ReturnIterator(iter) + dest := NewCountSketch() + dest.UnmarshalJSON(iter) + require.NoError(t, iter.Error()) + assert.Equal(t, NewCountSketch(), dest) +} + +func TestMarshalAndUnmarshalJSONCountSketch(t *testing.T) { + for name, src := range genTestEncodingValuesCountSketch() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + stream := json.BorrowStream(nil) + defer json.ReturnStream(stream) + src.MarshalJSON(stream) + require.NoError(t, stream.Error()) + + iter := json.BorrowIterator(stream.Buffer()) + defer json.ReturnIterator(iter) + dest := NewCountSketch() + dest.UnmarshalJSON(iter) + require.NoError(t, iter.Error()) + + assert.Equal(t, src, dest) + DeleteCountSketch(dest, true) + }) + } + } +} + +func TestMarshalAndUnmarshalProtoCountSketchFailing(t *testing.T) { + for name, buf := range genTestFailingUnmarshalProtoValuesCountSketch() { + t.Run(name, func(t *testing.T) { + dest := NewCountSketch() + require.Error(t, dest.UnmarshalProto(buf)) + }) + } +} + +func TestMarshalAndUnmarshalProtoCountSketchUnknown(t *testing.T) { + dest := NewCountSketch() + // message Test { required int64 field = 1313; } encoding { "field": "1234" } + require.NoError(t, dest.UnmarshalProto([]byte{0x88, 0x52, 0xD2, 0x09})) + assert.Equal(t, NewCountSketch(), dest) +} + +func TestMarshalAndUnmarshalProtoCountSketch(t *testing.T) { + for name, src := range genTestEncodingValuesCountSketch() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + buf := make([]byte, src.SizeProto()) + gotSize := src.MarshalProto(buf) + assert.Equal(t, len(buf), gotSize) + + dest := NewCountSketch() + require.NoError(t, dest.UnmarshalProto(buf)) + + assert.Equal(t, src, dest) + DeleteCountSketch(dest, true) + }) + } + } +} + +func TestMarshalAndUnmarshalProtoViaProtobufCountSketch(t *testing.T) { + for name, src := range genTestEncodingValuesCountSketch() { + t.Run(name, func(t *testing.T) { + buf := make([]byte, src.SizeProto()) + gotSize := src.MarshalProto(buf) + assert.Equal(t, len(buf), gotSize) + + goDest := &gootlpmetrics.CountSketch{} + require.NoError(t, proto.Unmarshal(buf, goDest)) + + goBuf, err := proto.Marshal(goDest) + require.NoError(t, err) + + dest := NewCountSketch() + require.NoError(t, dest.UnmarshalProto(goBuf)) + assert.Equal(t, src, dest) + }) + } +} + +func genTestFailingUnmarshalProtoValuesCountSketch() map[string][]byte { + return map[string][]byte{ + "invalid_field": {0x02}, + "DataPoints/wrong_wire_type": {0xc}, + "DataPoints/missing_value": {0xa}, + "AggregationTemporality/wrong_wire_type": {0x14}, + "AggregationTemporality/missing_value": {0x10}, + "Rows/wrong_wire_type": {0x1c}, + "Rows/missing_value": {0x18}, + "Cols/wrong_wire_type": {0x24}, + "Cols/missing_value": {0x20}, + } +} + +func genTestEncodingValuesCountSketch() map[string]*CountSketch { + return map[string]*CountSketch{ + "empty": NewCountSketch(), + "DataPoints/test": {DataPoints: []*CountSketchDataPoint{{}, GenTestCountSketchDataPoint()}}, + "AggregationTemporality/test": {AggregationTemporality: AggregationTemporality(13)}, + "Rows/test": {Rows: int32(13)}, + "Cols/test": {Cols: int32(13)}, + } +} diff --git a/opentelemetry-collector-patch/pdata/internal/generated_proto_countsketchdatapoint.go b/opentelemetry-collector-patch/pdata/internal/generated_proto_countsketchdatapoint.go index e82947c7..a9932794 100644 --- a/opentelemetry-collector-patch/pdata/internal/generated_proto_countsketchdatapoint.go +++ b/opentelemetry-collector-patch/pdata/internal/generated_proto_countsketchdatapoint.go @@ -9,7 +9,6 @@ package internal import ( "encoding/binary" "fmt" - "math" "sync" "go.opentelemetry.io/collector/pdata/internal/json" @@ -19,15 +18,12 @@ import ( // CountSketchDataPoint is a single data point that encodes frequency estimations using CountSketch. type CountSketchDataPoint struct { Attributes []KeyValue - SeriesID uint64 StartTimeUnixNano uint64 TimeUnixNano uint64 Sketch []byte Encoding CountSketchEncoding - Dimension string - Epsilon float64 - Delta float64 Flags uint32 + SeriesID uint64 } var ( @@ -66,6 +62,7 @@ func DeleteCountSketchDataPoint(orig *CountSketchDataPoint, nullable bool) { } func CopyCountSketchDataPoint(dest, src *CountSketchDataPoint) *CountSketchDataPoint { + // If copying to same object, just return. if src == dest { return dest } @@ -78,16 +75,19 @@ func CopyCountSketchDataPoint(dest, src *CountSketchDataPoint) *CountSketchDataP dest = NewCountSketchDataPoint() } dest.Attributes = CopyKeyValueSlice(dest.Attributes, src.Attributes) - dest.SeriesID = src.SeriesID + dest.StartTimeUnixNano = src.StartTimeUnixNano + dest.TimeUnixNano = src.TimeUnixNano + dest.Sketch = src.Sketch + dest.Encoding = src.Encoding - dest.Dimension = src.Dimension - dest.Epsilon = src.Epsilon - dest.Delta = src.Delta + dest.Flags = src.Flags + dest.SeriesID = src.SeriesID + return dest } @@ -97,6 +97,8 @@ func CopyCountSketchDataPointSlice(dest, src []CountSketchDataPoint) []CountSket newDest = make([]CountSketchDataPoint, len(src)) } else { newDest = dest[:len(src)] + // Cleanup the rest of the elements so GC can free the memory. + // This can happen when len(src) < len(dest) < cap(dest). for i := len(src); i < len(dest); i++ { DeleteCountSketchDataPoint(&dest[i], false) } @@ -111,16 +113,22 @@ func CopyCountSketchDataPointPtrSlice(dest, src []*CountSketchDataPoint) []*Coun var newDest []*CountSketchDataPoint if cap(dest) < len(src) { newDest = make([]*CountSketchDataPoint, len(src)) + // Copy old pointers to re-use. copy(newDest, dest) + // Add new pointers for missing elements from len(dest) to len(srt). for i := len(dest); i < len(src); i++ { newDest[i] = NewCountSketchDataPoint() } } else { newDest = dest[:len(src)] + // Cleanup the rest of the elements so GC can free the memory. + // This can happen when len(src) < len(dest) < cap(dest). for i := len(src); i < len(dest); i++ { DeleteCountSketchDataPoint(dest[i], true) dest[i] = nil } + // Add new pointers for missing elements. + // This can happen when len(dest) < len(src) < cap(dest). for i := len(dest); i < len(src); i++ { newDest[i] = NewCountSketchDataPoint() } @@ -148,10 +156,6 @@ func (orig *CountSketchDataPoint) MarshalJSON(dest *json.Stream) { } dest.WriteArrayEnd() } - if orig.SeriesID != uint64(0) { - dest.WriteObjectField("seriesID") - dest.WriteUint64(orig.SeriesID) - } if orig.StartTimeUnixNano != uint64(0) { dest.WriteObjectField("startTimeUnixNano") dest.WriteUint64(orig.StartTimeUnixNano) @@ -160,30 +164,24 @@ func (orig *CountSketchDataPoint) MarshalJSON(dest *json.Stream) { dest.WriteObjectField("timeUnixNano") dest.WriteUint64(orig.TimeUnixNano) } + if len(orig.Sketch) > 0 { dest.WriteObjectField("sketch") dest.WriteBytes(orig.Sketch) } + if int32(orig.Encoding) != 0 { dest.WriteObjectField("encoding") dest.WriteInt32(int32(orig.Encoding)) } - if orig.Dimension != "" { - dest.WriteObjectField("dimension") - dest.WriteString(orig.Dimension) - } - if orig.Epsilon != float64(0) { - dest.WriteObjectField("epsilon") - dest.WriteFloat64(orig.Epsilon) - } - if orig.Delta != float64(0) { - dest.WriteObjectField("delta") - dest.WriteFloat64(orig.Delta) - } if orig.Flags != uint32(0) { dest.WriteObjectField("flags") dest.WriteUint32(orig.Flags) } + if orig.SeriesID != uint64(0) { + dest.WriteObjectField("seriesID") + dest.WriteUint64(orig.SeriesID) + } dest.WriteObjectEnd() } @@ -196,8 +194,7 @@ func (orig *CountSketchDataPoint) UnmarshalJSON(iter *json.Iterator) { orig.Attributes = append(orig.Attributes, KeyValue{}) orig.Attributes[len(orig.Attributes)-1].UnmarshalJSON(iter) } - case "seriesID", "series_id": - orig.SeriesID = iter.ReadUint64() + case "startTimeUnixNano", "start_time_unix_nano": orig.StartTimeUnixNano = iter.ReadUint64() case "timeUnixNano", "time_unix_nano": @@ -206,14 +203,10 @@ func (orig *CountSketchDataPoint) UnmarshalJSON(iter *json.Iterator) { orig.Sketch = iter.ReadBytes() case "encoding": orig.Encoding = CountSketchEncoding(iter.ReadEnumValue(CountSketchEncoding_value)) - case "dimension": - orig.Dimension = iter.ReadString() - case "epsilon": - orig.Epsilon = iter.ReadFloat64() - case "delta": - orig.Delta = iter.ReadFloat64() case "flags": orig.Flags = iter.ReadUint32() + case "seriesID", "series_id": + orig.SeriesID = iter.ReadUint64() default: iter.Skip() } @@ -221,17 +214,6 @@ func (orig *CountSketchDataPoint) UnmarshalJSON(iter *json.Iterator) { } func (orig *CountSketchDataPoint) SizeProto() int { - // Field layout: - // 1=attributes (wire 2) - // 2=start_time_unix_nano (wire 1, I64) - // 3=time_unix_nano (wire 1, I64) - // 4=sketch (wire 2) - // 5=encoding (wire 0) - // 6=dimension (wire 2) - // 7=epsilon (wire 1, I64) - // 8=delta (wire 1, I64) - // 9=flags (wire 0) - // 10=series_id (wire 0) var n int var l int _ = l @@ -252,16 +234,6 @@ func (orig *CountSketchDataPoint) SizeProto() int { if orig.Encoding != 0 { n += 1 + proto.Sov(uint64(orig.Encoding)) } - l = len(orig.Dimension) - if l > 0 { - n += 1 + proto.Sov(uint64(l)) + l - } - if orig.Epsilon != 0 { - n += 9 - } - if orig.Delta != 0 { - n += 9 - } if orig.Flags != 0 { n += 1 + proto.Sov(uint64(orig.Flags)) } @@ -272,17 +244,6 @@ func (orig *CountSketchDataPoint) SizeProto() int { } func (orig *CountSketchDataPoint) MarshalProto(buf []byte) int { - // Wire tag bytes: - // 1=attributes: 0x0a - // 2=start_time_unix_nano: 0x11 - // 3=time_unix_nano: 0x19 - // 4=sketch: 0x22 - // 5=encoding: 0x28 - // 6=dimension: 0x32 - // 7=epsilon: 0x39 - // 8=delta: 0x41 - // 9=flags: 0x48 - // 10=series_id: 0x50 pos := len(buf) var l int _ = l @@ -291,12 +252,7 @@ func (orig *CountSketchDataPoint) MarshalProto(buf []byte) int { pos -= l pos = proto.EncodeVarint(buf, pos, uint64(l)) pos-- - buf[pos] = 0x0a - } - if orig.SeriesID != 0 { - pos = proto.EncodeVarint(buf, pos, uint64(orig.SeriesID)) - pos-- - buf[pos] = 0x50 + buf[pos] = 0xa } if orig.StartTimeUnixNano != 0 { pos -= 8 @@ -323,31 +279,16 @@ func (orig *CountSketchDataPoint) MarshalProto(buf []byte) int { pos-- buf[pos] = 0x28 } - l = len(orig.Dimension) - if l > 0 { - pos -= l - copy(buf[pos:], orig.Dimension) - pos = proto.EncodeVarint(buf, pos, uint64(l)) - pos-- - buf[pos] = 0x32 - } - if orig.Epsilon != 0 { - pos -= 8 - binary.LittleEndian.PutUint64(buf[pos:], math.Float64bits(orig.Epsilon)) - pos-- - buf[pos] = 0x39 - } - if orig.Delta != 0 { - pos -= 8 - binary.LittleEndian.PutUint64(buf[pos:], math.Float64bits(orig.Delta)) - pos-- - buf[pos] = 0x41 - } if orig.Flags != 0 { pos = proto.EncodeVarint(buf, pos, uint64(orig.Flags)) pos-- buf[pos] = 0x48 } + if orig.SeriesID != 0 { + pos = proto.EncodeVarint(buf, pos, uint64(orig.SeriesID)) + pos-- + buf[pos] = 0x50 + } return len(buf) - pos } @@ -359,6 +300,7 @@ func (orig *CountSketchDataPoint) UnmarshalProto(buf []byte) error { l := len(buf) pos := 0 for pos < l { + // If in a group parsing, move to the next tag. fieldNum, wireType, pos, err = proto.ConsumeTag(buf, pos) if err != nil { return err @@ -390,6 +332,7 @@ func (orig *CountSketchDataPoint) UnmarshalProto(buf []byte) error { if err != nil { return err } + orig.StartTimeUnixNano = uint64(num) case 3: @@ -401,6 +344,7 @@ func (orig *CountSketchDataPoint) UnmarshalProto(buf []byte) error { if err != nil { return err } + orig.TimeUnixNano = uint64(num) case 4: @@ -427,41 +371,8 @@ func (orig *CountSketchDataPoint) UnmarshalProto(buf []byte) error { if err != nil { return err } - orig.Encoding = CountSketchEncoding(num) - - case 6: - if wireType != proto.WireTypeLen { - return fmt.Errorf("proto: wrong wireType = %d for field Dimension", wireType) - } - var length int - length, pos, err = proto.ConsumeLen(buf, pos) - if err != nil { - return err - } - startPos := pos - length - orig.Dimension = string(buf[startPos:pos]) - case 7: - if wireType != proto.WireTypeI64 { - return fmt.Errorf("proto: wrong wireType = %d for field Epsilon", wireType) - } - var num uint64 - num, pos, err = proto.ConsumeI64(buf, pos) - if err != nil { - return err - } - orig.Epsilon = math.Float64frombits(num) - - case 8: - if wireType != proto.WireTypeI64 { - return fmt.Errorf("proto: wrong wireType = %d for field Delta", wireType) - } - var num uint64 - num, pos, err = proto.ConsumeI64(buf, pos) - if err != nil { - return err - } - orig.Delta = math.Float64frombits(num) + orig.Encoding = CountSketchEncoding(num) case 9: if wireType != proto.WireTypeVarint { @@ -472,6 +383,7 @@ func (orig *CountSketchDataPoint) UnmarshalProto(buf []byte) error { if err != nil { return err } + orig.Flags = uint32(num) case 10: @@ -483,8 +395,8 @@ func (orig *CountSketchDataPoint) UnmarshalProto(buf []byte) error { if err != nil { return err } - orig.SeriesID = uint64(num) + orig.SeriesID = uint64(num) default: pos, err = proto.ConsumeUnknown(buf, pos, wireType) if err != nil { @@ -498,15 +410,12 @@ func (orig *CountSketchDataPoint) UnmarshalProto(buf []byte) error { func GenTestCountSketchDataPoint() *CountSketchDataPoint { orig := NewCountSketchDataPoint() orig.Attributes = []KeyValue{{}, *GenTestKeyValue()} - orig.SeriesID = uint64(13) orig.StartTimeUnixNano = uint64(13) orig.TimeUnixNano = uint64(13) orig.Sketch = []byte{1, 2, 3} - orig.Encoding = CountSketchEncoding(1) - orig.Dimension = "row" - orig.Epsilon = float64(0.001) - orig.Delta = float64(0.01) + orig.Encoding = CountSketchEncoding(13) orig.Flags = uint32(13) + orig.SeriesID = uint64(13) return orig } diff --git a/opentelemetry-collector-patch/pdata/internal/generated_proto_countsketchdatapoint_test.go b/opentelemetry-collector-patch/pdata/internal/generated_proto_countsketchdatapoint_test.go new file mode 100644 index 00000000..5e303543 --- /dev/null +++ b/opentelemetry-collector-patch/pdata/internal/generated_proto_countsketchdatapoint_test.go @@ -0,0 +1,220 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package internal + +import ( + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + gootlpmetrics "go.opentelemetry.io/proto/slim/otlp/metrics/v1" + "google.golang.org/protobuf/proto" + + "go.opentelemetry.io/collector/featuregate" + "go.opentelemetry.io/collector/pdata/internal/json" +) + +func TestCopyCountSketchDataPoint(t *testing.T) { + for name, src := range genTestEncodingValuesCountSketchDataPoint() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + dest := NewCountSketchDataPoint() + CopyCountSketchDataPoint(dest, src) + assert.Equal(t, src, dest) + CopyCountSketchDataPoint(dest, dest) + assert.Equal(t, src, dest) + }) + } + } +} + +func TestCopyCountSketchDataPointSlice(t *testing.T) { + src := []CountSketchDataPoint{} + dest := []CountSketchDataPoint{} + // Test CopyTo empty + dest = CopyCountSketchDataPointSlice(dest, src) + assert.Equal(t, []CountSketchDataPoint{}, dest) + + // Test CopyTo larger slice + src = GenTestCountSketchDataPointSlice() + dest = CopyCountSketchDataPointSlice(dest, src) + assert.Equal(t, GenTestCountSketchDataPointSlice(), dest) + + // Test CopyTo same size slice + dest = CopyCountSketchDataPointSlice(dest, src) + assert.Equal(t, GenTestCountSketchDataPointSlice(), dest) + + // Test CopyTo smaller size slice + dest = CopyCountSketchDataPointSlice(dest, []CountSketchDataPoint{}) + assert.Len(t, dest, 0) + + // Test CopyTo larger slice with enough capacity + dest = CopyCountSketchDataPointSlice(dest, src) + assert.Equal(t, GenTestCountSketchDataPointSlice(), dest) +} + +func TestCopyCountSketchDataPointPtrSlice(t *testing.T) { + src := []*CountSketchDataPoint{} + dest := []*CountSketchDataPoint{} + // Test CopyTo empty + dest = CopyCountSketchDataPointPtrSlice(dest, src) + assert.Equal(t, []*CountSketchDataPoint{}, dest) + + // Test CopyTo larger slice + src = GenTestCountSketchDataPointPtrSlice() + dest = CopyCountSketchDataPointPtrSlice(dest, src) + assert.Equal(t, GenTestCountSketchDataPointPtrSlice(), dest) + + // Test CopyTo same size slice + dest = CopyCountSketchDataPointPtrSlice(dest, src) + assert.Equal(t, GenTestCountSketchDataPointPtrSlice(), dest) + + // Test CopyTo smaller size slice + dest = CopyCountSketchDataPointPtrSlice(dest, []*CountSketchDataPoint{}) + assert.Len(t, dest, 0) + + // Test CopyTo larger slice with enough capacity + dest = CopyCountSketchDataPointPtrSlice(dest, src) + assert.Equal(t, GenTestCountSketchDataPointPtrSlice(), dest) +} + +func TestMarshalAndUnmarshalJSONCountSketchDataPointUnknown(t *testing.T) { + iter := json.BorrowIterator([]byte(`{"unknown": "string"}`)) + defer json.ReturnIterator(iter) + dest := NewCountSketchDataPoint() + dest.UnmarshalJSON(iter) + require.NoError(t, iter.Error()) + assert.Equal(t, NewCountSketchDataPoint(), dest) +} + +func TestMarshalAndUnmarshalJSONCountSketchDataPoint(t *testing.T) { + for name, src := range genTestEncodingValuesCountSketchDataPoint() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + stream := json.BorrowStream(nil) + defer json.ReturnStream(stream) + src.MarshalJSON(stream) + require.NoError(t, stream.Error()) + + iter := json.BorrowIterator(stream.Buffer()) + defer json.ReturnIterator(iter) + dest := NewCountSketchDataPoint() + dest.UnmarshalJSON(iter) + require.NoError(t, iter.Error()) + + assert.Equal(t, src, dest) + DeleteCountSketchDataPoint(dest, true) + }) + } + } +} + +func TestMarshalAndUnmarshalProtoCountSketchDataPointFailing(t *testing.T) { + for name, buf := range genTestFailingUnmarshalProtoValuesCountSketchDataPoint() { + t.Run(name, func(t *testing.T) { + dest := NewCountSketchDataPoint() + require.Error(t, dest.UnmarshalProto(buf)) + }) + } +} + +func TestMarshalAndUnmarshalProtoCountSketchDataPointUnknown(t *testing.T) { + dest := NewCountSketchDataPoint() + // message Test { required int64 field = 1313; } encoding { "field": "1234" } + require.NoError(t, dest.UnmarshalProto([]byte{0x88, 0x52, 0xD2, 0x09})) + assert.Equal(t, NewCountSketchDataPoint(), dest) +} + +func TestMarshalAndUnmarshalProtoCountSketchDataPoint(t *testing.T) { + for name, src := range genTestEncodingValuesCountSketchDataPoint() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + buf := make([]byte, src.SizeProto()) + gotSize := src.MarshalProto(buf) + assert.Equal(t, len(buf), gotSize) + + dest := NewCountSketchDataPoint() + require.NoError(t, dest.UnmarshalProto(buf)) + + assert.Equal(t, src, dest) + DeleteCountSketchDataPoint(dest, true) + }) + } + } +} + +func TestMarshalAndUnmarshalProtoViaProtobufCountSketchDataPoint(t *testing.T) { + for name, src := range genTestEncodingValuesCountSketchDataPoint() { + t.Run(name, func(t *testing.T) { + buf := make([]byte, src.SizeProto()) + gotSize := src.MarshalProto(buf) + assert.Equal(t, len(buf), gotSize) + + goDest := &gootlpmetrics.CountSketchDataPoint{} + require.NoError(t, proto.Unmarshal(buf, goDest)) + + goBuf, err := proto.Marshal(goDest) + require.NoError(t, err) + + dest := NewCountSketchDataPoint() + require.NoError(t, dest.UnmarshalProto(goBuf)) + assert.Equal(t, src, dest) + }) + } +} + +func genTestFailingUnmarshalProtoValuesCountSketchDataPoint() map[string][]byte { + return map[string][]byte{ + "invalid_field": {0x02}, + "Attributes/wrong_wire_type": {0xc}, + "Attributes/missing_value": {0xa}, + "StartTimeUnixNano/wrong_wire_type": {0x14}, + "StartTimeUnixNano/missing_value": {0x11}, + "TimeUnixNano/wrong_wire_type": {0x1c}, + "TimeUnixNano/missing_value": {0x19}, + "Sketch/wrong_wire_type": {0x24}, + "Sketch/missing_value": {0x22}, + "Encoding/wrong_wire_type": {0x2c}, + "Encoding/missing_value": {0x28}, + "Flags/wrong_wire_type": {0x4c}, + "Flags/missing_value": {0x48}, + "SeriesID/wrong_wire_type": {0x54}, + "SeriesID/missing_value": {0x50}, + } +} + +func genTestEncodingValuesCountSketchDataPoint() map[string]*CountSketchDataPoint { + return map[string]*CountSketchDataPoint{ + "empty": NewCountSketchDataPoint(), + "Attributes/test": {Attributes: []KeyValue{{}, *GenTestKeyValue()}}, + "StartTimeUnixNano/test": {StartTimeUnixNano: uint64(13)}, + "TimeUnixNano/test": {TimeUnixNano: uint64(13)}, + "Sketch/test": {Sketch: []byte{1, 2, 3}}, + "Encoding/test": {Encoding: CountSketchEncoding(13)}, + "Flags/test": {Flags: uint32(13)}, + "SeriesID/test": {SeriesID: uint64(13)}, + } +} diff --git a/opentelemetry-collector-patch/pdata/internal/generated_proto_ddsketch.go b/opentelemetry-collector-patch/pdata/internal/generated_proto_ddsketch.go index c887563e..e61012ce 100644 --- a/opentelemetry-collector-patch/pdata/internal/generated_proto_ddsketch.go +++ b/opentelemetry-collector-patch/pdata/internal/generated_proto_ddsketch.go @@ -7,7 +7,9 @@ package internal import ( + "encoding/binary" "fmt" + "math" "sync" "go.opentelemetry.io/collector/pdata/internal/json" @@ -19,6 +21,7 @@ import ( type DDSketch struct { DataPoints []*DDSketchDataPoint AggregationTemporality AggregationTemporality + RelativeAccuracy float64 } var ( @@ -73,6 +76,8 @@ func CopyDDSketch(dest, src *DDSketch) *DDSketch { dest.AggregationTemporality = src.AggregationTemporality + dest.RelativeAccuracy = src.RelativeAccuracy + return dest } @@ -146,6 +151,10 @@ func (orig *DDSketch) MarshalJSON(dest *json.Stream) { dest.WriteObjectField("aggregationTemporality") dest.WriteInt32(int32(orig.AggregationTemporality)) } + if orig.RelativeAccuracy != float64(0) { + dest.WriteObjectField("relativeAccuracy") + dest.WriteFloat64(orig.RelativeAccuracy) + } dest.WriteObjectEnd() } @@ -161,6 +170,8 @@ func (orig *DDSketch) UnmarshalJSON(iter *json.Iterator) { case "aggregationTemporality", "aggregation_temporality": orig.AggregationTemporality = AggregationTemporality(iter.ReadEnumValue(AggregationTemporality_value)) + case "relativeAccuracy", "relative_accuracy": + orig.RelativeAccuracy = iter.ReadFloat64() default: iter.Skip() } @@ -178,6 +189,9 @@ func (orig *DDSketch) SizeProto() int { if orig.AggregationTemporality != 0 { n += 1 + proto.Sov(uint64(orig.AggregationTemporality)) } + if orig.RelativeAccuracy != 0 { + n += 9 + } return n } @@ -197,6 +211,12 @@ func (orig *DDSketch) MarshalProto(buf []byte) int { pos-- buf[pos] = 0x10 } + if orig.RelativeAccuracy != 0 { + pos -= 8 + binary.LittleEndian.PutUint64(buf[pos:], math.Float64bits(orig.RelativeAccuracy)) + pos-- + buf[pos] = 0x19 + } return len(buf) - pos } @@ -242,6 +262,18 @@ func (orig *DDSketch) UnmarshalProto(buf []byte) error { } orig.AggregationTemporality = AggregationTemporality(num) + + case 3: + if wireType != proto.WireTypeI64 { + return fmt.Errorf("proto: wrong wireType = %d for field RelativeAccuracy", wireType) + } + var num uint64 + num, pos, err = proto.ConsumeI64(buf, pos) + if err != nil { + return err + } + + orig.RelativeAccuracy = math.Float64frombits(num) default: pos, err = proto.ConsumeUnknown(buf, pos, wireType) if err != nil { @@ -256,6 +288,7 @@ func GenTestDDSketch() *DDSketch { orig := NewDDSketch() orig.DataPoints = []*DDSketchDataPoint{{}, GenTestDDSketchDataPoint()} orig.AggregationTemporality = AggregationTemporality(13) + orig.RelativeAccuracy = float64(3.1415926) return orig } diff --git a/opentelemetry-collector-patch/pdata/internal/generated_proto_ddsketch_test.go b/opentelemetry-collector-patch/pdata/internal/generated_proto_ddsketch_test.go index 6c359468..c5faf138 100644 --- a/opentelemetry-collector-patch/pdata/internal/generated_proto_ddsketch_test.go +++ b/opentelemetry-collector-patch/pdata/internal/generated_proto_ddsketch_test.go @@ -193,6 +193,8 @@ func genTestFailingUnmarshalProtoValuesDDSketch() map[string][]byte { "DataPoints/missing_value": {0xa}, "AggregationTemporality/wrong_wire_type": {0x14}, "AggregationTemporality/missing_value": {0x10}, + "RelativeAccuracy/wrong_wire_type": {0x1c}, + "RelativeAccuracy/missing_value": {0x19}, } } @@ -201,5 +203,6 @@ func genTestEncodingValuesDDSketch() map[string]*DDSketch { "empty": NewDDSketch(), "DataPoints/test": {DataPoints: []*DDSketchDataPoint{{}, GenTestDDSketchDataPoint()}}, "AggregationTemporality/test": {AggregationTemporality: AggregationTemporality(13)}, + "RelativeAccuracy/test": {RelativeAccuracy: float64(3.1415926)}, } } diff --git a/opentelemetry-collector-patch/pdata/internal/generated_proto_ddsketchdatapoint.go b/opentelemetry-collector-patch/pdata/internal/generated_proto_ddsketchdatapoint.go index d92126e4..5b248930 100644 --- a/opentelemetry-collector-patch/pdata/internal/generated_proto_ddsketchdatapoint.go +++ b/opentelemetry-collector-patch/pdata/internal/generated_proto_ddsketchdatapoint.go @@ -9,110 +9,18 @@ package internal import ( "encoding/binary" "fmt" - "math" "sync" "go.opentelemetry.io/collector/pdata/internal/json" "go.opentelemetry.io/collector/pdata/internal/proto" ) -func (m *DDSketchDataPoint) GetSum() any { - if m != nil { - return m.Sum - } - return nil -} - -type DDSketchDataPoint_SumAsDouble struct { - SumAsDouble float64 -} - -func (m *DDSketchDataPoint) GetSumAsDouble() float64 { - if v, ok := m.GetSum().(*DDSketchDataPoint_SumAsDouble); ok { - return v.SumAsDouble - } - return float64(0) -} - -type DDSketchDataPoint_SumAsInt struct { - SumAsInt int64 -} - -func (m *DDSketchDataPoint) GetSumAsInt() int64 { - if v, ok := m.GetSum().(*DDSketchDataPoint_SumAsInt); ok { - return v.SumAsInt - } - return int64(0) -} - -func (m *DDSketchDataPoint) GetMin() any { - if m != nil { - return m.Min - } - return nil -} - -type DDSketchDataPoint_MinAsDouble struct { - MinAsDouble float64 -} - -func (m *DDSketchDataPoint) GetMinAsDouble() float64 { - if v, ok := m.GetMin().(*DDSketchDataPoint_MinAsDouble); ok { - return v.MinAsDouble - } - return float64(0) -} - -type DDSketchDataPoint_MinAsInt struct { - MinAsInt int64 -} - -func (m *DDSketchDataPoint) GetMinAsInt() int64 { - if v, ok := m.GetMin().(*DDSketchDataPoint_MinAsInt); ok { - return v.MinAsInt - } - return int64(0) -} - -func (m *DDSketchDataPoint) GetMax() any { - if m != nil { - return m.Max - } - return nil -} - -type DDSketchDataPoint_MaxAsDouble struct { - MaxAsDouble float64 -} - -func (m *DDSketchDataPoint) GetMaxAsDouble() float64 { - if v, ok := m.GetMax().(*DDSketchDataPoint_MaxAsDouble); ok { - return v.MaxAsDouble - } - return float64(0) -} - -type DDSketchDataPoint_MaxAsInt struct { - MaxAsInt int64 -} - -func (m *DDSketchDataPoint) GetMaxAsInt() int64 { - if v, ok := m.GetMax().(*DDSketchDataPoint_MaxAsInt); ok { - return v.MaxAsInt - } - return int64(0) -} - // DDSketchDataPoint is a single data point that encodes a distribution using the DDSketch format. type DDSketchDataPoint struct { Attributes []KeyValue SeriesID uint64 StartTimeUnixNano uint64 TimeUnixNano uint64 - Count uint64 - Sum any - Min any - Max any Sketch []byte Encoding DDSketchEncoding Exemplars []Exemplar @@ -125,42 +33,6 @@ var ( return &DDSketchDataPoint{} }, } - - ProtoPoolDDSketchDataPoint_SumAsDouble = sync.Pool{ - New: func() any { - return &DDSketchDataPoint_SumAsDouble{} - }, - } - - ProtoPoolDDSketchDataPoint_SumAsInt = sync.Pool{ - New: func() any { - return &DDSketchDataPoint_SumAsInt{} - }, - } - - ProtoPoolDDSketchDataPoint_MinAsDouble = sync.Pool{ - New: func() any { - return &DDSketchDataPoint_MinAsDouble{} - }, - } - - ProtoPoolDDSketchDataPoint_MinAsInt = sync.Pool{ - New: func() any { - return &DDSketchDataPoint_MinAsInt{} - }, - } - - ProtoPoolDDSketchDataPoint_MaxAsDouble = sync.Pool{ - New: func() any { - return &DDSketchDataPoint_MaxAsDouble{} - }, - } - - ProtoPoolDDSketchDataPoint_MaxAsInt = sync.Pool{ - New: func() any { - return &DDSketchDataPoint_MaxAsInt{} - }, - } ) func NewDDSketchDataPoint() *DDSketchDataPoint { @@ -183,45 +55,6 @@ func DeleteDDSketchDataPoint(orig *DDSketchDataPoint, nullable bool) { for i := range orig.Attributes { DeleteKeyValue(&orig.Attributes[i], false) } - switch ov := orig.Sum.(type) { - case *DDSketchDataPoint_SumAsDouble: - if UseProtoPooling.IsEnabled() { - ov.SumAsDouble = float64(0) - ProtoPoolDDSketchDataPoint_SumAsDouble.Put(ov) - } - case *DDSketchDataPoint_SumAsInt: - if UseProtoPooling.IsEnabled() { - ov.SumAsInt = int64(0) - ProtoPoolDDSketchDataPoint_SumAsInt.Put(ov) - } - - } - switch ov := orig.Min.(type) { - case *DDSketchDataPoint_MinAsDouble: - if UseProtoPooling.IsEnabled() { - ov.MinAsDouble = float64(0) - ProtoPoolDDSketchDataPoint_MinAsDouble.Put(ov) - } - case *DDSketchDataPoint_MinAsInt: - if UseProtoPooling.IsEnabled() { - ov.MinAsInt = int64(0) - ProtoPoolDDSketchDataPoint_MinAsInt.Put(ov) - } - - } - switch ov := orig.Max.(type) { - case *DDSketchDataPoint_MaxAsDouble: - if UseProtoPooling.IsEnabled() { - ov.MaxAsDouble = float64(0) - ProtoPoolDDSketchDataPoint_MaxAsDouble.Put(ov) - } - case *DDSketchDataPoint_MaxAsInt: - if UseProtoPooling.IsEnabled() { - ov.MaxAsInt = int64(0) - ProtoPoolDDSketchDataPoint_MaxAsInt.Put(ov) - } - - } for i := range orig.Exemplars { DeleteExemplar(&orig.Exemplars[i], false) } @@ -253,74 +86,6 @@ func CopyDDSketchDataPoint(dest, src *DDSketchDataPoint) *DDSketchDataPoint { dest.TimeUnixNano = src.TimeUnixNano - dest.Count = src.Count - - switch t := src.Sum.(type) { - case *DDSketchDataPoint_SumAsDouble: - var ov *DDSketchDataPoint_SumAsDouble - if !UseProtoPooling.IsEnabled() { - ov = &DDSketchDataPoint_SumAsDouble{} - } else { - ov = ProtoPoolDDSketchDataPoint_SumAsDouble.Get().(*DDSketchDataPoint_SumAsDouble) - } - ov.SumAsDouble = t.SumAsDouble - dest.Sum = ov - case *DDSketchDataPoint_SumAsInt: - var ov *DDSketchDataPoint_SumAsInt - if !UseProtoPooling.IsEnabled() { - ov = &DDSketchDataPoint_SumAsInt{} - } else { - ov = ProtoPoolDDSketchDataPoint_SumAsInt.Get().(*DDSketchDataPoint_SumAsInt) - } - ov.SumAsInt = t.SumAsInt - dest.Sum = ov - default: - dest.Sum = nil - } - switch t := src.Min.(type) { - case *DDSketchDataPoint_MinAsDouble: - var ov *DDSketchDataPoint_MinAsDouble - if !UseProtoPooling.IsEnabled() { - ov = &DDSketchDataPoint_MinAsDouble{} - } else { - ov = ProtoPoolDDSketchDataPoint_MinAsDouble.Get().(*DDSketchDataPoint_MinAsDouble) - } - ov.MinAsDouble = t.MinAsDouble - dest.Min = ov - case *DDSketchDataPoint_MinAsInt: - var ov *DDSketchDataPoint_MinAsInt - if !UseProtoPooling.IsEnabled() { - ov = &DDSketchDataPoint_MinAsInt{} - } else { - ov = ProtoPoolDDSketchDataPoint_MinAsInt.Get().(*DDSketchDataPoint_MinAsInt) - } - ov.MinAsInt = t.MinAsInt - dest.Min = ov - default: - dest.Min = nil - } - switch t := src.Max.(type) { - case *DDSketchDataPoint_MaxAsDouble: - var ov *DDSketchDataPoint_MaxAsDouble - if !UseProtoPooling.IsEnabled() { - ov = &DDSketchDataPoint_MaxAsDouble{} - } else { - ov = ProtoPoolDDSketchDataPoint_MaxAsDouble.Get().(*DDSketchDataPoint_MaxAsDouble) - } - ov.MaxAsDouble = t.MaxAsDouble - dest.Max = ov - case *DDSketchDataPoint_MaxAsInt: - var ov *DDSketchDataPoint_MaxAsInt - if !UseProtoPooling.IsEnabled() { - ov = &DDSketchDataPoint_MaxAsInt{} - } else { - ov = ProtoPoolDDSketchDataPoint_MaxAsInt.Get().(*DDSketchDataPoint_MaxAsInt) - } - ov.MaxAsInt = t.MaxAsInt - dest.Max = ov - default: - dest.Max = nil - } dest.Sketch = src.Sketch dest.Encoding = src.Encoding @@ -409,34 +174,6 @@ func (orig *DDSketchDataPoint) MarshalJSON(dest *json.Stream) { dest.WriteObjectField("timeUnixNano") dest.WriteUint64(orig.TimeUnixNano) } - if orig.Count != uint64(0) { - dest.WriteObjectField("count") - dest.WriteUint64(orig.Count) - } - switch orig := orig.Sum.(type) { - case *DDSketchDataPoint_SumAsDouble: - dest.WriteObjectField("sumAsDouble") - dest.WriteFloat64(orig.SumAsDouble) - case *DDSketchDataPoint_SumAsInt: - dest.WriteObjectField("sumAsInt") - dest.WriteInt64(orig.SumAsInt) - } - switch orig := orig.Min.(type) { - case *DDSketchDataPoint_MinAsDouble: - dest.WriteObjectField("minAsDouble") - dest.WriteFloat64(orig.MinAsDouble) - case *DDSketchDataPoint_MinAsInt: - dest.WriteObjectField("minAsInt") - dest.WriteInt64(orig.MinAsInt) - } - switch orig := orig.Max.(type) { - case *DDSketchDataPoint_MaxAsDouble: - dest.WriteObjectField("maxAsDouble") - dest.WriteFloat64(orig.MaxAsDouble) - case *DDSketchDataPoint_MaxAsInt: - dest.WriteObjectField("maxAsInt") - dest.WriteInt64(orig.MaxAsInt) - } if len(orig.Sketch) > 0 { dest.WriteObjectField("sketch") @@ -480,81 +217,6 @@ func (orig *DDSketchDataPoint) UnmarshalJSON(iter *json.Iterator) { orig.StartTimeUnixNano = iter.ReadUint64() case "timeUnixNano", "time_unix_nano": orig.TimeUnixNano = iter.ReadUint64() - case "count": - orig.Count = iter.ReadUint64() - - case "sumAsDouble", "sum_as_double": - { - var ov *DDSketchDataPoint_SumAsDouble - if !UseProtoPooling.IsEnabled() { - ov = &DDSketchDataPoint_SumAsDouble{} - } else { - ov = ProtoPoolDDSketchDataPoint_SumAsDouble.Get().(*DDSketchDataPoint_SumAsDouble) - } - ov.SumAsDouble = iter.ReadFloat64() - orig.Sum = ov - } - - case "sumAsInt", "sum_as_int": - { - var ov *DDSketchDataPoint_SumAsInt - if !UseProtoPooling.IsEnabled() { - ov = &DDSketchDataPoint_SumAsInt{} - } else { - ov = ProtoPoolDDSketchDataPoint_SumAsInt.Get().(*DDSketchDataPoint_SumAsInt) - } - ov.SumAsInt = iter.ReadInt64() - orig.Sum = ov - } - - case "minAsDouble", "min_as_double": - { - var ov *DDSketchDataPoint_MinAsDouble - if !UseProtoPooling.IsEnabled() { - ov = &DDSketchDataPoint_MinAsDouble{} - } else { - ov = ProtoPoolDDSketchDataPoint_MinAsDouble.Get().(*DDSketchDataPoint_MinAsDouble) - } - ov.MinAsDouble = iter.ReadFloat64() - orig.Min = ov - } - - case "minAsInt", "min_as_int": - { - var ov *DDSketchDataPoint_MinAsInt - if !UseProtoPooling.IsEnabled() { - ov = &DDSketchDataPoint_MinAsInt{} - } else { - ov = ProtoPoolDDSketchDataPoint_MinAsInt.Get().(*DDSketchDataPoint_MinAsInt) - } - ov.MinAsInt = iter.ReadInt64() - orig.Min = ov - } - - case "maxAsDouble", "max_as_double": - { - var ov *DDSketchDataPoint_MaxAsDouble - if !UseProtoPooling.IsEnabled() { - ov = &DDSketchDataPoint_MaxAsDouble{} - } else { - ov = ProtoPoolDDSketchDataPoint_MaxAsDouble.Get().(*DDSketchDataPoint_MaxAsDouble) - } - ov.MaxAsDouble = iter.ReadFloat64() - orig.Max = ov - } - - case "maxAsInt", "max_as_int": - { - var ov *DDSketchDataPoint_MaxAsInt - if !UseProtoPooling.IsEnabled() { - ov = &DDSketchDataPoint_MaxAsInt{} - } else { - ov = ProtoPoolDDSketchDataPoint_MaxAsInt.Get().(*DDSketchDataPoint_MaxAsInt) - } - ov.MaxAsInt = iter.ReadInt64() - orig.Max = ov - } - case "sketch": orig.Sketch = iter.ReadBytes() case "encoding": @@ -590,36 +252,6 @@ func (orig *DDSketchDataPoint) SizeProto() int { if orig.TimeUnixNano != 0 { n += 9 } - if orig.Count != 0 { - n += 9 - } - switch orig := orig.Sum.(type) { - case nil: - _ = orig - break - case *DDSketchDataPoint_SumAsDouble: - n += 9 - case *DDSketchDataPoint_SumAsInt: - n += 9 - } - switch orig := orig.Min.(type) { - case nil: - _ = orig - break - case *DDSketchDataPoint_MinAsDouble: - n += 9 - case *DDSketchDataPoint_MinAsInt: - n += 9 - } - switch orig := orig.Max.(type) { - case nil: - _ = orig - break - case *DDSketchDataPoint_MaxAsDouble: - n += 9 - case *DDSketchDataPoint_MaxAsInt: - n += 9 - } l = len(orig.Sketch) if l > 0 { n += 1 + proto.Sov(uint64(l)) + l @@ -667,54 +299,6 @@ func (orig *DDSketchDataPoint) MarshalProto(buf []byte) int { pos-- buf[pos] = 0x19 } - if orig.Count != 0 { - pos -= 8 - binary.LittleEndian.PutUint64(buf[pos:], uint64(orig.Count)) - pos-- - buf[pos] = 0x21 - } - switch orig := orig.Sum.(type) { - case *DDSketchDataPoint_SumAsDouble: - pos -= 8 - binary.LittleEndian.PutUint64(buf[pos:], math.Float64bits(orig.SumAsDouble)) - pos-- - buf[pos] = 0x29 - - case *DDSketchDataPoint_SumAsInt: - pos -= 8 - binary.LittleEndian.PutUint64(buf[pos:], uint64(orig.SumAsInt)) - pos-- - buf[pos] = 0x61 - - } - switch orig := orig.Min.(type) { - case *DDSketchDataPoint_MinAsDouble: - pos -= 8 - binary.LittleEndian.PutUint64(buf[pos:], math.Float64bits(orig.MinAsDouble)) - pos-- - buf[pos] = 0x31 - - case *DDSketchDataPoint_MinAsInt: - pos -= 8 - binary.LittleEndian.PutUint64(buf[pos:], uint64(orig.MinAsInt)) - pos-- - buf[pos] = 0x69 - - } - switch orig := orig.Max.(type) { - case *DDSketchDataPoint_MaxAsDouble: - pos -= 8 - binary.LittleEndian.PutUint64(buf[pos:], math.Float64bits(orig.MaxAsDouble)) - pos-- - buf[pos] = 0x39 - - case *DDSketchDataPoint_MaxAsInt: - pos -= 8 - binary.LittleEndian.PutUint64(buf[pos:], uint64(orig.MaxAsInt)) - pos-- - buf[pos] = 0x71 - - } l = len(orig.Sketch) if l > 0 { pos -= l @@ -810,126 +394,6 @@ func (orig *DDSketchDataPoint) UnmarshalProto(buf []byte) error { orig.TimeUnixNano = uint64(num) - case 4: - if wireType != proto.WireTypeI64 { - return fmt.Errorf("proto: wrong wireType = %d for field Count", wireType) - } - var num uint64 - num, pos, err = proto.ConsumeI64(buf, pos) - if err != nil { - return err - } - - orig.Count = uint64(num) - - case 5: - if wireType != proto.WireTypeI64 { - return fmt.Errorf("proto: wrong wireType = %d for field SumAsDouble", wireType) - } - var num uint64 - num, pos, err = proto.ConsumeI64(buf, pos) - if err != nil { - return err - } - var ov *DDSketchDataPoint_SumAsDouble - if !UseProtoPooling.IsEnabled() { - ov = &DDSketchDataPoint_SumAsDouble{} - } else { - ov = ProtoPoolDDSketchDataPoint_SumAsDouble.Get().(*DDSketchDataPoint_SumAsDouble) - } - ov.SumAsDouble = math.Float64frombits(num) - orig.Sum = ov - - case 12: - if wireType != proto.WireTypeI64 { - return fmt.Errorf("proto: wrong wireType = %d for field SumAsInt", wireType) - } - var num uint64 - num, pos, err = proto.ConsumeI64(buf, pos) - if err != nil { - return err - } - var ov *DDSketchDataPoint_SumAsInt - if !UseProtoPooling.IsEnabled() { - ov = &DDSketchDataPoint_SumAsInt{} - } else { - ov = ProtoPoolDDSketchDataPoint_SumAsInt.Get().(*DDSketchDataPoint_SumAsInt) - } - ov.SumAsInt = int64(num) - orig.Sum = ov - - case 6: - if wireType != proto.WireTypeI64 { - return fmt.Errorf("proto: wrong wireType = %d for field MinAsDouble", wireType) - } - var num uint64 - num, pos, err = proto.ConsumeI64(buf, pos) - if err != nil { - return err - } - var ov *DDSketchDataPoint_MinAsDouble - if !UseProtoPooling.IsEnabled() { - ov = &DDSketchDataPoint_MinAsDouble{} - } else { - ov = ProtoPoolDDSketchDataPoint_MinAsDouble.Get().(*DDSketchDataPoint_MinAsDouble) - } - ov.MinAsDouble = math.Float64frombits(num) - orig.Min = ov - - case 13: - if wireType != proto.WireTypeI64 { - return fmt.Errorf("proto: wrong wireType = %d for field MinAsInt", wireType) - } - var num uint64 - num, pos, err = proto.ConsumeI64(buf, pos) - if err != nil { - return err - } - var ov *DDSketchDataPoint_MinAsInt - if !UseProtoPooling.IsEnabled() { - ov = &DDSketchDataPoint_MinAsInt{} - } else { - ov = ProtoPoolDDSketchDataPoint_MinAsInt.Get().(*DDSketchDataPoint_MinAsInt) - } - ov.MinAsInt = int64(num) - orig.Min = ov - - case 7: - if wireType != proto.WireTypeI64 { - return fmt.Errorf("proto: wrong wireType = %d for field MaxAsDouble", wireType) - } - var num uint64 - num, pos, err = proto.ConsumeI64(buf, pos) - if err != nil { - return err - } - var ov *DDSketchDataPoint_MaxAsDouble - if !UseProtoPooling.IsEnabled() { - ov = &DDSketchDataPoint_MaxAsDouble{} - } else { - ov = ProtoPoolDDSketchDataPoint_MaxAsDouble.Get().(*DDSketchDataPoint_MaxAsDouble) - } - ov.MaxAsDouble = math.Float64frombits(num) - orig.Max = ov - - case 14: - if wireType != proto.WireTypeI64 { - return fmt.Errorf("proto: wrong wireType = %d for field MaxAsInt", wireType) - } - var num uint64 - num, pos, err = proto.ConsumeI64(buf, pos) - if err != nil { - return err - } - var ov *DDSketchDataPoint_MaxAsInt - if !UseProtoPooling.IsEnabled() { - ov = &DDSketchDataPoint_MaxAsInt{} - } else { - ov = ProtoPoolDDSketchDataPoint_MaxAsInt.Get().(*DDSketchDataPoint_MaxAsInt) - } - ov.MaxAsInt = int64(num) - orig.Max = ov - case 8: if wireType != proto.WireTypeLen { return fmt.Errorf("proto: wrong wireType = %d for field Sketch", wireType) @@ -1000,10 +464,6 @@ func GenTestDDSketchDataPoint() *DDSketchDataPoint { orig.SeriesID = uint64(13) orig.StartTimeUnixNano = uint64(13) orig.TimeUnixNano = uint64(13) - orig.Count = uint64(13) - orig.Sum = &DDSketchDataPoint_SumAsDouble{SumAsDouble: float64(3.1415926)} - orig.Min = &DDSketchDataPoint_MinAsDouble{MinAsDouble: float64(3.1415926)} - orig.Max = &DDSketchDataPoint_MaxAsDouble{MaxAsDouble: float64(3.1415926)} orig.Sketch = []byte{1, 2, 3} orig.Encoding = DDSketchEncoding(13) orig.Exemplars = []Exemplar{{}, *GenTestExemplar()} diff --git a/opentelemetry-collector-patch/pdata/internal/generated_proto_ddsketchdatapoint_test.go b/opentelemetry-collector-patch/pdata/internal/generated_proto_ddsketchdatapoint_test.go index c8523be7..64106ba3 100644 --- a/opentelemetry-collector-patch/pdata/internal/generated_proto_ddsketchdatapoint_test.go +++ b/opentelemetry-collector-patch/pdata/internal/generated_proto_ddsketchdatapoint_test.go @@ -197,34 +197,14 @@ func genTestFailingUnmarshalProtoValuesDDSketchDataPoint() map[string][]byte { "StartTimeUnixNano/missing_value": {0x11}, "TimeUnixNano/wrong_wire_type": {0x1c}, "TimeUnixNano/missing_value": {0x19}, - "Count/wrong_wire_type": {0x24}, - "Count/missing_value": {0x21}, - - "SumAsDouble/wrong_wire_type": {0x2c}, - "SumAsDouble/missing_value": {0x29}, - - "SumAsInt/wrong_wire_type": {0x64}, - "SumAsInt/missing_value": {0x61}, - - "MinAsDouble/wrong_wire_type": {0x34}, - "MinAsDouble/missing_value": {0x31}, - - "MinAsInt/wrong_wire_type": {0x6c}, - "MinAsInt/missing_value": {0x69}, - - "MaxAsDouble/wrong_wire_type": {0x3c}, - "MaxAsDouble/missing_value": {0x39}, - - "MaxAsInt/wrong_wire_type": {0x74}, - "MaxAsInt/missing_value": {0x71}, - "Sketch/wrong_wire_type": {0x44}, - "Sketch/missing_value": {0x42}, - "Encoding/wrong_wire_type": {0x54}, - "Encoding/missing_value": {0x50}, - "Exemplars/wrong_wire_type": {0x5c}, - "Exemplars/missing_value": {0x5a}, - "Flags/wrong_wire_type": {0x7c}, - "Flags/missing_value": {0x78}, + "Sketch/wrong_wire_type": {0x44}, + "Sketch/missing_value": {0x42}, + "Encoding/wrong_wire_type": {0x54}, + "Encoding/missing_value": {0x50}, + "Exemplars/wrong_wire_type": {0x5c}, + "Exemplars/missing_value": {0x5a}, + "Flags/wrong_wire_type": {0x7c}, + "Flags/missing_value": {0x78}, } } @@ -235,19 +215,6 @@ func genTestEncodingValuesDDSketchDataPoint() map[string]*DDSketchDataPoint { "SeriesID/test": {SeriesID: uint64(13)}, "StartTimeUnixNano/test": {StartTimeUnixNano: uint64(13)}, "TimeUnixNano/test": {TimeUnixNano: uint64(13)}, - "Count/test": {Count: uint64(13)}, - "SumAsDouble/default": {Sum: &DDSketchDataPoint_SumAsDouble{SumAsDouble: float64(0)}}, - "SumAsDouble/test": {Sum: &DDSketchDataPoint_SumAsDouble{SumAsDouble: float64(3.1415926)}}, - "SumAsInt/default": {Sum: &DDSketchDataPoint_SumAsInt{SumAsInt: int64(0)}}, - "SumAsInt/test": {Sum: &DDSketchDataPoint_SumAsInt{SumAsInt: int64(13)}}, - "MinAsDouble/default": {Min: &DDSketchDataPoint_MinAsDouble{MinAsDouble: float64(0)}}, - "MinAsDouble/test": {Min: &DDSketchDataPoint_MinAsDouble{MinAsDouble: float64(3.1415926)}}, - "MinAsInt/default": {Min: &DDSketchDataPoint_MinAsInt{MinAsInt: int64(0)}}, - "MinAsInt/test": {Min: &DDSketchDataPoint_MinAsInt{MinAsInt: int64(13)}}, - "MaxAsDouble/default": {Max: &DDSketchDataPoint_MaxAsDouble{MaxAsDouble: float64(0)}}, - "MaxAsDouble/test": {Max: &DDSketchDataPoint_MaxAsDouble{MaxAsDouble: float64(3.1415926)}}, - "MaxAsInt/default": {Max: &DDSketchDataPoint_MaxAsInt{MaxAsInt: int64(0)}}, - "MaxAsInt/test": {Max: &DDSketchDataPoint_MaxAsInt{MaxAsInt: int64(13)}}, "Sketch/test": {Sketch: []byte{1, 2, 3}}, "Encoding/test": {Encoding: DDSketchEncoding(13)}, "Exemplars/test": {Exemplars: []Exemplar{{}, *GenTestExemplar()}}, diff --git a/opentelemetry-collector-patch/pdata/internal/generated_proto_hllsketch.go b/opentelemetry-collector-patch/pdata/internal/generated_proto_hllsketch.go index c9598ec4..aa224349 100644 --- a/opentelemetry-collector-patch/pdata/internal/generated_proto_hllsketch.go +++ b/opentelemetry-collector-patch/pdata/internal/generated_proto_hllsketch.go @@ -18,6 +18,7 @@ import ( type HLLSketch struct { DataPoints []*HLLSketchDataPoint AggregationTemporality AggregationTemporality + Precision uint32 } var ( @@ -56,6 +57,7 @@ func DeleteHLLSketch(orig *HLLSketch, nullable bool) { } func CopyHLLSketch(dest, src *HLLSketch) *HLLSketch { + // If copying to same object, just return. if src == dest { return dest } @@ -71,6 +73,8 @@ func CopyHLLSketch(dest, src *HLLSketch) *HLLSketch { dest.AggregationTemporality = src.AggregationTemporality + dest.Precision = src.Precision + return dest } @@ -80,6 +84,8 @@ func CopyHLLSketchSlice(dest, src []HLLSketch) []HLLSketch { newDest = make([]HLLSketch, len(src)) } else { newDest = dest[:len(src)] + // Cleanup the rest of the elements so GC can free the memory. + // This can happen when len(src) < len(dest) < cap(dest). for i := len(src); i < len(dest); i++ { DeleteHLLSketch(&dest[i], false) } @@ -94,16 +100,22 @@ func CopyHLLSketchPtrSlice(dest, src []*HLLSketch) []*HLLSketch { var newDest []*HLLSketch if cap(dest) < len(src) { newDest = make([]*HLLSketch, len(src)) + // Copy old pointers to re-use. copy(newDest, dest) + // Add new pointers for missing elements from len(dest) to len(srt). for i := len(dest); i < len(src); i++ { newDest[i] = NewHLLSketch() } } else { newDest = dest[:len(src)] + // Cleanup the rest of the elements so GC can free the memory. + // This can happen when len(src) < len(dest) < cap(dest). for i := len(src); i < len(dest); i++ { DeleteHLLSketch(dest[i], true) dest[i] = nil } + // Add new pointers for missing elements. + // This can happen when len(dest) < len(src) < cap(dest). for i := len(dest); i < len(src); i++ { newDest[i] = NewHLLSketch() } @@ -136,6 +148,10 @@ func (orig *HLLSketch) MarshalJSON(dest *json.Stream) { dest.WriteObjectField("aggregationTemporality") dest.WriteInt32(int32(orig.AggregationTemporality)) } + if orig.Precision != uint32(0) { + dest.WriteObjectField("precision") + dest.WriteUint32(orig.Precision) + } dest.WriteObjectEnd() } @@ -151,6 +167,8 @@ func (orig *HLLSketch) UnmarshalJSON(iter *json.Iterator) { case "aggregationTemporality", "aggregation_temporality": orig.AggregationTemporality = AggregationTemporality(iter.ReadEnumValue(AggregationTemporality_value)) + case "precision": + orig.Precision = iter.ReadUint32() default: iter.Skip() } @@ -168,6 +186,9 @@ func (orig *HLLSketch) SizeProto() int { if orig.AggregationTemporality != 0 { n += 1 + proto.Sov(uint64(orig.AggregationTemporality)) } + if orig.Precision != 0 { + n += 1 + proto.Sov(uint64(orig.Precision)) + } return n } @@ -187,6 +208,11 @@ func (orig *HLLSketch) MarshalProto(buf []byte) int { pos-- buf[pos] = 0x10 } + if orig.Precision != 0 { + pos = proto.EncodeVarint(buf, pos, uint64(orig.Precision)) + pos-- + buf[pos] = 0x18 + } return len(buf) - pos } @@ -198,6 +224,7 @@ func (orig *HLLSketch) UnmarshalProto(buf []byte) error { l := len(buf) pos := 0 for pos < l { + // If in a group parsing, move to the next tag. fieldNum, wireType, pos, err = proto.ConsumeTag(buf, pos) if err != nil { return err @@ -231,6 +258,18 @@ func (orig *HLLSketch) UnmarshalProto(buf []byte) error { } orig.AggregationTemporality = AggregationTemporality(num) + + case 3: + if wireType != proto.WireTypeVarint { + return fmt.Errorf("proto: wrong wireType = %d for field Precision", wireType) + } + var num uint64 + num, pos, err = proto.ConsumeVarint(buf, pos) + if err != nil { + return err + } + + orig.Precision = uint32(num) default: pos, err = proto.ConsumeUnknown(buf, pos, wireType) if err != nil { @@ -245,6 +284,7 @@ func GenTestHLLSketch() *HLLSketch { orig := NewHLLSketch() orig.DataPoints = []*HLLSketchDataPoint{{}, GenTestHLLSketchDataPoint()} orig.AggregationTemporality = AggregationTemporality(13) + orig.Precision = uint32(13) return orig } diff --git a/opentelemetry-collector-patch/pdata/internal/generated_proto_hllsketch_test.go b/opentelemetry-collector-patch/pdata/internal/generated_proto_hllsketch_test.go new file mode 100644 index 00000000..cd3db211 --- /dev/null +++ b/opentelemetry-collector-patch/pdata/internal/generated_proto_hllsketch_test.go @@ -0,0 +1,208 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package internal + +import ( + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + gootlpmetrics "go.opentelemetry.io/proto/slim/otlp/metrics/v1" + "google.golang.org/protobuf/proto" + + "go.opentelemetry.io/collector/featuregate" + "go.opentelemetry.io/collector/pdata/internal/json" +) + +func TestCopyHLLSketch(t *testing.T) { + for name, src := range genTestEncodingValuesHLLSketch() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + dest := NewHLLSketch() + CopyHLLSketch(dest, src) + assert.Equal(t, src, dest) + CopyHLLSketch(dest, dest) + assert.Equal(t, src, dest) + }) + } + } +} + +func TestCopyHLLSketchSlice(t *testing.T) { + src := []HLLSketch{} + dest := []HLLSketch{} + // Test CopyTo empty + dest = CopyHLLSketchSlice(dest, src) + assert.Equal(t, []HLLSketch{}, dest) + + // Test CopyTo larger slice + src = GenTestHLLSketchSlice() + dest = CopyHLLSketchSlice(dest, src) + assert.Equal(t, GenTestHLLSketchSlice(), dest) + + // Test CopyTo same size slice + dest = CopyHLLSketchSlice(dest, src) + assert.Equal(t, GenTestHLLSketchSlice(), dest) + + // Test CopyTo smaller size slice + dest = CopyHLLSketchSlice(dest, []HLLSketch{}) + assert.Len(t, dest, 0) + + // Test CopyTo larger slice with enough capacity + dest = CopyHLLSketchSlice(dest, src) + assert.Equal(t, GenTestHLLSketchSlice(), dest) +} + +func TestCopyHLLSketchPtrSlice(t *testing.T) { + src := []*HLLSketch{} + dest := []*HLLSketch{} + // Test CopyTo empty + dest = CopyHLLSketchPtrSlice(dest, src) + assert.Equal(t, []*HLLSketch{}, dest) + + // Test CopyTo larger slice + src = GenTestHLLSketchPtrSlice() + dest = CopyHLLSketchPtrSlice(dest, src) + assert.Equal(t, GenTestHLLSketchPtrSlice(), dest) + + // Test CopyTo same size slice + dest = CopyHLLSketchPtrSlice(dest, src) + assert.Equal(t, GenTestHLLSketchPtrSlice(), dest) + + // Test CopyTo smaller size slice + dest = CopyHLLSketchPtrSlice(dest, []*HLLSketch{}) + assert.Len(t, dest, 0) + + // Test CopyTo larger slice with enough capacity + dest = CopyHLLSketchPtrSlice(dest, src) + assert.Equal(t, GenTestHLLSketchPtrSlice(), dest) +} + +func TestMarshalAndUnmarshalJSONHLLSketchUnknown(t *testing.T) { + iter := json.BorrowIterator([]byte(`{"unknown": "string"}`)) + defer json.ReturnIterator(iter) + dest := NewHLLSketch() + dest.UnmarshalJSON(iter) + require.NoError(t, iter.Error()) + assert.Equal(t, NewHLLSketch(), dest) +} + +func TestMarshalAndUnmarshalJSONHLLSketch(t *testing.T) { + for name, src := range genTestEncodingValuesHLLSketch() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + stream := json.BorrowStream(nil) + defer json.ReturnStream(stream) + src.MarshalJSON(stream) + require.NoError(t, stream.Error()) + + iter := json.BorrowIterator(stream.Buffer()) + defer json.ReturnIterator(iter) + dest := NewHLLSketch() + dest.UnmarshalJSON(iter) + require.NoError(t, iter.Error()) + + assert.Equal(t, src, dest) + DeleteHLLSketch(dest, true) + }) + } + } +} + +func TestMarshalAndUnmarshalProtoHLLSketchFailing(t *testing.T) { + for name, buf := range genTestFailingUnmarshalProtoValuesHLLSketch() { + t.Run(name, func(t *testing.T) { + dest := NewHLLSketch() + require.Error(t, dest.UnmarshalProto(buf)) + }) + } +} + +func TestMarshalAndUnmarshalProtoHLLSketchUnknown(t *testing.T) { + dest := NewHLLSketch() + // message Test { required int64 field = 1313; } encoding { "field": "1234" } + require.NoError(t, dest.UnmarshalProto([]byte{0x88, 0x52, 0xD2, 0x09})) + assert.Equal(t, NewHLLSketch(), dest) +} + +func TestMarshalAndUnmarshalProtoHLLSketch(t *testing.T) { + for name, src := range genTestEncodingValuesHLLSketch() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + buf := make([]byte, src.SizeProto()) + gotSize := src.MarshalProto(buf) + assert.Equal(t, len(buf), gotSize) + + dest := NewHLLSketch() + require.NoError(t, dest.UnmarshalProto(buf)) + + assert.Equal(t, src, dest) + DeleteHLLSketch(dest, true) + }) + } + } +} + +func TestMarshalAndUnmarshalProtoViaProtobufHLLSketch(t *testing.T) { + for name, src := range genTestEncodingValuesHLLSketch() { + t.Run(name, func(t *testing.T) { + buf := make([]byte, src.SizeProto()) + gotSize := src.MarshalProto(buf) + assert.Equal(t, len(buf), gotSize) + + goDest := &gootlpmetrics.HLLSketch{} + require.NoError(t, proto.Unmarshal(buf, goDest)) + + goBuf, err := proto.Marshal(goDest) + require.NoError(t, err) + + dest := NewHLLSketch() + require.NoError(t, dest.UnmarshalProto(goBuf)) + assert.Equal(t, src, dest) + }) + } +} + +func genTestFailingUnmarshalProtoValuesHLLSketch() map[string][]byte { + return map[string][]byte{ + "invalid_field": {0x02}, + "DataPoints/wrong_wire_type": {0xc}, + "DataPoints/missing_value": {0xa}, + "AggregationTemporality/wrong_wire_type": {0x14}, + "AggregationTemporality/missing_value": {0x10}, + "Precision/wrong_wire_type": {0x1c}, + "Precision/missing_value": {0x18}, + } +} + +func genTestEncodingValuesHLLSketch() map[string]*HLLSketch { + return map[string]*HLLSketch{ + "empty": NewHLLSketch(), + "DataPoints/test": {DataPoints: []*HLLSketchDataPoint{{}, GenTestHLLSketchDataPoint()}}, + "AggregationTemporality/test": {AggregationTemporality: AggregationTemporality(13)}, + "Precision/test": {Precision: uint32(13)}, + } +} diff --git a/opentelemetry-collector-patch/pdata/internal/generated_proto_hllsketchdatapoint.go b/opentelemetry-collector-patch/pdata/internal/generated_proto_hllsketchdatapoint.go index 9fec3306..420bedad 100644 --- a/opentelemetry-collector-patch/pdata/internal/generated_proto_hllsketchdatapoint.go +++ b/opentelemetry-collector-patch/pdata/internal/generated_proto_hllsketchdatapoint.go @@ -18,15 +18,12 @@ import ( // HLLSketchDataPoint is a single data point that encodes cardinality estimations using HyperLogLog. type HLLSketchDataPoint struct { Attributes []KeyValue - SeriesID uint64 StartTimeUnixNano uint64 TimeUnixNano uint64 - Count uint64 - Cardinality uint64 Sketch []byte Encoding HLLSketchEncoding - Precision uint32 Flags uint32 + SeriesID uint64 } var ( @@ -65,6 +62,7 @@ func DeleteHLLSketchDataPoint(orig *HLLSketchDataPoint, nullable bool) { } func CopyHLLSketchDataPoint(dest, src *HLLSketchDataPoint) *HLLSketchDataPoint { + // If copying to same object, just return. if src == dest { return dest } @@ -77,16 +75,19 @@ func CopyHLLSketchDataPoint(dest, src *HLLSketchDataPoint) *HLLSketchDataPoint { dest = NewHLLSketchDataPoint() } dest.Attributes = CopyKeyValueSlice(dest.Attributes, src.Attributes) - dest.SeriesID = src.SeriesID + dest.StartTimeUnixNano = src.StartTimeUnixNano + dest.TimeUnixNano = src.TimeUnixNano - dest.Count = src.Count - dest.Cardinality = src.Cardinality + dest.Sketch = src.Sketch + dest.Encoding = src.Encoding - dest.Precision = src.Precision + dest.Flags = src.Flags + dest.SeriesID = src.SeriesID + return dest } @@ -96,6 +97,8 @@ func CopyHLLSketchDataPointSlice(dest, src []HLLSketchDataPoint) []HLLSketchData newDest = make([]HLLSketchDataPoint, len(src)) } else { newDest = dest[:len(src)] + // Cleanup the rest of the elements so GC can free the memory. + // This can happen when len(src) < len(dest) < cap(dest). for i := len(src); i < len(dest); i++ { DeleteHLLSketchDataPoint(&dest[i], false) } @@ -110,16 +113,22 @@ func CopyHLLSketchDataPointPtrSlice(dest, src []*HLLSketchDataPoint) []*HLLSketc var newDest []*HLLSketchDataPoint if cap(dest) < len(src) { newDest = make([]*HLLSketchDataPoint, len(src)) + // Copy old pointers to re-use. copy(newDest, dest) + // Add new pointers for missing elements from len(dest) to len(srt). for i := len(dest); i < len(src); i++ { newDest[i] = NewHLLSketchDataPoint() } } else { newDest = dest[:len(src)] + // Cleanup the rest of the elements so GC can free the memory. + // This can happen when len(src) < len(dest) < cap(dest). for i := len(src); i < len(dest); i++ { DeleteHLLSketchDataPoint(dest[i], true) dest[i] = nil } + // Add new pointers for missing elements. + // This can happen when len(dest) < len(src) < cap(dest). for i := len(dest); i < len(src); i++ { newDest[i] = NewHLLSketchDataPoint() } @@ -147,10 +156,6 @@ func (orig *HLLSketchDataPoint) MarshalJSON(dest *json.Stream) { } dest.WriteArrayEnd() } - if orig.SeriesID != uint64(0) { - dest.WriteObjectField("seriesID") - dest.WriteUint64(orig.SeriesID) - } if orig.StartTimeUnixNano != uint64(0) { dest.WriteObjectField("startTimeUnixNano") dest.WriteUint64(orig.StartTimeUnixNano) @@ -159,30 +164,24 @@ func (orig *HLLSketchDataPoint) MarshalJSON(dest *json.Stream) { dest.WriteObjectField("timeUnixNano") dest.WriteUint64(orig.TimeUnixNano) } - if orig.Count != uint64(0) { - dest.WriteObjectField("count") - dest.WriteUint64(orig.Count) - } - if orig.Cardinality != uint64(0) { - dest.WriteObjectField("cardinality") - dest.WriteUint64(orig.Cardinality) - } + if len(orig.Sketch) > 0 { dest.WriteObjectField("sketch") dest.WriteBytes(orig.Sketch) } + if int32(orig.Encoding) != 0 { dest.WriteObjectField("encoding") dest.WriteInt32(int32(orig.Encoding)) } - if orig.Precision != uint32(0) { - dest.WriteObjectField("precision") - dest.WriteUint32(orig.Precision) - } if orig.Flags != uint32(0) { dest.WriteObjectField("flags") dest.WriteUint32(orig.Flags) } + if orig.SeriesID != uint64(0) { + dest.WriteObjectField("seriesID") + dest.WriteUint64(orig.SeriesID) + } dest.WriteObjectEnd() } @@ -195,24 +194,19 @@ func (orig *HLLSketchDataPoint) UnmarshalJSON(iter *json.Iterator) { orig.Attributes = append(orig.Attributes, KeyValue{}) orig.Attributes[len(orig.Attributes)-1].UnmarshalJSON(iter) } - case "seriesID", "series_id": - orig.SeriesID = iter.ReadUint64() + case "startTimeUnixNano", "start_time_unix_nano": orig.StartTimeUnixNano = iter.ReadUint64() case "timeUnixNano", "time_unix_nano": orig.TimeUnixNano = iter.ReadUint64() - case "count": - orig.Count = iter.ReadUint64() - case "cardinality": - orig.Cardinality = iter.ReadUint64() case "sketch": orig.Sketch = iter.ReadBytes() case "encoding": orig.Encoding = HLLSketchEncoding(iter.ReadEnumValue(HLLSketchEncoding_value)) - case "precision": - orig.Precision = iter.ReadUint32() case "flags": orig.Flags = iter.ReadUint32() + case "seriesID", "series_id": + orig.SeriesID = iter.ReadUint64() default: iter.Skip() } @@ -220,17 +214,6 @@ func (orig *HLLSketchDataPoint) UnmarshalJSON(iter *json.Iterator) { } func (orig *HLLSketchDataPoint) SizeProto() int { - // Field layout: - // 1=attributes (wire 2) - // 2=start_time_unix_nano (wire 1, I64) - // 3=time_unix_nano (wire 1, I64) - // 4=count (wire 0) - // 5=cardinality (wire 0) - // 6=sketch (wire 2) - // 7=encoding (wire 0) - // 8=precision (wire 0) - // 9=flags (wire 0) - // 10=series_id (wire 0) var n int var l int _ = l @@ -244,12 +227,6 @@ func (orig *HLLSketchDataPoint) SizeProto() int { if orig.TimeUnixNano != 0 { n += 9 } - if orig.Count != 0 { - n += 1 + proto.Sov(uint64(orig.Count)) - } - if orig.Cardinality != 0 { - n += 1 + proto.Sov(uint64(orig.Cardinality)) - } l = len(orig.Sketch) if l > 0 { n += 1 + proto.Sov(uint64(l)) + l @@ -257,9 +234,6 @@ func (orig *HLLSketchDataPoint) SizeProto() int { if orig.Encoding != 0 { n += 1 + proto.Sov(uint64(orig.Encoding)) } - if orig.Precision != 0 { - n += 1 + proto.Sov(uint64(orig.Precision)) - } if orig.Flags != 0 { n += 1 + proto.Sov(uint64(orig.Flags)) } @@ -270,17 +244,6 @@ func (orig *HLLSketchDataPoint) SizeProto() int { } func (orig *HLLSketchDataPoint) MarshalProto(buf []byte) int { - // Wire tag bytes: - // 1=attributes: 0x0a - // 2=start_time_unix_nano: 0x11 - // 3=time_unix_nano: 0x19 - // 4=count: 0x20 - // 5=cardinality: 0x28 - // 6=sketch: 0x32 - // 7=encoding: 0x38 - // 8=precision: 0x40 - // 9=flags: 0x48 - // 10=series_id: 0x50 pos := len(buf) var l int _ = l @@ -289,12 +252,7 @@ func (orig *HLLSketchDataPoint) MarshalProto(buf []byte) int { pos -= l pos = proto.EncodeVarint(buf, pos, uint64(l)) pos-- - buf[pos] = 0x0a - } - if orig.SeriesID != 0 { - pos = proto.EncodeVarint(buf, pos, uint64(orig.SeriesID)) - pos-- - buf[pos] = 0x50 + buf[pos] = 0xa } if orig.StartTimeUnixNano != 0 { pos -= 8 @@ -308,16 +266,6 @@ func (orig *HLLSketchDataPoint) MarshalProto(buf []byte) int { pos-- buf[pos] = 0x19 } - if orig.Count != 0 { - pos = proto.EncodeVarint(buf, pos, uint64(orig.Count)) - pos-- - buf[pos] = 0x20 - } - if orig.Cardinality != 0 { - pos = proto.EncodeVarint(buf, pos, uint64(orig.Cardinality)) - pos-- - buf[pos] = 0x28 - } l = len(orig.Sketch) if l > 0 { pos -= l @@ -331,16 +279,16 @@ func (orig *HLLSketchDataPoint) MarshalProto(buf []byte) int { pos-- buf[pos] = 0x38 } - if orig.Precision != 0 { - pos = proto.EncodeVarint(buf, pos, uint64(orig.Precision)) - pos-- - buf[pos] = 0x40 - } if orig.Flags != 0 { pos = proto.EncodeVarint(buf, pos, uint64(orig.Flags)) pos-- buf[pos] = 0x48 } + if orig.SeriesID != 0 { + pos = proto.EncodeVarint(buf, pos, uint64(orig.SeriesID)) + pos-- + buf[pos] = 0x50 + } return len(buf) - pos } @@ -352,6 +300,7 @@ func (orig *HLLSketchDataPoint) UnmarshalProto(buf []byte) error { l := len(buf) pos := 0 for pos < l { + // If in a group parsing, move to the next tag. fieldNum, wireType, pos, err = proto.ConsumeTag(buf, pos) if err != nil { return err @@ -383,6 +332,7 @@ func (orig *HLLSketchDataPoint) UnmarshalProto(buf []byte) error { if err != nil { return err } + orig.StartTimeUnixNano = uint64(num) case 3: @@ -394,29 +344,8 @@ func (orig *HLLSketchDataPoint) UnmarshalProto(buf []byte) error { if err != nil { return err } - orig.TimeUnixNano = uint64(num) - - case 4: - if wireType != proto.WireTypeVarint { - return fmt.Errorf("proto: wrong wireType = %d for field Count", wireType) - } - var num uint64 - num, pos, err = proto.ConsumeVarint(buf, pos) - if err != nil { - return err - } - orig.Count = uint64(num) - case 5: - if wireType != proto.WireTypeVarint { - return fmt.Errorf("proto: wrong wireType = %d for field Cardinality", wireType) - } - var num uint64 - num, pos, err = proto.ConsumeVarint(buf, pos) - if err != nil { - return err - } - orig.Cardinality = uint64(num) + orig.TimeUnixNano = uint64(num) case 6: if wireType != proto.WireTypeLen { @@ -442,18 +371,8 @@ func (orig *HLLSketchDataPoint) UnmarshalProto(buf []byte) error { if err != nil { return err } - orig.Encoding = HLLSketchEncoding(num) - case 8: - if wireType != proto.WireTypeVarint { - return fmt.Errorf("proto: wrong wireType = %d for field Precision", wireType) - } - var num uint64 - num, pos, err = proto.ConsumeVarint(buf, pos) - if err != nil { - return err - } - orig.Precision = uint32(num) + orig.Encoding = HLLSketchEncoding(num) case 9: if wireType != proto.WireTypeVarint { @@ -464,6 +383,7 @@ func (orig *HLLSketchDataPoint) UnmarshalProto(buf []byte) error { if err != nil { return err } + orig.Flags = uint32(num) case 10: @@ -475,8 +395,8 @@ func (orig *HLLSketchDataPoint) UnmarshalProto(buf []byte) error { if err != nil { return err } - orig.SeriesID = uint64(num) + orig.SeriesID = uint64(num) default: pos, err = proto.ConsumeUnknown(buf, pos, wireType) if err != nil { @@ -490,15 +410,12 @@ func (orig *HLLSketchDataPoint) UnmarshalProto(buf []byte) error { func GenTestHLLSketchDataPoint() *HLLSketchDataPoint { orig := NewHLLSketchDataPoint() orig.Attributes = []KeyValue{{}, *GenTestKeyValue()} - orig.SeriesID = uint64(13) orig.StartTimeUnixNano = uint64(13) orig.TimeUnixNano = uint64(13) - orig.Count = uint64(1000) - orig.Cardinality = uint64(500) orig.Sketch = []byte{1, 2, 3} - orig.Encoding = HLLSketchEncoding(1) - orig.Precision = uint32(14) + orig.Encoding = HLLSketchEncoding(13) orig.Flags = uint32(13) + orig.SeriesID = uint64(13) return orig } diff --git a/opentelemetry-collector-patch/pdata/internal/generated_proto_hllsketchdatapoint_test.go b/opentelemetry-collector-patch/pdata/internal/generated_proto_hllsketchdatapoint_test.go new file mode 100644 index 00000000..15f5e7f2 --- /dev/null +++ b/opentelemetry-collector-patch/pdata/internal/generated_proto_hllsketchdatapoint_test.go @@ -0,0 +1,220 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package internal + +import ( + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + gootlpmetrics "go.opentelemetry.io/proto/slim/otlp/metrics/v1" + "google.golang.org/protobuf/proto" + + "go.opentelemetry.io/collector/featuregate" + "go.opentelemetry.io/collector/pdata/internal/json" +) + +func TestCopyHLLSketchDataPoint(t *testing.T) { + for name, src := range genTestEncodingValuesHLLSketchDataPoint() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + dest := NewHLLSketchDataPoint() + CopyHLLSketchDataPoint(dest, src) + assert.Equal(t, src, dest) + CopyHLLSketchDataPoint(dest, dest) + assert.Equal(t, src, dest) + }) + } + } +} + +func TestCopyHLLSketchDataPointSlice(t *testing.T) { + src := []HLLSketchDataPoint{} + dest := []HLLSketchDataPoint{} + // Test CopyTo empty + dest = CopyHLLSketchDataPointSlice(dest, src) + assert.Equal(t, []HLLSketchDataPoint{}, dest) + + // Test CopyTo larger slice + src = GenTestHLLSketchDataPointSlice() + dest = CopyHLLSketchDataPointSlice(dest, src) + assert.Equal(t, GenTestHLLSketchDataPointSlice(), dest) + + // Test CopyTo same size slice + dest = CopyHLLSketchDataPointSlice(dest, src) + assert.Equal(t, GenTestHLLSketchDataPointSlice(), dest) + + // Test CopyTo smaller size slice + dest = CopyHLLSketchDataPointSlice(dest, []HLLSketchDataPoint{}) + assert.Len(t, dest, 0) + + // Test CopyTo larger slice with enough capacity + dest = CopyHLLSketchDataPointSlice(dest, src) + assert.Equal(t, GenTestHLLSketchDataPointSlice(), dest) +} + +func TestCopyHLLSketchDataPointPtrSlice(t *testing.T) { + src := []*HLLSketchDataPoint{} + dest := []*HLLSketchDataPoint{} + // Test CopyTo empty + dest = CopyHLLSketchDataPointPtrSlice(dest, src) + assert.Equal(t, []*HLLSketchDataPoint{}, dest) + + // Test CopyTo larger slice + src = GenTestHLLSketchDataPointPtrSlice() + dest = CopyHLLSketchDataPointPtrSlice(dest, src) + assert.Equal(t, GenTestHLLSketchDataPointPtrSlice(), dest) + + // Test CopyTo same size slice + dest = CopyHLLSketchDataPointPtrSlice(dest, src) + assert.Equal(t, GenTestHLLSketchDataPointPtrSlice(), dest) + + // Test CopyTo smaller size slice + dest = CopyHLLSketchDataPointPtrSlice(dest, []*HLLSketchDataPoint{}) + assert.Len(t, dest, 0) + + // Test CopyTo larger slice with enough capacity + dest = CopyHLLSketchDataPointPtrSlice(dest, src) + assert.Equal(t, GenTestHLLSketchDataPointPtrSlice(), dest) +} + +func TestMarshalAndUnmarshalJSONHLLSketchDataPointUnknown(t *testing.T) { + iter := json.BorrowIterator([]byte(`{"unknown": "string"}`)) + defer json.ReturnIterator(iter) + dest := NewHLLSketchDataPoint() + dest.UnmarshalJSON(iter) + require.NoError(t, iter.Error()) + assert.Equal(t, NewHLLSketchDataPoint(), dest) +} + +func TestMarshalAndUnmarshalJSONHLLSketchDataPoint(t *testing.T) { + for name, src := range genTestEncodingValuesHLLSketchDataPoint() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + stream := json.BorrowStream(nil) + defer json.ReturnStream(stream) + src.MarshalJSON(stream) + require.NoError(t, stream.Error()) + + iter := json.BorrowIterator(stream.Buffer()) + defer json.ReturnIterator(iter) + dest := NewHLLSketchDataPoint() + dest.UnmarshalJSON(iter) + require.NoError(t, iter.Error()) + + assert.Equal(t, src, dest) + DeleteHLLSketchDataPoint(dest, true) + }) + } + } +} + +func TestMarshalAndUnmarshalProtoHLLSketchDataPointFailing(t *testing.T) { + for name, buf := range genTestFailingUnmarshalProtoValuesHLLSketchDataPoint() { + t.Run(name, func(t *testing.T) { + dest := NewHLLSketchDataPoint() + require.Error(t, dest.UnmarshalProto(buf)) + }) + } +} + +func TestMarshalAndUnmarshalProtoHLLSketchDataPointUnknown(t *testing.T) { + dest := NewHLLSketchDataPoint() + // message Test { required int64 field = 1313; } encoding { "field": "1234" } + require.NoError(t, dest.UnmarshalProto([]byte{0x88, 0x52, 0xD2, 0x09})) + assert.Equal(t, NewHLLSketchDataPoint(), dest) +} + +func TestMarshalAndUnmarshalProtoHLLSketchDataPoint(t *testing.T) { + for name, src := range genTestEncodingValuesHLLSketchDataPoint() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + buf := make([]byte, src.SizeProto()) + gotSize := src.MarshalProto(buf) + assert.Equal(t, len(buf), gotSize) + + dest := NewHLLSketchDataPoint() + require.NoError(t, dest.UnmarshalProto(buf)) + + assert.Equal(t, src, dest) + DeleteHLLSketchDataPoint(dest, true) + }) + } + } +} + +func TestMarshalAndUnmarshalProtoViaProtobufHLLSketchDataPoint(t *testing.T) { + for name, src := range genTestEncodingValuesHLLSketchDataPoint() { + t.Run(name, func(t *testing.T) { + buf := make([]byte, src.SizeProto()) + gotSize := src.MarshalProto(buf) + assert.Equal(t, len(buf), gotSize) + + goDest := &gootlpmetrics.HLLSketchDataPoint{} + require.NoError(t, proto.Unmarshal(buf, goDest)) + + goBuf, err := proto.Marshal(goDest) + require.NoError(t, err) + + dest := NewHLLSketchDataPoint() + require.NoError(t, dest.UnmarshalProto(goBuf)) + assert.Equal(t, src, dest) + }) + } +} + +func genTestFailingUnmarshalProtoValuesHLLSketchDataPoint() map[string][]byte { + return map[string][]byte{ + "invalid_field": {0x02}, + "Attributes/wrong_wire_type": {0xc}, + "Attributes/missing_value": {0xa}, + "StartTimeUnixNano/wrong_wire_type": {0x14}, + "StartTimeUnixNano/missing_value": {0x11}, + "TimeUnixNano/wrong_wire_type": {0x1c}, + "TimeUnixNano/missing_value": {0x19}, + "Sketch/wrong_wire_type": {0x34}, + "Sketch/missing_value": {0x32}, + "Encoding/wrong_wire_type": {0x3c}, + "Encoding/missing_value": {0x38}, + "Flags/wrong_wire_type": {0x4c}, + "Flags/missing_value": {0x48}, + "SeriesID/wrong_wire_type": {0x54}, + "SeriesID/missing_value": {0x50}, + } +} + +func genTestEncodingValuesHLLSketchDataPoint() map[string]*HLLSketchDataPoint { + return map[string]*HLLSketchDataPoint{ + "empty": NewHLLSketchDataPoint(), + "Attributes/test": {Attributes: []KeyValue{{}, *GenTestKeyValue()}}, + "StartTimeUnixNano/test": {StartTimeUnixNano: uint64(13)}, + "TimeUnixNano/test": {TimeUnixNano: uint64(13)}, + "Sketch/test": {Sketch: []byte{1, 2, 3}}, + "Encoding/test": {Encoding: HLLSketchEncoding(13)}, + "Flags/test": {Flags: uint32(13)}, + "SeriesID/test": {SeriesID: uint64(13)}, + } +} diff --git a/opentelemetry-collector-patch/pdata/internal/generated_proto_kllsketch.go b/opentelemetry-collector-patch/pdata/internal/generated_proto_kllsketch.go index aa864775..2e0662d5 100644 --- a/opentelemetry-collector-patch/pdata/internal/generated_proto_kllsketch.go +++ b/opentelemetry-collector-patch/pdata/internal/generated_proto_kllsketch.go @@ -18,6 +18,7 @@ import ( type KLLSketch struct { DataPoints []*KLLSketchDataPoint AggregationTemporality AggregationTemporality + K uint32 } var ( @@ -56,6 +57,7 @@ func DeleteKLLSketch(orig *KLLSketch, nullable bool) { } func CopyKLLSketch(dest, src *KLLSketch) *KLLSketch { + // If copying to same object, just return. if src == dest { return dest } @@ -71,6 +73,8 @@ func CopyKLLSketch(dest, src *KLLSketch) *KLLSketch { dest.AggregationTemporality = src.AggregationTemporality + dest.K = src.K + return dest } @@ -80,6 +84,8 @@ func CopyKLLSketchSlice(dest, src []KLLSketch) []KLLSketch { newDest = make([]KLLSketch, len(src)) } else { newDest = dest[:len(src)] + // Cleanup the rest of the elements so GC can free the memory. + // This can happen when len(src) < len(dest) < cap(dest). for i := len(src); i < len(dest); i++ { DeleteKLLSketch(&dest[i], false) } @@ -94,16 +100,22 @@ func CopyKLLSketchPtrSlice(dest, src []*KLLSketch) []*KLLSketch { var newDest []*KLLSketch if cap(dest) < len(src) { newDest = make([]*KLLSketch, len(src)) + // Copy old pointers to re-use. copy(newDest, dest) + // Add new pointers for missing elements from len(dest) to len(srt). for i := len(dest); i < len(src); i++ { newDest[i] = NewKLLSketch() } } else { newDest = dest[:len(src)] + // Cleanup the rest of the elements so GC can free the memory. + // This can happen when len(src) < len(dest) < cap(dest). for i := len(src); i < len(dest); i++ { DeleteKLLSketch(dest[i], true) dest[i] = nil } + // Add new pointers for missing elements. + // This can happen when len(dest) < len(src) < cap(dest). for i := len(dest); i < len(src); i++ { newDest[i] = NewKLLSketch() } @@ -136,6 +148,10 @@ func (orig *KLLSketch) MarshalJSON(dest *json.Stream) { dest.WriteObjectField("aggregationTemporality") dest.WriteInt32(int32(orig.AggregationTemporality)) } + if orig.K != uint32(0) { + dest.WriteObjectField("k") + dest.WriteUint32(orig.K) + } dest.WriteObjectEnd() } @@ -151,6 +167,8 @@ func (orig *KLLSketch) UnmarshalJSON(iter *json.Iterator) { case "aggregationTemporality", "aggregation_temporality": orig.AggregationTemporality = AggregationTemporality(iter.ReadEnumValue(AggregationTemporality_value)) + case "k": + orig.K = iter.ReadUint32() default: iter.Skip() } @@ -168,6 +186,9 @@ func (orig *KLLSketch) SizeProto() int { if orig.AggregationTemporality != 0 { n += 1 + proto.Sov(uint64(orig.AggregationTemporality)) } + if orig.K != 0 { + n += 1 + proto.Sov(uint64(orig.K)) + } return n } @@ -187,6 +208,11 @@ func (orig *KLLSketch) MarshalProto(buf []byte) int { pos-- buf[pos] = 0x10 } + if orig.K != 0 { + pos = proto.EncodeVarint(buf, pos, uint64(orig.K)) + pos-- + buf[pos] = 0x18 + } return len(buf) - pos } @@ -198,6 +224,7 @@ func (orig *KLLSketch) UnmarshalProto(buf []byte) error { l := len(buf) pos := 0 for pos < l { + // If in a group parsing, move to the next tag. fieldNum, wireType, pos, err = proto.ConsumeTag(buf, pos) if err != nil { return err @@ -231,6 +258,18 @@ func (orig *KLLSketch) UnmarshalProto(buf []byte) error { } orig.AggregationTemporality = AggregationTemporality(num) + + case 3: + if wireType != proto.WireTypeVarint { + return fmt.Errorf("proto: wrong wireType = %d for field K", wireType) + } + var num uint64 + num, pos, err = proto.ConsumeVarint(buf, pos) + if err != nil { + return err + } + + orig.K = uint32(num) default: pos, err = proto.ConsumeUnknown(buf, pos, wireType) if err != nil { @@ -245,6 +284,7 @@ func GenTestKLLSketch() *KLLSketch { orig := NewKLLSketch() orig.DataPoints = []*KLLSketchDataPoint{{}, GenTestKLLSketchDataPoint()} orig.AggregationTemporality = AggregationTemporality(13) + orig.K = uint32(13) return orig } diff --git a/opentelemetry-collector-patch/pdata/internal/generated_proto_kllsketch_test.go b/opentelemetry-collector-patch/pdata/internal/generated_proto_kllsketch_test.go new file mode 100644 index 00000000..6cee5cb2 --- /dev/null +++ b/opentelemetry-collector-patch/pdata/internal/generated_proto_kllsketch_test.go @@ -0,0 +1,208 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package internal + +import ( + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + gootlpmetrics "go.opentelemetry.io/proto/slim/otlp/metrics/v1" + "google.golang.org/protobuf/proto" + + "go.opentelemetry.io/collector/featuregate" + "go.opentelemetry.io/collector/pdata/internal/json" +) + +func TestCopyKLLSketch(t *testing.T) { + for name, src := range genTestEncodingValuesKLLSketch() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + dest := NewKLLSketch() + CopyKLLSketch(dest, src) + assert.Equal(t, src, dest) + CopyKLLSketch(dest, dest) + assert.Equal(t, src, dest) + }) + } + } +} + +func TestCopyKLLSketchSlice(t *testing.T) { + src := []KLLSketch{} + dest := []KLLSketch{} + // Test CopyTo empty + dest = CopyKLLSketchSlice(dest, src) + assert.Equal(t, []KLLSketch{}, dest) + + // Test CopyTo larger slice + src = GenTestKLLSketchSlice() + dest = CopyKLLSketchSlice(dest, src) + assert.Equal(t, GenTestKLLSketchSlice(), dest) + + // Test CopyTo same size slice + dest = CopyKLLSketchSlice(dest, src) + assert.Equal(t, GenTestKLLSketchSlice(), dest) + + // Test CopyTo smaller size slice + dest = CopyKLLSketchSlice(dest, []KLLSketch{}) + assert.Len(t, dest, 0) + + // Test CopyTo larger slice with enough capacity + dest = CopyKLLSketchSlice(dest, src) + assert.Equal(t, GenTestKLLSketchSlice(), dest) +} + +func TestCopyKLLSketchPtrSlice(t *testing.T) { + src := []*KLLSketch{} + dest := []*KLLSketch{} + // Test CopyTo empty + dest = CopyKLLSketchPtrSlice(dest, src) + assert.Equal(t, []*KLLSketch{}, dest) + + // Test CopyTo larger slice + src = GenTestKLLSketchPtrSlice() + dest = CopyKLLSketchPtrSlice(dest, src) + assert.Equal(t, GenTestKLLSketchPtrSlice(), dest) + + // Test CopyTo same size slice + dest = CopyKLLSketchPtrSlice(dest, src) + assert.Equal(t, GenTestKLLSketchPtrSlice(), dest) + + // Test CopyTo smaller size slice + dest = CopyKLLSketchPtrSlice(dest, []*KLLSketch{}) + assert.Len(t, dest, 0) + + // Test CopyTo larger slice with enough capacity + dest = CopyKLLSketchPtrSlice(dest, src) + assert.Equal(t, GenTestKLLSketchPtrSlice(), dest) +} + +func TestMarshalAndUnmarshalJSONKLLSketchUnknown(t *testing.T) { + iter := json.BorrowIterator([]byte(`{"unknown": "string"}`)) + defer json.ReturnIterator(iter) + dest := NewKLLSketch() + dest.UnmarshalJSON(iter) + require.NoError(t, iter.Error()) + assert.Equal(t, NewKLLSketch(), dest) +} + +func TestMarshalAndUnmarshalJSONKLLSketch(t *testing.T) { + for name, src := range genTestEncodingValuesKLLSketch() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + stream := json.BorrowStream(nil) + defer json.ReturnStream(stream) + src.MarshalJSON(stream) + require.NoError(t, stream.Error()) + + iter := json.BorrowIterator(stream.Buffer()) + defer json.ReturnIterator(iter) + dest := NewKLLSketch() + dest.UnmarshalJSON(iter) + require.NoError(t, iter.Error()) + + assert.Equal(t, src, dest) + DeleteKLLSketch(dest, true) + }) + } + } +} + +func TestMarshalAndUnmarshalProtoKLLSketchFailing(t *testing.T) { + for name, buf := range genTestFailingUnmarshalProtoValuesKLLSketch() { + t.Run(name, func(t *testing.T) { + dest := NewKLLSketch() + require.Error(t, dest.UnmarshalProto(buf)) + }) + } +} + +func TestMarshalAndUnmarshalProtoKLLSketchUnknown(t *testing.T) { + dest := NewKLLSketch() + // message Test { required int64 field = 1313; } encoding { "field": "1234" } + require.NoError(t, dest.UnmarshalProto([]byte{0x88, 0x52, 0xD2, 0x09})) + assert.Equal(t, NewKLLSketch(), dest) +} + +func TestMarshalAndUnmarshalProtoKLLSketch(t *testing.T) { + for name, src := range genTestEncodingValuesKLLSketch() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + buf := make([]byte, src.SizeProto()) + gotSize := src.MarshalProto(buf) + assert.Equal(t, len(buf), gotSize) + + dest := NewKLLSketch() + require.NoError(t, dest.UnmarshalProto(buf)) + + assert.Equal(t, src, dest) + DeleteKLLSketch(dest, true) + }) + } + } +} + +func TestMarshalAndUnmarshalProtoViaProtobufKLLSketch(t *testing.T) { + for name, src := range genTestEncodingValuesKLLSketch() { + t.Run(name, func(t *testing.T) { + buf := make([]byte, src.SizeProto()) + gotSize := src.MarshalProto(buf) + assert.Equal(t, len(buf), gotSize) + + goDest := &gootlpmetrics.KLLSketch{} + require.NoError(t, proto.Unmarshal(buf, goDest)) + + goBuf, err := proto.Marshal(goDest) + require.NoError(t, err) + + dest := NewKLLSketch() + require.NoError(t, dest.UnmarshalProto(goBuf)) + assert.Equal(t, src, dest) + }) + } +} + +func genTestFailingUnmarshalProtoValuesKLLSketch() map[string][]byte { + return map[string][]byte{ + "invalid_field": {0x02}, + "DataPoints/wrong_wire_type": {0xc}, + "DataPoints/missing_value": {0xa}, + "AggregationTemporality/wrong_wire_type": {0x14}, + "AggregationTemporality/missing_value": {0x10}, + "K/wrong_wire_type": {0x1c}, + "K/missing_value": {0x18}, + } +} + +func genTestEncodingValuesKLLSketch() map[string]*KLLSketch { + return map[string]*KLLSketch{ + "empty": NewKLLSketch(), + "DataPoints/test": {DataPoints: []*KLLSketchDataPoint{{}, GenTestKLLSketchDataPoint()}}, + "AggregationTemporality/test": {AggregationTemporality: AggregationTemporality(13)}, + "K/test": {K: uint32(13)}, + } +} diff --git a/opentelemetry-collector-patch/pdata/internal/generated_proto_kllsketchdatapoint.go b/opentelemetry-collector-patch/pdata/internal/generated_proto_kllsketchdatapoint.go index 25291c14..699a7ab1 100644 --- a/opentelemetry-collector-patch/pdata/internal/generated_proto_kllsketchdatapoint.go +++ b/opentelemetry-collector-patch/pdata/internal/generated_proto_kllsketchdatapoint.go @@ -9,7 +9,6 @@ package internal import ( "encoding/binary" "fmt" - "math" "sync" "go.opentelemetry.io/collector/pdata/internal/json" @@ -19,16 +18,12 @@ import ( // KLLSketchDataPoint is a single data point that encodes a distribution using the KLL sketch format. type KLLSketchDataPoint struct { Attributes []KeyValue - SeriesID uint64 StartTimeUnixNano uint64 TimeUnixNano uint64 - Count uint64 - Sum float64 - Min float64 - Max float64 Sketch []byte Encoding KLLSketchEncoding Flags uint32 + SeriesID uint64 } var ( @@ -67,6 +62,7 @@ func DeleteKLLSketchDataPoint(orig *KLLSketchDataPoint, nullable bool) { } func CopyKLLSketchDataPoint(dest, src *KLLSketchDataPoint) *KLLSketchDataPoint { + // If copying to same object, just return. if src == dest { return dest } @@ -80,26 +76,18 @@ func CopyKLLSketchDataPoint(dest, src *KLLSketchDataPoint) *KLLSketchDataPoint { } dest.Attributes = CopyKeyValueSlice(dest.Attributes, src.Attributes) - dest.SeriesID = src.SeriesID - dest.StartTimeUnixNano = src.StartTimeUnixNano dest.TimeUnixNano = src.TimeUnixNano - dest.Count = src.Count - - dest.Sum = src.Sum - - dest.Min = src.Min - - dest.Max = src.Max - dest.Sketch = src.Sketch dest.Encoding = src.Encoding dest.Flags = src.Flags + dest.SeriesID = src.SeriesID + return dest } @@ -109,6 +97,8 @@ func CopyKLLSketchDataPointSlice(dest, src []KLLSketchDataPoint) []KLLSketchData newDest = make([]KLLSketchDataPoint, len(src)) } else { newDest = dest[:len(src)] + // Cleanup the rest of the elements so GC can free the memory. + // This can happen when len(src) < len(dest) < cap(dest). for i := len(src); i < len(dest); i++ { DeleteKLLSketchDataPoint(&dest[i], false) } @@ -123,16 +113,22 @@ func CopyKLLSketchDataPointPtrSlice(dest, src []*KLLSketchDataPoint) []*KLLSketc var newDest []*KLLSketchDataPoint if cap(dest) < len(src) { newDest = make([]*KLLSketchDataPoint, len(src)) + // Copy old pointers to re-use. copy(newDest, dest) + // Add new pointers for missing elements from len(dest) to len(srt). for i := len(dest); i < len(src); i++ { newDest[i] = NewKLLSketchDataPoint() } } else { newDest = dest[:len(src)] + // Cleanup the rest of the elements so GC can free the memory. + // This can happen when len(src) < len(dest) < cap(dest). for i := len(src); i < len(dest); i++ { DeleteKLLSketchDataPoint(dest[i], true) dest[i] = nil } + // Add new pointers for missing elements. + // This can happen when len(dest) < len(src) < cap(dest). for i := len(dest); i < len(src); i++ { newDest[i] = NewKLLSketchDataPoint() } @@ -160,10 +156,6 @@ func (orig *KLLSketchDataPoint) MarshalJSON(dest *json.Stream) { } dest.WriteArrayEnd() } - if orig.SeriesID != uint64(0) { - dest.WriteObjectField("seriesID") - dest.WriteUint64(orig.SeriesID) - } if orig.StartTimeUnixNano != uint64(0) { dest.WriteObjectField("startTimeUnixNano") dest.WriteUint64(orig.StartTimeUnixNano) @@ -172,26 +164,12 @@ func (orig *KLLSketchDataPoint) MarshalJSON(dest *json.Stream) { dest.WriteObjectField("timeUnixNano") dest.WriteUint64(orig.TimeUnixNano) } - if orig.Count != uint64(0) { - dest.WriteObjectField("count") - dest.WriteUint64(orig.Count) - } - if orig.Sum != float64(0) { - dest.WriteObjectField("sum") - dest.WriteFloat64(orig.Sum) - } - if orig.Min != float64(0) { - dest.WriteObjectField("min") - dest.WriteFloat64(orig.Min) - } - if orig.Max != float64(0) { - dest.WriteObjectField("max") - dest.WriteFloat64(orig.Max) - } + if len(orig.Sketch) > 0 { dest.WriteObjectField("sketch") dest.WriteBytes(orig.Sketch) } + if int32(orig.Encoding) != 0 { dest.WriteObjectField("encoding") dest.WriteInt32(int32(orig.Encoding)) @@ -200,6 +178,10 @@ func (orig *KLLSketchDataPoint) MarshalJSON(dest *json.Stream) { dest.WriteObjectField("flags") dest.WriteUint32(orig.Flags) } + if orig.SeriesID != uint64(0) { + dest.WriteObjectField("seriesID") + dest.WriteUint64(orig.SeriesID) + } dest.WriteObjectEnd() } @@ -213,26 +195,18 @@ func (orig *KLLSketchDataPoint) UnmarshalJSON(iter *json.Iterator) { orig.Attributes[len(orig.Attributes)-1].UnmarshalJSON(iter) } - case "seriesID", "series_id": - orig.SeriesID = iter.ReadUint64() case "startTimeUnixNano", "start_time_unix_nano": orig.StartTimeUnixNano = iter.ReadUint64() case "timeUnixNano", "time_unix_nano": orig.TimeUnixNano = iter.ReadUint64() - case "count": - orig.Count = iter.ReadUint64() - case "sum": - orig.Sum = iter.ReadFloat64() - case "min": - orig.Min = iter.ReadFloat64() - case "max": - orig.Max = iter.ReadFloat64() case "sketch": orig.Sketch = iter.ReadBytes() case "encoding": orig.Encoding = KLLSketchEncoding(iter.ReadEnumValue(KLLSketchEncoding_value)) case "flags": orig.Flags = iter.ReadUint32() + case "seriesID", "series_id": + orig.SeriesID = iter.ReadUint64() default: iter.Skip() } @@ -253,18 +227,6 @@ func (orig *KLLSketchDataPoint) SizeProto() int { if orig.TimeUnixNano != 0 { n += 9 } - if orig.Count != 0 { - n += 9 - } - if orig.Sum != 0 { - n += 9 - } - if orig.Min != 0 { - n += 9 - } - if orig.Max != 0 { - n += 9 - } l = len(orig.Sketch) if l > 0 { n += 1 + proto.Sov(uint64(l)) + l @@ -282,18 +244,6 @@ func (orig *KLLSketchDataPoint) SizeProto() int { } func (orig *KLLSketchDataPoint) MarshalProto(buf []byte) int { - // Field numbers: - // 1=attributes (wire 2, tag=0x0a) - // 2=start_time_unix_nano (wire 1, tag=0x11) - // 3=time_unix_nano (wire 1, tag=0x19) - // 4=count (wire 1, tag=0x21) - // 5=sum (wire 1, tag=0x29) - // 6=min (wire 1, tag=0x31) - // 7=max (wire 1, tag=0x39) - // 8=sketch (wire 2, tag=0x42) - // 9=encoding (wire 0, tag=0x48) - // 10=flags (wire 0, tag=0x50) - // 11=series_id (wire 0, tag=0x58) pos := len(buf) var l int _ = l @@ -302,12 +252,7 @@ func (orig *KLLSketchDataPoint) MarshalProto(buf []byte) int { pos -= l pos = proto.EncodeVarint(buf, pos, uint64(l)) pos-- - buf[pos] = 0x0a - } - if orig.SeriesID != 0 { - pos = proto.EncodeVarint(buf, pos, uint64(orig.SeriesID)) - pos-- - buf[pos] = 0x58 + buf[pos] = 0xa } if orig.StartTimeUnixNano != 0 { pos -= 8 @@ -321,30 +266,6 @@ func (orig *KLLSketchDataPoint) MarshalProto(buf []byte) int { pos-- buf[pos] = 0x19 } - if orig.Count != 0 { - pos -= 8 - binary.LittleEndian.PutUint64(buf[pos:], uint64(orig.Count)) - pos-- - buf[pos] = 0x21 - } - if orig.Sum != 0 { - pos -= 8 - binary.LittleEndian.PutUint64(buf[pos:], math.Float64bits(orig.Sum)) - pos-- - buf[pos] = 0x29 - } - if orig.Min != 0 { - pos -= 8 - binary.LittleEndian.PutUint64(buf[pos:], math.Float64bits(orig.Min)) - pos-- - buf[pos] = 0x31 - } - if orig.Max != 0 { - pos -= 8 - binary.LittleEndian.PutUint64(buf[pos:], math.Float64bits(orig.Max)) - pos-- - buf[pos] = 0x39 - } l = len(orig.Sketch) if l > 0 { pos -= l @@ -363,6 +284,11 @@ func (orig *KLLSketchDataPoint) MarshalProto(buf []byte) int { pos-- buf[pos] = 0x50 } + if orig.SeriesID != 0 { + pos = proto.EncodeVarint(buf, pos, uint64(orig.SeriesID)) + pos-- + buf[pos] = 0x58 + } return len(buf) - pos } @@ -374,6 +300,7 @@ func (orig *KLLSketchDataPoint) UnmarshalProto(buf []byte) error { l := len(buf) pos := 0 for pos < l { + // If in a group parsing, move to the next tag. fieldNum, wireType, pos, err = proto.ConsumeTag(buf, pos) if err != nil { return err @@ -405,6 +332,7 @@ func (orig *KLLSketchDataPoint) UnmarshalProto(buf []byte) error { if err != nil { return err } + orig.StartTimeUnixNano = uint64(num) case 3: @@ -416,51 +344,8 @@ func (orig *KLLSketchDataPoint) UnmarshalProto(buf []byte) error { if err != nil { return err } - orig.TimeUnixNano = uint64(num) - case 4: - if wireType != proto.WireTypeI64 { - return fmt.Errorf("proto: wrong wireType = %d for field Count", wireType) - } - var num uint64 - num, pos, err = proto.ConsumeI64(buf, pos) - if err != nil { - return err - } - orig.Count = uint64(num) - - case 5: - if wireType != proto.WireTypeI64 { - return fmt.Errorf("proto: wrong wireType = %d for field Sum", wireType) - } - var num uint64 - num, pos, err = proto.ConsumeI64(buf, pos) - if err != nil { - return err - } - orig.Sum = math.Float64frombits(num) - - case 6: - if wireType != proto.WireTypeI64 { - return fmt.Errorf("proto: wrong wireType = %d for field Min", wireType) - } - var num uint64 - num, pos, err = proto.ConsumeI64(buf, pos) - if err != nil { - return err - } - orig.Min = math.Float64frombits(num) - - case 7: - if wireType != proto.WireTypeI64 { - return fmt.Errorf("proto: wrong wireType = %d for field Max", wireType) - } - var num uint64 - num, pos, err = proto.ConsumeI64(buf, pos) - if err != nil { - return err - } - orig.Max = math.Float64frombits(num) + orig.TimeUnixNano = uint64(num) case 8: if wireType != proto.WireTypeLen { @@ -486,6 +371,7 @@ func (orig *KLLSketchDataPoint) UnmarshalProto(buf []byte) error { if err != nil { return err } + orig.Encoding = KLLSketchEncoding(num) case 10: @@ -497,6 +383,7 @@ func (orig *KLLSketchDataPoint) UnmarshalProto(buf []byte) error { if err != nil { return err } + orig.Flags = uint32(num) case 11: @@ -508,8 +395,8 @@ func (orig *KLLSketchDataPoint) UnmarshalProto(buf []byte) error { if err != nil { return err } - orig.SeriesID = uint64(num) + orig.SeriesID = uint64(num) default: pos, err = proto.ConsumeUnknown(buf, pos, wireType) if err != nil { @@ -523,16 +410,12 @@ func (orig *KLLSketchDataPoint) UnmarshalProto(buf []byte) error { func GenTestKLLSketchDataPoint() *KLLSketchDataPoint { orig := NewKLLSketchDataPoint() orig.Attributes = []KeyValue{{}, *GenTestKeyValue()} - orig.SeriesID = uint64(13) orig.StartTimeUnixNano = uint64(13) orig.TimeUnixNano = uint64(13) - orig.Count = uint64(13) - orig.Sum = float64(3.1415926) - orig.Min = float64(1.0) - orig.Max = float64(13.0) orig.Sketch = []byte{1, 2, 3} - orig.Encoding = KLLSketchEncoding(1) + orig.Encoding = KLLSketchEncoding(13) orig.Flags = uint32(13) + orig.SeriesID = uint64(13) return orig } diff --git a/opentelemetry-collector-patch/pdata/internal/generated_proto_kllsketchdatapoint_test.go b/opentelemetry-collector-patch/pdata/internal/generated_proto_kllsketchdatapoint_test.go new file mode 100644 index 00000000..1bb17958 --- /dev/null +++ b/opentelemetry-collector-patch/pdata/internal/generated_proto_kllsketchdatapoint_test.go @@ -0,0 +1,220 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package internal + +import ( + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + gootlpmetrics "go.opentelemetry.io/proto/slim/otlp/metrics/v1" + "google.golang.org/protobuf/proto" + + "go.opentelemetry.io/collector/featuregate" + "go.opentelemetry.io/collector/pdata/internal/json" +) + +func TestCopyKLLSketchDataPoint(t *testing.T) { + for name, src := range genTestEncodingValuesKLLSketchDataPoint() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + dest := NewKLLSketchDataPoint() + CopyKLLSketchDataPoint(dest, src) + assert.Equal(t, src, dest) + CopyKLLSketchDataPoint(dest, dest) + assert.Equal(t, src, dest) + }) + } + } +} + +func TestCopyKLLSketchDataPointSlice(t *testing.T) { + src := []KLLSketchDataPoint{} + dest := []KLLSketchDataPoint{} + // Test CopyTo empty + dest = CopyKLLSketchDataPointSlice(dest, src) + assert.Equal(t, []KLLSketchDataPoint{}, dest) + + // Test CopyTo larger slice + src = GenTestKLLSketchDataPointSlice() + dest = CopyKLLSketchDataPointSlice(dest, src) + assert.Equal(t, GenTestKLLSketchDataPointSlice(), dest) + + // Test CopyTo same size slice + dest = CopyKLLSketchDataPointSlice(dest, src) + assert.Equal(t, GenTestKLLSketchDataPointSlice(), dest) + + // Test CopyTo smaller size slice + dest = CopyKLLSketchDataPointSlice(dest, []KLLSketchDataPoint{}) + assert.Len(t, dest, 0) + + // Test CopyTo larger slice with enough capacity + dest = CopyKLLSketchDataPointSlice(dest, src) + assert.Equal(t, GenTestKLLSketchDataPointSlice(), dest) +} + +func TestCopyKLLSketchDataPointPtrSlice(t *testing.T) { + src := []*KLLSketchDataPoint{} + dest := []*KLLSketchDataPoint{} + // Test CopyTo empty + dest = CopyKLLSketchDataPointPtrSlice(dest, src) + assert.Equal(t, []*KLLSketchDataPoint{}, dest) + + // Test CopyTo larger slice + src = GenTestKLLSketchDataPointPtrSlice() + dest = CopyKLLSketchDataPointPtrSlice(dest, src) + assert.Equal(t, GenTestKLLSketchDataPointPtrSlice(), dest) + + // Test CopyTo same size slice + dest = CopyKLLSketchDataPointPtrSlice(dest, src) + assert.Equal(t, GenTestKLLSketchDataPointPtrSlice(), dest) + + // Test CopyTo smaller size slice + dest = CopyKLLSketchDataPointPtrSlice(dest, []*KLLSketchDataPoint{}) + assert.Len(t, dest, 0) + + // Test CopyTo larger slice with enough capacity + dest = CopyKLLSketchDataPointPtrSlice(dest, src) + assert.Equal(t, GenTestKLLSketchDataPointPtrSlice(), dest) +} + +func TestMarshalAndUnmarshalJSONKLLSketchDataPointUnknown(t *testing.T) { + iter := json.BorrowIterator([]byte(`{"unknown": "string"}`)) + defer json.ReturnIterator(iter) + dest := NewKLLSketchDataPoint() + dest.UnmarshalJSON(iter) + require.NoError(t, iter.Error()) + assert.Equal(t, NewKLLSketchDataPoint(), dest) +} + +func TestMarshalAndUnmarshalJSONKLLSketchDataPoint(t *testing.T) { + for name, src := range genTestEncodingValuesKLLSketchDataPoint() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + stream := json.BorrowStream(nil) + defer json.ReturnStream(stream) + src.MarshalJSON(stream) + require.NoError(t, stream.Error()) + + iter := json.BorrowIterator(stream.Buffer()) + defer json.ReturnIterator(iter) + dest := NewKLLSketchDataPoint() + dest.UnmarshalJSON(iter) + require.NoError(t, iter.Error()) + + assert.Equal(t, src, dest) + DeleteKLLSketchDataPoint(dest, true) + }) + } + } +} + +func TestMarshalAndUnmarshalProtoKLLSketchDataPointFailing(t *testing.T) { + for name, buf := range genTestFailingUnmarshalProtoValuesKLLSketchDataPoint() { + t.Run(name, func(t *testing.T) { + dest := NewKLLSketchDataPoint() + require.Error(t, dest.UnmarshalProto(buf)) + }) + } +} + +func TestMarshalAndUnmarshalProtoKLLSketchDataPointUnknown(t *testing.T) { + dest := NewKLLSketchDataPoint() + // message Test { required int64 field = 1313; } encoding { "field": "1234" } + require.NoError(t, dest.UnmarshalProto([]byte{0x88, 0x52, 0xD2, 0x09})) + assert.Equal(t, NewKLLSketchDataPoint(), dest) +} + +func TestMarshalAndUnmarshalProtoKLLSketchDataPoint(t *testing.T) { + for name, src := range genTestEncodingValuesKLLSketchDataPoint() { + for _, pooling := range []bool{true, false} { + t.Run(name+"/Pooling="+strconv.FormatBool(pooling), func(t *testing.T) { + prevPooling := UseProtoPooling.IsEnabled() + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), pooling)) + defer func() { + require.NoError(t, featuregate.GlobalRegistry().Set(UseProtoPooling.ID(), prevPooling)) + }() + + buf := make([]byte, src.SizeProto()) + gotSize := src.MarshalProto(buf) + assert.Equal(t, len(buf), gotSize) + + dest := NewKLLSketchDataPoint() + require.NoError(t, dest.UnmarshalProto(buf)) + + assert.Equal(t, src, dest) + DeleteKLLSketchDataPoint(dest, true) + }) + } + } +} + +func TestMarshalAndUnmarshalProtoViaProtobufKLLSketchDataPoint(t *testing.T) { + for name, src := range genTestEncodingValuesKLLSketchDataPoint() { + t.Run(name, func(t *testing.T) { + buf := make([]byte, src.SizeProto()) + gotSize := src.MarshalProto(buf) + assert.Equal(t, len(buf), gotSize) + + goDest := &gootlpmetrics.KLLSketchDataPoint{} + require.NoError(t, proto.Unmarshal(buf, goDest)) + + goBuf, err := proto.Marshal(goDest) + require.NoError(t, err) + + dest := NewKLLSketchDataPoint() + require.NoError(t, dest.UnmarshalProto(goBuf)) + assert.Equal(t, src, dest) + }) + } +} + +func genTestFailingUnmarshalProtoValuesKLLSketchDataPoint() map[string][]byte { + return map[string][]byte{ + "invalid_field": {0x02}, + "Attributes/wrong_wire_type": {0xc}, + "Attributes/missing_value": {0xa}, + "StartTimeUnixNano/wrong_wire_type": {0x14}, + "StartTimeUnixNano/missing_value": {0x11}, + "TimeUnixNano/wrong_wire_type": {0x1c}, + "TimeUnixNano/missing_value": {0x19}, + "Sketch/wrong_wire_type": {0x44}, + "Sketch/missing_value": {0x42}, + "Encoding/wrong_wire_type": {0x4c}, + "Encoding/missing_value": {0x48}, + "Flags/wrong_wire_type": {0x54}, + "Flags/missing_value": {0x50}, + "SeriesID/wrong_wire_type": {0x5c}, + "SeriesID/missing_value": {0x58}, + } +} + +func genTestEncodingValuesKLLSketchDataPoint() map[string]*KLLSketchDataPoint { + return map[string]*KLLSketchDataPoint{ + "empty": NewKLLSketchDataPoint(), + "Attributes/test": {Attributes: []KeyValue{{}, *GenTestKeyValue()}}, + "StartTimeUnixNano/test": {StartTimeUnixNano: uint64(13)}, + "TimeUnixNano/test": {TimeUnixNano: uint64(13)}, + "Sketch/test": {Sketch: []byte{1, 2, 3}}, + "Encoding/test": {Encoding: KLLSketchEncoding(13)}, + "Flags/test": {Flags: uint32(13)}, + "SeriesID/test": {SeriesID: uint64(13)}, + } +} diff --git a/opentelemetry-collector-patch/pdata/internal/generated_proto_metric.go b/opentelemetry-collector-patch/pdata/internal/generated_proto_metric.go index 1f8d8ae9..40156127 100644 --- a/opentelemetry-collector-patch/pdata/internal/generated_proto_metric.go +++ b/opentelemetry-collector-patch/pdata/internal/generated_proto_metric.go @@ -76,6 +76,17 @@ func (m *Metric) GetDDSketch() *DDSketch { return nil } +type Metric_Summary struct { + Summary *Summary +} + +func (m *Metric) GetSummary() *Summary { + if v, ok := m.GetData().(*Metric_Summary); ok { + return v.Summary + } + return nil +} + type Metric_KLLSketch struct { KLLSketch *KLLSketch } @@ -120,17 +131,6 @@ func (m *Metric) GetHLLSketch() *HLLSketch { return nil } -type Metric_Summary struct { - Summary *Summary -} - -func (m *Metric) GetSummary() *Summary { - if v, ok := m.GetData().(*Metric_Summary); ok { - return v.Summary - } - return nil -} - // Metric represents one metric as a collection of datapoints. // See Metric definition in OTLP: https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/metrics/v1/metrics.proto type Metric struct { @@ -178,6 +178,12 @@ var ( }, } + ProtoPoolMetric_Summary = sync.Pool{ + New: func() any { + return &Metric_Summary{} + }, + } + ProtoPoolMetric_KLLSketch = sync.Pool{ New: func() any { return &Metric_KLLSketch{} @@ -201,12 +207,6 @@ var ( return &Metric_HLLSketch{} }, } - - ProtoPoolMetric_Summary = sync.Pool{ - New: func() any { - return &Metric_Summary{} - }, - } ) func NewMetric() *Metric { @@ -247,6 +247,10 @@ func DeleteMetric(orig *Metric, nullable bool) { DeleteDDSketch(ov.DDSketch, true) ov.DDSketch = nil ProtoPoolMetric_DDSketch.Put(ov) + case *Metric_Summary: + DeleteSummary(ov.Summary, true) + ov.Summary = nil + ProtoPoolMetric_Summary.Put(ov) case *Metric_KLLSketch: DeleteKLLSketch(ov.KLLSketch, true) ov.KLLSketch = nil @@ -263,10 +267,6 @@ func DeleteMetric(orig *Metric, nullable bool) { DeleteHLLSketch(ov.HLLSketch, true) ov.HLLSketch = nil ProtoPoolMetric_HLLSketch.Put(ov) - case *Metric_Summary: - DeleteSummary(ov.Summary, true) - ov.Summary = nil - ProtoPoolMetric_Summary.Put(ov) } for i := range orig.Metadata { @@ -354,6 +354,17 @@ func CopyMetric(dest, src *Metric) *Metric { CopyDDSketch(ov.DDSketch, t.DDSketch) dest.Data = ov + case *Metric_Summary: + var ov *Metric_Summary + if !UseProtoPooling.IsEnabled() { + ov = &Metric_Summary{} + } else { + ov = ProtoPoolMetric_Summary.Get().(*Metric_Summary) + } + ov.Summary = NewSummary() + CopySummary(ov.Summary, t.Summary) + dest.Data = ov + case *Metric_KLLSketch: var ov *Metric_KLLSketch if !UseProtoPooling.IsEnabled() { @@ -398,17 +409,6 @@ func CopyMetric(dest, src *Metric) *Metric { CopyHLLSketch(ov.HLLSketch, t.HLLSketch) dest.Data = ov - case *Metric_Summary: - var ov *Metric_Summary - if !UseProtoPooling.IsEnabled() { - ov = &Metric_Summary{} - } else { - ov = ProtoPoolMetric_Summary.Get().(*Metric_Summary) - } - ov.Summary = NewSummary() - CopySummary(ov.Summary, t.Summary) - dest.Data = ov - default: dest.Data = nil } @@ -510,6 +510,11 @@ func (orig *Metric) MarshalJSON(dest *json.Stream) { dest.WriteObjectField("dDSketch") orig.DDSketch.MarshalJSON(dest) } + case *Metric_Summary: + if orig.Summary != nil { + dest.WriteObjectField("summary") + orig.Summary.MarshalJSON(dest) + } case *Metric_KLLSketch: if orig.KLLSketch != nil { dest.WriteObjectField("kLLSketch") @@ -530,11 +535,6 @@ func (orig *Metric) MarshalJSON(dest *json.Stream) { dest.WriteObjectField("hLLSketch") orig.HLLSketch.MarshalJSON(dest) } - case *Metric_Summary: - if orig.Summary != nil { - dest.WriteObjectField("summary") - orig.Summary.MarshalJSON(dest) - } } if len(orig.Metadata) > 0 { dest.WriteObjectField("metadata") @@ -625,6 +625,19 @@ func (orig *Metric) UnmarshalJSON(iter *json.Iterator) { orig.Data = ov } + case "summary": + { + var ov *Metric_Summary + if !UseProtoPooling.IsEnabled() { + ov = &Metric_Summary{} + } else { + ov = ProtoPoolMetric_Summary.Get().(*Metric_Summary) + } + ov.Summary = NewSummary() + ov.Summary.UnmarshalJSON(iter) + orig.Data = ov + } + case "kLLSketch", "kll_sketch": { var ov *Metric_KLLSketch @@ -677,19 +690,6 @@ func (orig *Metric) UnmarshalJSON(iter *json.Iterator) { orig.Data = ov } - case "summary": - { - var ov *Metric_Summary - if !UseProtoPooling.IsEnabled() { - ov = &Metric_Summary{} - } else { - ov = ProtoPoolMetric_Summary.Get().(*Metric_Summary) - } - ov.Summary = NewSummary() - ov.Summary.UnmarshalJSON(iter) - orig.Data = ov - } - case "metadata": for iter.ReadArray() { orig.Metadata = append(orig.Metadata, KeyValue{}) @@ -747,6 +747,11 @@ func (orig *Metric) SizeProto() int { l = orig.DDSketch.SizeProto() n += 1 + proto.Sov(uint64(l)) + l } + case *Metric_Summary: + if orig.Summary != nil { + l = orig.Summary.SizeProto() + n += 1 + proto.Sov(uint64(l)) + l + } case *Metric_KLLSketch: if orig.KLLSketch != nil { l = orig.KLLSketch.SizeProto() @@ -767,11 +772,6 @@ func (orig *Metric) SizeProto() int { l = orig.HLLSketch.SizeProto() n += 2 + proto.Sov(uint64(l)) + l } - case *Metric_Summary: - if orig.Summary != nil { - l = orig.Summary.SizeProto() - n += 1 + proto.Sov(uint64(l)) + l - } } for i := range orig.Metadata { l = orig.Metadata[i].SizeProto() @@ -849,6 +849,14 @@ func (orig *Metric) MarshalProto(buf []byte) int { pos-- buf[pos] = 0x6a } + case *Metric_Summary: + if orig.Summary != nil { + l = orig.Summary.MarshalProto(buf[:pos]) + pos -= l + pos = proto.EncodeVarint(buf, pos, uint64(l)) + pos-- + buf[pos] = 0x5a + } case *Metric_KLLSketch: if orig.KLLSketch != nil { l = orig.KLLSketch.MarshalProto(buf[:pos]) @@ -870,26 +878,20 @@ func (orig *Metric) MarshalProto(buf []byte) int { l = orig.CountMinSketch.MarshalProto(buf[:pos]) pos -= l pos = proto.EncodeVarint(buf, pos, uint64(l)) - pos -= 2 + pos-- + buf[pos] = 0x1 + pos-- buf[pos] = 0x82 - buf[pos+1] = 0x01 } case *Metric_HLLSketch: if orig.HLLSketch != nil { l = orig.HLLSketch.MarshalProto(buf[:pos]) pos -= l pos = proto.EncodeVarint(buf, pos, uint64(l)) - pos -= 2 - buf[pos] = 0x8a - buf[pos+1] = 0x01 - } - case *Metric_Summary: - if orig.Summary != nil { - l = orig.Summary.MarshalProto(buf[:pos]) - pos -= l - pos = proto.EncodeVarint(buf, pos, uint64(l)) pos-- - buf[pos] = 0x5a + buf[pos] = 0x1 + pos-- + buf[pos] = 0x8a } } for i := len(orig.Metadata) - 1; i >= 0; i-- { @@ -1068,6 +1070,29 @@ func (orig *Metric) UnmarshalProto(buf []byte) error { } orig.Data = ov + case 11: + if wireType != proto.WireTypeLen { + return fmt.Errorf("proto: wrong wireType = %d for field Summary", wireType) + } + var length int + length, pos, err = proto.ConsumeLen(buf, pos) + if err != nil { + return err + } + startPos := pos - length + var ov *Metric_Summary + if !UseProtoPooling.IsEnabled() { + ov = &Metric_Summary{} + } else { + ov = ProtoPoolMetric_Summary.Get().(*Metric_Summary) + } + ov.Summary = NewSummary() + err = ov.Summary.UnmarshalProto(buf[startPos:pos]) + if err != nil { + return err + } + orig.Data = ov + case 14: if wireType != proto.WireTypeLen { return fmt.Errorf("proto: wrong wireType = %d for field KLLSketch", wireType) @@ -1160,29 +1185,6 @@ func (orig *Metric) UnmarshalProto(buf []byte) error { } orig.Data = ov - case 11: - if wireType != proto.WireTypeLen { - return fmt.Errorf("proto: wrong wireType = %d for field Summary", wireType) - } - var length int - length, pos, err = proto.ConsumeLen(buf, pos) - if err != nil { - return err - } - startPos := pos - length - var ov *Metric_Summary - if !UseProtoPooling.IsEnabled() { - ov = &Metric_Summary{} - } else { - ov = ProtoPoolMetric_Summary.Get().(*Metric_Summary) - } - ov.Summary = NewSummary() - err = ov.Summary.UnmarshalProto(buf[startPos:pos]) - if err != nil { - return err - } - orig.Data = ov - case 12: if wireType != proto.WireTypeLen { return fmt.Errorf("proto: wrong wireType = %d for field Metadata", wireType) diff --git a/opentelemetry-collector-patch/pdata/internal/generated_proto_metric_test.go b/opentelemetry-collector-patch/pdata/internal/generated_proto_metric_test.go index e7292a1a..d199ef78 100644 --- a/opentelemetry-collector-patch/pdata/internal/generated_proto_metric_test.go +++ b/opentelemetry-collector-patch/pdata/internal/generated_proto_metric_test.go @@ -211,10 +211,22 @@ func genTestFailingUnmarshalProtoValuesMetric() map[string][]byte { "DDSketch/wrong_wire_type": {0x6c}, "DDSketch/missing_value": {0x6a}, - "Summary/wrong_wire_type": {0x5c}, - "Summary/missing_value": {0x5a}, - "Metadata/wrong_wire_type": {0x64}, - "Metadata/missing_value": {0x62}, + "Summary/wrong_wire_type": {0x5c}, + "Summary/missing_value": {0x5a}, + + "KLLSketch/wrong_wire_type": {0x74}, + "KLLSketch/missing_value": {0x72}, + + "CountSketch/wrong_wire_type": {0x7c}, + "CountSketch/missing_value": {0x7a}, + + "CountMinSketch/wrong_wire_type": {0x84, 0x1}, + "CountMinSketch/missing_value": {0x82, 0x1}, + + "HLLSketch/wrong_wire_type": {0x8c, 0x1}, + "HLLSketch/missing_value": {0x8a, 0x1}, + "Metadata/wrong_wire_type": {0x64}, + "Metadata/missing_value": {0x62}, } } @@ -236,6 +248,14 @@ func genTestEncodingValuesMetric() map[string]*Metric { "DDSketch/test": {Data: &Metric_DDSketch{DDSketch: GenTestDDSketch()}}, "Summary/default": {Data: &Metric_Summary{Summary: &Summary{}}}, "Summary/test": {Data: &Metric_Summary{Summary: GenTestSummary()}}, + "KLLSketch/default": {Data: &Metric_KLLSketch{KLLSketch: &KLLSketch{}}}, + "KLLSketch/test": {Data: &Metric_KLLSketch{KLLSketch: GenTestKLLSketch()}}, + "CountSketch/default": {Data: &Metric_CountSketch{CountSketch: &CountSketch{}}}, + "CountSketch/test": {Data: &Metric_CountSketch{CountSketch: GenTestCountSketch()}}, + "CountMinSketch/default": {Data: &Metric_CountMinSketch{CountMinSketch: &CountMinSketch{}}}, + "CountMinSketch/test": {Data: &Metric_CountMinSketch{CountMinSketch: GenTestCountMinSketch()}}, + "HLLSketch/default": {Data: &Metric_HLLSketch{HLLSketch: &HLLSketch{}}}, + "HLLSketch/test": {Data: &Metric_HLLSketch{HLLSketch: GenTestHLLSketch()}}, "Metadata/test": {Metadata: []KeyValue{{}, *GenTestKeyValue()}}, } } diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_countminsketch.go b/opentelemetry-collector-patch/pdata/pmetric/generated_countminsketch.go index e9084e92..4e127ff6 100644 --- a/opentelemetry-collector-patch/pdata/pmetric/generated_countminsketch.go +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_countminsketch.go @@ -10,7 +10,7 @@ import ( "go.opentelemetry.io/collector/pdata/internal" ) -// CountMinSketch represents the type of a metric encoded with the Count-Min Sketch data structure. +// CountMinSketch represents the type of a metric encoded using the Count-Min Sketch frequency estimation algorithm. // // This is a reference type, if passed by value and callee modifies it the // caller will see the modification. @@ -39,6 +39,7 @@ func NewCountMinSketch() CountMinSketch { func (ms CountMinSketch) MoveTo(dest CountMinSketch) { ms.state.AssertMutable() dest.state.AssertMutable() + // If they point to the same data, they are the same, nothing to do. if ms.orig == dest.orig { return } @@ -62,6 +63,28 @@ func (ms CountMinSketch) SetAggregationTemporality(v AggregationTemporality) { ms.orig.AggregationTemporality = internal.AggregationTemporality(v) } +// Rows returns the rows associated with this CountMinSketch. +func (ms CountMinSketch) Rows() int32 { + return ms.orig.Rows +} + +// SetRows replaces the rows associated with this CountMinSketch. +func (ms CountMinSketch) SetRows(v int32) { + ms.state.AssertMutable() + ms.orig.Rows = v +} + +// Cols returns the cols associated with this CountMinSketch. +func (ms CountMinSketch) Cols() int32 { + return ms.orig.Cols +} + +// SetCols replaces the cols associated with this CountMinSketch. +func (ms CountMinSketch) SetCols(v int32) { + ms.state.AssertMutable() + ms.orig.Cols = v +} + // CopyTo copies all properties from the current struct overriding the destination. func (ms CountMinSketch) CopyTo(dest CountMinSketch) { dest.state.AssertMutable() diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_countminsketch_test.go b/opentelemetry-collector-patch/pdata/pmetric/generated_countminsketch_test.go new file mode 100644 index 00000000..f6f37c9f --- /dev/null +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_countminsketch_test.go @@ -0,0 +1,81 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package pmetric + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "go.opentelemetry.io/collector/pdata/internal" +) + +func TestCountMinSketch_MoveTo(t *testing.T) { + ms := generateTestCountMinSketch() + dest := NewCountMinSketch() + ms.MoveTo(dest) + assert.Equal(t, NewCountMinSketch(), ms) + assert.Equal(t, generateTestCountMinSketch(), dest) + dest.MoveTo(dest) + assert.Equal(t, generateTestCountMinSketch(), dest) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { ms.MoveTo(newCountMinSketch(internal.NewCountMinSketch(), sharedState)) }) + assert.Panics(t, func() { newCountMinSketch(internal.NewCountMinSketch(), sharedState).MoveTo(dest) }) +} + +func TestCountMinSketch_CopyTo(t *testing.T) { + ms := NewCountMinSketch() + orig := NewCountMinSketch() + orig.CopyTo(ms) + assert.Equal(t, orig, ms) + orig = generateTestCountMinSketch() + orig.CopyTo(ms) + assert.Equal(t, orig, ms) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { ms.CopyTo(newCountMinSketch(internal.NewCountMinSketch(), sharedState)) }) +} + +func TestCountMinSketch_DataPoints(t *testing.T) { + ms := NewCountMinSketch() + assert.Equal(t, NewCountMinSketchDataPointSlice(), ms.DataPoints()) + ms.orig.DataPoints = internal.GenTestCountMinSketchDataPointPtrSlice() + assert.Equal(t, generateTestCountMinSketchDataPointSlice(), ms.DataPoints()) +} + +func TestCountMinSketch_AggregationTemporality(t *testing.T) { + ms := NewCountMinSketch() + assert.Equal(t, AggregationTemporality(internal.AggregationTemporality(0)), ms.AggregationTemporality()) + testValAggregationTemporality := AggregationTemporality(internal.AggregationTemporality(1)) + ms.SetAggregationTemporality(testValAggregationTemporality) + assert.Equal(t, testValAggregationTemporality, ms.AggregationTemporality()) +} + +func TestCountMinSketch_Rows(t *testing.T) { + ms := NewCountMinSketch() + assert.Equal(t, int32(0), ms.Rows()) + ms.SetRows(int32(13)) + assert.Equal(t, int32(13), ms.Rows()) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { newCountMinSketch(internal.NewCountMinSketch(), sharedState).SetRows(int32(13)) }) +} + +func TestCountMinSketch_Cols(t *testing.T) { + ms := NewCountMinSketch() + assert.Equal(t, int32(0), ms.Cols()) + ms.SetCols(int32(13)) + assert.Equal(t, int32(13), ms.Cols()) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { newCountMinSketch(internal.NewCountMinSketch(), sharedState).SetCols(int32(13)) }) +} + +func generateTestCountMinSketch() CountMinSketch { + return newCountMinSketch(internal.GenTestCountMinSketch(), internal.NewState()) +} diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_countminsketchdatapoint.go b/opentelemetry-collector-patch/pdata/pmetric/generated_countminsketchdatapoint.go index 936773bd..dedff1e6 100644 --- a/opentelemetry-collector-patch/pdata/pmetric/generated_countminsketchdatapoint.go +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_countminsketchdatapoint.go @@ -40,6 +40,7 @@ func NewCountMinSketchDataPoint() CountMinSketchDataPoint { func (ms CountMinSketchDataPoint) MoveTo(dest CountMinSketchDataPoint) { ms.state.AssertMutable() dest.state.AssertMutable() + // If they point to the same data, they are the same, nothing to do. if ms.orig == dest.orig { return } @@ -52,17 +53,6 @@ func (ms CountMinSketchDataPoint) Attributes() pcommon.Map { return pcommon.Map(internal.NewMapWrapper(&ms.orig.Attributes, ms.state)) } -// SeriesID returns the seriesid associated with this CountMinSketchDataPoint. -func (ms CountMinSketchDataPoint) SeriesID() uint64 { - return ms.orig.SeriesID -} - -// SetSeriesID replaces the seriesid associated with this CountMinSketchDataPoint. -func (ms CountMinSketchDataPoint) SetSeriesID(v uint64) { - ms.state.AssertMutable() - ms.orig.SeriesID = v -} - // StartTimestamp returns the starttimestamp associated with this CountMinSketchDataPoint. func (ms CountMinSketchDataPoint) StartTimestamp() pcommon.Timestamp { return pcommon.Timestamp(ms.orig.StartTimeUnixNano) @@ -85,17 +75,6 @@ func (ms CountMinSketchDataPoint) SetTimestamp(v pcommon.Timestamp) { ms.orig.TimeUnixNano = uint64(v) } -// SampleCount returns the samplecount associated with this CountMinSketchDataPoint. -func (ms CountMinSketchDataPoint) SampleCount() uint64 { - return ms.orig.SampleCount -} - -// SetSampleCount replaces the samplecount associated with this CountMinSketchDataPoint. -func (ms CountMinSketchDataPoint) SetSampleCount(v uint64) { - ms.state.AssertMutable() - ms.orig.SampleCount = v -} - // Sketch returns the sketch associated with this CountMinSketchDataPoint. func (ms CountMinSketchDataPoint) Sketch() []byte { return ms.orig.Sketch @@ -118,28 +97,6 @@ func (ms CountMinSketchDataPoint) SetEncoding(v CountMinSketchEncoding) { ms.orig.Encoding = internal.CountMinSketchEncoding(v) } -// Rows returns the rows associated with this CountMinSketchDataPoint. -func (ms CountMinSketchDataPoint) Rows() int32 { - return ms.orig.Rows -} - -// SetRows replaces the rows associated with this CountMinSketchDataPoint. -func (ms CountMinSketchDataPoint) SetRows(v int32) { - ms.state.AssertMutable() - ms.orig.Rows = v -} - -// Cols returns the cols associated with this CountMinSketchDataPoint. -func (ms CountMinSketchDataPoint) Cols() int32 { - return ms.orig.Cols -} - -// SetCols replaces the cols associated with this CountMinSketchDataPoint. -func (ms CountMinSketchDataPoint) SetCols(v int32) { - ms.state.AssertMutable() - ms.orig.Cols = v -} - // Flags returns the flags associated with this CountMinSketchDataPoint. func (ms CountMinSketchDataPoint) Flags() DataPointFlags { return DataPointFlags(ms.orig.Flags) @@ -151,6 +108,17 @@ func (ms CountMinSketchDataPoint) SetFlags(v DataPointFlags) { ms.orig.Flags = uint32(v) } +// SeriesID returns the seriesid associated with this CountMinSketchDataPoint. +func (ms CountMinSketchDataPoint) SeriesID() uint64 { + return ms.orig.SeriesID +} + +// SetSeriesID replaces the seriesid associated with this CountMinSketchDataPoint. +func (ms CountMinSketchDataPoint) SetSeriesID(v uint64) { + ms.state.AssertMutable() + ms.orig.SeriesID = v +} + // CopyTo copies all properties from the current struct overriding the destination. func (ms CountMinSketchDataPoint) CopyTo(dest CountMinSketchDataPoint) { dest.state.AssertMutable() diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_countminsketchdatapoint_test.go b/opentelemetry-collector-patch/pdata/pmetric/generated_countminsketchdatapoint_test.go new file mode 100644 index 00000000..2c014393 --- /dev/null +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_countminsketchdatapoint_test.go @@ -0,0 +1,110 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package pmetric + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "go.opentelemetry.io/collector/pdata/internal" + "go.opentelemetry.io/collector/pdata/pcommon" +) + +func TestCountMinSketchDataPoint_MoveTo(t *testing.T) { + ms := generateTestCountMinSketchDataPoint() + dest := NewCountMinSketchDataPoint() + ms.MoveTo(dest) + assert.Equal(t, NewCountMinSketchDataPoint(), ms) + assert.Equal(t, generateTestCountMinSketchDataPoint(), dest) + dest.MoveTo(dest) + assert.Equal(t, generateTestCountMinSketchDataPoint(), dest) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { ms.MoveTo(newCountMinSketchDataPoint(internal.NewCountMinSketchDataPoint(), sharedState)) }) + assert.Panics(t, func() { newCountMinSketchDataPoint(internal.NewCountMinSketchDataPoint(), sharedState).MoveTo(dest) }) +} + +func TestCountMinSketchDataPoint_CopyTo(t *testing.T) { + ms := NewCountMinSketchDataPoint() + orig := NewCountMinSketchDataPoint() + orig.CopyTo(ms) + assert.Equal(t, orig, ms) + orig = generateTestCountMinSketchDataPoint() + orig.CopyTo(ms) + assert.Equal(t, orig, ms) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { ms.CopyTo(newCountMinSketchDataPoint(internal.NewCountMinSketchDataPoint(), sharedState)) }) +} + +func TestCountMinSketchDataPoint_Attributes(t *testing.T) { + ms := NewCountMinSketchDataPoint() + assert.Equal(t, pcommon.NewMap(), ms.Attributes()) + ms.orig.Attributes = internal.GenTestKeyValueSlice() + assert.Equal(t, pcommon.Map(internal.GenTestMapWrapper()), ms.Attributes()) +} + +func TestCountMinSketchDataPoint_StartTimestamp(t *testing.T) { + ms := NewCountMinSketchDataPoint() + assert.Equal(t, pcommon.Timestamp(0), ms.StartTimestamp()) + testValStartTimestamp := pcommon.Timestamp(1234567890) + ms.SetStartTimestamp(testValStartTimestamp) + assert.Equal(t, testValStartTimestamp, ms.StartTimestamp()) +} + +func TestCountMinSketchDataPoint_Timestamp(t *testing.T) { + ms := NewCountMinSketchDataPoint() + assert.Equal(t, pcommon.Timestamp(0), ms.Timestamp()) + testValTimestamp := pcommon.Timestamp(1234567890) + ms.SetTimestamp(testValTimestamp) + assert.Equal(t, testValTimestamp, ms.Timestamp()) +} + +func TestCountMinSketchDataPoint_Sketch(t *testing.T) { + ms := NewCountMinSketchDataPoint() + assert.Equal(t, nil, ms.Sketch()) + ms.SetSketch([]byte{1, 2, 3}) + assert.Equal(t, []byte{1, 2, 3}, ms.Sketch()) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { + newCountMinSketchDataPoint(internal.NewCountMinSketchDataPoint(), sharedState).SetSketch([]byte{1, 2, 3}) + }) +} + +func TestCountMinSketchDataPoint_Encoding(t *testing.T) { + ms := NewCountMinSketchDataPoint() + assert.Equal(t, CountMinSketchEncoding(internal.CountMinSketchEncoding(0)), ms.Encoding()) + testValEncoding := CountMinSketchEncoding(internal.CountMinSketchEncoding(1)) + ms.SetEncoding(testValEncoding) + assert.Equal(t, testValEncoding, ms.Encoding()) +} + +func TestCountMinSketchDataPoint_Flags(t *testing.T) { + ms := NewCountMinSketchDataPoint() + assert.Equal(t, DataPointFlags(0), ms.Flags()) + testValFlags := DataPointFlags(1) + ms.SetFlags(testValFlags) + assert.Equal(t, testValFlags, ms.Flags()) +} + +func TestCountMinSketchDataPoint_SeriesID(t *testing.T) { + ms := NewCountMinSketchDataPoint() + assert.Equal(t, uint64(0), ms.SeriesID()) + ms.SetSeriesID(uint64(13)) + assert.Equal(t, uint64(13), ms.SeriesID()) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { + newCountMinSketchDataPoint(internal.NewCountMinSketchDataPoint(), sharedState).SetSeriesID(uint64(13)) + }) +} + +func generateTestCountMinSketchDataPoint() CountMinSketchDataPoint { + return newCountMinSketchDataPoint(internal.GenTestCountMinSketchDataPoint(), internal.NewState()) +} diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_countminsketchdatapointslice.go b/opentelemetry-collector-patch/pdata/pmetric/generated_countminsketchdatapointslice.go index 2e69b3ad..9183b555 100644 --- a/opentelemetry-collector-patch/pdata/pmetric/generated_countminsketchdatapointslice.go +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_countminsketchdatapointslice.go @@ -29,7 +29,7 @@ func newCountMinSketchDataPointSlice(orig *[]*internal.CountMinSketchDataPoint, return CountMinSketchDataPointSlice{orig: orig, state: state} } -// NewCountMinSketchDataPointSlice creates a CountMinSketchDataPointSlice with 0 elements. +// NewCountMinSketchDataPointSlice creates a CountMinSketchDataPointSliceWrapper with 0 elements. // Can use "EnsureCapacity" to initialize with a given capacity. func NewCountMinSketchDataPointSlice() CountMinSketchDataPointSlice { orig := []*internal.CountMinSketchDataPoint(nil) @@ -37,16 +37,29 @@ func NewCountMinSketchDataPointSlice() CountMinSketchDataPointSlice { } // Len returns the number of elements in the slice. +// +// Returns "0" for a newly instance created with "NewCountMinSketchDataPointSlice()". func (es CountMinSketchDataPointSlice) Len() int { return len(*es.orig) } // At returns the element at the given index. +// +// This function is used mostly for iterating over all the values in the slice: +// +// for i := 0; i < es.Len(); i++ { +// e := es.At(i) +// ... // Do something with the element +// } func (es CountMinSketchDataPointSlice) At(i int) CountMinSketchDataPoint { return newCountMinSketchDataPoint((*es.orig)[i], es.state) } // All returns an iterator over index-value pairs in the slice. +// +// for i, v := range es.All() { +// ... // Do something with index-value pair +// } func (es CountMinSketchDataPointSlice) All() iter.Seq2[int, CountMinSketchDataPoint] { return func(yield func(int, CountMinSketchDataPoint) bool) { for i := 0; i < es.Len(); i++ { @@ -58,6 +71,17 @@ func (es CountMinSketchDataPointSlice) All() iter.Seq2[int, CountMinSketchDataPo } // EnsureCapacity is an operation that ensures the slice has at least the specified capacity. +// 1. If the newCap <= cap then no change in capacity. +// 2. If the newCap > cap then the slice capacity will be expanded to equal newCap. +// +// Here is how a new CountMinSketchDataPointSlice can be initialized: +// +// es := NewCountMinSketchDataPointSlice() +// es.EnsureCapacity(4) +// for i := 0; i < 4; i++ { +// e := es.AppendEmpty() +// // Here should set all the values for e. +// } func (es CountMinSketchDataPointSlice) EnsureCapacity(newCap int) { es.state.AssertMutable() oldCap := cap(*es.orig) @@ -83,10 +107,12 @@ func (es CountMinSketchDataPointSlice) AppendEmpty() CountMinSketchDataPoint { func (es CountMinSketchDataPointSlice) MoveAndAppendTo(dest CountMinSketchDataPointSlice) { es.state.AssertMutable() dest.state.AssertMutable() + // If they point to the same data, they are the same, nothing to do. if es.orig == dest.orig { return } if *dest.orig == nil { + // We can simply move the entire vector and avoid any allocations. *dest.orig = *es.orig } else { *dest.orig = append(*dest.orig, *es.orig...) @@ -103,13 +129,16 @@ func (es CountMinSketchDataPointSlice) RemoveIf(f func(CountMinSketchDataPoint) if f(es.At(i)) { internal.DeleteCountMinSketchDataPoint((*es.orig)[i], true) (*es.orig)[i] = nil + continue } if newLen == i { + // Nothing to move, element is at the right place. newLen++ continue } (*es.orig)[newLen] = (*es.orig)[i] + // Cannot delete here since we just move the data(or pointer to data) to a different position in the slice. (*es.orig)[i] = nil newLen++ } @@ -126,7 +155,8 @@ func (es CountMinSketchDataPointSlice) CopyTo(dest CountMinSketchDataPointSlice) } // Sort sorts the CountMinSketchDataPoint elements within CountMinSketchDataPointSlice given the -// provided less function. +// provided less function so that two instances of CountMinSketchDataPointSlice +// can be compared. func (es CountMinSketchDataPointSlice) Sort(less func(a, b CountMinSketchDataPoint) bool) { es.state.AssertMutable() sort.SliceStable(*es.orig, func(i, j int) bool { return less(es.At(i), es.At(j)) }) diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_countminsketchdatapointslice_test.go b/opentelemetry-collector-patch/pdata/pmetric/generated_countminsketchdatapointslice_test.go new file mode 100644 index 00000000..be10301a --- /dev/null +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_countminsketchdatapointslice_test.go @@ -0,0 +1,166 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package pmetric + +import ( + "testing" + "unsafe" + + "github.com/stretchr/testify/assert" + + "go.opentelemetry.io/collector/pdata/internal" +) + +func TestCountMinSketchDataPointSlice(t *testing.T) { + es := NewCountMinSketchDataPointSlice() + assert.Equal(t, 0, es.Len()) + es = newCountMinSketchDataPointSlice(&[]*internal.CountMinSketchDataPoint{}, internal.NewState()) + assert.Equal(t, 0, es.Len()) + + emptyVal := NewCountMinSketchDataPoint() + testVal := generateTestCountMinSketchDataPoint() + for i := 0; i < 7; i++ { + es.AppendEmpty() + assert.Equal(t, emptyVal, es.At(i)) + (*es.orig)[i] = internal.GenTestCountMinSketchDataPoint() + assert.Equal(t, testVal, es.At(i)) + } + assert.Equal(t, 7, es.Len()) +} + +func TestCountMinSketchDataPointSliceReadOnly(t *testing.T) { + sharedState := internal.NewState() + sharedState.MarkReadOnly() + es := newCountMinSketchDataPointSlice(&[]*internal.CountMinSketchDataPoint{}, sharedState) + assert.Equal(t, 0, es.Len()) + assert.Panics(t, func() { es.AppendEmpty() }) + assert.Panics(t, func() { es.EnsureCapacity(2) }) + es2 := NewCountMinSketchDataPointSlice() + es.CopyTo(es2) + assert.Panics(t, func() { es2.CopyTo(es) }) + assert.Panics(t, func() { es.MoveAndAppendTo(es2) }) + assert.Panics(t, func() { es2.MoveAndAppendTo(es) }) +} + +func TestCountMinSketchDataPointSlice_CopyTo(t *testing.T) { + dest := NewCountMinSketchDataPointSlice() + src := generateTestCountMinSketchDataPointSlice() + src.CopyTo(dest) + assert.Equal(t, generateTestCountMinSketchDataPointSlice(), dest) + dest.CopyTo(dest) + assert.Equal(t, generateTestCountMinSketchDataPointSlice(), dest) +} + +func TestCountMinSketchDataPointSlice_EnsureCapacity(t *testing.T) { + es := generateTestCountMinSketchDataPointSlice() + + // Test ensure smaller capacity. + const ensureSmallLen = 4 + es.EnsureCapacity(ensureSmallLen) + assert.Less(t, ensureSmallLen, es.Len()) + assert.Equal(t, es.Len(), cap(*es.orig)) + assert.Equal(t, generateTestCountMinSketchDataPointSlice(), es) + + // Test ensure larger capacity + const ensureLargeLen = 9 + es.EnsureCapacity(ensureLargeLen) + assert.Less(t, generateTestCountMinSketchDataPointSlice().Len(), ensureLargeLen) + assert.Equal(t, ensureLargeLen, cap(*es.orig)) + assert.Equal(t, generateTestCountMinSketchDataPointSlice(), es) +} + +func TestCountMinSketchDataPointSlice_MoveAndAppendTo(t *testing.T) { + // Test MoveAndAppendTo to empty + expectedSlice := generateTestCountMinSketchDataPointSlice() + dest := NewCountMinSketchDataPointSlice() + src := generateTestCountMinSketchDataPointSlice() + src.MoveAndAppendTo(dest) + assert.Equal(t, generateTestCountMinSketchDataPointSlice(), dest) + assert.Equal(t, 0, src.Len()) + assert.Equal(t, expectedSlice.Len(), dest.Len()) + + // Test MoveAndAppendTo empty slice + src.MoveAndAppendTo(dest) + assert.Equal(t, generateTestCountMinSketchDataPointSlice(), dest) + assert.Equal(t, 0, src.Len()) + assert.Equal(t, expectedSlice.Len(), dest.Len()) + + // Test MoveAndAppendTo not empty slice + generateTestCountMinSketchDataPointSlice().MoveAndAppendTo(dest) + assert.Equal(t, 2*expectedSlice.Len(), dest.Len()) + for i := 0; i < expectedSlice.Len(); i++ { + assert.Equal(t, expectedSlice.At(i), dest.At(i)) + assert.Equal(t, expectedSlice.At(i), dest.At(i+expectedSlice.Len())) + } + + dest.MoveAndAppendTo(dest) + assert.Equal(t, 2*expectedSlice.Len(), dest.Len()) + for i := 0; i < expectedSlice.Len(); i++ { + assert.Equal(t, expectedSlice.At(i), dest.At(i)) + assert.Equal(t, expectedSlice.At(i), dest.At(i+expectedSlice.Len())) + } +} + +func TestCountMinSketchDataPointSlice_RemoveIf(t *testing.T) { + // Test RemoveIf on empty slice + emptySlice := NewCountMinSketchDataPointSlice() + emptySlice.RemoveIf(func(el CountMinSketchDataPoint) bool { + t.Fail() + return false + }) + + // Test RemoveIf + filtered := generateTestCountMinSketchDataPointSlice() + pos := 0 + filtered.RemoveIf(func(el CountMinSketchDataPoint) bool { + pos++ + return pos%2 == 1 + }) + assert.Equal(t, 2, filtered.Len()) +} + +func TestCountMinSketchDataPointSlice_RemoveIfAll(t *testing.T) { + got := generateTestCountMinSketchDataPointSlice() + got.RemoveIf(func(el CountMinSketchDataPoint) bool { + return true + }) + assert.Equal(t, 0, got.Len()) +} + +func TestCountMinSketchDataPointSliceAll(t *testing.T) { + ms := generateTestCountMinSketchDataPointSlice() + assert.NotEmpty(t, ms.Len()) + + var c int + for i, v := range ms.All() { + assert.Equal(t, ms.At(i), v, "element should match") + c++ + } + assert.Equal(t, ms.Len(), c, "All elements should have been visited") +} + +func TestCountMinSketchDataPointSlice_Sort(t *testing.T) { + es := generateTestCountMinSketchDataPointSlice() + es.Sort(func(a, b CountMinSketchDataPoint) bool { + return uintptr(unsafe.Pointer(a.orig)) < uintptr(unsafe.Pointer(b.orig)) + }) + for i := 1; i < es.Len(); i++ { + assert.Less(t, uintptr(unsafe.Pointer(es.At(i-1).orig)), uintptr(unsafe.Pointer(es.At(i).orig))) + } + es.Sort(func(a, b CountMinSketchDataPoint) bool { + return uintptr(unsafe.Pointer(a.orig)) > uintptr(unsafe.Pointer(b.orig)) + }) + for i := 1; i < es.Len(); i++ { + assert.Greater(t, uintptr(unsafe.Pointer(es.At(i-1).orig)), uintptr(unsafe.Pointer(es.At(i).orig))) + } +} + +func generateTestCountMinSketchDataPointSlice() CountMinSketchDataPointSlice { + ms := NewCountMinSketchDataPointSlice() + *ms.orig = internal.GenTestCountMinSketchDataPointPtrSlice() + return ms +} diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_countsketch.go b/opentelemetry-collector-patch/pdata/pmetric/generated_countsketch.go index a8b3d16b..21e263a1 100644 --- a/opentelemetry-collector-patch/pdata/pmetric/generated_countsketch.go +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_countsketch.go @@ -10,7 +10,7 @@ import ( "go.opentelemetry.io/collector/pdata/internal" ) -// CountSketch represents the type of a metric encoded with the CountSketch data structure. +// CountSketch represents the type of a metric encoded using the CountSketch frequency estimation algorithm. // // This is a reference type, if passed by value and callee modifies it the // caller will see the modification. @@ -39,6 +39,7 @@ func NewCountSketch() CountSketch { func (ms CountSketch) MoveTo(dest CountSketch) { ms.state.AssertMutable() dest.state.AssertMutable() + // If they point to the same data, they are the same, nothing to do. if ms.orig == dest.orig { return } @@ -62,6 +63,28 @@ func (ms CountSketch) SetAggregationTemporality(v AggregationTemporality) { ms.orig.AggregationTemporality = internal.AggregationTemporality(v) } +// Rows returns the rows associated with this CountSketch. +func (ms CountSketch) Rows() int32 { + return ms.orig.Rows +} + +// SetRows replaces the rows associated with this CountSketch. +func (ms CountSketch) SetRows(v int32) { + ms.state.AssertMutable() + ms.orig.Rows = v +} + +// Cols returns the cols associated with this CountSketch. +func (ms CountSketch) Cols() int32 { + return ms.orig.Cols +} + +// SetCols replaces the cols associated with this CountSketch. +func (ms CountSketch) SetCols(v int32) { + ms.state.AssertMutable() + ms.orig.Cols = v +} + // CopyTo copies all properties from the current struct overriding the destination. func (ms CountSketch) CopyTo(dest CountSketch) { dest.state.AssertMutable() diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_countsketch_test.go b/opentelemetry-collector-patch/pdata/pmetric/generated_countsketch_test.go new file mode 100644 index 00000000..820a9a03 --- /dev/null +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_countsketch_test.go @@ -0,0 +1,81 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package pmetric + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "go.opentelemetry.io/collector/pdata/internal" +) + +func TestCountSketch_MoveTo(t *testing.T) { + ms := generateTestCountSketch() + dest := NewCountSketch() + ms.MoveTo(dest) + assert.Equal(t, NewCountSketch(), ms) + assert.Equal(t, generateTestCountSketch(), dest) + dest.MoveTo(dest) + assert.Equal(t, generateTestCountSketch(), dest) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { ms.MoveTo(newCountSketch(internal.NewCountSketch(), sharedState)) }) + assert.Panics(t, func() { newCountSketch(internal.NewCountSketch(), sharedState).MoveTo(dest) }) +} + +func TestCountSketch_CopyTo(t *testing.T) { + ms := NewCountSketch() + orig := NewCountSketch() + orig.CopyTo(ms) + assert.Equal(t, orig, ms) + orig = generateTestCountSketch() + orig.CopyTo(ms) + assert.Equal(t, orig, ms) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { ms.CopyTo(newCountSketch(internal.NewCountSketch(), sharedState)) }) +} + +func TestCountSketch_DataPoints(t *testing.T) { + ms := NewCountSketch() + assert.Equal(t, NewCountSketchDataPointSlice(), ms.DataPoints()) + ms.orig.DataPoints = internal.GenTestCountSketchDataPointPtrSlice() + assert.Equal(t, generateTestCountSketchDataPointSlice(), ms.DataPoints()) +} + +func TestCountSketch_AggregationTemporality(t *testing.T) { + ms := NewCountSketch() + assert.Equal(t, AggregationTemporality(internal.AggregationTemporality(0)), ms.AggregationTemporality()) + testValAggregationTemporality := AggregationTemporality(internal.AggregationTemporality(1)) + ms.SetAggregationTemporality(testValAggregationTemporality) + assert.Equal(t, testValAggregationTemporality, ms.AggregationTemporality()) +} + +func TestCountSketch_Rows(t *testing.T) { + ms := NewCountSketch() + assert.Equal(t, int32(0), ms.Rows()) + ms.SetRows(int32(13)) + assert.Equal(t, int32(13), ms.Rows()) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { newCountSketch(internal.NewCountSketch(), sharedState).SetRows(int32(13)) }) +} + +func TestCountSketch_Cols(t *testing.T) { + ms := NewCountSketch() + assert.Equal(t, int32(0), ms.Cols()) + ms.SetCols(int32(13)) + assert.Equal(t, int32(13), ms.Cols()) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { newCountSketch(internal.NewCountSketch(), sharedState).SetCols(int32(13)) }) +} + +func generateTestCountSketch() CountSketch { + return newCountSketch(internal.GenTestCountSketch(), internal.NewState()) +} diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_countsketchdatapoint.go b/opentelemetry-collector-patch/pdata/pmetric/generated_countsketchdatapoint.go index c7dea832..ec18080b 100644 --- a/opentelemetry-collector-patch/pdata/pmetric/generated_countsketchdatapoint.go +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_countsketchdatapoint.go @@ -40,6 +40,7 @@ func NewCountSketchDataPoint() CountSketchDataPoint { func (ms CountSketchDataPoint) MoveTo(dest CountSketchDataPoint) { ms.state.AssertMutable() dest.state.AssertMutable() + // If they point to the same data, they are the same, nothing to do. if ms.orig == dest.orig { return } @@ -52,17 +53,6 @@ func (ms CountSketchDataPoint) Attributes() pcommon.Map { return pcommon.Map(internal.NewMapWrapper(&ms.orig.Attributes, ms.state)) } -// SeriesID returns the seriesid associated with this CountSketchDataPoint. -func (ms CountSketchDataPoint) SeriesID() uint64 { - return ms.orig.SeriesID -} - -// SetSeriesID replaces the seriesid associated with this CountSketchDataPoint. -func (ms CountSketchDataPoint) SetSeriesID(v uint64) { - ms.state.AssertMutable() - ms.orig.SeriesID = v -} - // StartTimestamp returns the starttimestamp associated with this CountSketchDataPoint. func (ms CountSketchDataPoint) StartTimestamp() pcommon.Timestamp { return pcommon.Timestamp(ms.orig.StartTimeUnixNano) @@ -107,39 +97,6 @@ func (ms CountSketchDataPoint) SetEncoding(v CountSketchEncoding) { ms.orig.Encoding = internal.CountSketchEncoding(v) } -// Dimension returns the dimension associated with this CountSketchDataPoint. -func (ms CountSketchDataPoint) Dimension() string { - return ms.orig.Dimension -} - -// SetDimension replaces the dimension associated with this CountSketchDataPoint. -func (ms CountSketchDataPoint) SetDimension(v string) { - ms.state.AssertMutable() - ms.orig.Dimension = v -} - -// Epsilon returns the epsilon associated with this CountSketchDataPoint. -func (ms CountSketchDataPoint) Epsilon() float64 { - return ms.orig.Epsilon -} - -// SetEpsilon replaces the epsilon associated with this CountSketchDataPoint. -func (ms CountSketchDataPoint) SetEpsilon(v float64) { - ms.state.AssertMutable() - ms.orig.Epsilon = v -} - -// Delta returns the delta associated with this CountSketchDataPoint. -func (ms CountSketchDataPoint) Delta() float64 { - return ms.orig.Delta -} - -// SetDelta replaces the delta associated with this CountSketchDataPoint. -func (ms CountSketchDataPoint) SetDelta(v float64) { - ms.state.AssertMutable() - ms.orig.Delta = v -} - // Flags returns the flags associated with this CountSketchDataPoint. func (ms CountSketchDataPoint) Flags() DataPointFlags { return DataPointFlags(ms.orig.Flags) @@ -151,6 +108,17 @@ func (ms CountSketchDataPoint) SetFlags(v DataPointFlags) { ms.orig.Flags = uint32(v) } +// SeriesID returns the seriesid associated with this CountSketchDataPoint. +func (ms CountSketchDataPoint) SeriesID() uint64 { + return ms.orig.SeriesID +} + +// SetSeriesID replaces the seriesid associated with this CountSketchDataPoint. +func (ms CountSketchDataPoint) SetSeriesID(v uint64) { + ms.state.AssertMutable() + ms.orig.SeriesID = v +} + // CopyTo copies all properties from the current struct overriding the destination. func (ms CountSketchDataPoint) CopyTo(dest CountSketchDataPoint) { dest.state.AssertMutable() diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_countsketchdatapoint_test.go b/opentelemetry-collector-patch/pdata/pmetric/generated_countsketchdatapoint_test.go new file mode 100644 index 00000000..268751f0 --- /dev/null +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_countsketchdatapoint_test.go @@ -0,0 +1,110 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package pmetric + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "go.opentelemetry.io/collector/pdata/internal" + "go.opentelemetry.io/collector/pdata/pcommon" +) + +func TestCountSketchDataPoint_MoveTo(t *testing.T) { + ms := generateTestCountSketchDataPoint() + dest := NewCountSketchDataPoint() + ms.MoveTo(dest) + assert.Equal(t, NewCountSketchDataPoint(), ms) + assert.Equal(t, generateTestCountSketchDataPoint(), dest) + dest.MoveTo(dest) + assert.Equal(t, generateTestCountSketchDataPoint(), dest) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { ms.MoveTo(newCountSketchDataPoint(internal.NewCountSketchDataPoint(), sharedState)) }) + assert.Panics(t, func() { newCountSketchDataPoint(internal.NewCountSketchDataPoint(), sharedState).MoveTo(dest) }) +} + +func TestCountSketchDataPoint_CopyTo(t *testing.T) { + ms := NewCountSketchDataPoint() + orig := NewCountSketchDataPoint() + orig.CopyTo(ms) + assert.Equal(t, orig, ms) + orig = generateTestCountSketchDataPoint() + orig.CopyTo(ms) + assert.Equal(t, orig, ms) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { ms.CopyTo(newCountSketchDataPoint(internal.NewCountSketchDataPoint(), sharedState)) }) +} + +func TestCountSketchDataPoint_Attributes(t *testing.T) { + ms := NewCountSketchDataPoint() + assert.Equal(t, pcommon.NewMap(), ms.Attributes()) + ms.orig.Attributes = internal.GenTestKeyValueSlice() + assert.Equal(t, pcommon.Map(internal.GenTestMapWrapper()), ms.Attributes()) +} + +func TestCountSketchDataPoint_StartTimestamp(t *testing.T) { + ms := NewCountSketchDataPoint() + assert.Equal(t, pcommon.Timestamp(0), ms.StartTimestamp()) + testValStartTimestamp := pcommon.Timestamp(1234567890) + ms.SetStartTimestamp(testValStartTimestamp) + assert.Equal(t, testValStartTimestamp, ms.StartTimestamp()) +} + +func TestCountSketchDataPoint_Timestamp(t *testing.T) { + ms := NewCountSketchDataPoint() + assert.Equal(t, pcommon.Timestamp(0), ms.Timestamp()) + testValTimestamp := pcommon.Timestamp(1234567890) + ms.SetTimestamp(testValTimestamp) + assert.Equal(t, testValTimestamp, ms.Timestamp()) +} + +func TestCountSketchDataPoint_Sketch(t *testing.T) { + ms := NewCountSketchDataPoint() + assert.Equal(t, nil, ms.Sketch()) + ms.SetSketch([]byte{1, 2, 3}) + assert.Equal(t, []byte{1, 2, 3}, ms.Sketch()) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { + newCountSketchDataPoint(internal.NewCountSketchDataPoint(), sharedState).SetSketch([]byte{1, 2, 3}) + }) +} + +func TestCountSketchDataPoint_Encoding(t *testing.T) { + ms := NewCountSketchDataPoint() + assert.Equal(t, CountSketchEncoding(internal.CountSketchEncoding(0)), ms.Encoding()) + testValEncoding := CountSketchEncoding(internal.CountSketchEncoding(1)) + ms.SetEncoding(testValEncoding) + assert.Equal(t, testValEncoding, ms.Encoding()) +} + +func TestCountSketchDataPoint_Flags(t *testing.T) { + ms := NewCountSketchDataPoint() + assert.Equal(t, DataPointFlags(0), ms.Flags()) + testValFlags := DataPointFlags(1) + ms.SetFlags(testValFlags) + assert.Equal(t, testValFlags, ms.Flags()) +} + +func TestCountSketchDataPoint_SeriesID(t *testing.T) { + ms := NewCountSketchDataPoint() + assert.Equal(t, uint64(0), ms.SeriesID()) + ms.SetSeriesID(uint64(13)) + assert.Equal(t, uint64(13), ms.SeriesID()) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { + newCountSketchDataPoint(internal.NewCountSketchDataPoint(), sharedState).SetSeriesID(uint64(13)) + }) +} + +func generateTestCountSketchDataPoint() CountSketchDataPoint { + return newCountSketchDataPoint(internal.GenTestCountSketchDataPoint(), internal.NewState()) +} diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_countsketchdatapointslice.go b/opentelemetry-collector-patch/pdata/pmetric/generated_countsketchdatapointslice.go index 98413522..1c56034e 100644 --- a/opentelemetry-collector-patch/pdata/pmetric/generated_countsketchdatapointslice.go +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_countsketchdatapointslice.go @@ -29,7 +29,7 @@ func newCountSketchDataPointSlice(orig *[]*internal.CountSketchDataPoint, state return CountSketchDataPointSlice{orig: orig, state: state} } -// NewCountSketchDataPointSlice creates a CountSketchDataPointSlice with 0 elements. +// NewCountSketchDataPointSlice creates a CountSketchDataPointSliceWrapper with 0 elements. // Can use "EnsureCapacity" to initialize with a given capacity. func NewCountSketchDataPointSlice() CountSketchDataPointSlice { orig := []*internal.CountSketchDataPoint(nil) @@ -37,16 +37,29 @@ func NewCountSketchDataPointSlice() CountSketchDataPointSlice { } // Len returns the number of elements in the slice. +// +// Returns "0" for a newly instance created with "NewCountSketchDataPointSlice()". func (es CountSketchDataPointSlice) Len() int { return len(*es.orig) } // At returns the element at the given index. +// +// This function is used mostly for iterating over all the values in the slice: +// +// for i := 0; i < es.Len(); i++ { +// e := es.At(i) +// ... // Do something with the element +// } func (es CountSketchDataPointSlice) At(i int) CountSketchDataPoint { return newCountSketchDataPoint((*es.orig)[i], es.state) } // All returns an iterator over index-value pairs in the slice. +// +// for i, v := range es.All() { +// ... // Do something with index-value pair +// } func (es CountSketchDataPointSlice) All() iter.Seq2[int, CountSketchDataPoint] { return func(yield func(int, CountSketchDataPoint) bool) { for i := 0; i < es.Len(); i++ { @@ -58,6 +71,17 @@ func (es CountSketchDataPointSlice) All() iter.Seq2[int, CountSketchDataPoint] { } // EnsureCapacity is an operation that ensures the slice has at least the specified capacity. +// 1. If the newCap <= cap then no change in capacity. +// 2. If the newCap > cap then the slice capacity will be expanded to equal newCap. +// +// Here is how a new CountSketchDataPointSlice can be initialized: +// +// es := NewCountSketchDataPointSlice() +// es.EnsureCapacity(4) +// for i := 0; i < 4; i++ { +// e := es.AppendEmpty() +// // Here should set all the values for e. +// } func (es CountSketchDataPointSlice) EnsureCapacity(newCap int) { es.state.AssertMutable() oldCap := cap(*es.orig) @@ -83,10 +107,12 @@ func (es CountSketchDataPointSlice) AppendEmpty() CountSketchDataPoint { func (es CountSketchDataPointSlice) MoveAndAppendTo(dest CountSketchDataPointSlice) { es.state.AssertMutable() dest.state.AssertMutable() + // If they point to the same data, they are the same, nothing to do. if es.orig == dest.orig { return } if *dest.orig == nil { + // We can simply move the entire vector and avoid any allocations. *dest.orig = *es.orig } else { *dest.orig = append(*dest.orig, *es.orig...) @@ -103,13 +129,16 @@ func (es CountSketchDataPointSlice) RemoveIf(f func(CountSketchDataPoint) bool) if f(es.At(i)) { internal.DeleteCountSketchDataPoint((*es.orig)[i], true) (*es.orig)[i] = nil + continue } if newLen == i { + // Nothing to move, element is at the right place. newLen++ continue } (*es.orig)[newLen] = (*es.orig)[i] + // Cannot delete here since we just move the data(or pointer to data) to a different position in the slice. (*es.orig)[i] = nil newLen++ } @@ -126,7 +155,8 @@ func (es CountSketchDataPointSlice) CopyTo(dest CountSketchDataPointSlice) { } // Sort sorts the CountSketchDataPoint elements within CountSketchDataPointSlice given the -// provided less function. +// provided less function so that two instances of CountSketchDataPointSlice +// can be compared. func (es CountSketchDataPointSlice) Sort(less func(a, b CountSketchDataPoint) bool) { es.state.AssertMutable() sort.SliceStable(*es.orig, func(i, j int) bool { return less(es.At(i), es.At(j)) }) diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_countsketchdatapointslice_test.go b/opentelemetry-collector-patch/pdata/pmetric/generated_countsketchdatapointslice_test.go new file mode 100644 index 00000000..f0a1d6f7 --- /dev/null +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_countsketchdatapointslice_test.go @@ -0,0 +1,166 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package pmetric + +import ( + "testing" + "unsafe" + + "github.com/stretchr/testify/assert" + + "go.opentelemetry.io/collector/pdata/internal" +) + +func TestCountSketchDataPointSlice(t *testing.T) { + es := NewCountSketchDataPointSlice() + assert.Equal(t, 0, es.Len()) + es = newCountSketchDataPointSlice(&[]*internal.CountSketchDataPoint{}, internal.NewState()) + assert.Equal(t, 0, es.Len()) + + emptyVal := NewCountSketchDataPoint() + testVal := generateTestCountSketchDataPoint() + for i := 0; i < 7; i++ { + es.AppendEmpty() + assert.Equal(t, emptyVal, es.At(i)) + (*es.orig)[i] = internal.GenTestCountSketchDataPoint() + assert.Equal(t, testVal, es.At(i)) + } + assert.Equal(t, 7, es.Len()) +} + +func TestCountSketchDataPointSliceReadOnly(t *testing.T) { + sharedState := internal.NewState() + sharedState.MarkReadOnly() + es := newCountSketchDataPointSlice(&[]*internal.CountSketchDataPoint{}, sharedState) + assert.Equal(t, 0, es.Len()) + assert.Panics(t, func() { es.AppendEmpty() }) + assert.Panics(t, func() { es.EnsureCapacity(2) }) + es2 := NewCountSketchDataPointSlice() + es.CopyTo(es2) + assert.Panics(t, func() { es2.CopyTo(es) }) + assert.Panics(t, func() { es.MoveAndAppendTo(es2) }) + assert.Panics(t, func() { es2.MoveAndAppendTo(es) }) +} + +func TestCountSketchDataPointSlice_CopyTo(t *testing.T) { + dest := NewCountSketchDataPointSlice() + src := generateTestCountSketchDataPointSlice() + src.CopyTo(dest) + assert.Equal(t, generateTestCountSketchDataPointSlice(), dest) + dest.CopyTo(dest) + assert.Equal(t, generateTestCountSketchDataPointSlice(), dest) +} + +func TestCountSketchDataPointSlice_EnsureCapacity(t *testing.T) { + es := generateTestCountSketchDataPointSlice() + + // Test ensure smaller capacity. + const ensureSmallLen = 4 + es.EnsureCapacity(ensureSmallLen) + assert.Less(t, ensureSmallLen, es.Len()) + assert.Equal(t, es.Len(), cap(*es.orig)) + assert.Equal(t, generateTestCountSketchDataPointSlice(), es) + + // Test ensure larger capacity + const ensureLargeLen = 9 + es.EnsureCapacity(ensureLargeLen) + assert.Less(t, generateTestCountSketchDataPointSlice().Len(), ensureLargeLen) + assert.Equal(t, ensureLargeLen, cap(*es.orig)) + assert.Equal(t, generateTestCountSketchDataPointSlice(), es) +} + +func TestCountSketchDataPointSlice_MoveAndAppendTo(t *testing.T) { + // Test MoveAndAppendTo to empty + expectedSlice := generateTestCountSketchDataPointSlice() + dest := NewCountSketchDataPointSlice() + src := generateTestCountSketchDataPointSlice() + src.MoveAndAppendTo(dest) + assert.Equal(t, generateTestCountSketchDataPointSlice(), dest) + assert.Equal(t, 0, src.Len()) + assert.Equal(t, expectedSlice.Len(), dest.Len()) + + // Test MoveAndAppendTo empty slice + src.MoveAndAppendTo(dest) + assert.Equal(t, generateTestCountSketchDataPointSlice(), dest) + assert.Equal(t, 0, src.Len()) + assert.Equal(t, expectedSlice.Len(), dest.Len()) + + // Test MoveAndAppendTo not empty slice + generateTestCountSketchDataPointSlice().MoveAndAppendTo(dest) + assert.Equal(t, 2*expectedSlice.Len(), dest.Len()) + for i := 0; i < expectedSlice.Len(); i++ { + assert.Equal(t, expectedSlice.At(i), dest.At(i)) + assert.Equal(t, expectedSlice.At(i), dest.At(i+expectedSlice.Len())) + } + + dest.MoveAndAppendTo(dest) + assert.Equal(t, 2*expectedSlice.Len(), dest.Len()) + for i := 0; i < expectedSlice.Len(); i++ { + assert.Equal(t, expectedSlice.At(i), dest.At(i)) + assert.Equal(t, expectedSlice.At(i), dest.At(i+expectedSlice.Len())) + } +} + +func TestCountSketchDataPointSlice_RemoveIf(t *testing.T) { + // Test RemoveIf on empty slice + emptySlice := NewCountSketchDataPointSlice() + emptySlice.RemoveIf(func(el CountSketchDataPoint) bool { + t.Fail() + return false + }) + + // Test RemoveIf + filtered := generateTestCountSketchDataPointSlice() + pos := 0 + filtered.RemoveIf(func(el CountSketchDataPoint) bool { + pos++ + return pos%2 == 1 + }) + assert.Equal(t, 2, filtered.Len()) +} + +func TestCountSketchDataPointSlice_RemoveIfAll(t *testing.T) { + got := generateTestCountSketchDataPointSlice() + got.RemoveIf(func(el CountSketchDataPoint) bool { + return true + }) + assert.Equal(t, 0, got.Len()) +} + +func TestCountSketchDataPointSliceAll(t *testing.T) { + ms := generateTestCountSketchDataPointSlice() + assert.NotEmpty(t, ms.Len()) + + var c int + for i, v := range ms.All() { + assert.Equal(t, ms.At(i), v, "element should match") + c++ + } + assert.Equal(t, ms.Len(), c, "All elements should have been visited") +} + +func TestCountSketchDataPointSlice_Sort(t *testing.T) { + es := generateTestCountSketchDataPointSlice() + es.Sort(func(a, b CountSketchDataPoint) bool { + return uintptr(unsafe.Pointer(a.orig)) < uintptr(unsafe.Pointer(b.orig)) + }) + for i := 1; i < es.Len(); i++ { + assert.Less(t, uintptr(unsafe.Pointer(es.At(i-1).orig)), uintptr(unsafe.Pointer(es.At(i).orig))) + } + es.Sort(func(a, b CountSketchDataPoint) bool { + return uintptr(unsafe.Pointer(a.orig)) > uintptr(unsafe.Pointer(b.orig)) + }) + for i := 1; i < es.Len(); i++ { + assert.Greater(t, uintptr(unsafe.Pointer(es.At(i-1).orig)), uintptr(unsafe.Pointer(es.At(i).orig))) + } +} + +func generateTestCountSketchDataPointSlice() CountSketchDataPointSlice { + ms := NewCountSketchDataPointSlice() + *ms.orig = internal.GenTestCountSketchDataPointPtrSlice() + return ms +} diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_ddsketch.go b/opentelemetry-collector-patch/pdata/pmetric/generated_ddsketch.go index ca589bcf..5b71b39b 100644 --- a/opentelemetry-collector-patch/pdata/pmetric/generated_ddsketch.go +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_ddsketch.go @@ -64,6 +64,17 @@ func (ms DDSketch) SetAggregationTemporality(v AggregationTemporality) { ms.orig.AggregationTemporality = internal.AggregationTemporality(v) } +// RelativeAccuracy returns the relativeaccuracy associated with this DDSketch. +func (ms DDSketch) RelativeAccuracy() float64 { + return ms.orig.RelativeAccuracy +} + +// SetRelativeAccuracy replaces the relativeaccuracy associated with this DDSketch. +func (ms DDSketch) SetRelativeAccuracy(v float64) { + ms.state.AssertMutable() + ms.orig.RelativeAccuracy = v +} + // CopyTo copies all properties from the current struct overriding the destination. func (ms DDSketch) CopyTo(dest DDSketch) { dest.state.AssertMutable() diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_ddsketchdatapoint.go b/opentelemetry-collector-patch/pdata/pmetric/generated_ddsketchdatapoint.go index 93f3552b..76f09736 100644 --- a/opentelemetry-collector-patch/pdata/pmetric/generated_ddsketchdatapoint.go +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_ddsketchdatapoint.go @@ -86,161 +86,6 @@ func (ms DDSketchDataPoint) SetTimestamp(v pcommon.Timestamp) { ms.orig.TimeUnixNano = uint64(v) } -// Count returns the count associated with this DDSketchDataPoint. -func (ms DDSketchDataPoint) Count() uint64 { - return ms.orig.Count -} - -// SetCount replaces the count associated with this DDSketchDataPoint. -func (ms DDSketchDataPoint) SetCount(v uint64) { - ms.state.AssertMutable() - ms.orig.Count = v -} - -// SumType returns the type of the sum for this DDSketchDataPoint. -// Calling this function on zero-initialized DDSketchDataPoint will cause a panic. -func (ms DDSketchDataPoint) SumType() DDSketchDataPointSumType { - switch ms.orig.Sum.(type) { - case *internal.DDSketchDataPoint_SumAsDouble: - return DDSketchDataPointSumTypeDouble - case *internal.DDSketchDataPoint_SumAsInt: - return DDSketchDataPointSumTypeInt - } - return DDSketchDataPointSumTypeEmpty -} - -// DoubleSum returns the double associated with this DDSketchDataPoint. -func (ms DDSketchDataPoint) DoubleSum() float64 { - return ms.orig.GetSumAsDouble() -} - -// SetDoubleSum replaces the double associated with this DDSketchDataPoint. -func (ms DDSketchDataPoint) SetDoubleSum(v float64) { - ms.state.AssertMutable() - var ov *internal.DDSketchDataPoint_SumAsDouble - if !internal.UseProtoPooling.IsEnabled() { - ov = &internal.DDSketchDataPoint_SumAsDouble{} - } else { - ov = internal.ProtoPoolDDSketchDataPoint_SumAsDouble.Get().(*internal.DDSketchDataPoint_SumAsDouble) - } - ov.SumAsDouble = v - ms.orig.Sum = ov -} - -// IntSum returns the int associated with this DDSketchDataPoint. -func (ms DDSketchDataPoint) IntSum() int64 { - return ms.orig.GetSumAsInt() -} - -// SetIntSum replaces the int associated with this DDSketchDataPoint. -func (ms DDSketchDataPoint) SetIntSum(v int64) { - ms.state.AssertMutable() - var ov *internal.DDSketchDataPoint_SumAsInt - if !internal.UseProtoPooling.IsEnabled() { - ov = &internal.DDSketchDataPoint_SumAsInt{} - } else { - ov = internal.ProtoPoolDDSketchDataPoint_SumAsInt.Get().(*internal.DDSketchDataPoint_SumAsInt) - } - ov.SumAsInt = v - ms.orig.Sum = ov -} - -// MinType returns the type of the min for this DDSketchDataPoint. -// Calling this function on zero-initialized DDSketchDataPoint will cause a panic. -func (ms DDSketchDataPoint) MinType() DDSketchDataPointMinType { - switch ms.orig.Min.(type) { - case *internal.DDSketchDataPoint_MinAsDouble: - return DDSketchDataPointMinTypeDouble - case *internal.DDSketchDataPoint_MinAsInt: - return DDSketchDataPointMinTypeInt - } - return DDSketchDataPointMinTypeEmpty -} - -// DoubleMin returns the double associated with this DDSketchDataPoint. -func (ms DDSketchDataPoint) DoubleMin() float64 { - return ms.orig.GetMinAsDouble() -} - -// SetDoubleMin replaces the double associated with this DDSketchDataPoint. -func (ms DDSketchDataPoint) SetDoubleMin(v float64) { - ms.state.AssertMutable() - var ov *internal.DDSketchDataPoint_MinAsDouble - if !internal.UseProtoPooling.IsEnabled() { - ov = &internal.DDSketchDataPoint_MinAsDouble{} - } else { - ov = internal.ProtoPoolDDSketchDataPoint_MinAsDouble.Get().(*internal.DDSketchDataPoint_MinAsDouble) - } - ov.MinAsDouble = v - ms.orig.Min = ov -} - -// IntMin returns the int associated with this DDSketchDataPoint. -func (ms DDSketchDataPoint) IntMin() int64 { - return ms.orig.GetMinAsInt() -} - -// SetIntMin replaces the int associated with this DDSketchDataPoint. -func (ms DDSketchDataPoint) SetIntMin(v int64) { - ms.state.AssertMutable() - var ov *internal.DDSketchDataPoint_MinAsInt - if !internal.UseProtoPooling.IsEnabled() { - ov = &internal.DDSketchDataPoint_MinAsInt{} - } else { - ov = internal.ProtoPoolDDSketchDataPoint_MinAsInt.Get().(*internal.DDSketchDataPoint_MinAsInt) - } - ov.MinAsInt = v - ms.orig.Min = ov -} - -// MaxType returns the type of the max for this DDSketchDataPoint. -// Calling this function on zero-initialized DDSketchDataPoint will cause a panic. -func (ms DDSketchDataPoint) MaxType() DDSketchDataPointMaxType { - switch ms.orig.Max.(type) { - case *internal.DDSketchDataPoint_MaxAsDouble: - return DDSketchDataPointMaxTypeDouble - case *internal.DDSketchDataPoint_MaxAsInt: - return DDSketchDataPointMaxTypeInt - } - return DDSketchDataPointMaxTypeEmpty -} - -// DoubleMax returns the double associated with this DDSketchDataPoint. -func (ms DDSketchDataPoint) DoubleMax() float64 { - return ms.orig.GetMaxAsDouble() -} - -// SetDoubleMax replaces the double associated with this DDSketchDataPoint. -func (ms DDSketchDataPoint) SetDoubleMax(v float64) { - ms.state.AssertMutable() - var ov *internal.DDSketchDataPoint_MaxAsDouble - if !internal.UseProtoPooling.IsEnabled() { - ov = &internal.DDSketchDataPoint_MaxAsDouble{} - } else { - ov = internal.ProtoPoolDDSketchDataPoint_MaxAsDouble.Get().(*internal.DDSketchDataPoint_MaxAsDouble) - } - ov.MaxAsDouble = v - ms.orig.Max = ov -} - -// IntMax returns the int associated with this DDSketchDataPoint. -func (ms DDSketchDataPoint) IntMax() int64 { - return ms.orig.GetMaxAsInt() -} - -// SetIntMax replaces the int associated with this DDSketchDataPoint. -func (ms DDSketchDataPoint) SetIntMax(v int64) { - ms.state.AssertMutable() - var ov *internal.DDSketchDataPoint_MaxAsInt - if !internal.UseProtoPooling.IsEnabled() { - ov = &internal.DDSketchDataPoint_MaxAsInt{} - } else { - ov = internal.ProtoPoolDDSketchDataPoint_MaxAsInt.Get().(*internal.DDSketchDataPoint_MaxAsInt) - } - ov.MaxAsInt = v - ms.orig.Max = ov -} - // Sketch returns the sketch associated with this DDSketchDataPoint. func (ms DDSketchDataPoint) Sketch() []byte { return ms.orig.Sketch diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_hllsketch.go b/opentelemetry-collector-patch/pdata/pmetric/generated_hllsketch.go index b90c2a4b..d18f06c7 100644 --- a/opentelemetry-collector-patch/pdata/pmetric/generated_hllsketch.go +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_hllsketch.go @@ -10,7 +10,7 @@ import ( "go.opentelemetry.io/collector/pdata/internal" ) -// HLLSketch represents the type of a metric encoded with the HyperLogLog sketch data structure. +// HLLSketch represents the type of a metric encoded using HyperLogLog cardinality estimation. // // This is a reference type, if passed by value and callee modifies it the // caller will see the modification. @@ -39,6 +39,7 @@ func NewHLLSketch() HLLSketch { func (ms HLLSketch) MoveTo(dest HLLSketch) { ms.state.AssertMutable() dest.state.AssertMutable() + // If they point to the same data, they are the same, nothing to do. if ms.orig == dest.orig { return } @@ -62,6 +63,17 @@ func (ms HLLSketch) SetAggregationTemporality(v AggregationTemporality) { ms.orig.AggregationTemporality = internal.AggregationTemporality(v) } +// Precision returns the precision associated with this HLLSketch. +func (ms HLLSketch) Precision() uint32 { + return ms.orig.Precision +} + +// SetPrecision replaces the precision associated with this HLLSketch. +func (ms HLLSketch) SetPrecision(v uint32) { + ms.state.AssertMutable() + ms.orig.Precision = v +} + // CopyTo copies all properties from the current struct overriding the destination. func (ms HLLSketch) CopyTo(dest HLLSketch) { dest.state.AssertMutable() diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_hllsketch_test.go b/opentelemetry-collector-patch/pdata/pmetric/generated_hllsketch_test.go new file mode 100644 index 00000000..67c340e5 --- /dev/null +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_hllsketch_test.go @@ -0,0 +1,71 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package pmetric + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "go.opentelemetry.io/collector/pdata/internal" +) + +func TestHLLSketch_MoveTo(t *testing.T) { + ms := generateTestHLLSketch() + dest := NewHLLSketch() + ms.MoveTo(dest) + assert.Equal(t, NewHLLSketch(), ms) + assert.Equal(t, generateTestHLLSketch(), dest) + dest.MoveTo(dest) + assert.Equal(t, generateTestHLLSketch(), dest) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { ms.MoveTo(newHLLSketch(internal.NewHLLSketch(), sharedState)) }) + assert.Panics(t, func() { newHLLSketch(internal.NewHLLSketch(), sharedState).MoveTo(dest) }) +} + +func TestHLLSketch_CopyTo(t *testing.T) { + ms := NewHLLSketch() + orig := NewHLLSketch() + orig.CopyTo(ms) + assert.Equal(t, orig, ms) + orig = generateTestHLLSketch() + orig.CopyTo(ms) + assert.Equal(t, orig, ms) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { ms.CopyTo(newHLLSketch(internal.NewHLLSketch(), sharedState)) }) +} + +func TestHLLSketch_DataPoints(t *testing.T) { + ms := NewHLLSketch() + assert.Equal(t, NewHLLSketchDataPointSlice(), ms.DataPoints()) + ms.orig.DataPoints = internal.GenTestHLLSketchDataPointPtrSlice() + assert.Equal(t, generateTestHLLSketchDataPointSlice(), ms.DataPoints()) +} + +func TestHLLSketch_AggregationTemporality(t *testing.T) { + ms := NewHLLSketch() + assert.Equal(t, AggregationTemporality(internal.AggregationTemporality(0)), ms.AggregationTemporality()) + testValAggregationTemporality := AggregationTemporality(internal.AggregationTemporality(1)) + ms.SetAggregationTemporality(testValAggregationTemporality) + assert.Equal(t, testValAggregationTemporality, ms.AggregationTemporality()) +} + +func TestHLLSketch_Precision(t *testing.T) { + ms := NewHLLSketch() + assert.Equal(t, uint32(0), ms.Precision()) + ms.SetPrecision(uint32(13)) + assert.Equal(t, uint32(13), ms.Precision()) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { newHLLSketch(internal.NewHLLSketch(), sharedState).SetPrecision(uint32(13)) }) +} + +func generateTestHLLSketch() HLLSketch { + return newHLLSketch(internal.GenTestHLLSketch(), internal.NewState()) +} diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_hllsketchdatapoint.go b/opentelemetry-collector-patch/pdata/pmetric/generated_hllsketchdatapoint.go index 56b5badf..23771f64 100644 --- a/opentelemetry-collector-patch/pdata/pmetric/generated_hllsketchdatapoint.go +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_hllsketchdatapoint.go @@ -40,6 +40,7 @@ func NewHLLSketchDataPoint() HLLSketchDataPoint { func (ms HLLSketchDataPoint) MoveTo(dest HLLSketchDataPoint) { ms.state.AssertMutable() dest.state.AssertMutable() + // If they point to the same data, they are the same, nothing to do. if ms.orig == dest.orig { return } @@ -52,17 +53,6 @@ func (ms HLLSketchDataPoint) Attributes() pcommon.Map { return pcommon.Map(internal.NewMapWrapper(&ms.orig.Attributes, ms.state)) } -// SeriesID returns the seriesid associated with this HLLSketchDataPoint. -func (ms HLLSketchDataPoint) SeriesID() uint64 { - return ms.orig.SeriesID -} - -// SetSeriesID replaces the seriesid associated with this HLLSketchDataPoint. -func (ms HLLSketchDataPoint) SetSeriesID(v uint64) { - ms.state.AssertMutable() - ms.orig.SeriesID = v -} - // StartTimestamp returns the starttimestamp associated with this HLLSketchDataPoint. func (ms HLLSketchDataPoint) StartTimestamp() pcommon.Timestamp { return pcommon.Timestamp(ms.orig.StartTimeUnixNano) @@ -85,28 +75,6 @@ func (ms HLLSketchDataPoint) SetTimestamp(v pcommon.Timestamp) { ms.orig.TimeUnixNano = uint64(v) } -// Count returns the count associated with this HLLSketchDataPoint. -func (ms HLLSketchDataPoint) Count() uint64 { - return ms.orig.Count -} - -// SetCount replaces the count associated with this HLLSketchDataPoint. -func (ms HLLSketchDataPoint) SetCount(v uint64) { - ms.state.AssertMutable() - ms.orig.Count = v -} - -// Cardinality returns the cardinality associated with this HLLSketchDataPoint. -func (ms HLLSketchDataPoint) Cardinality() uint64 { - return ms.orig.Cardinality -} - -// SetCardinality replaces the cardinality associated with this HLLSketchDataPoint. -func (ms HLLSketchDataPoint) SetCardinality(v uint64) { - ms.state.AssertMutable() - ms.orig.Cardinality = v -} - // Sketch returns the sketch associated with this HLLSketchDataPoint. func (ms HLLSketchDataPoint) Sketch() []byte { return ms.orig.Sketch @@ -129,17 +97,6 @@ func (ms HLLSketchDataPoint) SetEncoding(v HLLSketchEncoding) { ms.orig.Encoding = internal.HLLSketchEncoding(v) } -// Precision returns the precision associated with this HLLSketchDataPoint. -func (ms HLLSketchDataPoint) Precision() uint32 { - return ms.orig.Precision -} - -// SetPrecision replaces the precision associated with this HLLSketchDataPoint. -func (ms HLLSketchDataPoint) SetPrecision(v uint32) { - ms.state.AssertMutable() - ms.orig.Precision = v -} - // Flags returns the flags associated with this HLLSketchDataPoint. func (ms HLLSketchDataPoint) Flags() DataPointFlags { return DataPointFlags(ms.orig.Flags) @@ -151,6 +108,17 @@ func (ms HLLSketchDataPoint) SetFlags(v DataPointFlags) { ms.orig.Flags = uint32(v) } +// SeriesID returns the seriesid associated with this HLLSketchDataPoint. +func (ms HLLSketchDataPoint) SeriesID() uint64 { + return ms.orig.SeriesID +} + +// SetSeriesID replaces the seriesid associated with this HLLSketchDataPoint. +func (ms HLLSketchDataPoint) SetSeriesID(v uint64) { + ms.state.AssertMutable() + ms.orig.SeriesID = v +} + // CopyTo copies all properties from the current struct overriding the destination. func (ms HLLSketchDataPoint) CopyTo(dest HLLSketchDataPoint) { dest.state.AssertMutable() diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_hllsketchdatapoint_test.go b/opentelemetry-collector-patch/pdata/pmetric/generated_hllsketchdatapoint_test.go new file mode 100644 index 00000000..fcbeba13 --- /dev/null +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_hllsketchdatapoint_test.go @@ -0,0 +1,108 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package pmetric + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "go.opentelemetry.io/collector/pdata/internal" + "go.opentelemetry.io/collector/pdata/pcommon" +) + +func TestHLLSketchDataPoint_MoveTo(t *testing.T) { + ms := generateTestHLLSketchDataPoint() + dest := NewHLLSketchDataPoint() + ms.MoveTo(dest) + assert.Equal(t, NewHLLSketchDataPoint(), ms) + assert.Equal(t, generateTestHLLSketchDataPoint(), dest) + dest.MoveTo(dest) + assert.Equal(t, generateTestHLLSketchDataPoint(), dest) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { ms.MoveTo(newHLLSketchDataPoint(internal.NewHLLSketchDataPoint(), sharedState)) }) + assert.Panics(t, func() { newHLLSketchDataPoint(internal.NewHLLSketchDataPoint(), sharedState).MoveTo(dest) }) +} + +func TestHLLSketchDataPoint_CopyTo(t *testing.T) { + ms := NewHLLSketchDataPoint() + orig := NewHLLSketchDataPoint() + orig.CopyTo(ms) + assert.Equal(t, orig, ms) + orig = generateTestHLLSketchDataPoint() + orig.CopyTo(ms) + assert.Equal(t, orig, ms) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { ms.CopyTo(newHLLSketchDataPoint(internal.NewHLLSketchDataPoint(), sharedState)) }) +} + +func TestHLLSketchDataPoint_Attributes(t *testing.T) { + ms := NewHLLSketchDataPoint() + assert.Equal(t, pcommon.NewMap(), ms.Attributes()) + ms.orig.Attributes = internal.GenTestKeyValueSlice() + assert.Equal(t, pcommon.Map(internal.GenTestMapWrapper()), ms.Attributes()) +} + +func TestHLLSketchDataPoint_StartTimestamp(t *testing.T) { + ms := NewHLLSketchDataPoint() + assert.Equal(t, pcommon.Timestamp(0), ms.StartTimestamp()) + testValStartTimestamp := pcommon.Timestamp(1234567890) + ms.SetStartTimestamp(testValStartTimestamp) + assert.Equal(t, testValStartTimestamp, ms.StartTimestamp()) +} + +func TestHLLSketchDataPoint_Timestamp(t *testing.T) { + ms := NewHLLSketchDataPoint() + assert.Equal(t, pcommon.Timestamp(0), ms.Timestamp()) + testValTimestamp := pcommon.Timestamp(1234567890) + ms.SetTimestamp(testValTimestamp) + assert.Equal(t, testValTimestamp, ms.Timestamp()) +} + +func TestHLLSketchDataPoint_Sketch(t *testing.T) { + ms := NewHLLSketchDataPoint() + assert.Equal(t, nil, ms.Sketch()) + ms.SetSketch([]byte{1, 2, 3}) + assert.Equal(t, []byte{1, 2, 3}, ms.Sketch()) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { + newHLLSketchDataPoint(internal.NewHLLSketchDataPoint(), sharedState).SetSketch([]byte{1, 2, 3}) + }) +} + +func TestHLLSketchDataPoint_Encoding(t *testing.T) { + ms := NewHLLSketchDataPoint() + assert.Equal(t, HLLSketchEncoding(internal.HLLSketchEncoding(0)), ms.Encoding()) + testValEncoding := HLLSketchEncoding(internal.HLLSketchEncoding(1)) + ms.SetEncoding(testValEncoding) + assert.Equal(t, testValEncoding, ms.Encoding()) +} + +func TestHLLSketchDataPoint_Flags(t *testing.T) { + ms := NewHLLSketchDataPoint() + assert.Equal(t, DataPointFlags(0), ms.Flags()) + testValFlags := DataPointFlags(1) + ms.SetFlags(testValFlags) + assert.Equal(t, testValFlags, ms.Flags()) +} + +func TestHLLSketchDataPoint_SeriesID(t *testing.T) { + ms := NewHLLSketchDataPoint() + assert.Equal(t, uint64(0), ms.SeriesID()) + ms.SetSeriesID(uint64(13)) + assert.Equal(t, uint64(13), ms.SeriesID()) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { newHLLSketchDataPoint(internal.NewHLLSketchDataPoint(), sharedState).SetSeriesID(uint64(13)) }) +} + +func generateTestHLLSketchDataPoint() HLLSketchDataPoint { + return newHLLSketchDataPoint(internal.GenTestHLLSketchDataPoint(), internal.NewState()) +} diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_hllsketchdatapointslice.go b/opentelemetry-collector-patch/pdata/pmetric/generated_hllsketchdatapointslice.go index bbdabf07..181352c5 100644 --- a/opentelemetry-collector-patch/pdata/pmetric/generated_hllsketchdatapointslice.go +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_hllsketchdatapointslice.go @@ -29,7 +29,7 @@ func newHLLSketchDataPointSlice(orig *[]*internal.HLLSketchDataPoint, state *int return HLLSketchDataPointSlice{orig: orig, state: state} } -// NewHLLSketchDataPointSlice creates a HLLSketchDataPointSlice with 0 elements. +// NewHLLSketchDataPointSlice creates a HLLSketchDataPointSliceWrapper with 0 elements. // Can use "EnsureCapacity" to initialize with a given capacity. func NewHLLSketchDataPointSlice() HLLSketchDataPointSlice { orig := []*internal.HLLSketchDataPoint(nil) @@ -37,16 +37,29 @@ func NewHLLSketchDataPointSlice() HLLSketchDataPointSlice { } // Len returns the number of elements in the slice. +// +// Returns "0" for a newly instance created with "NewHLLSketchDataPointSlice()". func (es HLLSketchDataPointSlice) Len() int { return len(*es.orig) } // At returns the element at the given index. +// +// This function is used mostly for iterating over all the values in the slice: +// +// for i := 0; i < es.Len(); i++ { +// e := es.At(i) +// ... // Do something with the element +// } func (es HLLSketchDataPointSlice) At(i int) HLLSketchDataPoint { return newHLLSketchDataPoint((*es.orig)[i], es.state) } // All returns an iterator over index-value pairs in the slice. +// +// for i, v := range es.All() { +// ... // Do something with index-value pair +// } func (es HLLSketchDataPointSlice) All() iter.Seq2[int, HLLSketchDataPoint] { return func(yield func(int, HLLSketchDataPoint) bool) { for i := 0; i < es.Len(); i++ { @@ -58,6 +71,17 @@ func (es HLLSketchDataPointSlice) All() iter.Seq2[int, HLLSketchDataPoint] { } // EnsureCapacity is an operation that ensures the slice has at least the specified capacity. +// 1. If the newCap <= cap then no change in capacity. +// 2. If the newCap > cap then the slice capacity will be expanded to equal newCap. +// +// Here is how a new HLLSketchDataPointSlice can be initialized: +// +// es := NewHLLSketchDataPointSlice() +// es.EnsureCapacity(4) +// for i := 0; i < 4; i++ { +// e := es.AppendEmpty() +// // Here should set all the values for e. +// } func (es HLLSketchDataPointSlice) EnsureCapacity(newCap int) { es.state.AssertMutable() oldCap := cap(*es.orig) @@ -83,10 +107,12 @@ func (es HLLSketchDataPointSlice) AppendEmpty() HLLSketchDataPoint { func (es HLLSketchDataPointSlice) MoveAndAppendTo(dest HLLSketchDataPointSlice) { es.state.AssertMutable() dest.state.AssertMutable() + // If they point to the same data, they are the same, nothing to do. if es.orig == dest.orig { return } if *dest.orig == nil { + // We can simply move the entire vector and avoid any allocations. *dest.orig = *es.orig } else { *dest.orig = append(*dest.orig, *es.orig...) @@ -103,13 +129,16 @@ func (es HLLSketchDataPointSlice) RemoveIf(f func(HLLSketchDataPoint) bool) { if f(es.At(i)) { internal.DeleteHLLSketchDataPoint((*es.orig)[i], true) (*es.orig)[i] = nil + continue } if newLen == i { + // Nothing to move, element is at the right place. newLen++ continue } (*es.orig)[newLen] = (*es.orig)[i] + // Cannot delete here since we just move the data(or pointer to data) to a different position in the slice. (*es.orig)[i] = nil newLen++ } @@ -126,7 +155,8 @@ func (es HLLSketchDataPointSlice) CopyTo(dest HLLSketchDataPointSlice) { } // Sort sorts the HLLSketchDataPoint elements within HLLSketchDataPointSlice given the -// provided less function. +// provided less function so that two instances of HLLSketchDataPointSlice +// can be compared. func (es HLLSketchDataPointSlice) Sort(less func(a, b HLLSketchDataPoint) bool) { es.state.AssertMutable() sort.SliceStable(*es.orig, func(i, j int) bool { return less(es.At(i), es.At(j)) }) diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_hllsketchdatapointslice_test.go b/opentelemetry-collector-patch/pdata/pmetric/generated_hllsketchdatapointslice_test.go new file mode 100644 index 00000000..24690633 --- /dev/null +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_hllsketchdatapointslice_test.go @@ -0,0 +1,166 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package pmetric + +import ( + "testing" + "unsafe" + + "github.com/stretchr/testify/assert" + + "go.opentelemetry.io/collector/pdata/internal" +) + +func TestHLLSketchDataPointSlice(t *testing.T) { + es := NewHLLSketchDataPointSlice() + assert.Equal(t, 0, es.Len()) + es = newHLLSketchDataPointSlice(&[]*internal.HLLSketchDataPoint{}, internal.NewState()) + assert.Equal(t, 0, es.Len()) + + emptyVal := NewHLLSketchDataPoint() + testVal := generateTestHLLSketchDataPoint() + for i := 0; i < 7; i++ { + es.AppendEmpty() + assert.Equal(t, emptyVal, es.At(i)) + (*es.orig)[i] = internal.GenTestHLLSketchDataPoint() + assert.Equal(t, testVal, es.At(i)) + } + assert.Equal(t, 7, es.Len()) +} + +func TestHLLSketchDataPointSliceReadOnly(t *testing.T) { + sharedState := internal.NewState() + sharedState.MarkReadOnly() + es := newHLLSketchDataPointSlice(&[]*internal.HLLSketchDataPoint{}, sharedState) + assert.Equal(t, 0, es.Len()) + assert.Panics(t, func() { es.AppendEmpty() }) + assert.Panics(t, func() { es.EnsureCapacity(2) }) + es2 := NewHLLSketchDataPointSlice() + es.CopyTo(es2) + assert.Panics(t, func() { es2.CopyTo(es) }) + assert.Panics(t, func() { es.MoveAndAppendTo(es2) }) + assert.Panics(t, func() { es2.MoveAndAppendTo(es) }) +} + +func TestHLLSketchDataPointSlice_CopyTo(t *testing.T) { + dest := NewHLLSketchDataPointSlice() + src := generateTestHLLSketchDataPointSlice() + src.CopyTo(dest) + assert.Equal(t, generateTestHLLSketchDataPointSlice(), dest) + dest.CopyTo(dest) + assert.Equal(t, generateTestHLLSketchDataPointSlice(), dest) +} + +func TestHLLSketchDataPointSlice_EnsureCapacity(t *testing.T) { + es := generateTestHLLSketchDataPointSlice() + + // Test ensure smaller capacity. + const ensureSmallLen = 4 + es.EnsureCapacity(ensureSmallLen) + assert.Less(t, ensureSmallLen, es.Len()) + assert.Equal(t, es.Len(), cap(*es.orig)) + assert.Equal(t, generateTestHLLSketchDataPointSlice(), es) + + // Test ensure larger capacity + const ensureLargeLen = 9 + es.EnsureCapacity(ensureLargeLen) + assert.Less(t, generateTestHLLSketchDataPointSlice().Len(), ensureLargeLen) + assert.Equal(t, ensureLargeLen, cap(*es.orig)) + assert.Equal(t, generateTestHLLSketchDataPointSlice(), es) +} + +func TestHLLSketchDataPointSlice_MoveAndAppendTo(t *testing.T) { + // Test MoveAndAppendTo to empty + expectedSlice := generateTestHLLSketchDataPointSlice() + dest := NewHLLSketchDataPointSlice() + src := generateTestHLLSketchDataPointSlice() + src.MoveAndAppendTo(dest) + assert.Equal(t, generateTestHLLSketchDataPointSlice(), dest) + assert.Equal(t, 0, src.Len()) + assert.Equal(t, expectedSlice.Len(), dest.Len()) + + // Test MoveAndAppendTo empty slice + src.MoveAndAppendTo(dest) + assert.Equal(t, generateTestHLLSketchDataPointSlice(), dest) + assert.Equal(t, 0, src.Len()) + assert.Equal(t, expectedSlice.Len(), dest.Len()) + + // Test MoveAndAppendTo not empty slice + generateTestHLLSketchDataPointSlice().MoveAndAppendTo(dest) + assert.Equal(t, 2*expectedSlice.Len(), dest.Len()) + for i := 0; i < expectedSlice.Len(); i++ { + assert.Equal(t, expectedSlice.At(i), dest.At(i)) + assert.Equal(t, expectedSlice.At(i), dest.At(i+expectedSlice.Len())) + } + + dest.MoveAndAppendTo(dest) + assert.Equal(t, 2*expectedSlice.Len(), dest.Len()) + for i := 0; i < expectedSlice.Len(); i++ { + assert.Equal(t, expectedSlice.At(i), dest.At(i)) + assert.Equal(t, expectedSlice.At(i), dest.At(i+expectedSlice.Len())) + } +} + +func TestHLLSketchDataPointSlice_RemoveIf(t *testing.T) { + // Test RemoveIf on empty slice + emptySlice := NewHLLSketchDataPointSlice() + emptySlice.RemoveIf(func(el HLLSketchDataPoint) bool { + t.Fail() + return false + }) + + // Test RemoveIf + filtered := generateTestHLLSketchDataPointSlice() + pos := 0 + filtered.RemoveIf(func(el HLLSketchDataPoint) bool { + pos++ + return pos%2 == 1 + }) + assert.Equal(t, 2, filtered.Len()) +} + +func TestHLLSketchDataPointSlice_RemoveIfAll(t *testing.T) { + got := generateTestHLLSketchDataPointSlice() + got.RemoveIf(func(el HLLSketchDataPoint) bool { + return true + }) + assert.Equal(t, 0, got.Len()) +} + +func TestHLLSketchDataPointSliceAll(t *testing.T) { + ms := generateTestHLLSketchDataPointSlice() + assert.NotEmpty(t, ms.Len()) + + var c int + for i, v := range ms.All() { + assert.Equal(t, ms.At(i), v, "element should match") + c++ + } + assert.Equal(t, ms.Len(), c, "All elements should have been visited") +} + +func TestHLLSketchDataPointSlice_Sort(t *testing.T) { + es := generateTestHLLSketchDataPointSlice() + es.Sort(func(a, b HLLSketchDataPoint) bool { + return uintptr(unsafe.Pointer(a.orig)) < uintptr(unsafe.Pointer(b.orig)) + }) + for i := 1; i < es.Len(); i++ { + assert.Less(t, uintptr(unsafe.Pointer(es.At(i-1).orig)), uintptr(unsafe.Pointer(es.At(i).orig))) + } + es.Sort(func(a, b HLLSketchDataPoint) bool { + return uintptr(unsafe.Pointer(a.orig)) > uintptr(unsafe.Pointer(b.orig)) + }) + for i := 1; i < es.Len(); i++ { + assert.Greater(t, uintptr(unsafe.Pointer(es.At(i-1).orig)), uintptr(unsafe.Pointer(es.At(i).orig))) + } +} + +func generateTestHLLSketchDataPointSlice() HLLSketchDataPointSlice { + ms := NewHLLSketchDataPointSlice() + *ms.orig = internal.GenTestHLLSketchDataPointPtrSlice() + return ms +} diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_kllsketch.go b/opentelemetry-collector-patch/pdata/pmetric/generated_kllsketch.go index d7cf806f..07758e1b 100644 --- a/opentelemetry-collector-patch/pdata/pmetric/generated_kllsketch.go +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_kllsketch.go @@ -10,7 +10,7 @@ import ( "go.opentelemetry.io/collector/pdata/internal" ) -// KLLSketch represents the type of a metric encoded with the KLL quantile sketch data structure. +// KLLSketch represents the type of a metric encoded using the KLL quantile sketch algorithm. // // This is a reference type, if passed by value and callee modifies it the // caller will see the modification. @@ -39,6 +39,7 @@ func NewKLLSketch() KLLSketch { func (ms KLLSketch) MoveTo(dest KLLSketch) { ms.state.AssertMutable() dest.state.AssertMutable() + // If they point to the same data, they are the same, nothing to do. if ms.orig == dest.orig { return } @@ -62,6 +63,17 @@ func (ms KLLSketch) SetAggregationTemporality(v AggregationTemporality) { ms.orig.AggregationTemporality = internal.AggregationTemporality(v) } +// K returns the k associated with this KLLSketch. +func (ms KLLSketch) K() uint32 { + return ms.orig.K +} + +// SetK replaces the k associated with this KLLSketch. +func (ms KLLSketch) SetK(v uint32) { + ms.state.AssertMutable() + ms.orig.K = v +} + // CopyTo copies all properties from the current struct overriding the destination. func (ms KLLSketch) CopyTo(dest KLLSketch) { dest.state.AssertMutable() diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_kllsketch_test.go b/opentelemetry-collector-patch/pdata/pmetric/generated_kllsketch_test.go new file mode 100644 index 00000000..76180504 --- /dev/null +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_kllsketch_test.go @@ -0,0 +1,71 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package pmetric + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "go.opentelemetry.io/collector/pdata/internal" +) + +func TestKLLSketch_MoveTo(t *testing.T) { + ms := generateTestKLLSketch() + dest := NewKLLSketch() + ms.MoveTo(dest) + assert.Equal(t, NewKLLSketch(), ms) + assert.Equal(t, generateTestKLLSketch(), dest) + dest.MoveTo(dest) + assert.Equal(t, generateTestKLLSketch(), dest) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { ms.MoveTo(newKLLSketch(internal.NewKLLSketch(), sharedState)) }) + assert.Panics(t, func() { newKLLSketch(internal.NewKLLSketch(), sharedState).MoveTo(dest) }) +} + +func TestKLLSketch_CopyTo(t *testing.T) { + ms := NewKLLSketch() + orig := NewKLLSketch() + orig.CopyTo(ms) + assert.Equal(t, orig, ms) + orig = generateTestKLLSketch() + orig.CopyTo(ms) + assert.Equal(t, orig, ms) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { ms.CopyTo(newKLLSketch(internal.NewKLLSketch(), sharedState)) }) +} + +func TestKLLSketch_DataPoints(t *testing.T) { + ms := NewKLLSketch() + assert.Equal(t, NewKLLSketchDataPointSlice(), ms.DataPoints()) + ms.orig.DataPoints = internal.GenTestKLLSketchDataPointPtrSlice() + assert.Equal(t, generateTestKLLSketchDataPointSlice(), ms.DataPoints()) +} + +func TestKLLSketch_AggregationTemporality(t *testing.T) { + ms := NewKLLSketch() + assert.Equal(t, AggregationTemporality(internal.AggregationTemporality(0)), ms.AggregationTemporality()) + testValAggregationTemporality := AggregationTemporality(internal.AggregationTemporality(1)) + ms.SetAggregationTemporality(testValAggregationTemporality) + assert.Equal(t, testValAggregationTemporality, ms.AggregationTemporality()) +} + +func TestKLLSketch_K(t *testing.T) { + ms := NewKLLSketch() + assert.Equal(t, uint32(0), ms.K()) + ms.SetK(uint32(13)) + assert.Equal(t, uint32(13), ms.K()) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { newKLLSketch(internal.NewKLLSketch(), sharedState).SetK(uint32(13)) }) +} + +func generateTestKLLSketch() KLLSketch { + return newKLLSketch(internal.GenTestKLLSketch(), internal.NewState()) +} diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_kllsketchdatapoint.go b/opentelemetry-collector-patch/pdata/pmetric/generated_kllsketchdatapoint.go index 42126cd4..00149e11 100644 --- a/opentelemetry-collector-patch/pdata/pmetric/generated_kllsketchdatapoint.go +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_kllsketchdatapoint.go @@ -40,6 +40,7 @@ func NewKLLSketchDataPoint() KLLSketchDataPoint { func (ms KLLSketchDataPoint) MoveTo(dest KLLSketchDataPoint) { ms.state.AssertMutable() dest.state.AssertMutable() + // If they point to the same data, they are the same, nothing to do. if ms.orig == dest.orig { return } @@ -52,17 +53,6 @@ func (ms KLLSketchDataPoint) Attributes() pcommon.Map { return pcommon.Map(internal.NewMapWrapper(&ms.orig.Attributes, ms.state)) } -// SeriesID returns the seriesid associated with this KLLSketchDataPoint. -func (ms KLLSketchDataPoint) SeriesID() uint64 { - return ms.orig.SeriesID -} - -// SetSeriesID replaces the seriesid associated with this KLLSketchDataPoint. -func (ms KLLSketchDataPoint) SetSeriesID(v uint64) { - ms.state.AssertMutable() - ms.orig.SeriesID = v -} - // StartTimestamp returns the starttimestamp associated with this KLLSketchDataPoint. func (ms KLLSketchDataPoint) StartTimestamp() pcommon.Timestamp { return pcommon.Timestamp(ms.orig.StartTimeUnixNano) @@ -85,50 +75,6 @@ func (ms KLLSketchDataPoint) SetTimestamp(v pcommon.Timestamp) { ms.orig.TimeUnixNano = uint64(v) } -// Count returns the count associated with this KLLSketchDataPoint. -func (ms KLLSketchDataPoint) Count() uint64 { - return ms.orig.Count -} - -// SetCount replaces the count associated with this KLLSketchDataPoint. -func (ms KLLSketchDataPoint) SetCount(v uint64) { - ms.state.AssertMutable() - ms.orig.Count = v -} - -// Sum returns the sum associated with this KLLSketchDataPoint. -func (ms KLLSketchDataPoint) Sum() float64 { - return ms.orig.Sum -} - -// SetSum replaces the sum associated with this KLLSketchDataPoint. -func (ms KLLSketchDataPoint) SetSum(v float64) { - ms.state.AssertMutable() - ms.orig.Sum = v -} - -// Min returns the min associated with this KLLSketchDataPoint. -func (ms KLLSketchDataPoint) Min() float64 { - return ms.orig.Min -} - -// SetMin replaces the min associated with this KLLSketchDataPoint. -func (ms KLLSketchDataPoint) SetMin(v float64) { - ms.state.AssertMutable() - ms.orig.Min = v -} - -// Max returns the max associated with this KLLSketchDataPoint. -func (ms KLLSketchDataPoint) Max() float64 { - return ms.orig.Max -} - -// SetMax replaces the max associated with this KLLSketchDataPoint. -func (ms KLLSketchDataPoint) SetMax(v float64) { - ms.state.AssertMutable() - ms.orig.Max = v -} - // Sketch returns the sketch associated with this KLLSketchDataPoint. func (ms KLLSketchDataPoint) Sketch() []byte { return ms.orig.Sketch @@ -162,6 +108,17 @@ func (ms KLLSketchDataPoint) SetFlags(v DataPointFlags) { ms.orig.Flags = uint32(v) } +// SeriesID returns the seriesid associated with this KLLSketchDataPoint. +func (ms KLLSketchDataPoint) SeriesID() uint64 { + return ms.orig.SeriesID +} + +// SetSeriesID replaces the seriesid associated with this KLLSketchDataPoint. +func (ms KLLSketchDataPoint) SetSeriesID(v uint64) { + ms.state.AssertMutable() + ms.orig.SeriesID = v +} + // CopyTo copies all properties from the current struct overriding the destination. func (ms KLLSketchDataPoint) CopyTo(dest KLLSketchDataPoint) { dest.state.AssertMutable() diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_kllsketchdatapoint_test.go b/opentelemetry-collector-patch/pdata/pmetric/generated_kllsketchdatapoint_test.go new file mode 100644 index 00000000..1ea91cd4 --- /dev/null +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_kllsketchdatapoint_test.go @@ -0,0 +1,108 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package pmetric + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "go.opentelemetry.io/collector/pdata/internal" + "go.opentelemetry.io/collector/pdata/pcommon" +) + +func TestKLLSketchDataPoint_MoveTo(t *testing.T) { + ms := generateTestKLLSketchDataPoint() + dest := NewKLLSketchDataPoint() + ms.MoveTo(dest) + assert.Equal(t, NewKLLSketchDataPoint(), ms) + assert.Equal(t, generateTestKLLSketchDataPoint(), dest) + dest.MoveTo(dest) + assert.Equal(t, generateTestKLLSketchDataPoint(), dest) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { ms.MoveTo(newKLLSketchDataPoint(internal.NewKLLSketchDataPoint(), sharedState)) }) + assert.Panics(t, func() { newKLLSketchDataPoint(internal.NewKLLSketchDataPoint(), sharedState).MoveTo(dest) }) +} + +func TestKLLSketchDataPoint_CopyTo(t *testing.T) { + ms := NewKLLSketchDataPoint() + orig := NewKLLSketchDataPoint() + orig.CopyTo(ms) + assert.Equal(t, orig, ms) + orig = generateTestKLLSketchDataPoint() + orig.CopyTo(ms) + assert.Equal(t, orig, ms) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { ms.CopyTo(newKLLSketchDataPoint(internal.NewKLLSketchDataPoint(), sharedState)) }) +} + +func TestKLLSketchDataPoint_Attributes(t *testing.T) { + ms := NewKLLSketchDataPoint() + assert.Equal(t, pcommon.NewMap(), ms.Attributes()) + ms.orig.Attributes = internal.GenTestKeyValueSlice() + assert.Equal(t, pcommon.Map(internal.GenTestMapWrapper()), ms.Attributes()) +} + +func TestKLLSketchDataPoint_StartTimestamp(t *testing.T) { + ms := NewKLLSketchDataPoint() + assert.Equal(t, pcommon.Timestamp(0), ms.StartTimestamp()) + testValStartTimestamp := pcommon.Timestamp(1234567890) + ms.SetStartTimestamp(testValStartTimestamp) + assert.Equal(t, testValStartTimestamp, ms.StartTimestamp()) +} + +func TestKLLSketchDataPoint_Timestamp(t *testing.T) { + ms := NewKLLSketchDataPoint() + assert.Equal(t, pcommon.Timestamp(0), ms.Timestamp()) + testValTimestamp := pcommon.Timestamp(1234567890) + ms.SetTimestamp(testValTimestamp) + assert.Equal(t, testValTimestamp, ms.Timestamp()) +} + +func TestKLLSketchDataPoint_Sketch(t *testing.T) { + ms := NewKLLSketchDataPoint() + assert.Equal(t, nil, ms.Sketch()) + ms.SetSketch([]byte{1, 2, 3}) + assert.Equal(t, []byte{1, 2, 3}, ms.Sketch()) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { + newKLLSketchDataPoint(internal.NewKLLSketchDataPoint(), sharedState).SetSketch([]byte{1, 2, 3}) + }) +} + +func TestKLLSketchDataPoint_Encoding(t *testing.T) { + ms := NewKLLSketchDataPoint() + assert.Equal(t, KLLSketchEncoding(internal.KLLSketchEncoding(0)), ms.Encoding()) + testValEncoding := KLLSketchEncoding(internal.KLLSketchEncoding(1)) + ms.SetEncoding(testValEncoding) + assert.Equal(t, testValEncoding, ms.Encoding()) +} + +func TestKLLSketchDataPoint_Flags(t *testing.T) { + ms := NewKLLSketchDataPoint() + assert.Equal(t, DataPointFlags(0), ms.Flags()) + testValFlags := DataPointFlags(1) + ms.SetFlags(testValFlags) + assert.Equal(t, testValFlags, ms.Flags()) +} + +func TestKLLSketchDataPoint_SeriesID(t *testing.T) { + ms := NewKLLSketchDataPoint() + assert.Equal(t, uint64(0), ms.SeriesID()) + ms.SetSeriesID(uint64(13)) + assert.Equal(t, uint64(13), ms.SeriesID()) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { newKLLSketchDataPoint(internal.NewKLLSketchDataPoint(), sharedState).SetSeriesID(uint64(13)) }) +} + +func generateTestKLLSketchDataPoint() KLLSketchDataPoint { + return newKLLSketchDataPoint(internal.GenTestKLLSketchDataPoint(), internal.NewState()) +} diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_kllsketchdatapointslice.go b/opentelemetry-collector-patch/pdata/pmetric/generated_kllsketchdatapointslice.go index 3deafc7a..7a993bb6 100644 --- a/opentelemetry-collector-patch/pdata/pmetric/generated_kllsketchdatapointslice.go +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_kllsketchdatapointslice.go @@ -29,7 +29,7 @@ func newKLLSketchDataPointSlice(orig *[]*internal.KLLSketchDataPoint, state *int return KLLSketchDataPointSlice{orig: orig, state: state} } -// NewKLLSketchDataPointSlice creates a KLLSketchDataPointSlice with 0 elements. +// NewKLLSketchDataPointSlice creates a KLLSketchDataPointSliceWrapper with 0 elements. // Can use "EnsureCapacity" to initialize with a given capacity. func NewKLLSketchDataPointSlice() KLLSketchDataPointSlice { orig := []*internal.KLLSketchDataPoint(nil) @@ -37,16 +37,29 @@ func NewKLLSketchDataPointSlice() KLLSketchDataPointSlice { } // Len returns the number of elements in the slice. +// +// Returns "0" for a newly instance created with "NewKLLSketchDataPointSlice()". func (es KLLSketchDataPointSlice) Len() int { return len(*es.orig) } // At returns the element at the given index. +// +// This function is used mostly for iterating over all the values in the slice: +// +// for i := 0; i < es.Len(); i++ { +// e := es.At(i) +// ... // Do something with the element +// } func (es KLLSketchDataPointSlice) At(i int) KLLSketchDataPoint { return newKLLSketchDataPoint((*es.orig)[i], es.state) } // All returns an iterator over index-value pairs in the slice. +// +// for i, v := range es.All() { +// ... // Do something with index-value pair +// } func (es KLLSketchDataPointSlice) All() iter.Seq2[int, KLLSketchDataPoint] { return func(yield func(int, KLLSketchDataPoint) bool) { for i := 0; i < es.Len(); i++ { @@ -58,6 +71,17 @@ func (es KLLSketchDataPointSlice) All() iter.Seq2[int, KLLSketchDataPoint] { } // EnsureCapacity is an operation that ensures the slice has at least the specified capacity. +// 1. If the newCap <= cap then no change in capacity. +// 2. If the newCap > cap then the slice capacity will be expanded to equal newCap. +// +// Here is how a new KLLSketchDataPointSlice can be initialized: +// +// es := NewKLLSketchDataPointSlice() +// es.EnsureCapacity(4) +// for i := 0; i < 4; i++ { +// e := es.AppendEmpty() +// // Here should set all the values for e. +// } func (es KLLSketchDataPointSlice) EnsureCapacity(newCap int) { es.state.AssertMutable() oldCap := cap(*es.orig) @@ -83,10 +107,12 @@ func (es KLLSketchDataPointSlice) AppendEmpty() KLLSketchDataPoint { func (es KLLSketchDataPointSlice) MoveAndAppendTo(dest KLLSketchDataPointSlice) { es.state.AssertMutable() dest.state.AssertMutable() + // If they point to the same data, they are the same, nothing to do. if es.orig == dest.orig { return } if *dest.orig == nil { + // We can simply move the entire vector and avoid any allocations. *dest.orig = *es.orig } else { *dest.orig = append(*dest.orig, *es.orig...) @@ -103,13 +129,16 @@ func (es KLLSketchDataPointSlice) RemoveIf(f func(KLLSketchDataPoint) bool) { if f(es.At(i)) { internal.DeleteKLLSketchDataPoint((*es.orig)[i], true) (*es.orig)[i] = nil + continue } if newLen == i { + // Nothing to move, element is at the right place. newLen++ continue } (*es.orig)[newLen] = (*es.orig)[i] + // Cannot delete here since we just move the data(or pointer to data) to a different position in the slice. (*es.orig)[i] = nil newLen++ } @@ -126,7 +155,8 @@ func (es KLLSketchDataPointSlice) CopyTo(dest KLLSketchDataPointSlice) { } // Sort sorts the KLLSketchDataPoint elements within KLLSketchDataPointSlice given the -// provided less function. +// provided less function so that two instances of KLLSketchDataPointSlice +// can be compared. func (es KLLSketchDataPointSlice) Sort(less func(a, b KLLSketchDataPoint) bool) { es.state.AssertMutable() sort.SliceStable(*es.orig, func(i, j int) bool { return less(es.At(i), es.At(j)) }) diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_kllsketchdatapointslice_test.go b/opentelemetry-collector-patch/pdata/pmetric/generated_kllsketchdatapointslice_test.go new file mode 100644 index 00000000..879af4c9 --- /dev/null +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_kllsketchdatapointslice_test.go @@ -0,0 +1,166 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by "internal/cmd/pdatagen/main.go". DO NOT EDIT. +// To regenerate this file run "make genpdata". + +package pmetric + +import ( + "testing" + "unsafe" + + "github.com/stretchr/testify/assert" + + "go.opentelemetry.io/collector/pdata/internal" +) + +func TestKLLSketchDataPointSlice(t *testing.T) { + es := NewKLLSketchDataPointSlice() + assert.Equal(t, 0, es.Len()) + es = newKLLSketchDataPointSlice(&[]*internal.KLLSketchDataPoint{}, internal.NewState()) + assert.Equal(t, 0, es.Len()) + + emptyVal := NewKLLSketchDataPoint() + testVal := generateTestKLLSketchDataPoint() + for i := 0; i < 7; i++ { + es.AppendEmpty() + assert.Equal(t, emptyVal, es.At(i)) + (*es.orig)[i] = internal.GenTestKLLSketchDataPoint() + assert.Equal(t, testVal, es.At(i)) + } + assert.Equal(t, 7, es.Len()) +} + +func TestKLLSketchDataPointSliceReadOnly(t *testing.T) { + sharedState := internal.NewState() + sharedState.MarkReadOnly() + es := newKLLSketchDataPointSlice(&[]*internal.KLLSketchDataPoint{}, sharedState) + assert.Equal(t, 0, es.Len()) + assert.Panics(t, func() { es.AppendEmpty() }) + assert.Panics(t, func() { es.EnsureCapacity(2) }) + es2 := NewKLLSketchDataPointSlice() + es.CopyTo(es2) + assert.Panics(t, func() { es2.CopyTo(es) }) + assert.Panics(t, func() { es.MoveAndAppendTo(es2) }) + assert.Panics(t, func() { es2.MoveAndAppendTo(es) }) +} + +func TestKLLSketchDataPointSlice_CopyTo(t *testing.T) { + dest := NewKLLSketchDataPointSlice() + src := generateTestKLLSketchDataPointSlice() + src.CopyTo(dest) + assert.Equal(t, generateTestKLLSketchDataPointSlice(), dest) + dest.CopyTo(dest) + assert.Equal(t, generateTestKLLSketchDataPointSlice(), dest) +} + +func TestKLLSketchDataPointSlice_EnsureCapacity(t *testing.T) { + es := generateTestKLLSketchDataPointSlice() + + // Test ensure smaller capacity. + const ensureSmallLen = 4 + es.EnsureCapacity(ensureSmallLen) + assert.Less(t, ensureSmallLen, es.Len()) + assert.Equal(t, es.Len(), cap(*es.orig)) + assert.Equal(t, generateTestKLLSketchDataPointSlice(), es) + + // Test ensure larger capacity + const ensureLargeLen = 9 + es.EnsureCapacity(ensureLargeLen) + assert.Less(t, generateTestKLLSketchDataPointSlice().Len(), ensureLargeLen) + assert.Equal(t, ensureLargeLen, cap(*es.orig)) + assert.Equal(t, generateTestKLLSketchDataPointSlice(), es) +} + +func TestKLLSketchDataPointSlice_MoveAndAppendTo(t *testing.T) { + // Test MoveAndAppendTo to empty + expectedSlice := generateTestKLLSketchDataPointSlice() + dest := NewKLLSketchDataPointSlice() + src := generateTestKLLSketchDataPointSlice() + src.MoveAndAppendTo(dest) + assert.Equal(t, generateTestKLLSketchDataPointSlice(), dest) + assert.Equal(t, 0, src.Len()) + assert.Equal(t, expectedSlice.Len(), dest.Len()) + + // Test MoveAndAppendTo empty slice + src.MoveAndAppendTo(dest) + assert.Equal(t, generateTestKLLSketchDataPointSlice(), dest) + assert.Equal(t, 0, src.Len()) + assert.Equal(t, expectedSlice.Len(), dest.Len()) + + // Test MoveAndAppendTo not empty slice + generateTestKLLSketchDataPointSlice().MoveAndAppendTo(dest) + assert.Equal(t, 2*expectedSlice.Len(), dest.Len()) + for i := 0; i < expectedSlice.Len(); i++ { + assert.Equal(t, expectedSlice.At(i), dest.At(i)) + assert.Equal(t, expectedSlice.At(i), dest.At(i+expectedSlice.Len())) + } + + dest.MoveAndAppendTo(dest) + assert.Equal(t, 2*expectedSlice.Len(), dest.Len()) + for i := 0; i < expectedSlice.Len(); i++ { + assert.Equal(t, expectedSlice.At(i), dest.At(i)) + assert.Equal(t, expectedSlice.At(i), dest.At(i+expectedSlice.Len())) + } +} + +func TestKLLSketchDataPointSlice_RemoveIf(t *testing.T) { + // Test RemoveIf on empty slice + emptySlice := NewKLLSketchDataPointSlice() + emptySlice.RemoveIf(func(el KLLSketchDataPoint) bool { + t.Fail() + return false + }) + + // Test RemoveIf + filtered := generateTestKLLSketchDataPointSlice() + pos := 0 + filtered.RemoveIf(func(el KLLSketchDataPoint) bool { + pos++ + return pos%2 == 1 + }) + assert.Equal(t, 2, filtered.Len()) +} + +func TestKLLSketchDataPointSlice_RemoveIfAll(t *testing.T) { + got := generateTestKLLSketchDataPointSlice() + got.RemoveIf(func(el KLLSketchDataPoint) bool { + return true + }) + assert.Equal(t, 0, got.Len()) +} + +func TestKLLSketchDataPointSliceAll(t *testing.T) { + ms := generateTestKLLSketchDataPointSlice() + assert.NotEmpty(t, ms.Len()) + + var c int + for i, v := range ms.All() { + assert.Equal(t, ms.At(i), v, "element should match") + c++ + } + assert.Equal(t, ms.Len(), c, "All elements should have been visited") +} + +func TestKLLSketchDataPointSlice_Sort(t *testing.T) { + es := generateTestKLLSketchDataPointSlice() + es.Sort(func(a, b KLLSketchDataPoint) bool { + return uintptr(unsafe.Pointer(a.orig)) < uintptr(unsafe.Pointer(b.orig)) + }) + for i := 1; i < es.Len(); i++ { + assert.Less(t, uintptr(unsafe.Pointer(es.At(i-1).orig)), uintptr(unsafe.Pointer(es.At(i).orig))) + } + es.Sort(func(a, b KLLSketchDataPoint) bool { + return uintptr(unsafe.Pointer(a.orig)) > uintptr(unsafe.Pointer(b.orig)) + }) + for i := 1; i < es.Len(); i++ { + assert.Greater(t, uintptr(unsafe.Pointer(es.At(i-1).orig)), uintptr(unsafe.Pointer(es.At(i).orig))) + } +} + +func generateTestKLLSketchDataPointSlice() KLLSketchDataPointSlice { + ms := NewKLLSketchDataPointSlice() + *ms.orig = internal.GenTestKLLSketchDataPointPtrSlice() + return ms +} diff --git a/opentelemetry-collector-patch/pdata/pmetric/generated_metric_test.go b/opentelemetry-collector-patch/pdata/pmetric/generated_metric_test.go index 0ede692a..b79e3107 100644 --- a/opentelemetry-collector-patch/pdata/pmetric/generated_metric_test.go +++ b/opentelemetry-collector-patch/pdata/pmetric/generated_metric_test.go @@ -149,6 +149,54 @@ func TestMetric_Summary(t *testing.T) { assert.Panics(t, func() { newMetric(internal.NewMetric(), sharedState).SetEmptySummary() }) } +func TestMetric_KLLSketch(t *testing.T) { + ms := NewMetric() + ms.SetEmptyKLLSketch() + assert.Equal(t, NewKLLSketch(), ms.KLLSketch()) + ms.orig.GetData().(*internal.Metric_KLLSketch).KLLSketch = internal.GenTestKLLSketch() + assert.Equal(t, MetricTypeKLLSketch, ms.Type()) + assert.Equal(t, generateTestKLLSketch(), ms.KLLSketch()) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { newMetric(internal.NewMetric(), sharedState).SetEmptyKLLSketch() }) +} + +func TestMetric_CountSketch(t *testing.T) { + ms := NewMetric() + ms.SetEmptyCountSketch() + assert.Equal(t, NewCountSketch(), ms.CountSketch()) + ms.orig.GetData().(*internal.Metric_CountSketch).CountSketch = internal.GenTestCountSketch() + assert.Equal(t, MetricTypeCountSketch, ms.Type()) + assert.Equal(t, generateTestCountSketch(), ms.CountSketch()) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { newMetric(internal.NewMetric(), sharedState).SetEmptyCountSketch() }) +} + +func TestMetric_CountMinSketch(t *testing.T) { + ms := NewMetric() + ms.SetEmptyCountMinSketch() + assert.Equal(t, NewCountMinSketch(), ms.CountMinSketch()) + ms.orig.GetData().(*internal.Metric_CountMinSketch).CountMinSketch = internal.GenTestCountMinSketch() + assert.Equal(t, MetricTypeCountMinSketch, ms.Type()) + assert.Equal(t, generateTestCountMinSketch(), ms.CountMinSketch()) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { newMetric(internal.NewMetric(), sharedState).SetEmptyCountMinSketch() }) +} + +func TestMetric_HLLSketch(t *testing.T) { + ms := NewMetric() + ms.SetEmptyHLLSketch() + assert.Equal(t, NewHLLSketch(), ms.HLLSketch()) + ms.orig.GetData().(*internal.Metric_HLLSketch).HLLSketch = internal.GenTestHLLSketch() + assert.Equal(t, MetricTypeHLLSketch, ms.Type()) + assert.Equal(t, generateTestHLLSketch(), ms.HLLSketch()) + sharedState := internal.NewState() + sharedState.MarkReadOnly() + assert.Panics(t, func() { newMetric(internal.NewMetric(), sharedState).SetEmptyHLLSketch() }) +} + func TestMetric_Metadata(t *testing.T) { ms := NewMetric() assert.Equal(t, pcommon.NewMap(), ms.Metadata()) diff --git a/opentelemetry-go-patch/exporters/otlp/otlpmetric/otlpmetricgrpc/exporter.go b/opentelemetry-go-patch/exporters/otlp/otlpmetric/otlpmetricgrpc/exporter.go index c82e620b..5928aaf9 100644 --- a/opentelemetry-go-patch/exporters/otlp/otlpmetric/otlpmetricgrpc/exporter.go +++ b/opentelemetry-go-patch/exporters/otlp/otlpmetric/otlpmetricgrpc/exporter.go @@ -86,6 +86,7 @@ func (e *Exporter) Export(ctx context.Context, rm *metricdata.ResourceMetrics) e e.clientMu.Unlock() if resp != nil { applySeriesAssignments(e.seriesState, resp) + applyUnknownSeriesIds(e.seriesState, resp) } if upErr != nil { if err == nil { @@ -202,6 +203,36 @@ func applySeriesAssignments(dict *series.Dictionary, resp *colmetricpb.ExportMet dict.Apply(assignments) } +// applyUnknownSeriesIds reads response.UnknownSeriesIds (refactor-2026-05 +// addition; only present in patched proto builds) and evicts those sids +// from the local series dictionary. The next emission referencing those +// series will fall back to attribute-carrying mode and the receiver will +// re-resolve. This is the universal recovery primitive for sid-cache +// divergence (e.g., backend restart without persistence). +func applyUnknownSeriesIds(dict *series.Dictionary, resp *colmetricpb.ExportMetricsServiceResponse) { + if resp == nil { + return + } + // Reflective access keeps the patched-vs-upstream proto build switchable. + getUnknown := reflect.ValueOf(resp).MethodByName("GetUnknownSeriesIds") + if !getUnknown.IsValid() { + return + } + res := getUnknown.Call(nil) + if len(res) != 1 || res[0].Kind() != reflect.Slice || res[0].Len() == 0 { + return + } + sids := make([]uint64, 0, res[0].Len()) + for i := 0; i < res[0].Len(); i++ { + v := res[0].Index(i) + if v.Kind() != reflect.Uint64 { + continue + } + sids = append(sids, v.Uint()) + } + dict.EvictByID(sids) +} + func callStringMethod(v reflect.Value, name string) string { m := v.MethodByName(name) if !m.IsValid() { diff --git a/opentelemetry-go-patch/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/series/dictionary.go b/opentelemetry-go-patch/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/series/dictionary.go index 8cb40d60..db0f2579 100644 --- a/opentelemetry-go-patch/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/series/dictionary.go +++ b/opentelemetry-go-patch/exporters/otlp/otlpmetric/otlpmetricgrpc/internal/series/dictionary.go @@ -151,6 +151,38 @@ func (d *Dictionary) Apply(assignments []Assignment) { d.mu.Unlock() } +// EvictByID removes any cached entries whose assigned series_id appears in +// the provided list. The next emission referencing the same logical series +// will fall back to attribute-carrying mode (sid=0 + populated attributes), +// triggering a fresh resolution from the receiver. +// +// This is the universal recovery path for sid-cache divergence — used when +// the receiver's response carries `unknown_series_ids` (refactor-2026-05). +// Implementation walks every (source, entry) pair and clears the seriesID +// cache slot if it matches a listed sid; the entry itself is retained so +// subsequent annotateMetric() can repopulate via the next response. +func (d *Dictionary) EvictByID(sids []uint64) { + if len(sids) == 0 { + return + } + want := make(map[uint64]struct{}, len(sids)) + for _, s := range sids { + want[s] = struct{}{} + } + d.mu.Lock() + defer d.mu.Unlock() + for _, src := range d.sources { + src.mu.Lock() + for _, entry := range src.entries { + if _, hit := want[entry.seriesID]; hit { + entry.seriesID = 0 + entry.registered = false + } + } + src.mu.Unlock() + } +} + func (d *Dictionary) getOrCreateSource(key string) *sourceState { d.mu.Lock() defer d.mu.Unlock() diff --git a/opentelemetry-proto-patch/gen/go/go.opentelemetry.io/proto/otlp/collector/metrics/v1/metrics_service.pb.go b/opentelemetry-proto-patch/gen/go/go.opentelemetry.io/proto/otlp/collector/metrics/v1/metrics_service.pb.go index 2c8c76ae..5c534819 100644 --- a/opentelemetry-proto-patch/gen/go/go.opentelemetry.io/proto/otlp/collector/metrics/v1/metrics_service.pb.go +++ b/opentelemetry-proto-patch/gen/go/go.opentelemetry.io/proto/otlp/collector/metrics/v1/metrics_service.pb.go @@ -117,6 +117,16 @@ type ExportMetricsServiceResponse struct { // assignments and omit the associated attributes on subsequent exports once // a series_id is known. SeriesAssignments []*SeriesAssignment `protobuf:"bytes,2,rep,name=series_assignments,json=seriesAssignments,proto3" json:"series_assignments,omitempty"` + // Refactor-2026-05: sids the receiver did NOT recognize in this Export. + // Sent when the receiver's series-id cache has diverged from the + // sender's (e.g., after backend restart without persistence, or any + // sid-cache eviction). Senders MUST evict the listed sids from their + // local cache; subsequent emits MUST re-attach `attributes` so the + // receiver can re-resolve and return fresh SeriesAssignments. This + // field is the universal recovery primitive — same code path that + // handles cold-start bootstrap also handles every cache-divergence + // failure mode. + UnknownSeriesIds []uint64 `protobuf:"varint,3,rep,packed,name=unknown_series_ids,json=unknownSeriesIds,proto3" json:"unknown_series_ids,omitempty"` } func (x *ExportMetricsServiceResponse) Reset() { @@ -165,6 +175,13 @@ func (x *ExportMetricsServiceResponse) GetSeriesAssignments() []*SeriesAssignmen return nil } +func (x *ExportMetricsServiceResponse) GetUnknownSeriesIds() []uint64 { + if x != nil { + return x.UnknownSeriesIds + } + return nil +} + type ExportMetricsPartialSuccess struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -356,7 +373,7 @@ var file_opentelemetry_proto_collector_metrics_v1_metrics_service_proto_rawDesc 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, 0x0f, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4d, 0x65, 0x74, 0x72, - 0x69, 0x63, 0x73, 0x22, 0xf9, 0x01, 0x0a, 0x1c, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x4d, 0x65, + 0x69, 0x63, 0x73, 0x22, 0xa7, 0x02, 0x0a, 0x1c, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6e, 0x0a, 0x0f, 0x70, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x45, 0x2e, @@ -371,51 +388,54 @@ var file_opentelemetry_proto_collector_metrics_v1_metrics_service_proto_rawDesc 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x72, 0x69, 0x65, 0x73, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x11, 0x73, 0x65, - 0x72, 0x69, 0x65, 0x73, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x22, - 0x74, 0x0a, 0x1b, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, - 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x30, - 0x0a, 0x14, 0x72, 0x65, 0x6a, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x5f, - 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x12, 0x72, 0x65, - 0x6a, 0x65, 0x63, 0x74, 0x65, 0x64, 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x73, - 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, - 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0xe8, 0x01, 0x0a, 0x10, 0x53, 0x65, 0x72, 0x69, 0x65, 0x73, - 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x72, 0x65, - 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0b, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1b, 0x0a, - 0x09, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x08, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x65, - 0x74, 0x72, 0x69, 0x63, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0a, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, - 0x65, 0x74, 0x72, 0x69, 0x63, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0a, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x54, 0x79, 0x70, 0x65, 0x12, 0x35, 0x0a, 0x16, - 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x5f, 0x66, 0x69, 0x6e, 0x67, 0x65, - 0x72, 0x70, 0x72, 0x69, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x15, 0x61, 0x74, - 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x46, 0x69, 0x6e, 0x67, 0x65, 0x72, 0x70, 0x72, - 0x69, 0x6e, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x69, 0x65, 0x73, 0x5f, 0x69, 0x64, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x73, 0x65, 0x72, 0x69, 0x65, 0x73, 0x49, 0x64, - 0x32, 0xac, 0x01, 0x0a, 0x0e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x12, 0x99, 0x01, 0x0a, 0x06, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x45, - 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x6d, - 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, - 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x46, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, - 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6c, 0x6c, - 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, - 0x2e, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, - 0xa4, 0x01, 0x0a, 0x2b, 0x69, 0x6f, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, - 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6c, 0x6c, 0x65, - 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x42, - 0x13, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, - 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x33, 0x67, 0x6f, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, - 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x69, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x2f, 0x6f, 0x74, 0x6c, 0x70, 0x2f, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, - 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2f, 0x76, 0x31, 0xaa, 0x02, 0x28, 0x4f, 0x70, - 0x65, 0x6e, 0x54, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x50, 0x72, 0x6f, 0x74, - 0x6f, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4d, 0x65, 0x74, 0x72, - 0x69, 0x63, 0x73, 0x2e, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x72, 0x69, 0x65, 0x73, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, + 0x2c, 0x0a, 0x12, 0x75, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x5f, 0x73, 0x65, 0x72, 0x69, 0x65, + 0x73, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x04, 0x52, 0x10, 0x75, 0x6e, 0x6b, + 0x6e, 0x6f, 0x77, 0x6e, 0x53, 0x65, 0x72, 0x69, 0x65, 0x73, 0x49, 0x64, 0x73, 0x22, 0x74, 0x0a, + 0x1b, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x50, 0x61, + 0x72, 0x74, 0x69, 0x61, 0x6c, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x30, 0x0a, 0x14, + 0x72, 0x65, 0x6a, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x70, 0x6f, + 0x69, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x12, 0x72, 0x65, 0x6a, 0x65, + 0x63, 0x74, 0x65, 0x64, 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x12, 0x23, + 0x0a, 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x22, 0xe8, 0x01, 0x0a, 0x10, 0x53, 0x65, 0x72, 0x69, 0x65, 0x73, 0x41, 0x73, + 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x72, 0x65, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, + 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1b, 0x0a, 0x09, 0x73, + 0x63, 0x6f, 0x70, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, + 0x73, 0x63, 0x6f, 0x70, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x65, 0x74, 0x72, + 0x69, 0x63, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6d, + 0x65, 0x74, 0x72, 0x69, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x65, 0x74, + 0x72, 0x69, 0x63, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, + 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x54, 0x79, 0x70, 0x65, 0x12, 0x35, 0x0a, 0x16, 0x61, 0x74, + 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x5f, 0x66, 0x69, 0x6e, 0x67, 0x65, 0x72, 0x70, + 0x72, 0x69, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x15, 0x61, 0x74, 0x74, 0x72, + 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x46, 0x69, 0x6e, 0x67, 0x65, 0x72, 0x70, 0x72, 0x69, 0x6e, + 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x69, 0x65, 0x73, 0x5f, 0x69, 0x64, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x73, 0x65, 0x72, 0x69, 0x65, 0x73, 0x49, 0x64, 0x32, 0xac, + 0x01, 0x0a, 0x0e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x12, 0x99, 0x01, 0x0a, 0x06, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x45, 0x2e, 0x6f, + 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x6d, 0x65, 0x74, + 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x4d, 0x65, + 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x46, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, + 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, + 0x74, 0x6f, 0x72, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x45, + 0x78, 0x70, 0x6f, 0x72, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0xa4, 0x01, + 0x0a, 0x2b, 0x69, 0x6f, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, + 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, + 0x6f, 0x72, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x42, 0x13, 0x4d, + 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x72, 0x6f, + 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x33, 0x67, 0x6f, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, + 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x69, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, + 0x6f, 0x74, 0x6c, 0x70, 0x2f, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2f, 0x6d, + 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2f, 0x76, 0x31, 0xaa, 0x02, 0x28, 0x4f, 0x70, 0x65, 0x6e, + 0x54, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x2e, + 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, + 0x73, 0x2e, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/opentelemetry-proto-patch/gen/go/go.opentelemetry.io/proto/otlp/metrics/v1/metrics.pb.go b/opentelemetry-proto-patch/gen/go/go.opentelemetry.io/proto/otlp/metrics/v1/metrics.pb.go index 145d319e..8f9a290e 100644 --- a/opentelemetry-proto-patch/gen/go/go.opentelemetry.io/proto/otlp/metrics/v1/metrics.pb.go +++ b/opentelemetry-proto-patch/gen/go/go.opentelemetry.io/proto/otlp/metrics/v1/metrics.pb.go @@ -1323,6 +1323,12 @@ type DDSketch struct { // aggregation_temporality describes if the aggregator reports delta changes // since last report time, or cumulative changes since a fixed start time. AggregationTemporality AggregationTemporality `protobuf:"varint,2,opt,name=aggregation_temporality,json=aggregationTemporality,proto3,enum=opentelemetry.proto.metrics.v1.AggregationTemporality" json:"aggregation_temporality,omitempty"` + // Refactor-2026-05: sketch instance config — DDSketch's relative + // accuracy (alpha). Sent ONCE per Metric emit (not per DataPoint), + // since this is a per-sketch-instance parameter, not per-window + // mutable state. Backend reads this into + // SketchInstanceMetadata.sketch_config on first sight and caches. + RelativeAccuracy float64 `protobuf:"fixed64,3,opt,name=relative_accuracy,json=relativeAccuracy,proto3" json:"relative_accuracy,omitempty"` } func (x *DDSketch) Reset() { @@ -1371,6 +1377,13 @@ func (x *DDSketch) GetAggregationTemporality() AggregationTemporality { return AggregationTemporality_AGGREGATION_TEMPORALITY_UNSPECIFIED } +func (x *DDSketch) GetRelativeAccuracy() float64 { + if x != nil { + return x.RelativeAccuracy + } + return 0 +} + // Summary metric data are used to convey quantile summaries, // a Prometheus (see: https://prometheus.io/docs/concepts/metric_types/#summary) // and OpenMetrics (see: https://github.com/prometheus/OpenMetrics/blob/4dbf6075567ab43296eed941037c12951faafb92/protos/prometheus.proto#L45) @@ -1430,6 +1443,11 @@ func (x *Summary) GetDataPoints() []*SummaryDataPoint { } // DDSketchDataPoint is a single data point that encodes a distribution using the DDSketch format. +// +// Refactor-2026-05: removed precomputed/duplicated fields (count, sum, +// min, max) — they are derivable from the sketch payload, so storing +// them per-DataPoint duplicates state and risks cache-invalidation +// bugs. Field numbers reserved for forward/backward compat. type DDSketchDataPoint struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1439,44 +1457,15 @@ type DDSketchDataPoint struct { // where this point belongs. The list MUST be omitted (or present but empty) // whenever `series_id` is non-zero, and MUST be populated when `series_id` // is zero so the receiver can mint a new entry. - // Attribute keys MUST be unique (it is not allowed to have more than one - // attribute with the same key). - // The behavior of software that receives duplicated keys can be unpredictable. + // Attribute keys MUST be unique. After group-by rollup at the agent's + // sketch processor, `attributes.keys()` IS the group-by KEY set; + // `attributes` (as a map) is the group-by VALUES for this series. Attributes []*v11.KeyValue `protobuf:"bytes,9,rep,name=attributes,proto3" json:"attributes,omitempty"` // StartTimeUnixNano is optional but strongly encouraged, see the // the detailed comments above Metric. - // - // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January - // 1970. StartTimeUnixNano uint64 `protobuf:"fixed64,2,opt,name=start_time_unix_nano,json=startTimeUnixNano,proto3" json:"start_time_unix_nano,omitempty"` // TimeUnixNano is required, see the detailed comments above Metric. - // - // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January - // 1970. TimeUnixNano uint64 `protobuf:"fixed64,3,opt,name=time_unix_nano,json=timeUnixNano,proto3" json:"time_unix_nano,omitempty"` - // Count is the number of measurements that were aggregated into this sketch. - Count uint64 `protobuf:"fixed64,4,opt,name=count,proto3" json:"count,omitempty"` - // Sum is the total of all measurements aggregated for this sketch. - // At most one of the fields in this oneof may be set at a time. - // - // Types that are assignable to Sum: - // *DDSketchDataPoint_SumAsDouble - // *DDSketchDataPoint_SumAsInt - Sum isDDSketchDataPoint_Sum `protobuf_oneof:"sum"` - // Min is the minimum value over (start_time, end_time]. Optional. - // At most one of the fields in this oneof may be set at a time. - // - // Types that are assignable to Min: - // *DDSketchDataPoint_MinAsDouble - // *DDSketchDataPoint_MinAsInt - Min isDDSketchDataPoint_Min `protobuf_oneof:"min"` - // Max is the maximum value over (start_time, end_time]. Optional. - // At most one of the fields in this oneof may be set at a time. - // - // Types that are assignable to Max: - // *DDSketchDataPoint_MaxAsDouble - // *DDSketchDataPoint_MaxAsInt - Max isDDSketchDataPoint_Max `protobuf_oneof:"max"` // Serialized DDSketch payload. The encoding is described by the encoding field. Sketch []byte `protobuf:"bytes,8,opt,name=sketch,proto3" json:"sketch,omitempty"` // Encoding that describes how the sketch bytes should be decoded. @@ -1490,6 +1479,9 @@ type DDSketchDataPoint struct { // series_id references a collector-assigned series descriptor. Receivers MUST // reject the point unless exactly one of (`series_id` != 0, `attributes` populated) // is true. + // After Phase 5 (centralized resolver at asap-query-backend), this id + // is the BACKEND-assigned global identifier for `(metric_name, attributes)`, + // shared across all hops in the pipeline. SeriesId uint64 `protobuf:"varint,16,opt,name=series_id,json=seriesId,proto3" json:"series_id,omitempty"` } @@ -1546,76 +1538,6 @@ func (x *DDSketchDataPoint) GetTimeUnixNano() uint64 { return 0 } -func (x *DDSketchDataPoint) GetCount() uint64 { - if x != nil { - return x.Count - } - return 0 -} - -func (m *DDSketchDataPoint) GetSum() isDDSketchDataPoint_Sum { - if m != nil { - return m.Sum - } - return nil -} - -func (x *DDSketchDataPoint) GetSumAsDouble() float64 { - if x, ok := x.GetSum().(*DDSketchDataPoint_SumAsDouble); ok { - return x.SumAsDouble - } - return 0 -} - -func (x *DDSketchDataPoint) GetSumAsInt() int64 { - if x, ok := x.GetSum().(*DDSketchDataPoint_SumAsInt); ok { - return x.SumAsInt - } - return 0 -} - -func (m *DDSketchDataPoint) GetMin() isDDSketchDataPoint_Min { - if m != nil { - return m.Min - } - return nil -} - -func (x *DDSketchDataPoint) GetMinAsDouble() float64 { - if x, ok := x.GetMin().(*DDSketchDataPoint_MinAsDouble); ok { - return x.MinAsDouble - } - return 0 -} - -func (x *DDSketchDataPoint) GetMinAsInt() int64 { - if x, ok := x.GetMin().(*DDSketchDataPoint_MinAsInt); ok { - return x.MinAsInt - } - return 0 -} - -func (m *DDSketchDataPoint) GetMax() isDDSketchDataPoint_Max { - if m != nil { - return m.Max - } - return nil -} - -func (x *DDSketchDataPoint) GetMaxAsDouble() float64 { - if x, ok := x.GetMax().(*DDSketchDataPoint_MaxAsDouble); ok { - return x.MaxAsDouble - } - return 0 -} - -func (x *DDSketchDataPoint) GetMaxAsInt() int64 { - if x, ok := x.GetMax().(*DDSketchDataPoint_MaxAsInt); ok { - return x.MaxAsInt - } - return 0 -} - func (x *DDSketchDataPoint) GetSketch() []byte { if x != nil { return x.Sketch @@ -1651,65 +1573,17 @@ func (x *DDSketchDataPoint) GetSeriesId() uint64 { return 0 } -type isDDSketchDataPoint_Sum interface { - isDDSketchDataPoint_Sum() -} - -type DDSketchDataPoint_SumAsDouble struct { - SumAsDouble float64 `protobuf:"fixed64,5,opt,name=sum_as_double,json=sumAsDouble,proto3,oneof"` -} - -type DDSketchDataPoint_SumAsInt struct { - SumAsInt int64 `protobuf:"fixed64,12,opt,name=sum_as_int,json=sumAsInt,proto3,oneof"` -} - -func (*DDSketchDataPoint_SumAsDouble) isDDSketchDataPoint_Sum() {} - -func (*DDSketchDataPoint_SumAsInt) isDDSketchDataPoint_Sum() {} - -type isDDSketchDataPoint_Min interface { - isDDSketchDataPoint_Min() -} - -type DDSketchDataPoint_MinAsDouble struct { - MinAsDouble float64 `protobuf:"fixed64,6,opt,name=min_as_double,json=minAsDouble,proto3,oneof"` -} - -type DDSketchDataPoint_MinAsInt struct { - MinAsInt int64 `protobuf:"fixed64,13,opt,name=min_as_int,json=minAsInt,proto3,oneof"` -} - -func (*DDSketchDataPoint_MinAsDouble) isDDSketchDataPoint_Min() {} - -func (*DDSketchDataPoint_MinAsInt) isDDSketchDataPoint_Min() {} - -type isDDSketchDataPoint_Max interface { - isDDSketchDataPoint_Max() -} - -type DDSketchDataPoint_MaxAsDouble struct { - MaxAsDouble float64 `protobuf:"fixed64,7,opt,name=max_as_double,json=maxAsDouble,proto3,oneof"` -} - -type DDSketchDataPoint_MaxAsInt struct { - MaxAsInt int64 `protobuf:"fixed64,14,opt,name=max_as_int,json=maxAsInt,proto3,oneof"` -} - -func (*DDSketchDataPoint_MaxAsDouble) isDDSketchDataPoint_Max() {} - -func (*DDSketchDataPoint_MaxAsInt) isDDSketchDataPoint_Max() {} - // KLLSketch represents the type of a metric that encodes measurements using the KLL quantile sketch. type KLLSketch struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // The time series data points. - DataPoints []*KLLSketchDataPoint `protobuf:"bytes,1,rep,name=data_points,json=dataPoints,proto3" json:"data_points,omitempty"` - // aggregation_temporality describes if the aggregator reports delta changes - // since last report time, or cumulative changes since a fixed start time. + DataPoints []*KLLSketchDataPoint `protobuf:"bytes,1,rep,name=data_points,json=dataPoints,proto3" json:"data_points,omitempty"` AggregationTemporality AggregationTemporality `protobuf:"varint,2,opt,name=aggregation_temporality,json=aggregationTemporality,proto3,enum=opentelemetry.proto.metrics.v1.AggregationTemporality" json:"aggregation_temporality,omitempty"` + // Refactor-2026-05: KLL sketch parameter `k` — controls quantile + // error / space tradeoff. Sent once per Metric emit. + K uint32 `protobuf:"varint,3,opt,name=k,proto3" json:"k,omitempty"` } func (x *KLLSketch) Reset() { @@ -1758,7 +1632,16 @@ func (x *KLLSketch) GetAggregationTemporality() AggregationTemporality { return AggregationTemporality_AGGREGATION_TEMPORALITY_UNSPECIFIED } +func (x *KLLSketch) GetK() uint32 { + if x != nil { + return x.K + } + return 0 +} + // KLLSketchDataPoint is a single data point that encodes a distribution using the KLL sketch format. +// +// Refactor-2026-05: removed count, sum, min, max — derivable from sketch state. type KLLSketchDataPoint struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1767,10 +1650,6 @@ type KLLSketchDataPoint struct { Attributes []*v11.KeyValue `protobuf:"bytes,1,rep,name=attributes,proto3" json:"attributes,omitempty"` StartTimeUnixNano uint64 `protobuf:"fixed64,2,opt,name=start_time_unix_nano,json=startTimeUnixNano,proto3" json:"start_time_unix_nano,omitempty"` TimeUnixNano uint64 `protobuf:"fixed64,3,opt,name=time_unix_nano,json=timeUnixNano,proto3" json:"time_unix_nano,omitempty"` - Count uint64 `protobuf:"fixed64,4,opt,name=count,proto3" json:"count,omitempty"` - Sum float64 `protobuf:"fixed64,5,opt,name=sum,proto3" json:"sum,omitempty"` - Min float64 `protobuf:"fixed64,6,opt,name=min,proto3" json:"min,omitempty"` - Max float64 `protobuf:"fixed64,7,opt,name=max,proto3" json:"max,omitempty"` Sketch []byte `protobuf:"bytes,8,opt,name=sketch,proto3" json:"sketch,omitempty"` Encoding KLLSketchEncoding `protobuf:"varint,9,opt,name=encoding,proto3,enum=opentelemetry.proto.metrics.v1.KLLSketchEncoding" json:"encoding,omitempty"` Flags uint32 `protobuf:"varint,10,opt,name=flags,proto3" json:"flags,omitempty"` @@ -1830,34 +1709,6 @@ func (x *KLLSketchDataPoint) GetTimeUnixNano() uint64 { return 0 } -func (x *KLLSketchDataPoint) GetCount() uint64 { - if x != nil { - return x.Count - } - return 0 -} - -func (x *KLLSketchDataPoint) GetSum() float64 { - if x != nil { - return x.Sum - } - return 0 -} - -func (x *KLLSketchDataPoint) GetMin() float64 { - if x != nil { - return x.Min - } - return 0 -} - -func (x *KLLSketchDataPoint) GetMax() float64 { - if x != nil { - return x.Max - } - return 0 -} - func (x *KLLSketchDataPoint) GetSketch() []byte { if x != nil { return x.Sketch @@ -1892,11 +1743,13 @@ type CountSketch struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // The time series data points. - DataPoints []*CountSketchDataPoint `protobuf:"bytes,1,rep,name=data_points,json=dataPoints,proto3" json:"data_points,omitempty"` - // aggregation_temporality describes if the aggregator reports delta changes - // since last report time, or cumulative changes since a fixed start time. - AggregationTemporality AggregationTemporality `protobuf:"varint,2,opt,name=aggregation_temporality,json=aggregationTemporality,proto3,enum=opentelemetry.proto.metrics.v1.AggregationTemporality" json:"aggregation_temporality,omitempty"` + DataPoints []*CountSketchDataPoint `protobuf:"bytes,1,rep,name=data_points,json=dataPoints,proto3" json:"data_points,omitempty"` + AggregationTemporality AggregationTemporality `protobuf:"varint,2,opt,name=aggregation_temporality,json=aggregationTemporality,proto3,enum=opentelemetry.proto.metrics.v1.AggregationTemporality" json:"aggregation_temporality,omitempty"` + // Refactor-2026-05: CountSketch matrix dimensions. depth × width. + // (epsilon, delta) deriveable: eps = sqrt(e/cols) for L2 estimation; + // delta ≈ 1/(2^(rows/2)). Backend computes these for display. + Rows int32 `protobuf:"varint,3,opt,name=rows,proto3" json:"rows,omitempty"` // depth + Cols int32 `protobuf:"varint,4,opt,name=cols,proto3" json:"cols,omitempty"` // width } func (x *CountSketch) Reset() { @@ -1945,7 +1798,25 @@ func (x *CountSketch) GetAggregationTemporality() AggregationTemporality { return AggregationTemporality_AGGREGATION_TEMPORALITY_UNSPECIFIED } +func (x *CountSketch) GetRows() int32 { + if x != nil { + return x.Rows + } + return 0 +} + +func (x *CountSketch) GetCols() int32 { + if x != nil { + return x.Cols + } + return 0 +} + // CountSketchDataPoint is a single data point encoding frequency estimations. +// +// Refactor-2026-05: removed `dimension` (legacy partition-key +// descriptor — replaced by `attributes` keys = group-by labels); +// removed `epsilon`/`delta` (derivable from container's rows/cols). type CountSketchDataPoint struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1956,9 +1827,6 @@ type CountSketchDataPoint struct { TimeUnixNano uint64 `protobuf:"fixed64,3,opt,name=time_unix_nano,json=timeUnixNano,proto3" json:"time_unix_nano,omitempty"` Sketch []byte `protobuf:"bytes,4,opt,name=sketch,proto3" json:"sketch,omitempty"` Encoding CountSketchEncoding `protobuf:"varint,5,opt,name=encoding,proto3,enum=opentelemetry.proto.metrics.v1.CountSketchEncoding" json:"encoding,omitempty"` - Dimension string `protobuf:"bytes,6,opt,name=dimension,proto3" json:"dimension,omitempty"` - Epsilon float64 `protobuf:"fixed64,7,opt,name=epsilon,proto3" json:"epsilon,omitempty"` - Delta float64 `protobuf:"fixed64,8,opt,name=delta,proto3" json:"delta,omitempty"` Flags uint32 `protobuf:"varint,9,opt,name=flags,proto3" json:"flags,omitempty"` SeriesId uint64 `protobuf:"varint,10,opt,name=series_id,json=seriesId,proto3" json:"series_id,omitempty"` } @@ -2030,27 +1898,6 @@ func (x *CountSketchDataPoint) GetEncoding() CountSketchEncoding { return CountSketchEncoding_COUNT_SKETCH_ENCODING_UNSPECIFIED } -func (x *CountSketchDataPoint) GetDimension() string { - if x != nil { - return x.Dimension - } - return "" -} - -func (x *CountSketchDataPoint) GetEpsilon() float64 { - if x != nil { - return x.Epsilon - } - return 0 -} - -func (x *CountSketchDataPoint) GetDelta() float64 { - if x != nil { - return x.Delta - } - return 0 -} - func (x *CountSketchDataPoint) GetFlags() uint32 { if x != nil { return x.Flags @@ -2071,11 +1918,12 @@ type CountMinSketch struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // The time series data points. - DataPoints []*CountMinSketchDataPoint `protobuf:"bytes,1,rep,name=data_points,json=dataPoints,proto3" json:"data_points,omitempty"` - // aggregation_temporality describes if the aggregator reports delta changes - // since last report time, or cumulative changes since a fixed start time. - AggregationTemporality AggregationTemporality `protobuf:"varint,2,opt,name=aggregation_temporality,json=aggregationTemporality,proto3,enum=opentelemetry.proto.metrics.v1.AggregationTemporality" json:"aggregation_temporality,omitempty"` + DataPoints []*CountMinSketchDataPoint `protobuf:"bytes,1,rep,name=data_points,json=dataPoints,proto3" json:"data_points,omitempty"` + AggregationTemporality AggregationTemporality `protobuf:"varint,2,opt,name=aggregation_temporality,json=aggregationTemporality,proto3,enum=opentelemetry.proto.metrics.v1.AggregationTemporality" json:"aggregation_temporality,omitempty"` + // Refactor-2026-05: CMS matrix dimensions. depth × width. + // (epsilon, delta) deriveable: eps ≈ e/cols; delta ≈ exp(-rows). + Rows int32 `protobuf:"varint,3,opt,name=rows,proto3" json:"rows,omitempty"` // depth = ⌈ln(1/delta)⌉ + Cols int32 `protobuf:"varint,4,opt,name=cols,proto3" json:"cols,omitempty"` // width = ⌈e/epsilon⌉ } func (x *CountMinSketch) Reset() { @@ -2124,7 +1972,25 @@ func (x *CountMinSketch) GetAggregationTemporality() AggregationTemporality { return AggregationTemporality_AGGREGATION_TEMPORALITY_UNSPECIFIED } +func (x *CountMinSketch) GetRows() int32 { + if x != nil { + return x.Rows + } + return 0 +} + +func (x *CountMinSketch) GetCols() int32 { + if x != nil { + return x.Cols + } + return 0 +} + // CountMinSketchDataPoint is a single data point encoding frequency estimations. +// +// Refactor-2026-05: removed `sample_count` (derivable from sketch +// row sums); removed `rows`/`cols` (lifted to parent container, sent +// once per Metric emit instead of per DataPoint). type CountMinSketchDataPoint struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2133,11 +1999,8 @@ type CountMinSketchDataPoint struct { Attributes []*v11.KeyValue `protobuf:"bytes,1,rep,name=attributes,proto3" json:"attributes,omitempty"` StartTimeUnixNano uint64 `protobuf:"fixed64,2,opt,name=start_time_unix_nano,json=startTimeUnixNano,proto3" json:"start_time_unix_nano,omitempty"` TimeUnixNano uint64 `protobuf:"fixed64,3,opt,name=time_unix_nano,json=timeUnixNano,proto3" json:"time_unix_nano,omitempty"` - SampleCount uint64 `protobuf:"varint,4,opt,name=sample_count,json=sampleCount,proto3" json:"sample_count,omitempty"` Sketch []byte `protobuf:"bytes,5,opt,name=sketch,proto3" json:"sketch,omitempty"` Encoding CountMinSketchEncoding `protobuf:"varint,6,opt,name=encoding,proto3,enum=opentelemetry.proto.metrics.v1.CountMinSketchEncoding" json:"encoding,omitempty"` - Rows int32 `protobuf:"varint,7,opt,name=rows,proto3" json:"rows,omitempty"` - Cols int32 `protobuf:"varint,8,opt,name=cols,proto3" json:"cols,omitempty"` Flags uint32 `protobuf:"varint,9,opt,name=flags,proto3" json:"flags,omitempty"` SeriesId uint64 `protobuf:"varint,10,opt,name=series_id,json=seriesId,proto3" json:"series_id,omitempty"` } @@ -2195,13 +2058,6 @@ func (x *CountMinSketchDataPoint) GetTimeUnixNano() uint64 { return 0 } -func (x *CountMinSketchDataPoint) GetSampleCount() uint64 { - if x != nil { - return x.SampleCount - } - return 0 -} - func (x *CountMinSketchDataPoint) GetSketch() []byte { if x != nil { return x.Sketch @@ -2216,20 +2072,6 @@ func (x *CountMinSketchDataPoint) GetEncoding() CountMinSketchEncoding { return CountMinSketchEncoding_COUNT_MIN_SKETCH_ENCODING_UNSPECIFIED } -func (x *CountMinSketchDataPoint) GetRows() int32 { - if x != nil { - return x.Rows - } - return 0 -} - -func (x *CountMinSketchDataPoint) GetCols() int32 { - if x != nil { - return x.Cols - } - return 0 -} - func (x *CountMinSketchDataPoint) GetFlags() uint32 { if x != nil { return x.Flags @@ -2250,11 +2092,11 @@ type HLLSketch struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // The time series data points. - DataPoints []*HLLSketchDataPoint `protobuf:"bytes,1,rep,name=data_points,json=dataPoints,proto3" json:"data_points,omitempty"` - // aggregation_temporality describes if the aggregator reports delta changes - // since last report time, or cumulative changes since a fixed start time. + DataPoints []*HLLSketchDataPoint `protobuf:"bytes,1,rep,name=data_points,json=dataPoints,proto3" json:"data_points,omitempty"` AggregationTemporality AggregationTemporality `protobuf:"varint,2,opt,name=aggregation_temporality,json=aggregationTemporality,proto3,enum=opentelemetry.proto.metrics.v1.AggregationTemporality" json:"aggregation_temporality,omitempty"` + // Refactor-2026-05: HLL register-bucket precision (= log2(register count)). + // Standard error ≈ 1.04 / sqrt(2^precision). Sent once per Metric emit. + Precision uint32 `protobuf:"varint,3,opt,name=precision,proto3" json:"precision,omitempty"` } func (x *HLLSketch) Reset() { @@ -2303,7 +2145,19 @@ func (x *HLLSketch) GetAggregationTemporality() AggregationTemporality { return AggregationTemporality_AGGREGATION_TEMPORALITY_UNSPECIFIED } +func (x *HLLSketch) GetPrecision() uint32 { + if x != nil { + return x.Precision + } + return 0 +} + // HLLSketchDataPoint is a single data point encoding cardinality estimations. +// +// Refactor-2026-05: removed `count` (derivable from sketch register +// state), removed `cardinality` (precomputed cache of +// sketch.estimate() — let the receiver evaluate on demand to avoid +// cache drift). `precision` lifted to parent container. type HLLSketchDataPoint struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2312,11 +2166,8 @@ type HLLSketchDataPoint struct { Attributes []*v11.KeyValue `protobuf:"bytes,1,rep,name=attributes,proto3" json:"attributes,omitempty"` StartTimeUnixNano uint64 `protobuf:"fixed64,2,opt,name=start_time_unix_nano,json=startTimeUnixNano,proto3" json:"start_time_unix_nano,omitempty"` TimeUnixNano uint64 `protobuf:"fixed64,3,opt,name=time_unix_nano,json=timeUnixNano,proto3" json:"time_unix_nano,omitempty"` - Count uint64 `protobuf:"varint,4,opt,name=count,proto3" json:"count,omitempty"` - Cardinality uint64 `protobuf:"varint,5,opt,name=cardinality,proto3" json:"cardinality,omitempty"` Sketch []byte `protobuf:"bytes,6,opt,name=sketch,proto3" json:"sketch,omitempty"` Encoding HLLSketchEncoding `protobuf:"varint,7,opt,name=encoding,proto3,enum=opentelemetry.proto.metrics.v1.HLLSketchEncoding" json:"encoding,omitempty"` - Precision uint32 `protobuf:"varint,8,opt,name=precision,proto3" json:"precision,omitempty"` Flags uint32 `protobuf:"varint,9,opt,name=flags,proto3" json:"flags,omitempty"` SeriesId uint64 `protobuf:"varint,10,opt,name=series_id,json=seriesId,proto3" json:"series_id,omitempty"` } @@ -2374,20 +2225,6 @@ func (x *HLLSketchDataPoint) GetTimeUnixNano() uint64 { return 0 } -func (x *HLLSketchDataPoint) GetCount() uint64 { - if x != nil { - return x.Count - } - return 0 -} - -func (x *HLLSketchDataPoint) GetCardinality() uint64 { - if x != nil { - return x.Cardinality - } - return 0 -} - func (x *HLLSketchDataPoint) GetSketch() []byte { if x != nil { return x.Sketch @@ -2402,13 +2239,6 @@ func (x *HLLSketchDataPoint) GetEncoding() HLLSketchEncoding { return HLLSketchEncoding_HLL_SKETCH_ENCODING_UNSPECIFIED } -func (x *HLLSketchDataPoint) GetPrecision() uint32 { - if x != nil { - return x.Precision - } - return 0 -} - func (x *HLLSketchDataPoint) GetFlags() uint32 { if x != nil { return x.Flags @@ -3577,7 +3407,7 @@ var file_opentelemetry_proto_metrics_v1_metrics_proto_rawDesc = []byte{ 0x31, 0x2e, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x16, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x69, 0x74, 0x79, - 0x22, 0xcf, 0x01, 0x0a, 0x08, 0x44, 0x44, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x12, 0x52, 0x0a, + 0x22, 0xfc, 0x01, 0x0a, 0x08, 0x44, 0x44, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x12, 0x52, 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, @@ -3590,133 +3420,119 @@ var file_opentelemetry_proto_metrics_v1_metrics_proto_rawDesc = []byte{ 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x16, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x69, - 0x74, 0x79, 0x22, 0x5c, 0x0a, 0x07, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x51, 0x0a, - 0x0b, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, - 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, - 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x44, 0x61, 0x74, 0x61, 0x50, - 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x73, - 0x22, 0x97, 0x05, 0x0a, 0x11, 0x44, 0x44, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x44, 0x61, 0x74, - 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x47, 0x0a, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, - 0x75, 0x74, 0x65, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x70, 0x65, - 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x65, 0x79, 0x56, 0x61, - 0x6c, 0x75, 0x65, 0x52, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x12, - 0x2f, 0x0a, 0x14, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, - 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x06, 0x52, 0x11, 0x73, - 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, 0x4e, 0x61, 0x6e, 0x6f, - 0x12, 0x24, 0x0a, 0x0e, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6e, 0x61, - 0x6e, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x06, 0x52, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x55, 0x6e, - 0x69, 0x78, 0x4e, 0x61, 0x6e, 0x6f, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x06, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x24, 0x0a, 0x0d, - 0x73, 0x75, 0x6d, 0x5f, 0x61, 0x73, 0x5f, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x01, 0x48, 0x00, 0x52, 0x0b, 0x73, 0x75, 0x6d, 0x41, 0x73, 0x44, 0x6f, 0x75, 0x62, - 0x6c, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x75, 0x6d, 0x5f, 0x61, 0x73, 0x5f, 0x69, 0x6e, 0x74, - 0x18, 0x0c, 0x20, 0x01, 0x28, 0x10, 0x48, 0x00, 0x52, 0x08, 0x73, 0x75, 0x6d, 0x41, 0x73, 0x49, - 0x6e, 0x74, 0x12, 0x24, 0x0a, 0x0d, 0x6d, 0x69, 0x6e, 0x5f, 0x61, 0x73, 0x5f, 0x64, 0x6f, 0x75, - 0x62, 0x6c, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x01, 0x48, 0x01, 0x52, 0x0b, 0x6d, 0x69, 0x6e, - 0x41, 0x73, 0x44, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x6d, 0x69, 0x6e, 0x5f, - 0x61, 0x73, 0x5f, 0x69, 0x6e, 0x74, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x10, 0x48, 0x01, 0x52, 0x08, - 0x6d, 0x69, 0x6e, 0x41, 0x73, 0x49, 0x6e, 0x74, 0x12, 0x24, 0x0a, 0x0d, 0x6d, 0x61, 0x78, 0x5f, - 0x61, 0x73, 0x5f, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x01, 0x48, - 0x02, 0x52, 0x0b, 0x6d, 0x61, 0x78, 0x41, 0x73, 0x44, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x12, 0x1e, - 0x0a, 0x0a, 0x6d, 0x61, 0x78, 0x5f, 0x61, 0x73, 0x5f, 0x69, 0x6e, 0x74, 0x18, 0x0e, 0x20, 0x01, - 0x28, 0x10, 0x48, 0x02, 0x52, 0x08, 0x6d, 0x61, 0x78, 0x41, 0x73, 0x49, 0x6e, 0x74, 0x12, 0x16, - 0x0a, 0x06, 0x73, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, - 0x73, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x12, 0x4c, 0x0a, 0x08, 0x65, 0x6e, 0x63, 0x6f, 0x64, 0x69, - 0x6e, 0x67, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x30, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, + 0x74, 0x79, 0x12, 0x2b, 0x0a, 0x11, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x61, + 0x63, 0x63, 0x75, 0x72, 0x61, 0x63, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x52, 0x10, 0x72, + 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x41, 0x63, 0x63, 0x75, 0x72, 0x61, 0x63, 0x79, 0x22, + 0x5c, 0x0a, 0x07, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x51, 0x0a, 0x0b, 0x64, 0x61, + 0x74, 0x61, 0x5f, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x30, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, + 0x2e, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, + 0x74, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x22, 0xc4, 0x03, + 0x0a, 0x11, 0x44, 0x44, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, + 0x69, 0x6e, 0x74, 0x12, 0x47, 0x0a, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, + 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, + 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, + 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x65, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, + 0x52, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x12, 0x2f, 0x0a, 0x14, + 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, + 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x06, 0x52, 0x11, 0x73, 0x74, 0x61, 0x72, + 0x74, 0x54, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, 0x4e, 0x61, 0x6e, 0x6f, 0x12, 0x24, 0x0a, + 0x0e, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x06, 0x52, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, 0x4e, + 0x61, 0x6e, 0x6f, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x18, 0x08, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x06, 0x73, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x12, 0x4c, 0x0a, 0x08, 0x65, + 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x30, 0x2e, + 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x44, + 0x44, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x52, + 0x08, 0x65, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x46, 0x0a, 0x09, 0x65, 0x78, 0x65, + 0x6d, 0x70, 0x6c, 0x61, 0x72, 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6f, + 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x72, 0x52, 0x09, 0x65, 0x78, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x72, + 0x73, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x69, 0x65, + 0x73, 0x5f, 0x69, 0x64, 0x18, 0x10, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x73, 0x65, 0x72, 0x69, + 0x65, 0x73, 0x49, 0x64, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, + 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x4a, 0x04, 0x08, 0x06, 0x10, 0x07, 0x4a, 0x04, 0x08, 0x07, + 0x10, 0x08, 0x4a, 0x04, 0x08, 0x0c, 0x10, 0x0d, 0x4a, 0x04, 0x08, 0x0d, 0x10, 0x0e, 0x4a, 0x04, + 0x08, 0x0e, 0x10, 0x0f, 0x22, 0xdf, 0x01, 0x0a, 0x09, 0x4b, 0x4c, 0x4c, 0x53, 0x6b, 0x65, 0x74, + 0x63, 0x68, 0x12, 0x53, 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x70, 0x6f, 0x69, 0x6e, 0x74, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, + 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, + 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x4c, 0x4c, 0x53, 0x6b, 0x65, 0x74, + 0x63, 0x68, 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x0a, 0x64, 0x61, 0x74, + 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x12, 0x6f, 0x0a, 0x17, 0x61, 0x67, 0x67, 0x72, 0x65, + 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x69, + 0x74, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, - 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x44, 0x53, 0x6b, 0x65, 0x74, + 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x69, 0x74, 0x79, + 0x52, 0x16, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x65, 0x6d, + 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x0c, 0x0a, 0x01, 0x6b, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x01, 0x6b, 0x22, 0xe6, 0x02, 0x0a, 0x12, 0x4b, 0x4c, 0x4c, 0x53, 0x6b, + 0x65, 0x74, 0x63, 0x68, 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x47, 0x0a, + 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, + 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x2e, 0x4b, 0x65, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0a, 0x61, 0x74, 0x74, 0x72, + 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x12, 0x2f, 0x0a, 0x14, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, + 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x06, 0x52, 0x11, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x55, + 0x6e, 0x69, 0x78, 0x4e, 0x61, 0x6e, 0x6f, 0x12, 0x24, 0x0a, 0x0e, 0x74, 0x69, 0x6d, 0x65, 0x5f, + 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x06, 0x52, + 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, 0x4e, 0x61, 0x6e, 0x6f, 0x12, 0x16, 0x0a, + 0x06, 0x73, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x73, + 0x6b, 0x65, 0x74, 0x63, 0x68, 0x12, 0x4d, 0x0a, 0x08, 0x65, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, + 0x67, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x31, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, + 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, + 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x4c, 0x4c, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x08, 0x65, 0x6e, 0x63, 0x6f, - 0x64, 0x69, 0x6e, 0x67, 0x12, 0x46, 0x0a, 0x09, 0x65, 0x78, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x72, - 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, + 0x64, 0x69, 0x6e, 0x67, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, 0x0a, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x65, + 0x72, 0x69, 0x65, 0x73, 0x5f, 0x69, 0x64, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x73, + 0x65, 0x72, 0x69, 0x65, 0x73, 0x49, 0x64, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x4a, 0x04, 0x08, + 0x05, 0x10, 0x06, 0x4a, 0x04, 0x08, 0x06, 0x10, 0x07, 0x4a, 0x04, 0x08, 0x07, 0x10, 0x08, 0x22, + 0xfd, 0x01, 0x0a, 0x0b, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x12, + 0x55, 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, + 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, + 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x53, 0x6b, 0x65, 0x74, 0x63, + 0x68, 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, + 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x12, 0x6f, 0x0a, 0x17, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x69, 0x74, + 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, - 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x65, 0x6d, 0x70, 0x6c, 0x61, - 0x72, 0x52, 0x09, 0x65, 0x78, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x72, 0x73, 0x12, 0x14, 0x0a, 0x05, - 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x66, 0x6c, 0x61, - 0x67, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x69, 0x65, 0x73, 0x5f, 0x69, 0x64, 0x18, - 0x10, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x73, 0x65, 0x72, 0x69, 0x65, 0x73, 0x49, 0x64, 0x42, - 0x05, 0x0a, 0x03, 0x73, 0x75, 0x6d, 0x42, 0x05, 0x0a, 0x03, 0x6d, 0x69, 0x6e, 0x42, 0x05, 0x0a, - 0x03, 0x6d, 0x61, 0x78, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x22, 0xd1, 0x01, 0x0a, 0x09, 0x4b, - 0x4c, 0x4c, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x12, 0x53, 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, - 0x5f, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x32, 0x2e, - 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x4b, - 0x4c, 0x4c, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, - 0x74, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x12, 0x6f, 0x0a, - 0x17, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x65, 0x6d, - 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, - 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, - 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x65, 0x6d, 0x70, 0x6f, - 0x72, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x16, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x54, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x22, 0x9a, - 0x03, 0x0a, 0x12, 0x4b, 0x4c, 0x4c, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x44, 0x61, 0x74, 0x61, - 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x47, 0x0a, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, - 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x70, 0x65, 0x6e, - 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, - 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x65, 0x79, 0x56, 0x61, 0x6c, - 0x75, 0x65, 0x52, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x12, 0x2f, - 0x0a, 0x14, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, - 0x78, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x06, 0x52, 0x11, 0x73, 0x74, - 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, 0x4e, 0x61, 0x6e, 0x6f, 0x12, - 0x24, 0x0a, 0x0e, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, - 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x06, 0x52, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, - 0x78, 0x4e, 0x61, 0x6e, 0x6f, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x06, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x73, - 0x75, 0x6d, 0x18, 0x05, 0x20, 0x01, 0x28, 0x01, 0x52, 0x03, 0x73, 0x75, 0x6d, 0x12, 0x10, 0x0a, - 0x03, 0x6d, 0x69, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x01, 0x52, 0x03, 0x6d, 0x69, 0x6e, 0x12, - 0x10, 0x0a, 0x03, 0x6d, 0x61, 0x78, 0x18, 0x07, 0x20, 0x01, 0x28, 0x01, 0x52, 0x03, 0x6d, 0x61, - 0x78, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x18, 0x08, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x06, 0x73, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x12, 0x4d, 0x0a, 0x08, 0x65, 0x6e, 0x63, - 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x31, 0x2e, 0x6f, 0x70, - 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x4c, 0x4c, - 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x08, - 0x65, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6c, 0x61, 0x67, - 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x12, 0x1b, - 0x0a, 0x09, 0x73, 0x65, 0x72, 0x69, 0x65, 0x73, 0x5f, 0x69, 0x64, 0x18, 0x0b, 0x20, 0x01, 0x28, - 0x04, 0x52, 0x08, 0x73, 0x65, 0x72, 0x69, 0x65, 0x73, 0x49, 0x64, 0x22, 0xd5, 0x01, 0x0a, 0x0b, - 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x12, 0x55, 0x0a, 0x0b, 0x64, - 0x61, 0x74, 0x61, 0x5f, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x34, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, - 0x31, 0x2e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x44, 0x61, 0x74, - 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, - 0x74, 0x73, 0x12, 0x6f, 0x0a, 0x17, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x36, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, - 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, - 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x54, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x16, 0x61, 0x67, 0x67, - 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, - 0x69, 0x74, 0x79, 0x22, 0xa0, 0x03, 0x0a, 0x14, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x53, 0x6b, 0x65, - 0x74, 0x63, 0x68, 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x47, 0x0a, 0x0a, - 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x27, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x76, 0x31, - 0x2e, 0x4b, 0x65, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, - 0x62, 0x75, 0x74, 0x65, 0x73, 0x12, 0x2f, 0x0a, 0x14, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, - 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x06, 0x52, 0x11, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x55, 0x6e, - 0x69, 0x78, 0x4e, 0x61, 0x6e, 0x6f, 0x12, 0x24, 0x0a, 0x0e, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, - 0x6e, 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x06, 0x52, 0x0c, - 0x74, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, 0x4e, 0x61, 0x6e, 0x6f, 0x12, 0x16, 0x0a, 0x06, - 0x73, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x73, 0x6b, - 0x65, 0x74, 0x63, 0x68, 0x12, 0x4f, 0x0a, 0x08, 0x65, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x33, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, - 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x53, 0x6b, 0x65, - 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x08, 0x65, 0x6e, 0x63, - 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x69, 0x6d, 0x65, 0x6e, 0x73, 0x69, - 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x64, 0x69, 0x6d, 0x65, 0x6e, 0x73, - 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x70, 0x73, 0x69, 0x6c, 0x6f, 0x6e, 0x18, 0x07, - 0x20, 0x01, 0x28, 0x01, 0x52, 0x07, 0x65, 0x70, 0x73, 0x69, 0x6c, 0x6f, 0x6e, 0x12, 0x14, 0x0a, - 0x05, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x18, 0x08, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x64, 0x65, - 0x6c, 0x74, 0x61, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, 0x09, 0x20, 0x01, - 0x28, 0x0d, 0x52, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x65, 0x72, - 0x69, 0x65, 0x73, 0x5f, 0x69, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x73, 0x65, - 0x72, 0x69, 0x65, 0x73, 0x49, 0x64, 0x22, 0xdb, 0x01, 0x0a, 0x0e, 0x43, 0x6f, 0x75, 0x6e, 0x74, + 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x52, + 0x16, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x65, 0x6d, 0x70, + 0x6f, 0x72, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x63, + 0x6f, 0x6c, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x63, 0x6f, 0x6c, 0x73, 0x22, + 0xe4, 0x02, 0x0a, 0x14, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x44, + 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x47, 0x0a, 0x0a, 0x61, 0x74, 0x74, 0x72, + 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6f, + 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x65, 0x79, + 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, + 0x73, 0x12, 0x2f, 0x0a, 0x14, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, + 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x06, 0x52, + 0x11, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, 0x4e, 0x61, + 0x6e, 0x6f, 0x12, 0x24, 0x0a, 0x0e, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, + 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x06, 0x52, 0x0c, 0x74, 0x69, 0x6d, 0x65, + 0x55, 0x6e, 0x69, 0x78, 0x4e, 0x61, 0x6e, 0x6f, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x6b, 0x65, 0x74, + 0x63, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x73, 0x6b, 0x65, 0x74, 0x63, 0x68, + 0x12, 0x4f, 0x0a, 0x08, 0x65, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x0e, 0x32, 0x33, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, + 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, + 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x45, + 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x08, 0x65, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, + 0x67, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x69, 0x65, + 0x73, 0x5f, 0x69, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x73, 0x65, 0x72, 0x69, + 0x65, 0x73, 0x49, 0x64, 0x4a, 0x04, 0x08, 0x06, 0x10, 0x07, 0x4a, 0x04, 0x08, 0x07, 0x10, 0x08, + 0x4a, 0x04, 0x08, 0x08, 0x10, 0x09, 0x22, 0x83, 0x02, 0x0a, 0x0e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x4d, 0x69, 0x6e, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x12, 0x58, 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, @@ -3730,99 +3546,72 @@ var file_opentelemetry_proto_metrics_v1_metrics_proto_rawDesc = []byte{ 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x16, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, - 0x6c, 0x69, 0x74, 0x79, 0x22, 0xa3, 0x03, 0x0a, 0x17, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x4d, 0x69, - 0x6e, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, - 0x12, 0x47, 0x0a, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, - 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, - 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x65, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0a, 0x61, - 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x12, 0x2f, 0x0a, 0x14, 0x73, 0x74, 0x61, - 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, - 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x06, 0x52, 0x11, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, - 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, 0x4e, 0x61, 0x6e, 0x6f, 0x12, 0x24, 0x0a, 0x0e, 0x74, 0x69, - 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x06, 0x52, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, 0x4e, 0x61, 0x6e, 0x6f, - 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f, - 0x75, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x06, 0x73, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x12, 0x52, 0x0a, 0x08, 0x65, - 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, 0x2e, - 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x43, - 0x6f, 0x75, 0x6e, 0x74, 0x4d, 0x69, 0x6e, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, - 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x08, 0x65, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x12, - 0x12, 0x0a, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x72, - 0x6f, 0x77, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, 0x6c, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, - 0x05, 0x52, 0x04, 0x63, 0x6f, 0x6c, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, - 0x18, 0x09, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x12, 0x1b, 0x0a, - 0x09, 0x73, 0x65, 0x72, 0x69, 0x65, 0x73, 0x5f, 0x69, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x04, - 0x52, 0x08, 0x73, 0x65, 0x72, 0x69, 0x65, 0x73, 0x49, 0x64, 0x22, 0xd1, 0x01, 0x0a, 0x09, 0x48, - 0x4c, 0x4c, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x12, 0x53, 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, - 0x5f, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x32, 0x2e, - 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x48, - 0x4c, 0x4c, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, - 0x74, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x12, 0x6f, 0x0a, - 0x17, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x65, 0x6d, - 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, - 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, - 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x65, 0x6d, 0x70, 0x6f, - 0x72, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x16, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x54, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x22, 0xa4, - 0x03, 0x0a, 0x12, 0x48, 0x4c, 0x4c, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x44, 0x61, 0x74, 0x61, - 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x47, 0x0a, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, - 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x70, 0x65, 0x6e, - 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, - 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x65, 0x79, 0x56, 0x61, 0x6c, - 0x75, 0x65, 0x52, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x12, 0x2f, - 0x0a, 0x14, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, - 0x78, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x06, 0x52, 0x11, 0x73, 0x74, - 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, 0x4e, 0x61, 0x6e, 0x6f, 0x12, - 0x24, 0x0a, 0x0e, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, - 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x06, 0x52, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, - 0x78, 0x4e, 0x61, 0x6e, 0x6f, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x20, 0x0a, 0x0b, 0x63, - 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, - 0x52, 0x0b, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x16, 0x0a, - 0x06, 0x73, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x73, - 0x6b, 0x65, 0x74, 0x63, 0x68, 0x12, 0x4d, 0x0a, 0x08, 0x65, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, - 0x67, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x31, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, - 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, - 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x4c, 0x4c, 0x53, 0x6b, 0x65, 0x74, + 0x6c, 0x69, 0x74, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, 0x6c, 0x73, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x63, 0x6f, 0x6c, 0x73, 0x22, 0xea, 0x02, 0x0a, + 0x17, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x4d, 0x69, 0x6e, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x44, + 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x47, 0x0a, 0x0a, 0x61, 0x74, 0x74, 0x72, + 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6f, + 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x65, 0x79, + 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, + 0x73, 0x12, 0x2f, 0x0a, 0x14, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, + 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x06, 0x52, + 0x11, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, 0x4e, 0x61, + 0x6e, 0x6f, 0x12, 0x24, 0x0a, 0x0e, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, + 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x06, 0x52, 0x0c, 0x74, 0x69, 0x6d, 0x65, + 0x55, 0x6e, 0x69, 0x78, 0x4e, 0x61, 0x6e, 0x6f, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x6b, 0x65, 0x74, + 0x63, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x73, 0x6b, 0x65, 0x74, 0x63, 0x68, + 0x12, 0x52, 0x0a, 0x08, 0x65, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x18, 0x06, 0x20, 0x01, + 0x28, 0x0e, 0x32, 0x36, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, + 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, + 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x4d, 0x69, 0x6e, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x08, 0x65, 0x6e, 0x63, 0x6f, - 0x64, 0x69, 0x6e, 0x67, 0x12, 0x1c, 0x0a, 0x09, 0x70, 0x72, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, - 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x70, 0x72, 0x65, 0x63, 0x69, 0x73, 0x69, - 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, - 0x0d, 0x52, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x69, - 0x65, 0x73, 0x5f, 0x69, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x73, 0x65, 0x72, - 0x69, 0x65, 0x73, 0x49, 0x64, 0x22, 0xf3, 0x02, 0x0a, 0x0f, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, - 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x47, 0x0a, 0x0a, 0x61, 0x74, 0x74, - 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, + 0x64, 0x69, 0x6e, 0x67, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, 0x09, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x65, + 0x72, 0x69, 0x65, 0x73, 0x5f, 0x69, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x73, + 0x65, 0x72, 0x69, 0x65, 0x73, 0x49, 0x64, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x4a, 0x04, 0x08, + 0x07, 0x10, 0x08, 0x4a, 0x04, 0x08, 0x08, 0x10, 0x09, 0x22, 0xef, 0x01, 0x0a, 0x09, 0x48, 0x4c, + 0x4c, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x12, 0x53, 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x5f, + 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x6f, + 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x4c, + 0x4c, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, + 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x12, 0x6f, 0x0a, 0x17, + 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x65, 0x6d, 0x70, + 0x6f, 0x72, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x65, - 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, - 0x65, 0x73, 0x12, 0x2f, 0x0a, 0x14, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, - 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x06, - 0x52, 0x11, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, 0x4e, - 0x61, 0x6e, 0x6f, 0x12, 0x24, 0x0a, 0x0e, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, - 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x06, 0x52, 0x0c, 0x74, 0x69, 0x6d, - 0x65, 0x55, 0x6e, 0x69, 0x78, 0x4e, 0x61, 0x6e, 0x6f, 0x12, 0x1d, 0x0a, 0x09, 0x61, 0x73, 0x5f, - 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x01, 0x48, 0x00, 0x52, 0x08, - 0x61, 0x73, 0x44, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x12, 0x17, 0x0a, 0x06, 0x61, 0x73, 0x5f, 0x69, - 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x10, 0x48, 0x00, 0x52, 0x05, 0x61, 0x73, 0x49, 0x6e, - 0x74, 0x12, 0x46, 0x0a, 0x09, 0x65, 0x78, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x72, 0x73, 0x18, 0x05, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, - 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, - 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x72, 0x52, 0x09, - 0x65, 0x78, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x72, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6c, 0x61, - 0x67, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x12, - 0x1b, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x69, 0x65, 0x73, 0x5f, 0x69, 0x64, 0x18, 0x09, 0x20, 0x01, - 0x28, 0x04, 0x52, 0x08, 0x73, 0x65, 0x72, 0x69, 0x65, 0x73, 0x49, 0x64, 0x42, 0x07, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x22, 0xf6, 0x03, 0x0a, 0x12, - 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, + 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x41, + 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x65, 0x6d, 0x70, 0x6f, 0x72, + 0x61, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x16, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x54, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x1c, 0x0a, + 0x09, 0x70, 0x72, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x09, 0x70, 0x72, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0xe0, 0x02, 0x0a, 0x12, + 0x48, 0x4c, 0x4c, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, + 0x6e, 0x74, 0x12, 0x47, 0x0a, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, + 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, + 0x6d, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x65, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, + 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x12, 0x2f, 0x0a, 0x14, 0x73, + 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6e, + 0x61, 0x6e, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x06, 0x52, 0x11, 0x73, 0x74, 0x61, 0x72, 0x74, + 0x54, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, 0x4e, 0x61, 0x6e, 0x6f, 0x12, 0x24, 0x0a, 0x0e, + 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x06, 0x52, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, 0x4e, 0x61, + 0x6e, 0x6f, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x18, 0x06, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x06, 0x73, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x12, 0x4d, 0x0a, 0x08, 0x65, 0x6e, + 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x31, 0x2e, 0x6f, + 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x4c, + 0x4c, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x52, + 0x08, 0x65, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6c, 0x61, + 0x67, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x12, + 0x1b, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x69, 0x65, 0x73, 0x5f, 0x69, 0x64, 0x18, 0x0a, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x08, 0x73, 0x65, 0x72, 0x69, 0x65, 0x73, 0x49, 0x64, 0x4a, 0x04, 0x08, 0x04, + 0x10, 0x05, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x4a, 0x04, 0x08, 0x08, 0x10, 0x09, 0x22, 0xf3, + 0x02, 0x0a, 0x0f, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x47, 0x0a, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, - 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, + 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x65, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x12, 0x2f, 0x0a, 0x14, 0x73, @@ -3831,206 +3620,229 @@ var file_opentelemetry_proto_metrics_v1_metrics_proto_rawDesc = []byte{ 0x54, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, 0x4e, 0x61, 0x6e, 0x6f, 0x12, 0x24, 0x0a, 0x0e, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x06, 0x52, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, 0x4e, 0x61, - 0x6e, 0x6f, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x06, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x15, 0x0a, 0x03, 0x73, 0x75, 0x6d, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x01, 0x48, 0x00, 0x52, 0x03, 0x73, 0x75, 0x6d, 0x88, 0x01, 0x01, 0x12, - 0x23, 0x0a, 0x0d, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, - 0x18, 0x06, 0x20, 0x03, 0x28, 0x06, 0x52, 0x0c, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x43, 0x6f, - 0x75, 0x6e, 0x74, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x65, 0x78, 0x70, 0x6c, 0x69, 0x63, 0x69, 0x74, - 0x5f, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x01, 0x52, 0x0e, 0x65, - 0x78, 0x70, 0x6c, 0x69, 0x63, 0x69, 0x74, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x73, 0x12, 0x46, 0x0a, - 0x09, 0x65, 0x78, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x72, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, + 0x6e, 0x6f, 0x12, 0x1d, 0x0a, 0x09, 0x61, 0x73, 0x5f, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x01, 0x48, 0x00, 0x52, 0x08, 0x61, 0x73, 0x44, 0x6f, 0x75, 0x62, 0x6c, + 0x65, 0x12, 0x17, 0x0a, 0x06, 0x61, 0x73, 0x5f, 0x69, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x10, 0x48, 0x00, 0x52, 0x05, 0x61, 0x73, 0x49, 0x6e, 0x74, 0x12, 0x46, 0x0a, 0x09, 0x65, 0x78, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x72, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, + 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x45, + 0x78, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x72, 0x52, 0x09, 0x65, 0x78, 0x65, 0x6d, 0x70, 0x6c, 0x61, + 0x72, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x69, + 0x65, 0x73, 0x5f, 0x69, 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x73, 0x65, 0x72, + 0x69, 0x65, 0x73, 0x49, 0x64, 0x42, 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x4a, 0x04, + 0x08, 0x01, 0x10, 0x02, 0x22, 0xf6, 0x03, 0x0a, 0x12, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, + 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x47, 0x0a, 0x0a, 0x61, + 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x27, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, + 0x4b, 0x65, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, + 0x75, 0x74, 0x65, 0x73, 0x12, 0x2f, 0x0a, 0x14, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, + 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x06, 0x52, 0x11, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, + 0x78, 0x4e, 0x61, 0x6e, 0x6f, 0x12, 0x24, 0x0a, 0x0e, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, + 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x06, 0x52, 0x0c, 0x74, + 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, 0x4e, 0x61, 0x6e, 0x6f, 0x12, 0x14, 0x0a, 0x05, 0x63, + 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x06, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, + 0x74, 0x12, 0x15, 0x0a, 0x03, 0x73, 0x75, 0x6d, 0x18, 0x05, 0x20, 0x01, 0x28, 0x01, 0x48, 0x00, + 0x52, 0x03, 0x73, 0x75, 0x6d, 0x88, 0x01, 0x01, 0x12, 0x23, 0x0a, 0x0d, 0x62, 0x75, 0x63, 0x6b, + 0x65, 0x74, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x06, 0x52, + 0x0c, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x12, 0x27, 0x0a, + 0x0f, 0x65, 0x78, 0x70, 0x6c, 0x69, 0x63, 0x69, 0x74, 0x5f, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x73, + 0x18, 0x07, 0x20, 0x03, 0x28, 0x01, 0x52, 0x0e, 0x65, 0x78, 0x70, 0x6c, 0x69, 0x63, 0x69, 0x74, + 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x73, 0x12, 0x46, 0x0a, 0x09, 0x65, 0x78, 0x65, 0x6d, 0x70, 0x6c, + 0x61, 0x72, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6f, 0x70, 0x65, 0x6e, + 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, + 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x65, 0x6d, 0x70, + 0x6c, 0x61, 0x72, 0x52, 0x09, 0x65, 0x78, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x72, 0x73, 0x12, 0x14, + 0x0a, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x66, + 0x6c, 0x61, 0x67, 0x73, 0x12, 0x15, 0x0a, 0x03, 0x6d, 0x69, 0x6e, 0x18, 0x0b, 0x20, 0x01, 0x28, + 0x01, 0x48, 0x01, 0x52, 0x03, 0x6d, 0x69, 0x6e, 0x88, 0x01, 0x01, 0x12, 0x15, 0x0a, 0x03, 0x6d, + 0x61, 0x78, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x01, 0x48, 0x02, 0x52, 0x03, 0x6d, 0x61, 0x78, 0x88, + 0x01, 0x01, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x69, 0x65, 0x73, 0x5f, 0x69, 0x64, 0x18, + 0x0d, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x73, 0x65, 0x72, 0x69, 0x65, 0x73, 0x49, 0x64, 0x42, + 0x06, 0x0a, 0x04, 0x5f, 0x73, 0x75, 0x6d, 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x6d, 0x69, 0x6e, 0x42, + 0x06, 0x0a, 0x04, 0x5f, 0x6d, 0x61, 0x78, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x22, 0x97, 0x06, + 0x0a, 0x1d, 0x45, 0x78, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x48, 0x69, 0x73, + 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, + 0x47, 0x0a, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, + 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, + 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x65, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0a, 0x61, 0x74, + 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x12, 0x2f, 0x0a, 0x14, 0x73, 0x74, 0x61, 0x72, + 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x06, 0x52, 0x11, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, + 0x65, 0x55, 0x6e, 0x69, 0x78, 0x4e, 0x61, 0x6e, 0x6f, 0x12, 0x24, 0x0a, 0x0e, 0x74, 0x69, 0x6d, + 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x06, 0x52, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, 0x4e, 0x61, 0x6e, 0x6f, 0x12, + 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x06, 0x52, 0x05, + 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x15, 0x0a, 0x03, 0x73, 0x75, 0x6d, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x01, 0x48, 0x00, 0x52, 0x03, 0x73, 0x75, 0x6d, 0x88, 0x01, 0x01, 0x12, 0x14, 0x0a, 0x05, + 0x73, 0x63, 0x61, 0x6c, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x11, 0x52, 0x05, 0x73, 0x63, 0x61, + 0x6c, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x7a, 0x65, 0x72, 0x6f, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, + 0x18, 0x07, 0x20, 0x01, 0x28, 0x06, 0x52, 0x09, 0x7a, 0x65, 0x72, 0x6f, 0x43, 0x6f, 0x75, 0x6e, + 0x74, 0x12, 0x61, 0x0a, 0x08, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x76, 0x65, 0x18, 0x08, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x45, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, + 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, + 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, + 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, + 0x6e, 0x74, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x52, 0x08, 0x70, 0x6f, 0x73, 0x69, + 0x74, 0x69, 0x76, 0x65, 0x12, 0x61, 0x0a, 0x08, 0x6e, 0x65, 0x67, 0x61, 0x74, 0x69, 0x76, 0x65, + 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x45, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, + 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x74, + 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, + 0x69, 0x61, 0x6c, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, + 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x52, 0x08, 0x6e, + 0x65, 0x67, 0x61, 0x74, 0x69, 0x76, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, + 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x12, 0x46, 0x0a, + 0x09, 0x65, 0x78, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x72, 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x72, 0x52, 0x09, 0x65, 0x78, 0x65, 0x6d, - 0x70, 0x6c, 0x61, 0x72, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, 0x0a, - 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x12, 0x15, 0x0a, 0x03, 0x6d, - 0x69, 0x6e, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x01, 0x48, 0x01, 0x52, 0x03, 0x6d, 0x69, 0x6e, 0x88, - 0x01, 0x01, 0x12, 0x15, 0x0a, 0x03, 0x6d, 0x61, 0x78, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x01, 0x48, - 0x02, 0x52, 0x03, 0x6d, 0x61, 0x78, 0x88, 0x01, 0x01, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x65, 0x72, - 0x69, 0x65, 0x73, 0x5f, 0x69, 0x64, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x73, 0x65, - 0x72, 0x69, 0x65, 0x73, 0x49, 0x64, 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x73, 0x75, 0x6d, 0x42, 0x06, - 0x0a, 0x04, 0x5f, 0x6d, 0x69, 0x6e, 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x6d, 0x61, 0x78, 0x4a, 0x04, - 0x08, 0x01, 0x10, 0x02, 0x22, 0x97, 0x06, 0x0a, 0x1d, 0x45, 0x78, 0x70, 0x6f, 0x6e, 0x65, 0x6e, - 0x74, 0x69, 0x61, 0x6c, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x44, 0x61, 0x74, - 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x47, 0x0a, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, - 0x75, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x70, 0x65, - 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x65, 0x79, 0x56, 0x61, - 0x6c, 0x75, 0x65, 0x52, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x12, - 0x2f, 0x0a, 0x14, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, - 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x06, 0x52, 0x11, 0x73, - 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, 0x4e, 0x61, 0x6e, 0x6f, - 0x12, 0x24, 0x0a, 0x0e, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6e, 0x61, - 0x6e, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x06, 0x52, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x55, 0x6e, - 0x69, 0x78, 0x4e, 0x61, 0x6e, 0x6f, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x06, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x15, 0x0a, 0x03, - 0x73, 0x75, 0x6d, 0x18, 0x05, 0x20, 0x01, 0x28, 0x01, 0x48, 0x00, 0x52, 0x03, 0x73, 0x75, 0x6d, - 0x88, 0x01, 0x01, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x11, 0x52, 0x05, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x7a, 0x65, 0x72, - 0x6f, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x06, 0x52, 0x09, 0x7a, - 0x65, 0x72, 0x6f, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x61, 0x0a, 0x08, 0x70, 0x6f, 0x73, 0x69, - 0x74, 0x69, 0x76, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x45, 0x2e, 0x6f, 0x70, 0x65, - 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x70, 0x6f, - 0x6e, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, - 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, - 0x73, 0x52, 0x08, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x76, 0x65, 0x12, 0x61, 0x0a, 0x08, 0x6e, - 0x65, 0x67, 0x61, 0x74, 0x69, 0x76, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x45, 0x2e, + 0x70, 0x6c, 0x61, 0x72, 0x73, 0x12, 0x15, 0x0a, 0x03, 0x6d, 0x69, 0x6e, 0x18, 0x0c, 0x20, 0x01, + 0x28, 0x01, 0x48, 0x01, 0x52, 0x03, 0x6d, 0x69, 0x6e, 0x88, 0x01, 0x01, 0x12, 0x15, 0x0a, 0x03, + 0x6d, 0x61, 0x78, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x01, 0x48, 0x02, 0x52, 0x03, 0x6d, 0x61, 0x78, + 0x88, 0x01, 0x01, 0x12, 0x25, 0x0a, 0x0e, 0x7a, 0x65, 0x72, 0x6f, 0x5f, 0x74, 0x68, 0x72, 0x65, + 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0d, 0x7a, 0x65, 0x72, + 0x6f, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x65, + 0x72, 0x69, 0x65, 0x73, 0x5f, 0x69, 0x64, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x73, + 0x65, 0x72, 0x69, 0x65, 0x73, 0x49, 0x64, 0x1a, 0x46, 0x0a, 0x07, 0x42, 0x75, 0x63, 0x6b, 0x65, + 0x74, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x11, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x62, 0x75, + 0x63, 0x6b, 0x65, 0x74, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, + 0x04, 0x52, 0x0c, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x42, + 0x06, 0x0a, 0x04, 0x5f, 0x73, 0x75, 0x6d, 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x6d, 0x69, 0x6e, 0x42, + 0x06, 0x0a, 0x04, 0x5f, 0x6d, 0x61, 0x78, 0x22, 0xc3, 0x03, 0x0a, 0x10, 0x53, 0x75, 0x6d, 0x6d, + 0x61, 0x72, 0x79, 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x47, 0x0a, 0x0a, + 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x27, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x76, 0x31, + 0x2e, 0x4b, 0x65, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, + 0x62, 0x75, 0x74, 0x65, 0x73, 0x12, 0x2f, 0x0a, 0x14, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, + 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x06, 0x52, 0x11, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x55, 0x6e, + 0x69, 0x78, 0x4e, 0x61, 0x6e, 0x6f, 0x12, 0x24, 0x0a, 0x0e, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, + 0x6e, 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x06, 0x52, 0x0c, + 0x74, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, 0x4e, 0x61, 0x6e, 0x6f, 0x12, 0x14, 0x0a, 0x05, + 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x06, 0x52, 0x05, 0x63, 0x6f, 0x75, + 0x6e, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x75, 0x6d, 0x18, 0x05, 0x20, 0x01, 0x28, 0x01, 0x52, + 0x03, 0x73, 0x75, 0x6d, 0x12, 0x69, 0x0a, 0x0f, 0x71, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x6c, 0x65, + 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x45, - 0x78, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, - 0x72, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x2e, 0x42, 0x75, 0x63, - 0x6b, 0x65, 0x74, 0x73, 0x52, 0x08, 0x6e, 0x65, 0x67, 0x61, 0x74, 0x69, 0x76, 0x65, 0x12, 0x14, - 0x0a, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x66, - 0x6c, 0x61, 0x67, 0x73, 0x12, 0x46, 0x0a, 0x09, 0x65, 0x78, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x72, - 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, - 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, - 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x65, 0x6d, 0x70, 0x6c, 0x61, - 0x72, 0x52, 0x09, 0x65, 0x78, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x72, 0x73, 0x12, 0x15, 0x0a, 0x03, - 0x6d, 0x69, 0x6e, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x01, 0x48, 0x01, 0x52, 0x03, 0x6d, 0x69, 0x6e, - 0x88, 0x01, 0x01, 0x12, 0x15, 0x0a, 0x03, 0x6d, 0x61, 0x78, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x01, - 0x48, 0x02, 0x52, 0x03, 0x6d, 0x61, 0x78, 0x88, 0x01, 0x01, 0x12, 0x25, 0x0a, 0x0e, 0x7a, 0x65, - 0x72, 0x6f, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x0e, 0x20, 0x01, - 0x28, 0x01, 0x52, 0x0d, 0x7a, 0x65, 0x72, 0x6f, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, - 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x69, 0x65, 0x73, 0x5f, 0x69, 0x64, 0x18, 0x0f, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x73, 0x65, 0x72, 0x69, 0x65, 0x73, 0x49, 0x64, 0x1a, 0x46, - 0x0a, 0x07, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, - 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x11, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, - 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x5f, 0x63, 0x6f, 0x75, 0x6e, - 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x04, 0x52, 0x0c, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, - 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x73, 0x75, 0x6d, 0x42, 0x06, - 0x0a, 0x04, 0x5f, 0x6d, 0x69, 0x6e, 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x6d, 0x61, 0x78, 0x22, 0xc3, - 0x03, 0x0a, 0x10, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, - 0x69, 0x6e, 0x74, 0x12, 0x47, 0x0a, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, + 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x53, + 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x2e, + 0x56, 0x61, 0x6c, 0x75, 0x65, 0x41, 0x74, 0x51, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x6c, 0x65, 0x52, + 0x0e, 0x71, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x6c, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x12, + 0x14, 0x0a, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, + 0x66, 0x6c, 0x61, 0x67, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x69, 0x65, 0x73, 0x5f, + 0x69, 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x73, 0x65, 0x72, 0x69, 0x65, 0x73, + 0x49, 0x64, 0x1a, 0x43, 0x0a, 0x0f, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x41, 0x74, 0x51, 0x75, 0x61, + 0x6e, 0x74, 0x69, 0x6c, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x71, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x6c, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x08, 0x71, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x6c, + 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x22, 0x85, 0x02, + 0x0a, 0x08, 0x45, 0x78, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x72, 0x12, 0x58, 0x0a, 0x13, 0x66, 0x69, + 0x6c, 0x74, 0x65, 0x72, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x65, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, - 0x52, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x12, 0x2f, 0x0a, 0x14, - 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, - 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x06, 0x52, 0x11, 0x73, 0x74, 0x61, 0x72, - 0x74, 0x54, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, 0x4e, 0x61, 0x6e, 0x6f, 0x12, 0x24, 0x0a, - 0x0e, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x06, 0x52, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, 0x4e, - 0x61, 0x6e, 0x6f, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x06, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x75, 0x6d, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x01, 0x52, 0x03, 0x73, 0x75, 0x6d, 0x12, 0x69, 0x0a, 0x0f, 0x71, - 0x75, 0x61, 0x6e, 0x74, 0x69, 0x6c, 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x06, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, - 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, - 0x63, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x44, 0x61, 0x74, - 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x41, 0x74, 0x51, 0x75, - 0x61, 0x6e, 0x74, 0x69, 0x6c, 0x65, 0x52, 0x0e, 0x71, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x6c, 0x65, - 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, - 0x08, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x12, 0x1b, 0x0a, 0x09, - 0x73, 0x65, 0x72, 0x69, 0x65, 0x73, 0x5f, 0x69, 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, - 0x08, 0x73, 0x65, 0x72, 0x69, 0x65, 0x73, 0x49, 0x64, 0x1a, 0x43, 0x0a, 0x0f, 0x56, 0x61, 0x6c, - 0x75, 0x65, 0x41, 0x74, 0x51, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x6c, 0x65, 0x12, 0x1a, 0x0a, 0x08, - 0x71, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x08, - 0x71, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x4a, 0x04, - 0x08, 0x01, 0x10, 0x02, 0x22, 0x85, 0x02, 0x0a, 0x08, 0x45, 0x78, 0x65, 0x6d, 0x70, 0x6c, 0x61, - 0x72, 0x12, 0x58, 0x0a, 0x13, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x65, 0x64, 0x5f, 0x61, 0x74, - 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, - 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4b, - 0x65, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x12, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x65, - 0x64, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x74, - 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x06, 0x52, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, 0x4e, 0x61, 0x6e, - 0x6f, 0x12, 0x1d, 0x0a, 0x09, 0x61, 0x73, 0x5f, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x01, 0x48, 0x00, 0x52, 0x08, 0x61, 0x73, 0x44, 0x6f, 0x75, 0x62, 0x6c, 0x65, - 0x12, 0x17, 0x0a, 0x06, 0x61, 0x73, 0x5f, 0x69, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x10, - 0x48, 0x00, 0x52, 0x05, 0x61, 0x73, 0x49, 0x6e, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x73, 0x70, 0x61, - 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x73, 0x70, 0x61, 0x6e, - 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x74, 0x72, 0x61, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x74, 0x72, 0x61, 0x63, 0x65, 0x49, 0x64, 0x42, 0x07, 0x0a, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x2a, 0xb9, 0x01, 0x0a, - 0x10, 0x44, 0x44, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, - 0x67, 0x12, 0x21, 0x0a, 0x1d, 0x44, 0x44, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, - 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, - 0x45, 0x44, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x44, 0x44, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, - 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x50, 0x52, 0x4f, 0x54, 0x4f, 0x10, - 0x01, 0x12, 0x21, 0x0a, 0x1d, 0x44, 0x44, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, - 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x50, 0x52, 0x4f, 0x54, 0x4f, 0x5f, 0x44, 0x45, 0x4c, - 0x54, 0x41, 0x10, 0x02, 0x12, 0x1d, 0x0a, 0x19, 0x44, 0x44, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, - 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x4d, 0x53, 0x47, 0x50, 0x41, 0x43, - 0x4b, 0x10, 0x03, 0x12, 0x23, 0x0a, 0x1f, 0x44, 0x44, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, - 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x4d, 0x53, 0x47, 0x50, 0x41, 0x43, 0x4b, - 0x5f, 0x44, 0x45, 0x4c, 0x54, 0x41, 0x10, 0x04, 0x2a, 0x9f, 0x01, 0x0a, 0x11, 0x4b, 0x4c, 0x4c, - 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x23, - 0x0a, 0x1f, 0x4b, 0x4c, 0x4c, 0x5f, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, - 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, - 0x44, 0x10, 0x00, 0x12, 0x1d, 0x0a, 0x19, 0x4b, 0x4c, 0x4c, 0x5f, 0x53, 0x4b, 0x45, 0x54, 0x43, - 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x50, 0x52, 0x4f, 0x54, 0x4f, - 0x10, 0x01, 0x12, 0x1f, 0x0a, 0x1b, 0x4b, 0x4c, 0x4c, 0x5f, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, - 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x4d, 0x53, 0x47, 0x50, 0x41, 0x43, - 0x4b, 0x10, 0x03, 0x12, 0x25, 0x0a, 0x21, 0x4b, 0x4c, 0x4c, 0x5f, 0x53, 0x4b, 0x45, 0x54, 0x43, - 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x4d, 0x53, 0x47, 0x50, 0x41, - 0x43, 0x4b, 0x5f, 0x44, 0x45, 0x4c, 0x54, 0x41, 0x10, 0x04, 0x2a, 0xca, 0x01, 0x0a, 0x13, 0x43, - 0x6f, 0x75, 0x6e, 0x74, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x69, - 0x6e, 0x67, 0x12, 0x25, 0x0a, 0x21, 0x43, 0x4f, 0x55, 0x4e, 0x54, 0x5f, 0x53, 0x4b, 0x45, 0x54, - 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x55, 0x4e, 0x53, 0x50, - 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1f, 0x0a, 0x1b, 0x43, 0x4f, 0x55, - 0x4e, 0x54, 0x5f, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, - 0x4e, 0x47, 0x5f, 0x50, 0x52, 0x4f, 0x54, 0x4f, 0x10, 0x01, 0x12, 0x1f, 0x0a, 0x1b, 0x43, 0x4f, - 0x55, 0x4e, 0x54, 0x5f, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, - 0x49, 0x4e, 0x47, 0x5f, 0x44, 0x45, 0x4c, 0x54, 0x41, 0x10, 0x02, 0x12, 0x21, 0x0a, 0x1d, 0x43, - 0x4f, 0x55, 0x4e, 0x54, 0x5f, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, - 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x4d, 0x53, 0x47, 0x50, 0x41, 0x43, 0x4b, 0x10, 0x03, 0x12, 0x27, - 0x0a, 0x23, 0x43, 0x4f, 0x55, 0x4e, 0x54, 0x5f, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, - 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x4d, 0x53, 0x47, 0x50, 0x41, 0x43, 0x4b, 0x5f, - 0x44, 0x45, 0x4c, 0x54, 0x41, 0x10, 0x04, 0x2a, 0xe1, 0x01, 0x0a, 0x16, 0x43, 0x6f, 0x75, 0x6e, - 0x74, 0x4d, 0x69, 0x6e, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x69, - 0x6e, 0x67, 0x12, 0x29, 0x0a, 0x25, 0x43, 0x4f, 0x55, 0x4e, 0x54, 0x5f, 0x4d, 0x49, 0x4e, 0x5f, + 0x52, 0x12, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x65, 0x64, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, + 0x75, 0x74, 0x65, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, + 0x78, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x06, 0x52, 0x0c, 0x74, 0x69, + 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, 0x4e, 0x61, 0x6e, 0x6f, 0x12, 0x1d, 0x0a, 0x09, 0x61, 0x73, + 0x5f, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x48, 0x00, 0x52, + 0x08, 0x61, 0x73, 0x44, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x12, 0x17, 0x0a, 0x06, 0x61, 0x73, 0x5f, + 0x69, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x10, 0x48, 0x00, 0x52, 0x05, 0x61, 0x73, 0x49, + 0x6e, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x73, 0x70, 0x61, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x06, 0x73, 0x70, 0x61, 0x6e, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x74, + 0x72, 0x61, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x74, + 0x72, 0x61, 0x63, 0x65, 0x49, 0x64, 0x42, 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x4a, + 0x04, 0x08, 0x01, 0x10, 0x02, 0x2a, 0xb9, 0x01, 0x0a, 0x10, 0x44, 0x44, 0x53, 0x6b, 0x65, 0x74, + 0x63, 0x68, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x21, 0x0a, 0x1d, 0x44, 0x44, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, - 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x23, 0x0a, - 0x1f, 0x43, 0x4f, 0x55, 0x4e, 0x54, 0x5f, 0x4d, 0x49, 0x4e, 0x5f, 0x53, 0x4b, 0x45, 0x54, 0x43, - 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x50, 0x52, 0x4f, 0x54, 0x4f, - 0x10, 0x01, 0x12, 0x23, 0x0a, 0x1f, 0x43, 0x4f, 0x55, 0x4e, 0x54, 0x5f, 0x4d, 0x49, 0x4e, 0x5f, + 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1b, 0x0a, + 0x17, 0x44, 0x44, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, + 0x4e, 0x47, 0x5f, 0x50, 0x52, 0x4f, 0x54, 0x4f, 0x10, 0x01, 0x12, 0x21, 0x0a, 0x1d, 0x44, 0x44, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, - 0x44, 0x45, 0x4c, 0x54, 0x41, 0x10, 0x02, 0x12, 0x25, 0x0a, 0x21, 0x43, 0x4f, 0x55, 0x4e, 0x54, - 0x5f, 0x4d, 0x49, 0x4e, 0x5f, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, - 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x4d, 0x53, 0x47, 0x50, 0x41, 0x43, 0x4b, 0x10, 0x03, 0x12, 0x2b, - 0x0a, 0x27, 0x43, 0x4f, 0x55, 0x4e, 0x54, 0x5f, 0x4d, 0x49, 0x4e, 0x5f, 0x53, 0x4b, 0x45, 0x54, - 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x4d, 0x53, 0x47, 0x50, - 0x41, 0x43, 0x4b, 0x5f, 0x44, 0x45, 0x4c, 0x54, 0x41, 0x10, 0x04, 0x2a, 0xbe, 0x01, 0x0a, 0x11, - 0x48, 0x4c, 0x4c, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, - 0x67, 0x12, 0x23, 0x0a, 0x1f, 0x48, 0x4c, 0x4c, 0x5f, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, - 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, - 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1d, 0x0a, 0x19, 0x48, 0x4c, 0x4c, 0x5f, 0x53, 0x4b, - 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x50, 0x52, - 0x4f, 0x54, 0x4f, 0x10, 0x01, 0x12, 0x1d, 0x0a, 0x19, 0x48, 0x4c, 0x4c, 0x5f, 0x53, 0x4b, 0x45, + 0x50, 0x52, 0x4f, 0x54, 0x4f, 0x5f, 0x44, 0x45, 0x4c, 0x54, 0x41, 0x10, 0x02, 0x12, 0x1d, 0x0a, + 0x19, 0x44, 0x44, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, + 0x4e, 0x47, 0x5f, 0x4d, 0x53, 0x47, 0x50, 0x41, 0x43, 0x4b, 0x10, 0x03, 0x12, 0x23, 0x0a, 0x1f, + 0x44, 0x44, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, + 0x47, 0x5f, 0x4d, 0x53, 0x47, 0x50, 0x41, 0x43, 0x4b, 0x5f, 0x44, 0x45, 0x4c, 0x54, 0x41, 0x10, + 0x04, 0x2a, 0x9f, 0x01, 0x0a, 0x11, 0x4b, 0x4c, 0x4c, 0x53, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x45, + 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x23, 0x0a, 0x1f, 0x4b, 0x4c, 0x4c, 0x5f, 0x53, + 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x55, + 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1d, 0x0a, 0x19, + 0x4b, 0x4c, 0x4c, 0x5f, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, + 0x49, 0x4e, 0x47, 0x5f, 0x50, 0x52, 0x4f, 0x54, 0x4f, 0x10, 0x01, 0x12, 0x1f, 0x0a, 0x1b, 0x4b, + 0x4c, 0x4c, 0x5f, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, + 0x4e, 0x47, 0x5f, 0x4d, 0x53, 0x47, 0x50, 0x41, 0x43, 0x4b, 0x10, 0x03, 0x12, 0x25, 0x0a, 0x21, + 0x4b, 0x4c, 0x4c, 0x5f, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, + 0x49, 0x4e, 0x47, 0x5f, 0x4d, 0x53, 0x47, 0x50, 0x41, 0x43, 0x4b, 0x5f, 0x44, 0x45, 0x4c, 0x54, + 0x41, 0x10, 0x04, 0x2a, 0xca, 0x01, 0x0a, 0x13, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x53, 0x6b, 0x65, + 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x25, 0x0a, 0x21, 0x43, + 0x4f, 0x55, 0x4e, 0x54, 0x5f, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, + 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, + 0x10, 0x00, 0x12, 0x1f, 0x0a, 0x1b, 0x43, 0x4f, 0x55, 0x4e, 0x54, 0x5f, 0x53, 0x4b, 0x45, 0x54, + 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x50, 0x52, 0x4f, 0x54, + 0x4f, 0x10, 0x01, 0x12, 0x1f, 0x0a, 0x1b, 0x43, 0x4f, 0x55, 0x4e, 0x54, 0x5f, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x44, 0x45, 0x4c, - 0x54, 0x41, 0x10, 0x02, 0x12, 0x1f, 0x0a, 0x1b, 0x48, 0x4c, 0x4c, 0x5f, 0x53, 0x4b, 0x45, 0x54, - 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x4d, 0x53, 0x47, 0x50, - 0x41, 0x43, 0x4b, 0x10, 0x03, 0x12, 0x25, 0x0a, 0x21, 0x48, 0x4c, 0x4c, 0x5f, 0x53, 0x4b, 0x45, - 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x4d, 0x53, 0x47, - 0x50, 0x41, 0x43, 0x4b, 0x5f, 0x44, 0x45, 0x4c, 0x54, 0x41, 0x10, 0x04, 0x2a, 0x8c, 0x01, 0x0a, - 0x16, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x65, 0x6d, 0x70, - 0x6f, 0x72, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x27, 0x0a, 0x23, 0x41, 0x47, 0x47, 0x52, 0x45, - 0x47, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x45, 0x4d, 0x50, 0x4f, 0x52, 0x41, 0x4c, 0x49, - 0x54, 0x59, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, - 0x12, 0x21, 0x0a, 0x1d, 0x41, 0x47, 0x47, 0x52, 0x45, 0x47, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, - 0x54, 0x45, 0x4d, 0x50, 0x4f, 0x52, 0x41, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x44, 0x45, 0x4c, 0x54, - 0x41, 0x10, 0x01, 0x12, 0x26, 0x0a, 0x22, 0x41, 0x47, 0x47, 0x52, 0x45, 0x47, 0x41, 0x54, 0x49, - 0x4f, 0x4e, 0x5f, 0x54, 0x45, 0x4d, 0x50, 0x4f, 0x52, 0x41, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x43, - 0x55, 0x4d, 0x55, 0x4c, 0x41, 0x54, 0x49, 0x56, 0x45, 0x10, 0x02, 0x2a, 0x5e, 0x0a, 0x0e, 0x44, - 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x46, 0x6c, 0x61, 0x67, 0x73, 0x12, 0x1f, 0x0a, - 0x1b, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x50, 0x4f, 0x49, 0x4e, 0x54, 0x5f, 0x46, 0x4c, 0x41, 0x47, - 0x53, 0x5f, 0x44, 0x4f, 0x5f, 0x4e, 0x4f, 0x54, 0x5f, 0x55, 0x53, 0x45, 0x10, 0x00, 0x12, 0x2b, - 0x0a, 0x27, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x50, 0x4f, 0x49, 0x4e, 0x54, 0x5f, 0x46, 0x4c, 0x41, - 0x47, 0x53, 0x5f, 0x4e, 0x4f, 0x5f, 0x52, 0x45, 0x43, 0x4f, 0x52, 0x44, 0x45, 0x44, 0x5f, 0x56, - 0x41, 0x4c, 0x55, 0x45, 0x5f, 0x4d, 0x41, 0x53, 0x4b, 0x10, 0x01, 0x42, 0x7f, 0x0a, 0x21, 0x69, - 0x6f, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, - 0x42, 0x0c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, - 0x5a, 0x29, 0x67, 0x6f, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, - 0x72, 0x79, 0x2e, 0x69, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6f, 0x74, 0x6c, 0x70, - 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2f, 0x76, 0x31, 0xaa, 0x02, 0x1e, 0x4f, 0x70, - 0x65, 0x6e, 0x54, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x50, 0x72, 0x6f, 0x74, - 0x6f, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x54, 0x41, 0x10, 0x02, 0x12, 0x21, 0x0a, 0x1d, 0x43, 0x4f, 0x55, 0x4e, 0x54, 0x5f, 0x53, 0x4b, + 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x4d, 0x53, + 0x47, 0x50, 0x41, 0x43, 0x4b, 0x10, 0x03, 0x12, 0x27, 0x0a, 0x23, 0x43, 0x4f, 0x55, 0x4e, 0x54, + 0x5f, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, + 0x5f, 0x4d, 0x53, 0x47, 0x50, 0x41, 0x43, 0x4b, 0x5f, 0x44, 0x45, 0x4c, 0x54, 0x41, 0x10, 0x04, + 0x2a, 0xe1, 0x01, 0x0a, 0x16, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x4d, 0x69, 0x6e, 0x53, 0x6b, 0x65, + 0x74, 0x63, 0x68, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x29, 0x0a, 0x25, 0x43, + 0x4f, 0x55, 0x4e, 0x54, 0x5f, 0x4d, 0x49, 0x4e, 0x5f, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, + 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, + 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x23, 0x0a, 0x1f, 0x43, 0x4f, 0x55, 0x4e, 0x54, 0x5f, + 0x4d, 0x49, 0x4e, 0x5f, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, + 0x49, 0x4e, 0x47, 0x5f, 0x50, 0x52, 0x4f, 0x54, 0x4f, 0x10, 0x01, 0x12, 0x23, 0x0a, 0x1f, 0x43, + 0x4f, 0x55, 0x4e, 0x54, 0x5f, 0x4d, 0x49, 0x4e, 0x5f, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, + 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x44, 0x45, 0x4c, 0x54, 0x41, 0x10, 0x02, + 0x12, 0x25, 0x0a, 0x21, 0x43, 0x4f, 0x55, 0x4e, 0x54, 0x5f, 0x4d, 0x49, 0x4e, 0x5f, 0x53, 0x4b, + 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x4d, 0x53, + 0x47, 0x50, 0x41, 0x43, 0x4b, 0x10, 0x03, 0x12, 0x2b, 0x0a, 0x27, 0x43, 0x4f, 0x55, 0x4e, 0x54, + 0x5f, 0x4d, 0x49, 0x4e, 0x5f, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, + 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x4d, 0x53, 0x47, 0x50, 0x41, 0x43, 0x4b, 0x5f, 0x44, 0x45, 0x4c, + 0x54, 0x41, 0x10, 0x04, 0x2a, 0xbe, 0x01, 0x0a, 0x11, 0x48, 0x4c, 0x4c, 0x53, 0x6b, 0x65, 0x74, + 0x63, 0x68, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x23, 0x0a, 0x1f, 0x48, 0x4c, + 0x4c, 0x5f, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, + 0x47, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, + 0x1d, 0x0a, 0x19, 0x48, 0x4c, 0x4c, 0x5f, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, + 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x50, 0x52, 0x4f, 0x54, 0x4f, 0x10, 0x01, 0x12, 0x1d, + 0x0a, 0x19, 0x48, 0x4c, 0x4c, 0x5f, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, + 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x44, 0x45, 0x4c, 0x54, 0x41, 0x10, 0x02, 0x12, 0x1f, 0x0a, + 0x1b, 0x48, 0x4c, 0x4c, 0x5f, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, 0x4f, + 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x4d, 0x53, 0x47, 0x50, 0x41, 0x43, 0x4b, 0x10, 0x03, 0x12, 0x25, + 0x0a, 0x21, 0x48, 0x4c, 0x4c, 0x5f, 0x53, 0x4b, 0x45, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x43, + 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x4d, 0x53, 0x47, 0x50, 0x41, 0x43, 0x4b, 0x5f, 0x44, 0x45, + 0x4c, 0x54, 0x41, 0x10, 0x04, 0x2a, 0x8c, 0x01, 0x0a, 0x16, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x69, 0x74, 0x79, + 0x12, 0x27, 0x0a, 0x23, 0x41, 0x47, 0x47, 0x52, 0x45, 0x47, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, + 0x54, 0x45, 0x4d, 0x50, 0x4f, 0x52, 0x41, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x55, 0x4e, 0x53, 0x50, + 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x21, 0x0a, 0x1d, 0x41, 0x47, 0x47, + 0x52, 0x45, 0x47, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x45, 0x4d, 0x50, 0x4f, 0x52, 0x41, + 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x44, 0x45, 0x4c, 0x54, 0x41, 0x10, 0x01, 0x12, 0x26, 0x0a, 0x22, + 0x41, 0x47, 0x47, 0x52, 0x45, 0x47, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x45, 0x4d, 0x50, + 0x4f, 0x52, 0x41, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x43, 0x55, 0x4d, 0x55, 0x4c, 0x41, 0x54, 0x49, + 0x56, 0x45, 0x10, 0x02, 0x2a, 0x5e, 0x0a, 0x0e, 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, + 0x74, 0x46, 0x6c, 0x61, 0x67, 0x73, 0x12, 0x1f, 0x0a, 0x1b, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x50, + 0x4f, 0x49, 0x4e, 0x54, 0x5f, 0x46, 0x4c, 0x41, 0x47, 0x53, 0x5f, 0x44, 0x4f, 0x5f, 0x4e, 0x4f, + 0x54, 0x5f, 0x55, 0x53, 0x45, 0x10, 0x00, 0x12, 0x2b, 0x0a, 0x27, 0x44, 0x41, 0x54, 0x41, 0x5f, + 0x50, 0x4f, 0x49, 0x4e, 0x54, 0x5f, 0x46, 0x4c, 0x41, 0x47, 0x53, 0x5f, 0x4e, 0x4f, 0x5f, 0x52, + 0x45, 0x43, 0x4f, 0x52, 0x44, 0x45, 0x44, 0x5f, 0x56, 0x41, 0x4c, 0x55, 0x45, 0x5f, 0x4d, 0x41, + 0x53, 0x4b, 0x10, 0x01, 0x42, 0x7f, 0x0a, 0x21, 0x69, 0x6f, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x74, + 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, + 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x76, 0x31, 0x42, 0x0c, 0x4d, 0x65, 0x74, 0x72, 0x69, + 0x63, 0x73, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x29, 0x67, 0x6f, 0x2e, 0x6f, 0x70, + 0x65, 0x6e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x69, 0x6f, 0x2f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6f, 0x74, 0x6c, 0x70, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, + 0x73, 0x2f, 0x76, 0x31, 0xaa, 0x02, 0x1e, 0x4f, 0x70, 0x65, 0x6e, 0x54, 0x65, 0x6c, 0x65, 0x6d, + 0x65, 0x74, 0x72, 0x79, 0x2e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, + 0x63, 0x73, 0x2e, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -4480,14 +4292,6 @@ func file_opentelemetry_proto_metrics_v1_metrics_proto_init() { (*Metric_Countminsketch)(nil), (*Metric_Hllsketch)(nil), } - file_opentelemetry_proto_metrics_v1_metrics_proto_msgTypes[10].OneofWrappers = []interface{}{ - (*DDSketchDataPoint_SumAsDouble)(nil), - (*DDSketchDataPoint_SumAsInt)(nil), - (*DDSketchDataPoint_MinAsDouble)(nil), - (*DDSketchDataPoint_MinAsInt)(nil), - (*DDSketchDataPoint_MaxAsDouble)(nil), - (*DDSketchDataPoint_MaxAsInt)(nil), - } file_opentelemetry_proto_metrics_v1_metrics_proto_msgTypes[19].OneofWrappers = []interface{}{ (*NumberDataPoint_AsDouble)(nil), (*NumberDataPoint_AsInt)(nil), diff --git a/opentelemetry-proto-patch/opentelemetry/proto/collector/metrics/v1/metrics_service.proto b/opentelemetry-proto-patch/opentelemetry/proto/collector/metrics/v1/metrics_service.proto index a2e98e02..0c105abb 100644 --- a/opentelemetry-proto-patch/opentelemetry/proto/collector/metrics/v1/metrics_service.proto +++ b/opentelemetry-proto-patch/opentelemetry/proto/collector/metrics/v1/metrics_service.proto @@ -63,6 +63,17 @@ message ExportMetricsServiceResponse { // assignments and omit the associated attributes on subsequent exports once // a series_id is known. repeated SeriesAssignment series_assignments = 2; + + // Refactor-2026-05: sids the receiver did NOT recognize in this Export. + // Sent when the receiver's series-id cache has diverged from the + // sender's (e.g., after backend restart without persistence, or any + // sid-cache eviction). Senders MUST evict the listed sids from their + // local cache; subsequent emits MUST re-attach `attributes` so the + // receiver can re-resolve and return fresh SeriesAssignments. This + // field is the universal recovery primitive — same code path that + // handles cold-start bootstrap also handles every cache-divergence + // failure mode. + repeated uint64 unknown_series_ids = 3; } message ExportMetricsPartialSuccess { diff --git a/opentelemetry-proto-patch/opentelemetry/proto/metrics/v1/metrics.proto b/opentelemetry-proto-patch/opentelemetry/proto/metrics/v1/metrics.proto index 7d43ccec..cf5ee296 100644 --- a/opentelemetry-proto-patch/opentelemetry/proto/metrics/v1/metrics.proto +++ b/opentelemetry-proto-patch/opentelemetry/proto/metrics/v1/metrics.proto @@ -289,6 +289,13 @@ message DDSketch { // aggregation_temporality describes if the aggregator reports delta changes // since last report time, or cumulative changes since a fixed start time. AggregationTemporality aggregation_temporality = 2; + + // Refactor-2026-05: sketch instance config — DDSketch's relative + // accuracy (alpha). Sent ONCE per Metric emit (not per DataPoint), + // since this is a per-sketch-instance parameter, not per-window + // mutable state. Backend reads this into + // SketchInstanceMetadata.sketch_config on first sight and caches. + double relative_accuracy = 3; } // Summary metric data are used to convey quantile summaries, @@ -307,55 +314,31 @@ message Summary { } // DDSketchDataPoint is a single data point that encodes a distribution using the DDSketch format. +// +// Refactor-2026-05: removed precomputed/duplicated fields (count, sum, +// min, max) — they are derivable from the sketch payload, so storing +// them per-DataPoint duplicates state and risks cache-invalidation +// bugs. Field numbers reserved for forward/backward compat. message DDSketchDataPoint { - reserved 1; + reserved 1; // legacy "labels" (replaced by `attributes`) + reserved 4, 5, 6, 7, 12, 13, 14; // refactor-2026-05: count, sum oneof, min oneof, max oneof — derivable from sketch state // The set of key/value pairs that uniquely identify the timeseries from // where this point belongs. The list MUST be omitted (or present but empty) // whenever `series_id` is non-zero, and MUST be populated when `series_id` // is zero so the receiver can mint a new entry. - // Attribute keys MUST be unique (it is not allowed to have more than one - // attribute with the same key). - // The behavior of software that receives duplicated keys can be unpredictable. + // Attribute keys MUST be unique. After group-by rollup at the agent's + // sketch processor, `attributes.keys()` IS the group-by KEY set; + // `attributes` (as a map) is the group-by VALUES for this series. repeated opentelemetry.proto.common.v1.KeyValue attributes = 9; // StartTimeUnixNano is optional but strongly encouraged, see the // the detailed comments above Metric. - // - // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January - // 1970. fixed64 start_time_unix_nano = 2; // TimeUnixNano is required, see the detailed comments above Metric. - // - // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January - // 1970. fixed64 time_unix_nano = 3; - // Count is the number of measurements that were aggregated into this sketch. - fixed64 count = 4; - - // Sum is the total of all measurements aggregated for this sketch. - // At most one of the fields in this oneof may be set at a time. - oneof sum { - double sum_as_double = 5; - sfixed64 sum_as_int = 12; - } - - // Min is the minimum value over (start_time, end_time]. Optional. - // At most one of the fields in this oneof may be set at a time. - oneof min { - double min_as_double = 6; - sfixed64 min_as_int = 13; - } - - // Max is the maximum value over (start_time, end_time]. Optional. - // At most one of the fields in this oneof may be set at a time. - oneof max { - double max_as_double = 7; - sfixed64 max_as_int = 14; - } - // Serialized DDSketch payload. The encoding is described by the encoding field. bytes sketch = 8; @@ -373,6 +356,9 @@ message DDSketchDataPoint { // series_id references a collector-assigned series descriptor. Receivers MUST // reject the point unless exactly one of (`series_id` != 0, `attributes` populated) // is true. + // After Phase 5 (centralized resolver at asap-query-backend), this id + // is the BACKEND-assigned global identifier for `(metric_name, attributes)`, + // shared across all hops in the pipeline. uint64 series_id = 16; } @@ -394,23 +380,22 @@ enum DDSketchEncoding { // KLLSketch represents the type of a metric that encodes measurements using the KLL quantile sketch. message KLLSketch { - // The time series data points. repeated KLLSketchDataPoint data_points = 1; - - // aggregation_temporality describes if the aggregator reports delta changes - // since last report time, or cumulative changes since a fixed start time. AggregationTemporality aggregation_temporality = 2; + + // Refactor-2026-05: KLL sketch parameter `k` — controls quantile + // error / space tradeoff. Sent once per Metric emit. + uint32 k = 3; } // KLLSketchDataPoint is a single data point that encodes a distribution using the KLL sketch format. +// +// Refactor-2026-05: removed count, sum, min, max — derivable from sketch state. message KLLSketchDataPoint { + reserved 4, 5, 6, 7; // count, sum, min, max repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; fixed64 start_time_unix_nano = 2; fixed64 time_unix_nano = 3; - fixed64 count = 4; - double sum = 5; - double min = 6; - double max = 7; bytes sketch = 8; KLLSketchEncoding encoding = 9; uint32 flags = 10; @@ -432,24 +417,28 @@ enum KLLSketchEncoding { // CountSketch represents the type of a metric that encodes frequency estimations using CountSketch. message CountSketch { - // The time series data points. repeated CountSketchDataPoint data_points = 1; - - // aggregation_temporality describes if the aggregator reports delta changes - // since last report time, or cumulative changes since a fixed start time. AggregationTemporality aggregation_temporality = 2; + + // Refactor-2026-05: CountSketch matrix dimensions. depth × width. + // (epsilon, delta) deriveable: eps = sqrt(e/cols) for L2 estimation; + // delta ≈ 1/(2^(rows/2)). Backend computes these for display. + int32 rows = 3; // depth + int32 cols = 4; // width } // CountSketchDataPoint is a single data point encoding frequency estimations. +// +// Refactor-2026-05: removed `dimension` (legacy partition-key +// descriptor — replaced by `attributes` keys = group-by labels); +// removed `epsilon`/`delta` (derivable from container's rows/cols). message CountSketchDataPoint { + reserved 6, 7, 8; // dimension, epsilon, delta repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; fixed64 start_time_unix_nano = 2; fixed64 time_unix_nano = 3; bytes sketch = 4; CountSketchEncoding encoding = 5; - string dimension = 6; - double epsilon = 7; - double delta = 8; uint32 flags = 9; uint64 series_id = 10; } @@ -470,24 +459,27 @@ enum CountSketchEncoding { // CountMinSketch represents the type of a metric encoding frequency estimations using CountMinSketch. message CountMinSketch { - // The time series data points. repeated CountMinSketchDataPoint data_points = 1; - - // aggregation_temporality describes if the aggregator reports delta changes - // since last report time, or cumulative changes since a fixed start time. AggregationTemporality aggregation_temporality = 2; + + // Refactor-2026-05: CMS matrix dimensions. depth × width. + // (epsilon, delta) deriveable: eps ≈ e/cols; delta ≈ exp(-rows). + int32 rows = 3; // depth = ⌈ln(1/delta)⌉ + int32 cols = 4; // width = ⌈e/epsilon⌉ } // CountMinSketchDataPoint is a single data point encoding frequency estimations. +// +// Refactor-2026-05: removed `sample_count` (derivable from sketch +// row sums); removed `rows`/`cols` (lifted to parent container, sent +// once per Metric emit instead of per DataPoint). message CountMinSketchDataPoint { + reserved 4, 7, 8; // sample_count, rows, cols repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; fixed64 start_time_unix_nano = 2; fixed64 time_unix_nano = 3; - uint64 sample_count = 4; bytes sketch = 5; CountMinSketchEncoding encoding = 6; - int32 rows = 7; - int32 cols = 8; uint32 flags = 9; uint64 series_id = 10; } @@ -508,24 +500,27 @@ enum CountMinSketchEncoding { // HLLSketch represents the type of a metric that encodes cardinality estimations using HyperLogLog. message HLLSketch { - // The time series data points. repeated HLLSketchDataPoint data_points = 1; - - // aggregation_temporality describes if the aggregator reports delta changes - // since last report time, or cumulative changes since a fixed start time. AggregationTemporality aggregation_temporality = 2; + + // Refactor-2026-05: HLL register-bucket precision (= log2(register count)). + // Standard error ≈ 1.04 / sqrt(2^precision). Sent once per Metric emit. + uint32 precision = 3; } // HLLSketchDataPoint is a single data point encoding cardinality estimations. +// +// Refactor-2026-05: removed `count` (derivable from sketch register +// state), removed `cardinality` (precomputed cache of +// sketch.estimate() — let the receiver evaluate on demand to avoid +// cache drift). `precision` lifted to parent container. message HLLSketchDataPoint { + reserved 4, 5, 8; // count, cardinality, precision repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; fixed64 start_time_unix_nano = 2; fixed64 time_unix_nano = 3; - uint64 count = 4; - uint64 cardinality = 5; bytes sketch = 6; HLLSketchEncoding encoding = 7; - uint32 precision = 8; uint32 flags = 9; uint64 series_id = 10; }